From 3560924191a244513d0e1f777e408579f3a914b8 Mon Sep 17 00:00:00 2001 From: Catherine Date: Sat, 18 May 2024 18:32:58 +0000 Subject: [PATCH] Initial commit. --- .github/FUNDING.yml | 1 + .github/workflows/package.yml | 46 +++++++++ .gitignore | 6 ++ .gitmodules | 9 ++ CONTRIBUTING.md | 9 ++ Get-LLVM-Version.cmake | 2 + LICENSE.txt | 15 +++ build.sh | 180 ++++++++++++++++++++++++++++++++++ driverdriver.py | 71 ++++++++++++++ llvm-src | 1 + wasi-libc-src | 1 + 11 files changed, 341 insertions(+) create mode 100644 .github/FUNDING.yml create mode 100644 .github/workflows/package.yml create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 CONTRIBUTING.md create mode 100644 Get-LLVM-Version.cmake create mode 100644 LICENSE.txt create mode 100755 build.sh create mode 100755 driverdriver.py create mode 160000 llvm-src create mode 160000 wasi-libc-src diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..5e2b3f8 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +patreon: whitequark diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml new file mode 100644 index 0000000..9f02e9d --- /dev/null +++ b/.github/workflows/package.yml @@ -0,0 +1,46 @@ +on: [push, pull_request] +name: Build & publish +jobs: + build: + if: ${{ !contains(github.event.head_commit.message, 'skip ci') }} + runs-on: ubuntu-latest + env: + RELEASE_BRANCH: ${{ startsWith(github.event.ref, 'refs/heads/develop-') || startsWith(github.event.ref, 'refs/heads/release-') }} + steps: + - name: Check out source code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: true # not recursive! + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip build + sudo apt update + sudo apt-get install flex bison ccache + - name: Set up caching + uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: llvm-${{ hashFiles('llvm-src', 'build.sh') }} + restore-keys: | + llvm-${{ hashFiles('llvm-src', 'build.sh') }} + llvm- + - name: Set up ccache + run: | + ccache --max-size=10G -z + - name: Build WASM binaries + run: | + MAKEFLAGS=-j$(nproc) ./build.sh + - name: Upload WASM artifact + uses: actions/upload-artifact@v4 + with: + name: dist-prefix + path: wasi-prefix + # ... snip... + - name: Print ccache statistics + run: | + ccache -s diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ada5f3e --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +/wasi-sdk-*.* +*.wasm + +/*-build +/*-prefix +/Toolchain-WASI*.cmake diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..ce16965 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,9 @@ +[submodule "llvm-src"] + path = llvm-src + url = https://github.com/YoWASP/llvm-project +[submodule "wasi-sdk-src"] + path = wasi-sdk-src + url = https://github.com/WebAssembly/wasi-sdk +[submodule "wasi-libc-src"] + path = wasi-libc-src + url = https://github.com/WebAssembly/wasi-libc diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..08257e2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,9 @@ +YoWASP Clang packages +===================== + +Building +-------- + +The primary build environment for this repository is the `ubuntu-latest` GitHub CI runner; packages are built on every push and automatically published from the `release` branch to PyPI. + +To reduce maintenance overhead, the only development environment we will support for this repository is x86_64 Linux. diff --git a/Get-LLVM-Version.cmake b/Get-LLVM-Version.cmake new file mode 100644 index 0000000..1c9cf2b --- /dev/null +++ b/Get-LLVM-Version.cmake @@ -0,0 +1,2 @@ +include(llvm-src/cmake/Modules/LLVMVersion.cmake) +message(NOTICE ${LLVM_VERSION_MAJOR}) diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..428f07e --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,15 @@ +ISC License + +Copyright (C) Catherine + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..b305ac6 --- /dev/null +++ b/build.sh @@ -0,0 +1,180 @@ +#!/bin/sh -ex + +export SOURCE_DATE_EPOCH=$(git log -1 --format=%ct) + +WASI_SDK=wasi-sdk-22.0 +WASI_SDK_URL=https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-22/wasi-sdk-22.0-linux.tar.gz +if ! [ -d ${WASI_SDK} ]; then curl -L ${WASI_SDK_URL} | tar xzf -; fi +WASI_SDK_PATH=$(pwd)/${WASI_SDK} + +WASI_SYSROOT="--sysroot ${WASI_SDK_PATH}/share/wasi-sysroot" +WASI_TARGET="wasm32-wasip1" +WASI_CFLAGS="-flto" +WASI_LDFLAGS="-flto -Wl,--strip-all" +WASI_TARGET_LLVM="${WASI_TARGET}-threads" +WASI_CFLAGS_LLVM="${WASI_CFLAGS} -D_WASI_EMULATED_MMAN -pthread" +WASI_LDFLAGS_LLVM="${WASI_LDFLAGS} -lwasi-emulated-mman -Wl,--max-memory=4294967296" +# Compiling C++ code requires a lot of stack space and can overflow and corrupt the heap. +# (For example, `#include ` alone does it in a build with the default stack size.) +WASI_LDFLAGS_LLVM="${WASI_LDFLAGS_LLVM} -Wl,-z,stack-size=1048576 -Wl,--stack-first" + +# We need two toolchain files: one for the compiler itself (which needs threads at the moment since +# -DLLVM_ENABLE_THREADS=OFF is kind of broken), and one for the runtime libs. +cat >Toolchain-WASI.cmake <>Toolchain-WASI.cmake <>Toolchain-WASI-LLVM.cmake <&1) + +if ! [ -f llvm-tblgen-build/bin/llvm-tblgen -a -f llvm-tblgen-build/bin/clang-tblgen ]; then + mkdir -p llvm-tblgen-build + cmake -B llvm-tblgen-build -S llvm-src/llvm \ + -DLLVM_CCACHE_BUILD=ON \ + -DCMAKE_BUILD_TYPE=MinSizeRel \ + -DLLVM_BUILD_RUNTIME=OFF \ + -DLLVM_BUILD_TOOLS=OFF \ + -DLLVM_INCLUDE_UTILS=OFF \ + -DLLVM_INCLUDE_RUNTIMES=OFF \ + -DLLVM_INCLUDE_EXAMPLES=OFF \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_INCLUDE_BENCHMARKS=OFF \ + -DLLVM_INCLUDE_DOCS=OFF \ + -DLLVM_TARGETS_TO_BUILD=WebAssembly \ + -DLLVM_DEFAULT_TARGET_TRIPLE=wasm32-wasi \ + -DLLVM_ENABLE_PROJECTS="clang" \ + -DCLANG_BUILD_EXAMPLES=OFF \ + -DCLANG_BUILD_TOOLS=OFF \ + -DCLANG_INCLUDE_TESTS=OFF + cmake --build llvm-tblgen-build --target llvm-tblgen --target clang-tblgen +fi + +mkdir -p llvm-build +cmake -B llvm-build -S llvm-src/llvm \ + -DCMAKE_TOOLCHAIN_FILE=../Toolchain-WASI-LLVM.cmake \ + -DLLVM_CCACHE_BUILD=ON \ + -DLLVM_NATIVE_TOOL_DIR=$(pwd)/llvm-tblgen-build/bin \ + -DCMAKE_BUILD_TYPE=MinSizeRel \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_BUILD_SHARED_LIBS=OFF \ + -DLLVM_ENABLE_PIC=OFF \ + -DLLVM_BUILD_STATIC=ON \ + -DLLVM_ENABLE_THREADS=OFF \ + -DLLVM_BUILD_RUNTIME=OFF \ + -DLLVM_BUILD_TOOLS=OFF \ + -DLLVM_INCLUDE_UTILS=OFF \ + -DLLVM_BUILD_UTILS=OFF \ + -DLLVM_INCLUDE_RUNTIMES=OFF \ + -DLLVM_INCLUDE_EXAMPLES=OFF \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_INCLUDE_BENCHMARKS=OFF \ + -DLLVM_INCLUDE_DOCS=OFF \ + -DLLVM_TARGETS_TO_BUILD=WebAssembly \ + -DLLVM_DEFAULT_TARGET_TRIPLE=wasm32-wasip1 \ + -DLLVM_ENABLE_PROJECTS="clang;lld" \ + -DCLANG_ENABLE_ARCMT=OFF \ + -DCLANG_ENABLE_STATIC_ANALYZER=OFF \ + -DCLANG_INCLUDE_TESTS=OFF \ + -DCLANG_BUILD_TOOLS=OFF \ + -DCLANG_BUILD_EXAMPLES=OFF \ + -DCLANG_INCLUDE_DOCS=OFF \ + -DCLANG_LINKS_TO_CREATE="clang;clang++" \ + -DLLD_BUILD_TOOLS=OFF \ + -DCMAKE_INSTALL_PREFIX=llvm-prefix +# The "all" target still contains far too much stuff, even given all the options above, so build +# only Clang/LLD, explicitly. For the same reason using the "install" target is infeasible. +# I spent a while trying and it leads nowhere. +cmake --build llvm-build --target clang --target lld +cmake --build llvm-build --target install-core-resource-headers + +mkdir -p wasi-prefix/bin +cp llvm-build/bin/clang wasi-prefix/bin/clang.wasm +cp llvm-build/bin/wasm-ld wasi-prefix/bin/wasm-ld.wasm +cp -r llvm-prefix/lib/clang/${LLVM_VERSION_MAJOR}/* wasi-prefix/ + +# Options below heavily based on wasi-sdk. +mkdir -p compiler-rt-build +cmake -B compiler-rt-build -S llvm-src/compiler-rt \ + -DCMAKE_TOOLCHAIN_FILE=../Toolchain-WASI.cmake \ + -DCOMPILER_RT_BAREMETAL_BUILD=ON \ + -DCOMPILER_RT_BUILD_XRAY=OFF \ + -DCOMPILER_RT_INCLUDE_TESTS=OFF \ + -DCOMPILER_RT_HAS_FPIC_FLAG=OFF \ + -DCOMPILER_RT_ENABLE_IOS=OFF \ + -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON \ + -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=ON \ + -DCMAKE_INSTALL_PREFIX=wasi-prefix +cmake --build compiler-rt-build --target install + +make -C wasi-libc-src \ + CC=${WASI_SDK_PATH}/bin/clang \ + AR=${WASI_SDK_PATH}/bin/ar \ + NM=${WASI_SDK_PATH}/bin/nm \ + TARGET_TRIPLE=${WASI_TARGET} \ + SYSROOT=$(pwd)/wasi-prefix + +# Options below heavily based on wasi-sdk. +mkdir -p libcxx-build +cmake -B libcxx-build -S llvm-src/runtimes \ + -DCMAKE_TOOLCHAIN_FILE=../Toolchain-WASI.cmake \ + -DLLVM_ENABLE_RUNTIMES:STRING="libcxx;libcxxabi" \ + -DLIBCXX_ENABLE_THREADS:BOOL=OFF \ + -DLIBCXX_BUILD_EXTERNAL_THREAD_LIBRARY:BOOL=OFF \ + -DLIBCXX_ENABLE_SHARED:BOOL=OFF \ + -DLIBCXX_ENABLE_EXCEPTIONS:BOOL=OFF \ + -DLIBCXX_ENABLE_FILESYSTEM:BOOL=ON \ + -DLIBCXX_ENABLE_EXPERIMENTAL_LIBRARY:BOOL=OFF \ + -DLIBCXX_ENABLE_ABI_LINKER_SCRIPT:BOOL=OFF \ + -DLIBCXX_CXX_ABI=libcxxabi \ + -DLIBCXX_CXX_ABI_INCLUDE_PATHS=$(pwd)/llvm-src/libcxxabi/include \ + -DLIBCXX_HAS_MUSL_LIBC:BOOL=ON \ + -DLIBCXX_ABI_VERSION=2 \ + -DLIBCXXABI_ENABLE_THREADS:BOOL=OFF \ + -DLIBCXXABI_BUILD_EXTERNAL_THREAD_LIBRARY:BOOL=OFF \ + -DLIBCXXABI_ENABLE_PIC:BOOL=OFF \ + -DLIBCXXABI_ENABLE_SHARED:BOOL=OFF \ + -DLIBCXXABI_ENABLE_EXCEPTIONS:BOOL=OFF \ + -DLIBCXXABI_USE_LLVM_UNWINDER:BOOL=OFF \ + -DLIBCXXABI_SILENT_TERMINATE:BOOL=ON \ + -DLIBCXX_LIBDIR_SUFFIX=/${WASI_TARGET} \ + -DLIBCXXABI_LIBDIR_SUFFIX=/${WASI_TARGET} \ + -DCMAKE_INSTALL_PREFIX=wasi-prefix +cmake --build libcxx-build --target install + +# Crimees. For testing only! +cp driverdriver.py wasi-prefix/bin/ +ln -sf driverdriver.py wasi-prefix/bin/clang +ln -sf driverdriver.py wasi-prefix/bin/clang++ diff --git a/driverdriver.py b/driverdriver.py new file mode 100755 index 0000000..3ac574d --- /dev/null +++ b/driverdriver.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +import os +import sys +import shlex +import subprocess + +BIN_DIR = os.path.dirname(os.path.abspath(__file__)) +ROOT_DIR = os.path.dirname(BIN_DIR) + +LAUNCHER = [ + "wasmtime", "run", + "-S", "threads", + "--dir", f"{ROOT_DIR}::/wasi", + "--dir", "/", + "--dir", ".", +] +BINARIES = { + "": [f"{BIN_DIR}/clang.wasm"], + "clang": [f"{BIN_DIR}/clang.wasm"], + "clang++": [f"{BIN_DIR}/clang.wasm", "--driver-mode=g++"], + "wasm-ld": [f"{BIN_DIR}/wasm-ld.wasm"], +} + + +def run_command(args): + arg0, *args = args + return subprocess.check_call([*LAUNCHER, *BINARIES[os.path.basename(arg0)], *args]) + + +def run_driver(args): + arg0, *args = args + output = subprocess.check_output([ + *LAUNCHER, + *BINARIES[os.path.basename(arg0)], + "--sysroot", "/wasi", + "-resource-dir", "/wasi", + "-###", + *args, + ], + stderr=subprocess.STDOUT, + text=True, + ) + # horrific in-band signaling code. please do not hold me to account for writing this + state = 0 + commands = [] + for line in output.splitlines(): + if state == 0: + if not line.startswith( + ("clang", "Target:", "Thread model:", "InstalledDir:", "Build config:")): + state = 1 + if state == 1: + if line == " (in-process)": + pass + elif line.startswith(" \""): + commands.append(shlex.split(line)) + else: + state = 2 + if state == 2: + pass + if state == 1: # valid `-###` output recognized + for command in commands: + exit_code = run_command(command) + if exit_code != 0: + sys.exit(exit_code) + else: # something else, perhaps an error? just display it + print(output) + + +if __name__ == "__main__": + sys.exit(run_driver(sys.argv)) diff --git a/llvm-src b/llvm-src new file mode 160000 index 0000000..826c21d --- /dev/null +++ b/llvm-src @@ -0,0 +1 @@ +Subproject commit 826c21d6070bbb629edc525340f10c2ad068f4b8 diff --git a/wasi-libc-src b/wasi-libc-src new file mode 160000 index 0000000..b9e15a8 --- /dev/null +++ b/wasi-libc-src @@ -0,0 +1 @@ +Subproject commit b9e15a8af930603183eb13af62e193de0f9f9ee3