Merge pull request #1104 from bjorn3/build_system_refactor

Build system refactor

.github/workflows/main.yml

@@ -51,4 +51,13 @@ jobs:
         export COMPILE_RUNS=2
         export RUN_RUNS=2
 
-        ./test.sh --release
+        ./test.sh
+
+    - name: Package prebuilt cg_clif
+      run: tar cvfJ cg_clif.tar.xz build
+
+    - name: Upload prebuilt cg_clif
+      uses: actions/upload-artifact@v2
+      with:
+        name: cg_clif-${{ runner.os }}
+        path: cg_clif.tar.xz

.gitignore

@@ -6,7 +6,7 @@ perf.data
 perf.data.old
 *.events
 *.string*
-/build_sysroot/sysroot
+/build
 /build_sysroot/sysroot_src
 /rust
 /rand

Readme.md

@@ -2,7 +2,10 @@
 
 > ⚠⚠⚠ Certain kinds of FFI don't work yet. ⚠⚠⚠
 
-The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/master/cranelift). This has the potential to improve compilation times in debug mode. If your project doesn't use any of the things listed under "Not yet supported", it should work fine. If not please open an issue.
+The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/master/cranelift).
+This has the potential to improve compilation times in debug mode.
+If your project doesn't use any of the things listed under "Not yet supported", it should work fine.
+If not please open an issue.
 
 ## Building and testing
 
@@ -10,40 +13,45 @@ The goal of this project is to create an alternative codegen backend for the rus
 $ git clone https://github.com/bjorn3/rustc_codegen_cranelift.git
 $ cd rustc_codegen_cranelift
 $ ./prepare.sh # download and patch sysroot src and install hyperfine for benchmarking
-$ ./test.sh --release
+$ ./build.sh
 ```
 
-If you want to only build but not test you should replace the last command with:
+To run the test suite replace the last command with:
 
 ```bash
-$ cargo build --release
-$ ./build_sysroot/build_sysroot.sh
+$ ./test.sh
 ```
 
+This will implicitly build cg_clif too. Both `build.sh` and `test.sh` accept a `--debug` argument to
+build in debug mode.
+
+Alternatively you can download a pre built version from [GHA]. It is listed in the artifacts section
+of workflow runs. Unfortunately due to GHA restrictions you need to be logged in to access it.
+
+[GHA]: https://github.com/bjorn3/rustc_codegen_cranelift/actions?query=branch%3Amaster+event%3Apush+is%3Asuccess
+
 ## Usage
 
 rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects.
 
-Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `test.sh`).
+Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `build.sh` or `test.sh`).
 
 ### Cargo
 
 In the directory with your project (where you can do the usual `cargo build`), run:
 
 ```bash
-$ $cg_clif_dir/cargo.sh run
+$ $cg_clif_dir/build/cargo.sh run
 ```
 
 This should build and run your project with rustc_codegen_cranelift instead of the usual LLVM backend.
 
-If you compiled cg_clif in debug mode (aka you didn't pass `--release` to `./test.sh`) you should set `CHANNEL="debug"`.
-
 ### Rustc
 
 > You should prefer using the Cargo method.
 
 ```bash
-$ $cg_clif_dir/target/release/cg_clif my_crate.rs
+$ $cg_clif_dir/build/cg_clif my_crate.rs
 ```
 
 ### Jit mode
@@ -54,13 +62,13 @@ In jit mode cg_clif will immediately execute your code without creating an execu
 > The jit mode will probably need cargo integration to make this possible.
 
 ```bash
-$ $cg_clif_dir/cargo.sh jit
+$ $cg_clif_dir/build/cargo.sh jit
 ```
 
 or
 
 ```bash
-$ $cg_clif_dir/target/release/cg_clif --jit my_crate.rs
+$ $cg_clif_dir/build/cg_clif --jit my_crate.rs
 ```
 
 ### Shell
@@ -69,7 +77,7 @@ These are a few functions that allow you to easily run rust code from the shell
 
 ```bash
 function jit_naked() {
-    echo "$@" | $cg_clif_dir/target/release/cg_clif - --jit
+    echo "$@" | $cg_clif_dir/build/cg_clif - --jit
 }
 
 function jit() {

build.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+set -e
+
+# Settings
+export CHANNEL="release"
+build_sysroot=1
+target_dir='build'
+while [[ $# != 0 ]]; do
+    case $1 in
+        "--debug")
+            export CHANNEL="debug"
+            ;;
+        "--without-sysroot")
+            build_sysroot=0
+            ;;
+        "--target-dir")
+            target_dir=$2
+            shift
+            ;;
+        *)
+            echo "Unknown flag '$1'"
+            echo "Usage: ./build.sh [--debug] [--without-sysroot] [--target-dir DIR]"
+            ;;
+    esac
+    shift
+done
+
+# Build cg_clif
+export RUSTFLAGS="-Zrun_dsymutil=no"
+if [[ "$CHANNEL" == "release" ]]; then
+    cargo build --release
+else
+    cargo build
+fi
+
+rm -rf $target_dir
+mkdir $target_dir
+cp -a target/$CHANNEL/cg_clif{,_build_sysroot} target/$CHANNEL/*rustc_codegen_cranelift* $target_dir/
+cp -a rust-toolchain scripts/config.sh scripts/cargo.sh $target_dir
+
+if [[ "$build_sysroot" == "1" ]]; then
+    echo "[BUILD] sysroot"
+    export CG_CLIF_INCR_CACHE_DISABLED=1
+    dir=$(pwd)
+    cd $target_dir
+    time $dir/build_sysroot/build_sysroot.sh
+fi

build_sysroot/build_sysroot.sh

@@ -3,29 +3,25 @@
 # Requires the CHANNEL env var to be set to `debug` or `release.`
 
 set -e
-cd $(dirname "$0")
 
-if [ -z $CHANNEL ]; then
-export CHANNEL='release'
-fi
+source ./config.sh
 
-pushd ../ >/dev/null
-source ./scripts/config.sh
-popd >/dev/null
-
-# We expect the target dir in the default location. Guard against the user changing it.
-export CARGO_TARGET_DIR=target
-
-# Cleanup for previous run
-#     v Clean target dir except for build scripts and incremental cache
-rm -r target/*/{debug,release}/{build,deps,examples,libsysroot*,native} 2>/dev/null || true
-rm -r sysroot/ 2>/dev/null || true
+dir=$(pwd)
 
 # Use rustc with cg_clif as hotpluggable backend instead of the custom cg_clif driver so that
 # build scripts are still compiled using cg_llvm.
-export RUSTC=$(pwd)/../"target/"$CHANNEL"/cg_clif_build_sysroot"
+export RUSTC=$dir"/cg_clif_build_sysroot"
 export RUSTFLAGS=$RUSTFLAGS" --clif"
 
+cd $(dirname "$0")
+
+# Cleanup for previous run
+#     v Clean target dir except for build scripts and incremental cache
+#rm -r target/*/{debug,release}/{build,deps,examples,libsysroot*,native} 2>/dev/null || true
+
+# We expect the target dir in the default location. Guard against the user changing it.
+export CARGO_TARGET_DIR=target
+
 # Build libs
 export RUSTFLAGS="$RUSTFLAGS -Zforce-unstable-if-unmarked -Cpanic=abort"
 if [[ "$1" != "--debug" ]]; then
@@ -39,5 +35,5 @@ else
 fi
 
 # Copy files to sysroot
-mkdir -p sysroot/lib/rustlib/$TARGET_TRIPLE/lib/
-cp -r target/$TARGET_TRIPLE/$sysroot_channel/deps/* sysroot/lib/rustlib/$TARGET_TRIPLE/lib/
+mkdir -p $dir/sysroot/lib/rustlib/$TARGET_TRIPLE/lib/
+cp -a target/$TARGET_TRIPLE/$sysroot_channel/deps/* $dir/sysroot/lib/rustlib/$TARGET_TRIPLE/lib/

build_sysroot/prepare_sysroot_src.sh

@@ -12,7 +12,7 @@ fi
 
 rm -rf $DST_DIR
 mkdir -p $DST_DIR/library
-cp -r $SRC_DIR/library $DST_DIR/
+cp -a $SRC_DIR/library $DST_DIR/
 
 pushd $DST_DIR
 echo "[GIT] init"

clean_all.sh

@@ -1,5 +1,5 @@
 #!/bin/bash --verbose
 set -e
 
-rm -rf target/ build_sysroot/{sysroot/,sysroot_src/,target/} perf.data{,.old}
+rm -rf target/ build/ build_sysroot/{sysroot_src/,target/} perf.data{,.old}
 rm -rf rand/ regex/ simple-raytracer/

scripts/cargo.sh

@@ -1,19 +1,13 @@
 #!/bin/bash
 
-if [ -z $CHANNEL ]; then
-export CHANNEL='release'
-fi
-
-pushd $(dirname "$0") >/dev/null
-source scripts/config.sh
+dir=$(dirname "$0")
+source $dir/config.sh
 
 # read nightly compiler from rust-toolchain file
-TOOLCHAIN=$(cat rust-toolchain)
-
-popd >/dev/null
+TOOLCHAIN=$(cat $dir/rust-toolchain)
 
 cmd=$1
-shift
+shift || true
 
 if [[ "$cmd" = "jit" ]]; then
 cargo +${TOOLCHAIN} rustc $@ -- --jit

scripts/config.sh

@@ -39,18 +39,19 @@ echo
 export RUSTC_WRAPPER=
 fi
 
-export RUSTC=$(pwd)/"target/"$CHANNEL"/cg_clif"
+dir=$(cd $(dirname "$BASH_SOURCE"); pwd)
+
+export RUSTC=$dir"/cg_clif"
 export RUSTFLAGS=$linker
 export RUSTDOCFLAGS=$linker' -Ztrim-diagnostic-paths=no -Cpanic=abort -Zpanic-abort-tests '\
-'-Zcodegen-backend='$(pwd)'/target/'$CHANNEL'/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$(pwd)'/build_sysroot/sysroot'
+'-Zcodegen-backend='$dir'/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$dir'/sysroot'
 
 # FIXME remove once the atomic shim is gone
 if [[ `uname` == 'Darwin' ]]; then
    export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
 fi
 
-export LD_LIBRARY_PATH="$(pwd)/target/out:$(pwd)/build_sysroot/sysroot/lib/rustlib/"$TARGET_TRIPLE"/lib:\
-$(pwd)/target/"$CHANNEL":$(rustc --print sysroot)/lib"
+export LD_LIBRARY_PATH="$dir:$(rustc --print sysroot)/lib:$dir/target/out:$dir/sysroot/lib/rustlib/"$TARGET_TRIPLE"/lib"
 export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH
 
 export CG_CLIF_DISPLAY_CG_TIME=1

scripts/filter_profile.rs

@@ -1,9 +1,8 @@
 #!/bin/bash
 #![forbid(unsafe_code)]/* This line is ignored by bash
 # This block is ignored by rustc
-CHANNEL="release"
 pushd $(dirname "$0")/../
-source scripts/config.sh
+source build/config.sh
 popd
 PROFILE=$1 OUTPUT=$2 exec $RUSTC $RUSTFLAGS --jit $0
 #*/

scripts/rustup.sh

@@ -26,6 +26,15 @@ case $1 in
         git add rust-toolchain build_sysroot/Cargo.lock
         git commit -m "Rustup to $(rustc -V)"
         ;;
+    "push")
+	cg_clif=$(pwd)
+	pushd ../rust
+	branch=update_cg_clif-$(date +%Y-%m-%d)
+	git checkout -b $branch
+	git subtree pull --prefix=compiler/rustc_codegen_cranelift/ https://github.com/bjorn3/rustc_codegen_cranelift.git master
+	git push -u my $branch
+	popd
+	;;
     *)
         echo "Unknown command '$1'"
         echo "Usage: ./rustup.sh prepare|commit"

scripts/tests.sh

@@ -1,65 +1,71 @@
-function no_sysroot_tests() {
-    RUSTC=$RUSTC" "$RUSTFLAGS" -L crate=target/out --out-dir target/out -Cdebuginfo=2"
+#!/bin/bash
 
+set -e
+
+source build/config.sh
+export CG_CLIF_INCR_CACHE_DISABLED=1
+MY_RUSTC=$RUSTC" "$RUSTFLAGS" -L crate=target/out --out-dir target/out -Cdebuginfo=2"
+
+function no_sysroot_tests() {
     echo "[BUILD] mini_core"
-    $RUSTC example/mini_core.rs --crate-name mini_core --crate-type lib,dylib --target $TARGET_TRIPLE
+    $MY_RUSTC example/mini_core.rs --crate-name mini_core --crate-type lib,dylib --target $TARGET_TRIPLE
 
     echo "[BUILD] example"
-    $RUSTC example/example.rs --crate-type lib --target $TARGET_TRIPLE
+    $MY_RUSTC example/example.rs --crate-type lib --target $TARGET_TRIPLE
 
     if [[ "$JIT_SUPPORTED" = "1" ]]; then
         echo "[JIT] mini_core_hello_world"
-        CG_CLIF_JIT_ARGS="abc bcd" $RUSTC --jit example/mini_core_hello_world.rs --cfg jit --target $HOST_TRIPLE
+        CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC --jit example/mini_core_hello_world.rs --cfg jit --target $HOST_TRIPLE
     else
         echo "[JIT] mini_core_hello_world (skipped)"
     fi
 
     echo "[AOT] mini_core_hello_world"
-    $RUSTC example/mini_core_hello_world.rs --crate-name mini_core_hello_world --crate-type bin -g --target $TARGET_TRIPLE
+    $MY_RUSTC example/mini_core_hello_world.rs --crate-name mini_core_hello_world --crate-type bin -g --target $TARGET_TRIPLE
     $RUN_WRAPPER ./target/out/mini_core_hello_world abc bcd
     # (echo "break set -n main"; echo "run"; sleep 1; echo "si -c 10"; sleep 1; echo "frame variable") | lldb -- ./target/out/mini_core_hello_world abc bcd
 
     echo "[AOT] arbitrary_self_types_pointers_and_wrappers"
-    $RUSTC example/arbitrary_self_types_pointers_and_wrappers.rs --crate-name arbitrary_self_types_pointers_and_wrappers --crate-type bin --target $TARGET_TRIPLE
+    $MY_RUSTC example/arbitrary_self_types_pointers_and_wrappers.rs --crate-name arbitrary_self_types_pointers_and_wrappers --crate-type bin --target $TARGET_TRIPLE
     $RUN_WRAPPER ./target/out/arbitrary_self_types_pointers_and_wrappers
 }
 
 function base_sysroot_tests() {
     echo "[AOT] alloc_example"
-    $RUSTC example/alloc_example.rs --crate-type bin --target $TARGET_TRIPLE
+    $MY_RUSTC example/alloc_example.rs --crate-type bin --target $TARGET_TRIPLE
     $RUN_WRAPPER ./target/out/alloc_example
 
     if [[ "$JIT_SUPPORTED" = "1" ]]; then
         echo "[JIT] std_example"
-        $RUSTC --jit example/std_example.rs --target $HOST_TRIPLE
+        $MY_RUSTC --jit example/std_example.rs --target $HOST_TRIPLE
     else
         echo "[JIT] std_example (skipped)"
     fi
 
     echo "[AOT] dst_field_align"
     # FIXME Re-add -Zmir-opt-level=2 once rust-lang/rust#67529 is fixed.
-    $RUSTC example/dst-field-align.rs --crate-name dst_field_align --crate-type bin --target $TARGET_TRIPLE
+    $MY_RUSTC example/dst-field-align.rs --crate-name dst_field_align --crate-type bin --target $TARGET_TRIPLE
     $RUN_WRAPPER ./target/out/dst_field_align || (echo $?; false)
 
     echo "[AOT] std_example"
-    $RUSTC example/std_example.rs --crate-type bin --target $TARGET_TRIPLE
+    $MY_RUSTC example/std_example.rs --crate-type bin --target $TARGET_TRIPLE
     $RUN_WRAPPER ./target/out/std_example arg
 
     echo "[AOT] subslice-patterns-const-eval"
-    $RUSTC example/subslice-patterns-const-eval.rs --crate-type bin -Cpanic=abort --target $TARGET_TRIPLE
+    $MY_RUSTC example/subslice-patterns-const-eval.rs --crate-type bin -Cpanic=abort --target $TARGET_TRIPLE
     $RUN_WRAPPER ./target/out/subslice-patterns-const-eval
 
     echo "[AOT] track-caller-attribute"
-    $RUSTC example/track-caller-attribute.rs --crate-type bin -Cpanic=abort --target $TARGET_TRIPLE
+    $MY_RUSTC example/track-caller-attribute.rs --crate-type bin -Cpanic=abort --target $TARGET_TRIPLE
     $RUN_WRAPPER ./target/out/track-caller-attribute
 
     echo "[AOT] mod_bench"
-    $RUSTC example/mod_bench.rs --crate-type bin --target $TARGET_TRIPLE
+    $MY_RUSTC example/mod_bench.rs --crate-type bin --target $TARGET_TRIPLE
     $RUN_WRAPPER ./target/out/mod_bench
 
     pushd rand
     rm -r ./target || true
-    ../cargo.sh test --workspace
+    ../build/cargo.sh test --workspace
     popd
 }
 
@@ -69,7 +75,7 @@ function extended_sysroot_tests() {
         echo "[BENCH COMPILE] ebobby/simple-raytracer"
         hyperfine --runs ${RUN_RUNS:-10} --warmup 1 --prepare "cargo clean" \
         "RUSTC=rustc RUSTFLAGS='' cargo build" \
-        "../cargo.sh build"
+        "../build/cargo.sh build"
 
         echo "[BENCH RUN] ebobby/simple-raytracer"
         cp ./target/debug/main ./raytracer_cg_clif
@@ -85,18 +91,33 @@ function extended_sysroot_tests() {
     pushd build_sysroot/sysroot_src/library/core/tests
     echo "[TEST] libcore"
     rm -r ./target || true
-    ../../../../../cargo.sh test
+    ../../../../../build/cargo.sh test
     popd
 
     pushd regex
     echo "[TEST] rust-lang/regex example shootout-regex-dna"
-    ../cargo.sh clean
+    ../build/cargo.sh clean
     # Make sure `[codegen mono items] start` doesn't poison the diff
-    ../cargo.sh build --example shootout-regex-dna
-    cat examples/regexdna-input.txt | ../cargo.sh run --example shootout-regex-dna | grep -v "Spawned thread" > res.txt
+    ../build/cargo.sh build --example shootout-regex-dna
+    cat examples/regexdna-input.txt | ../build/cargo.sh run --example shootout-regex-dna | grep -v "Spawned thread" > res.txt
     diff -u res.txt examples/regexdna-output.txt
 
     echo "[TEST] rust-lang/regex tests"
-    ../cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options
+    ../build/cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q
     popd
 }
+
+case "$1" in
+    "no_sysroot")
+        no_sysroot_tests
+        ;;
+    "base_sysroot")
+        base_sysroot_tests
+        ;;
+    "extended_sysroot")
+        extended_sysroot_tests
+        ;;
+    *)
+        echo "unknown test suite"
+        ;;
+esac

src/bin/cg_clif.rs

@@ -31,11 +31,6 @@ impl rustc_driver::Callbacks for CraneliftPassesCallbacks {
                 .unwrap()
                 .parent()
                 .unwrap()
-                .parent()
-                .unwrap()
-                .parent()
-                .unwrap()
-                .join("build_sysroot")
                 .join("sysroot"),
         );
     }

test.sh

@@ -1,29 +1,15 @@
 #!/bin/bash
 set -e
 
-# Build cg_clif
 export RUSTFLAGS="-Zrun_dsymutil=no"
-if [[ "$1" == "--release" ]]; then
-    export CHANNEL='release'
-    cargo build --release
-else
-    export CHANNEL='debug'
-    cargo build --bin cg_clif
-fi
 
-# Config
-source scripts/config.sh
-source scripts/tests.sh
-export CG_CLIF_INCR_CACHE_DISABLED=1
+./build.sh --without-sysroot $@
 
-# Cleanup
 rm -r target/out || true
 
-no_sysroot_tests
+scripts/tests.sh no_sysroot
 
-echo "[BUILD] sysroot"
-time ./build_sysroot/build_sysroot.sh
+./build.sh $@
 
-base_sysroot_tests
-
-extended_sysroot_tests
+scripts/tests.sh base_sysroot
+scripts/tests.sh extended_sysroot