Workaround presence of LLVM library in stage0/lib

This commit works around the newly-introduced LLVM shared library.

This is needed such that llvm-config run from
librustc_llvm's build script can correctly locate it's own LLVM, not the
one in stage0/lib. The LLVM build system uses the DT_RUNPATH/RUNPATH
header within the llvm-config binary, which we want to use, but because
Cargo always adds the host compiler's "libdir" (stage0/lib in our
case) to the dynamic linker's search path, we weren't properly finding
the freshly-built LLVM in llvm/lib. By restoring the environment
variable setting the search path to what bootstrap sees, the problem is
resolved and librustc_llvm correctly links and finds the appropriate
LLVM.

Several run-make-fulldeps tests are also updated with similar handling.

src/bootstrap/builder.rs

@@ -21,7 +21,7 @@ use crate::install;
 use crate::native;
 use crate::test;
 use crate::tool;
-use crate::util::{add_lib_path, exe, libdir};
+use crate::util::{self, add_lib_path, exe, libdir};
 use crate::{Build, DocTests, Mode, GitRepo};
 
 pub use crate::Compiler;
@@ -791,6 +791,13 @@ impl<'a> Builder<'a> {
             .env("CARGO_TARGET_DIR", out_dir)
             .arg(cmd);
 
+        // See comment in librustc_llvm/build.rs for why this is necessary, largely llvm-config
+        // needs to not accidentally link to libLLVM in stage0/lib.
+        cargo.env("REAL_LIBRARY_PATH_VAR", &util::dylib_path_var());
+        if let Some(e) = env::var_os(util::dylib_path_var()) {
+            cargo.env("REAL_LIBRARY_PATH", e);
+        }
+
         if cmd != "install" {
             cargo.arg("--target")
                  .arg(target);

src/bootstrap/compile.rs

@@ -712,6 +712,7 @@ pub fn build_codegen_backend(builder: &Builder,
             if builder.is_rust_llvm(target) && backend != "emscripten" {
                 cargo.env("LLVM_RUSTLLVM", "1");
             }
+
             cargo.env("LLVM_CONFIG", &llvm_config);
             if backend != "emscripten" {
                 let target_config = builder.config.target_config.get(&target);

src/bootstrap/util.rs

@@ -70,7 +70,11 @@ pub fn dylib_path_var() -> &'static str {
 /// Parses the `dylib_path_var()` environment variable, returning a list of
 /// paths that are members of this lookup path.
 pub fn dylib_path() -> Vec<PathBuf> {
-    env::split_paths(&env::var_os(dylib_path_var()).unwrap_or_default()).collect()
+    let var = match env::var_os(dylib_path_var()) {
+        Some(v) => v,
+        None => return vec![],
+    };
+    env::split_paths(&var).collect()
 }
 
 /// `push` all components to `buf`. On windows, append `.exe` to the last component.

src/build_helper/lib.rs

@@ -23,6 +23,25 @@ macro_rules! t {
     };
 }
 
+// Because Cargo adds the compiler's dylib path to our library search path, llvm-config may
+// break: the dylib path for the compiler, as of this writing, contains a copy of the LLVM
+// shared library, which means that when our freshly built llvm-config goes to load it's
+// associated LLVM, it actually loads the compiler's LLVM. In particular when building the first
+// compiler (i.e., in stage 0) that's a problem, as the compiler's LLVM is likely different from
+// the one we want to use. As such, we restore the environment to what bootstrap saw. This isn't
+// perfect -- we might actually want to see something from Cargo's added library paths -- but
+// for now it works.
+pub fn restore_library_path() {
+    println!("cargo:rerun-if-env-changed=REAL_LIBRARY_PATH_VAR");
+    println!("cargo:rerun-if-env-changed=REAL_LIBRARY_PATH");
+    let key = env::var_os("REAL_LIBRARY_PATH_VAR").expect("REAL_LIBRARY_PATH_VAR");
+    if let Some(env) = env::var_os("REAL_LIBRARY_PATH") {
+        env::set_var(&key, &env);
+    } else {
+        env::remove_var(&key);
+    }
+}
+
 pub fn run(cmd: &mut Command) {
     println!("running: {:?}", cmd);
     run_silent(cmd);

src/librustc_asan/build.rs

@@ -8,6 +8,8 @@ use cmake::Config;
 
 fn main() {
     if let Some(llvm_config) = env::var_os("LLVM_CONFIG") {
+        build_helper::restore_library_path();
+
         let (native, target) = match sanitizer_lib_boilerplate("asan") {
             Ok(native) => native,
             _ => return,

src/librustc_llvm/build.rs

@@ -24,6 +24,8 @@ fn main() {
         return;
     }
 
+    build_helper::restore_library_path();
+
     let target = env::var("TARGET").expect("TARGET was not set");
     let llvm_config = env::var_os("LLVM_CONFIG")
         .map(PathBuf::from)

src/librustc_lsan/build.rs

@@ -8,6 +8,8 @@ use cmake::Config;
 
 fn main() {
     if let Some(llvm_config) = env::var_os("LLVM_CONFIG") {
+        build_helper::restore_library_path();
+
         let (native, target) = match sanitizer_lib_boilerplate("lsan") {
             Ok(native) => native,
             _ => return,

src/librustc_msan/build.rs

@@ -8,6 +8,8 @@ use cmake::Config;
 
 fn main() {
     if let Some(llvm_config) = env::var_os("LLVM_CONFIG") {
+        build_helper::restore_library_path();
+
         let (native, target) = match sanitizer_lib_boilerplate("msan") {
             Ok(native) => native,
             _ => return,

src/librustc_tsan/build.rs

@@ -8,6 +8,8 @@ use cmake::Config;
 
 fn main() {
     if let Some(llvm_config) = env::var_os("LLVM_CONFIG") {
+        build_helper::restore_library_path();
+
         let (native, target) = match sanitizer_lib_boilerplate("tsan") {
             Ok(native) => native,
             _ => return,

src/test/run-make-fulldeps/cross-lang-lto-upstream-rlibs/Makefile

@@ -9,7 +9,7 @@ all: staticlib.rs upstream.rs
 
 	# Check No LTO
 	$(RUSTC) staticlib.rs -Z cross-lang-lto -Ccodegen-units=1 -L. -o $(TMPDIR)/staticlib.a
-	(cd $(TMPDIR); llvm-ar x ./staticlib.a)
+	(cd $(TMPDIR); $(LD_LIB_PATH_ENVVAR)=$(REAL_LD_LIBRARY_PATH) llvm-ar x ./staticlib.a)
 	# Make sure the upstream object file was included
 	ls $(TMPDIR)/upstream.*.rcgu.o
 
@@ -19,5 +19,5 @@ all: staticlib.rs upstream.rs
 	# Check ThinLTO
 	$(RUSTC) upstream.rs -Z cross-lang-lto -Ccodegen-units=1 -Clto=thin
 	$(RUSTC) staticlib.rs -Z cross-lang-lto -Ccodegen-units=1 -Clto=thin -L. -o $(TMPDIR)/staticlib.a
-	(cd $(TMPDIR); llvm-ar x ./staticlib.a)
+	(cd $(TMPDIR); $(LD_LIB_PATH_ENVVAR)=$(REAL_LD_LIBRARY_PATH) llvm-ar x ./staticlib.a)
 	ls $(TMPDIR)/upstream.*.rcgu.o

src/test/run-make-fulldeps/cross-lang-lto/Makefile

@@ -5,8 +5,9 @@
 # LLVM bitcode files (as used by linker LTO plugins) when compiling with
 # -Z cross-lang-lto.
 
-ASSERT_IS_BITCODE_OBJ=llvm-bcanalyzer # this only succeeds for bitcode files
-EXTRACT_OBJS=(cd $(TMPDIR); rm -f ./*.o; llvm-ar x $(1))
+# this only succeeds for bitcode files
+ASSERT_IS_BITCODE_OBJ=($(LD_LIB_PATH_ENVVAR)=$(REAL_LD_LIBRARY_PATH) llvm-bcanalyzer $(1))
+EXTRACT_OBJS=(cd $(TMPDIR); rm -f ./*.o; $(LD_LIB_PATH_ENVVAR)=$(REAL_LD_LIBRARY_PATH) llvm-ar x $(1))
 
 BUILD_LIB=$(RUSTC) lib.rs -Copt-level=2 -Z cross-lang-lto=on -Ccodegen-units=1
 BUILD_EXE=$(RUSTC) main.rs -Copt-level=2 -Z cross-lang-lto=on -Ccodegen-units=1 --emit=obj
@@ -16,31 +17,31 @@ all: staticlib staticlib-fat-lto staticlib-thin-lto rlib exe cdylib rdylib
 staticlib: lib.rs
 	$(BUILD_LIB) --crate-type=staticlib -o $(TMPDIR)/liblib.a
 	$(call EXTRACT_OBJS, liblib.a)
-	for file in $(TMPDIR)/liblib.*.rcgu.o; do $(ASSERT_IS_BITCODE_OBJ) $$file; done
+	for file in $(TMPDIR)/liblib.*.rcgu.o; do $(call ASSERT_IS_BITCODE_OBJ, $$file); done
 
 staticlib-fat-lto: lib.rs
 	$(BUILD_LIB) --crate-type=staticlib -o $(TMPDIR)/liblib-fat-lto.a -Clto=fat
 	$(call EXTRACT_OBJS, liblib-fat-lto.a)
-	for file in $(TMPDIR)/liblib-fat-lto.*.rcgu.o; do $(ASSERT_IS_BITCODE_OBJ) $$file; done
+	for file in $(TMPDIR)/liblib-fat-lto.*.rcgu.o; do $(call ASSERT_IS_BITCODE_OBJ, $$file); done
 
 staticlib-thin-lto: lib.rs
 	$(BUILD_LIB) --crate-type=staticlib -o $(TMPDIR)/liblib-thin-lto.a -Clto=thin
 	$(call EXTRACT_OBJS, liblib-thin-lto.a)
-	for file in $(TMPDIR)/liblib-thin-lto.*.rcgu.o; do $(ASSERT_IS_BITCODE_OBJ) $$file; done
+	for file in $(TMPDIR)/liblib-thin-lto.*.rcgu.o; do $(call ASSERT_IS_BITCODE_OBJ, $$file); done
 
 rlib: lib.rs
 	$(BUILD_LIB) --crate-type=rlib -o $(TMPDIR)/liblib.rlib
 	$(call EXTRACT_OBJS, liblib.rlib)
-	for file in $(TMPDIR)/liblib.*.rcgu.o; do $(ASSERT_IS_BITCODE_OBJ) $$file; done
+	for file in $(TMPDIR)/liblib.*.rcgu.o; do $(call ASSERT_IS_BITCODE_OBJ, $$file); done
 
 cdylib: lib.rs
 	$(BUILD_LIB) --crate-type=cdylib --emit=obj -o $(TMPDIR)/cdylib.o
-	$(ASSERT_IS_BITCODE_OBJ) $(TMPDIR)/cdylib.o
+	$(call ASSERT_IS_BITCODE_OBJ, $(TMPDIR)/cdylib.o)
 
 rdylib: lib.rs
 	$(BUILD_LIB) --crate-type=dylib --emit=obj -o $(TMPDIR)/rdylib.o
-	$(ASSERT_IS_BITCODE_OBJ) $(TMPDIR)/rdylib.o
+	$(call ASSERT_IS_BITCODE_OBJ, $(TMPDIR)/rdylib.o)
 
 exe: lib.rs
 	$(BUILD_EXE) -o $(TMPDIR)/exe.o
-	$(ASSERT_IS_BITCODE_OBJ) $(TMPDIR)/exe.o
+	$(call ASSERT_IS_BITCODE_OBJ, $(TMPDIR)/exe.o)