From 61dd278ade42a2d6d88f1c7b2bf2ee3ffab3b132 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Mon, 2 May 2022 23:38:25 -0500 Subject: [PATCH 1/5] Move `download` functions from `native` to builder.rs This has no logic changes, just a move. --- src/bootstrap/builder.rs | 211 +++++++++++++++++++++++++++++++++++- src/bootstrap/native.rs | 225 ++------------------------------------- src/bootstrap/util.rs | 8 ++ 3 files changed, 222 insertions(+), 222 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 761bd66b505..a0ce200883d 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -2,13 +2,14 @@ use std::any::{type_name, Any}; use std::cell::{Cell, RefCell}; use std::collections::BTreeSet; use std::env; -use std::ffi::OsStr; +use std::ffi::{OsStr, OsString}; use std::fmt::{Debug, Write}; -use std::fs; +use std::fs::{self, File}; use std::hash::Hash; +use std::io::{BufRead, BufReader, ErrorKind}; use std::ops::Deref; use std::path::{Component, Path, PathBuf}; -use std::process::Command; +use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; use crate::cache::{Cache, Interned, INTERNER}; @@ -29,7 +30,8 @@ use crate::{Build, CLang, DocTests, GitRepo, Mode}; pub use crate::Compiler; // FIXME: replace with std::lazy after it gets stabilized and reaches beta -use once_cell::sync::Lazy; +use once_cell::sync::{Lazy, OnceCell}; +use xz2::bufread::XzDecoder; pub struct Builder<'a> { pub build: &'a Build, @@ -758,6 +760,207 @@ impl<'a> Builder<'a> { StepDescription::run(v, self, paths); } + /// Modifies the interpreter section of 'fname' to fix the dynamic linker, + /// or the RPATH section, to fix the dynamic library search path + /// + /// This is only required on NixOS and uses the PatchELF utility to + /// change the interpreter/RPATH of ELF executables. + /// + /// Please see https://nixos.org/patchelf.html for more information + pub(crate) fn fix_bin_or_dylib(&self, fname: &Path) { + // FIXME: cache NixOS detection? + match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() { + Err(_) => return, + Ok(output) if !output.status.success() => return, + Ok(output) => { + let mut s = output.stdout; + if s.last() == Some(&b'\n') { + s.pop(); + } + if s != b"Linux" { + return; + } + } + } + + // If the user has asked binaries to be patched for Nix, then + // don't check for NixOS or `/lib`, just continue to the patching. + // FIXME: shouldn't this take precedence over the `uname` check above? + if !self.config.patch_binaries_for_nix { + // Use `/etc/os-release` instead of `/etc/NIXOS`. + // The latter one does not exist on NixOS when using tmpfs as root. + const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""]; + let os_release = match File::open("/etc/os-release") { + Err(e) if e.kind() == ErrorKind::NotFound => return, + Err(e) => panic!("failed to access /etc/os-release: {}", e), + Ok(f) => f, + }; + if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) { + return; + } + if Path::new("/lib").exists() { + return; + } + } + + // At this point we're pretty sure the user is running NixOS or using Nix + println!("info: you seem to be using Nix. Attempting to patch {}", fname.display()); + + // Only build `.nix-deps` once. + static NIX_DEPS_DIR: OnceCell = OnceCell::new(); + let mut nix_build_succeeded = true; + let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| { + // Run `nix-build` to "build" each dependency (which will likely reuse + // the existing `/nix/store` copy, or at most download a pre-built copy). + // + // Importantly, we create a gc-root called `.nix-deps` in the `build/` + // directory, but still reference the actual `/nix/store` path in the rpath + // as it makes it significantly more robust against changes to the location of + // the `.nix-deps` location. + // + // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). + // zlib: Needed as a system dependency of `libLLVM-*.so`. + // patchelf: Needed for patching ELF binaries (see doc comment above). + let nix_deps_dir = self.out.join(".nix-deps"); + const NIX_EXPR: &str = " + with (import {}); + symlinkJoin { + name = \"rust-stage0-dependencies\"; + paths = [ + zlib + patchelf + stdenv.cc.bintools + ]; + } + "; + nix_build_succeeded = self.try_run(Command::new("nix-build").args(&[ + Path::new("-E"), + Path::new(NIX_EXPR), + Path::new("-o"), + &nix_deps_dir, + ])); + nix_deps_dir + }); + if !nix_build_succeeded { + return; + } + + let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf")); + let rpath_entries = { + // ORIGIN is a relative default, all binary and dynamic libraries we ship + // appear to have this (even when `../lib` is redundant). + // NOTE: there are only two paths here, delimited by a `:` + let mut entries = OsString::from("$ORIGIN/../lib:"); + entries.push(t!(fs::canonicalize(nix_deps_dir))); + entries.push("/lib"); + entries + }; + patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]); + if !fname.extension().map_or(false, |ext| ext == "so") { + // Finally, set the corret .interp for binaries + let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker"); + // FIXME: can we support utf8 here? `args` doesn't accept Vec, only OsString ... + let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path)))); + patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]); + } + + self.try_run(patchelf.arg(fname)); + } + + pub(crate) fn download_component(&self, base: &str, url: &str, dest_path: &Path) { + // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. + let tempfile = self.tempdir().join(dest_path.file_name().unwrap()); + // FIXME: support `do_verify` (only really needed for nightly rustfmt) + // FIXME: support non-utf8 paths? + self.download_with_retries(tempfile.to_str().unwrap(), &format!("{}/{}", base, url)); + t!(std::fs::rename(&tempfile, dest_path)); + } + + fn download_with_retries(&self, tempfile: &str, url: &str) { + println!("downloading {}", url); + // Try curl. If that fails and we are on windows, fallback to PowerShell. + if !self.check_run(Command::new("curl").args(&[ + "-#", + "-y", + "30", + "-Y", + "10", // timeout if speed is < 10 bytes/sec for > 30 seconds + "--connect-timeout", + "30", // timeout if cannot connect within 30 seconds + "--retry", + "3", + "-Sf", + "-o", + tempfile, + url, + ])) { + if self.build.build.contains("windows-msvc") { + println!("Fallback to PowerShell"); + for _ in 0..3 { + if self.try_run(Command::new("PowerShell.exe").args(&[ + "/nologo", + "-Command", + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", + &format!( + "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", + url, tempfile + ), + ])) { + return; + } + println!("\nspurious failure, trying again"); + } + } + std::process::exit(1); + } + } + + pub(crate) fn unpack(&self, tarball: &Path, dst: &Path) { + println!("extracting {} to {}", tarball.display(), dst.display()); + if !dst.exists() { + t!(fs::create_dir_all(dst)); + } + + // FIXME: will need to be a parameter once `download-rustc` is moved to rustbuild + const MATCH: &str = "rust-dev"; + + // `tarball` ends with `.tar.xz`; strip that suffix + // example: `rust-dev-nightly-x86_64-unknown-linux-gnu` + let uncompressed_filename = + Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap(); + let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap()); + + // decompress the file + let data = t!(File::open(tarball)); + let decompressor = XzDecoder::new(BufReader::new(data)); + + let mut tar = tar::Archive::new(decompressor); + for member in t!(tar.entries()) { + let mut member = t!(member); + let original_path = t!(member.path()).into_owned(); + // skip the top-level directory + if original_path == directory_prefix { + continue; + } + let mut short_path = t!(original_path.strip_prefix(directory_prefix)); + if !short_path.starts_with(MATCH) { + continue; + } + short_path = t!(short_path.strip_prefix(MATCH)); + let dst_path = dst.join(short_path); + self.verbose(&format!("extracting {} to {}", original_path.display(), dst.display())); + if !t!(member.unpack_in(dst)) { + panic!("path traversal attack ??"); + } + let src_path = dst.join(original_path); + if src_path.is_dir() && dst_path.exists() { + continue; + } + t!(fs::rename(src_path, dst_path)); + } + t!(fs::remove_dir_all(dst.join(directory_prefix))); + } + /// Obtain a compiler at a given stage and for a given host. Explicitly does /// not take `Compiler` since all `Compiler` instances are meant to be /// obtained through this function, since it ensures that they are valid diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 09b8cbe9014..5bb94f297c8 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -12,16 +12,13 @@ use std::env; use std::env::consts::EXE_EXTENSION; use std::ffi::{OsStr, OsString}; use std::fs::{self, File}; -use std::io::{self, BufRead, BufReader, ErrorKind}; +use std::io; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; - -use once_cell::sync::OnceCell; -use xz2::bufread::XzDecoder; +use std::process::Command; use crate::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::config::TargetSelection; -use crate::util::{self, exe, output, t, up_to_date}; +use crate::util::{self, exe, output, program_out_of_date, t, up_to_date}; use crate::{CLang, GitRepo}; pub struct Meta { @@ -151,13 +148,13 @@ pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) { if program_out_of_date(&llvm_stamp, &key) && !config.dry_run { download_ci_llvm(builder, &llvm_sha); for binary in ["llvm-config", "FileCheck"] { - fix_bin_or_dylib(builder, &llvm_root.join("bin").join(binary)); + builder.fix_bin_or_dylib(&llvm_root.join("bin").join(binary)); } let llvm_lib = llvm_root.join("lib"); for entry in t!(fs::read_dir(&llvm_lib)) { let lib = t!(entry).path(); if lib.extension().map_or(false, |ext| ext == "so") { - fix_bin_or_dylib(builder, &lib); + builder.fix_bin_or_dylib(&lib); } } t!(fs::write(llvm_stamp, key)); @@ -182,218 +179,10 @@ fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) { let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple); let tarball = rustc_cache.join(&filename); if !tarball.exists() { - download_component(builder, base, &format!("{}/{}", url, filename), &tarball); + builder.download_component(base, &format!("{}/{}", url, filename), &tarball); } let llvm_root = builder.config.ci_llvm_root(); - unpack(builder, &tarball, &llvm_root); -} - -/// Modifies the interpreter section of 'fname' to fix the dynamic linker, -/// or the RPATH section, to fix the dynamic library search path -/// -/// This is only required on NixOS and uses the PatchELF utility to -/// change the interpreter/RPATH of ELF executables. -/// -/// Please see https://nixos.org/patchelf.html for more information -fn fix_bin_or_dylib(builder: &Builder<'_>, fname: &Path) { - // FIXME: cache NixOS detection? - match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() { - Err(_) => return, - Ok(output) if !output.status.success() => return, - Ok(output) => { - let mut s = output.stdout; - if s.last() == Some(&b'\n') { - s.pop(); - } - if s != b"Linux" { - return; - } - } - } - - // If the user has asked binaries to be patched for Nix, then - // don't check for NixOS or `/lib`, just continue to the patching. - // FIXME: shouldn't this take precedence over the `uname` check above? - if !builder.config.patch_binaries_for_nix { - // Use `/etc/os-release` instead of `/etc/NIXOS`. - // The latter one does not exist on NixOS when using tmpfs as root. - const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""]; - let os_release = match File::open("/etc/os-release") { - Err(e) if e.kind() == ErrorKind::NotFound => return, - Err(e) => panic!("failed to access /etc/os-release: {}", e), - Ok(f) => f, - }; - if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) { - return; - } - if Path::new("/lib").exists() { - return; - } - } - - // At this point we're pretty sure the user is running NixOS or using Nix - println!("info: you seem to be using Nix. Attempting to patch {}", fname.display()); - - // Only build `.nix-deps` once. - static NIX_DEPS_DIR: OnceCell = OnceCell::new(); - let mut nix_build_succeeded = true; - let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| { - // Run `nix-build` to "build" each dependency (which will likely reuse - // the existing `/nix/store` copy, or at most download a pre-built copy). - // - // Importantly, we create a gc-root called `.nix-deps` in the `build/` - // directory, but still reference the actual `/nix/store` path in the rpath - // as it makes it significantly more robust against changes to the location of - // the `.nix-deps` location. - // - // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). - // zlib: Needed as a system dependency of `libLLVM-*.so`. - // patchelf: Needed for patching ELF binaries (see doc comment above). - let nix_deps_dir = builder.out.join(".nix-deps"); - const NIX_EXPR: &str = " - with (import {}); - symlinkJoin { - name = \"rust-stage0-dependencies\"; - paths = [ - zlib - patchelf - stdenv.cc.bintools - ]; - } - "; - nix_build_succeeded = builder.try_run(Command::new("nix-build").args(&[ - Path::new("-E"), - Path::new(NIX_EXPR), - Path::new("-o"), - &nix_deps_dir, - ])); - nix_deps_dir - }); - if !nix_build_succeeded { - return; - } - - let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf")); - let rpath_entries = { - // ORIGIN is a relative default, all binary and dynamic libraries we ship - // appear to have this (even when `../lib` is redundant). - // NOTE: there are only two paths here, delimited by a `:` - let mut entries = OsString::from("$ORIGIN/../lib:"); - entries.push(t!(fs::canonicalize(nix_deps_dir))); - entries.push("/lib"); - entries - }; - patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]); - if !fname.extension().map_or(false, |ext| ext == "so") { - // Finally, set the corret .interp for binaries - let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker"); - // FIXME: can we support utf8 here? `args` doesn't accept Vec, only OsString ... - let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path)))); - patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]); - } - - builder.try_run(patchelf.arg(fname)); -} - -fn download_component(builder: &Builder<'_>, base: &str, url: &str, dest_path: &Path) { - // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. - let tempfile = builder.tempdir().join(dest_path.file_name().unwrap()); - // FIXME: support `do_verify` (only really needed for nightly rustfmt) - // FIXME: support non-utf8 paths? - download_with_retries(builder, tempfile.to_str().unwrap(), &format!("{}/{}", base, url)); - t!(std::fs::rename(&tempfile, dest_path)); -} - -fn download_with_retries(builder: &Builder<'_>, tempfile: &str, url: &str) { - println!("downloading {}", url); - // Try curl. If that fails and we are on windows, fallback to PowerShell. - if !builder.check_run(Command::new("curl").args(&[ - "-#", - "-y", - "30", - "-Y", - "10", // timeout if speed is < 10 bytes/sec for > 30 seconds - "--connect-timeout", - "30", // timeout if cannot connect within 30 seconds - "--retry", - "3", - "-Sf", - "-o", - tempfile, - url, - ])) { - if builder.build.build.contains("windows-msvc") { - println!("Fallback to PowerShell"); - for _ in 0..3 { - if builder.try_run(Command::new("PowerShell.exe").args(&[ - "/nologo", - "-Command", - "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", - &format!( - "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", - url, tempfile - ), - ])) { - return; - } - println!("\nspurious failure, trying again"); - } - } - std::process::exit(1); - } -} - -fn unpack(builder: &Builder<'_>, tarball: &Path, dst: &Path) { - println!("extracting {} to {}", tarball.display(), dst.display()); - if !dst.exists() { - t!(fs::create_dir_all(dst)); - } - - // FIXME: will need to be a parameter once `download-rustc` is moved to rustbuild - const MATCH: &str = "rust-dev"; - - // `tarball` ends with `.tar.xz`; strip that suffix - // example: `rust-dev-nightly-x86_64-unknown-linux-gnu` - let uncompressed_filename = - Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap(); - let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap()); - - // decompress the file - let data = t!(File::open(tarball)); - let decompressor = XzDecoder::new(BufReader::new(data)); - - let mut tar = tar::Archive::new(decompressor); - for member in t!(tar.entries()) { - let mut member = t!(member); - let original_path = t!(member.path()).into_owned(); - // skip the top-level directory - if original_path == directory_prefix { - continue; - } - let mut short_path = t!(original_path.strip_prefix(directory_prefix)); - if !short_path.starts_with(MATCH) { - continue; - } - short_path = t!(short_path.strip_prefix(MATCH)); - let dst_path = dst.join(short_path); - builder.verbose(&format!("extracting {} to {}", original_path.display(), dst.display())); - if !t!(member.unpack_in(dst)) { - panic!("path traversal attack ??"); - } - let src_path = dst.join(original_path); - if src_path.is_dir() && dst_path.exists() { - continue; - } - t!(fs::rename(src_path, dst_path)); - } - t!(fs::remove_dir_all(dst.join(directory_prefix))); -} - -fn program_out_of_date(stamp: &Path, key: &str) -> bool { - if !stamp.exists() { - return true; - } - t!(fs::read_to_string(stamp)) != key + builder.unpack(&tarball, &llvm_root); } #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index 710b3588b84..b4691840618 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -115,6 +115,14 @@ impl Drop for TimeIt { } } +/// Used for download caching +pub(crate) fn program_out_of_date(stamp: &Path, key: &str) -> bool { + if !stamp.exists() { + return true; + } + t!(fs::read_to_string(stamp)) != key +} + /// Symlinks two directories, using junctions on Windows and normal symlinks on /// Unix. pub fn symlink_dir(config: &Config, src: &Path, dest: &Path) -> io::Result<()> { From 53bf24c82ba2102c84949ed2c2ed94fa972b4c2a Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Tue, 3 May 2022 13:29:37 -0500 Subject: [PATCH 2/5] Move download-rustc from bootstrap.py to rustbuild - Remove download-rustc handling from bootstrap.py - Allow a custom `pattern` in `builder.unpack()` - Only download rustc once another part of bootstrap depends on it. This is somewhat necessary since the download functions rely on having a full `Builder`, which isn't available until after config parsing finishes. --- src/bootstrap/bootstrap.py | 103 ++++++++++---------------- src/bootstrap/builder.rs | 15 ++-- src/bootstrap/compile.rs | 14 ++-- src/bootstrap/config.rs | 146 ++++++++++++++++++++++++++++++++++++- src/bootstrap/native.rs | 2 +- src/bootstrap/test.rs | 4 +- src/bootstrap/tool.rs | 2 +- 7 files changed, 200 insertions(+), 86 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 916c8e5d187..eee90ed2c1c 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -446,7 +446,7 @@ class RustBuild(object): self.nix_deps_dir = None self.rustc_commit = None - def download_toolchain(self, stage0=True, rustc_channel=None): + def download_toolchain(self, rustc_channel=None): """Fetch the build system for Rust, written in Rust This method will build a cache directory, then it will fetch the @@ -458,35 +458,26 @@ class RustBuild(object): """ if rustc_channel is None: rustc_channel = self.stage0_compiler.version - bin_root = self.bin_root(stage0) + bin_root = self.bin_root() key = self.stage0_compiler.date - if not stage0: - key += str(self.rustc_commit) - if self.rustc(stage0).startswith(bin_root) and \ - (not os.path.exists(self.rustc(stage0)) or - self.program_out_of_date(self.rustc_stamp(stage0), key)): + if self.rustc().startswith(bin_root) and \ + (not os.path.exists(self.rustc()) or + self.program_out_of_date(self.rustc_stamp(), key)): if os.path.exists(bin_root): shutil.rmtree(bin_root) tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' filename = "rust-std-{}-{}{}".format( rustc_channel, self.build, tarball_suffix) pattern = "rust-std-{}".format(self.build) - self._download_component_helper(filename, pattern, tarball_suffix, stage0) + self._download_component_helper(filename, pattern, tarball_suffix) filename = "rustc-{}-{}{}".format(rustc_channel, self.build, tarball_suffix) - self._download_component_helper(filename, "rustc", tarball_suffix, stage0) - # download-rustc doesn't need its own cargo, it can just use beta's. - if stage0: - filename = "cargo-{}-{}{}".format(rustc_channel, self.build, - tarball_suffix) - self._download_component_helper(filename, "cargo", tarball_suffix) - self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root)) - else: - filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix) - self._download_component_helper( - filename, "rustc-dev", tarball_suffix, stage0 - ) + self._download_component_helper(filename, "rustc", tarball_suffix) + filename = "cargo-{}-{}{}".format(rustc_channel, self.build, + tarball_suffix) + self._download_component_helper(filename, "cargo", tarball_suffix) + self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root)) self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root)) self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root)) @@ -494,7 +485,7 @@ class RustBuild(object): for lib in os.listdir(lib_dir): if lib.endswith(".so"): self.fix_bin_or_dylib(os.path.join(lib_dir, lib)) - with output(self.rustc_stamp(stage0)) as rust_stamp: + with output(self.rustc_stamp()) as rust_stamp: rust_stamp.write(key) if self.rustfmt() and self.rustfmt().startswith(bin_root) and ( @@ -518,24 +509,17 @@ class RustBuild(object): rustfmt_stamp.write(self.stage0_rustfmt.channel()) def _download_component_helper( - self, filename, pattern, tarball_suffix, stage0=True, key=None + self, filename, pattern, tarball_suffix, key=None ): if key is None: - if stage0: - key = self.stage0_compiler.date - else: - key = self.rustc_commit + key = self.stage0_compiler.date cache_dst = os.path.join(self.build_dir, "cache") rustc_cache = os.path.join(cache_dst, key) if not os.path.exists(rustc_cache): os.makedirs(rustc_cache) - if stage0: - base = self._download_url - url = "dist/{}".format(key) - else: - base = "https://ci-artifacts.rust-lang.org" - url = "rustc-builds/{}".format(self.rustc_commit) + base = self._download_url + url = "dist/{}".format(key) tarball = os.path.join(rustc_cache, filename) if not os.path.exists(tarball): get( @@ -544,9 +528,9 @@ class RustBuild(object): tarball, self.checksums_sha256, verbose=self.verbose, - do_verify=stage0, + do_verify=True, ) - unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose) + unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose) def fix_bin_or_dylib(self, fname): """Modifies the interpreter section of 'fname' to fix the dynamic linker, @@ -689,17 +673,15 @@ class RustBuild(object): # FIXME: support downloading artifacts from the beta channel self.download_toolchain(False, "nightly") - def rustc_stamp(self, stage0): + def rustc_stamp(self): """Return the path for .rustc-stamp at the given stage >>> rb = RustBuild() >>> rb.build_dir = "build" - >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp") - True - >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp") + >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp") True """ - return os.path.join(self.bin_root(stage0), '.rustc-stamp') + return os.path.join(self.bin_root(), '.rustc-stamp') def rustfmt_stamp(self): """Return the path for .rustfmt-stamp @@ -709,7 +691,7 @@ class RustBuild(object): >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp") True """ - return os.path.join(self.bin_root(True), '.rustfmt-stamp') + return os.path.join(self.bin_root(), '.rustfmt-stamp') def program_out_of_date(self, stamp_path, key): """Check if the given program stamp is out of date""" @@ -718,26 +700,21 @@ class RustBuild(object): with open(stamp_path, 'r') as stamp: return key != stamp.read() - def bin_root(self, stage0): + def bin_root(self): """Return the binary root directory for the given stage >>> rb = RustBuild() >>> rb.build_dir = "build" - >>> rb.bin_root(True) == os.path.join("build", "stage0") - True - >>> rb.bin_root(False) == os.path.join("build", "ci-rustc") + >>> rb.bin_root() == os.path.join("build", "stage0") True When the 'build' property is given should be a nested directory: >>> rb.build = "devel" - >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0") + >>> rb.bin_root() == os.path.join("build", "devel", "stage0") True """ - if stage0: - subdir = "stage0" - else: - subdir = "ci-rustc" + subdir = "stage0" return os.path.join(self.build_dir, self.build, subdir) def get_toml(self, key, section=None): @@ -785,9 +762,9 @@ class RustBuild(object): """Return config path for cargo""" return self.program_config('cargo') - def rustc(self, stage0): + def rustc(self): """Return config path for rustc""" - return self.program_config('rustc', stage0) + return self.program_config('rustc') def rustfmt(self): """Return config path for rustfmt""" @@ -795,7 +772,7 @@ class RustBuild(object): return None return self.program_config('rustfmt') - def program_config(self, program, stage0=True): + def program_config(self, program): """Return config path for the given program at the given stage >>> rb = RustBuild() @@ -803,19 +780,15 @@ class RustBuild(object): >>> rb.program_config('rustc') 'rustc' >>> rb.config_toml = '' - >>> cargo_path = rb.program_config('cargo', True) - >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True), - ... "bin", "cargo") - True - >>> cargo_path = rb.program_config('cargo', False) - >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False), + >>> cargo_path = rb.program_config('cargo') + >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(), ... "bin", "cargo") True """ config = self.get_toml(program) if config: return os.path.expanduser(config) - return os.path.join(self.bin_root(stage0), "bin", "{}{}".format( + return os.path.join(self.bin_root(), "bin", "{}{}".format( program, self.exe_suffix())) @staticmethod @@ -871,14 +844,14 @@ class RustBuild(object): if "CARGO_BUILD_TARGET" in env: del env["CARGO_BUILD_TARGET"] env["CARGO_TARGET_DIR"] = build_dir - env["RUSTC"] = self.rustc(True) - env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \ + env["RUSTC"] = self.rustc() + env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ (os.pathsep + env["LD_LIBRARY_PATH"]) \ if "LD_LIBRARY_PATH" in env else "" - env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \ + env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ (os.pathsep + env["DYLD_LIBRARY_PATH"]) \ if "DYLD_LIBRARY_PATH" in env else "" - env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \ + env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ (os.pathsep + env["LIBRARY_PATH"]) \ if "LIBRARY_PATH" in env else "" @@ -900,7 +873,7 @@ class RustBuild(object): if self.get_toml("deny-warnings", "rust") != "false": env["RUSTFLAGS"] += " -Dwarnings" - env["PATH"] = os.path.join(self.bin_root(True), "bin") + \ + env["PATH"] = os.path.join(self.bin_root(), "bin") + \ os.pathsep + env["PATH"] if not os.path.isfile(self.cargo()): raise Exception("no cargo executable found at `{}`".format( @@ -1172,7 +1145,7 @@ def bootstrap(help_triggered): # Fetch/build the bootstrap build.download_toolchain() # Download the master compiler if `download-rustc` is set - build.maybe_download_ci_toolchain() + # build.maybe_download_ci_toolchain() sys.stdout.flush() build.ensure_vendored() build.build_bootstrap() diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index a0ce200883d..94918cb318f 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -915,15 +915,12 @@ impl<'a> Builder<'a> { } } - pub(crate) fn unpack(&self, tarball: &Path, dst: &Path) { + pub(crate) fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) { println!("extracting {} to {}", tarball.display(), dst.display()); if !dst.exists() { t!(fs::create_dir_all(dst)); } - // FIXME: will need to be a parameter once `download-rustc` is moved to rustbuild - const MATCH: &str = "rust-dev"; - // `tarball` ends with `.tar.xz`; strip that suffix // example: `rust-dev-nightly-x86_64-unknown-linux-gnu` let uncompressed_filename = @@ -943,10 +940,10 @@ impl<'a> Builder<'a> { continue; } let mut short_path = t!(original_path.strip_prefix(directory_prefix)); - if !short_path.starts_with(MATCH) { + if !short_path.starts_with(pattern) { continue; } - short_path = t!(short_path.strip_prefix(MATCH)); + short_path = t!(short_path.strip_prefix(pattern)); let dst_path = dst.join(short_path); self.verbose(&format!("extracting {} to {}", original_path.display(), dst.display())); if !t!(member.unpack_in(dst)) { @@ -1022,7 +1019,7 @@ impl<'a> Builder<'a> { .join("lib"); // Avoid deleting the rustlib/ directory we just copied // (in `impl Step for Sysroot`). - if !builder.config.download_rustc { + if !builder.download_rustc() { let _ = fs::remove_dir_all(&sysroot); t!(fs::create_dir_all(&sysroot)); } @@ -1179,6 +1176,10 @@ impl<'a> Builder<'a> { Config::llvm_link_shared(self) } + pub(crate) fn download_rustc(&self) -> bool { + Config::download_rustc(self) + } + /// Prepares an invocation of `cargo` to be run. /// /// This will create a `Command` that represents a pending execution of diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 0b430f64e1e..f1960e39a41 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -42,8 +42,10 @@ impl Step for Std { fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { // When downloading stage1, the standard library has already been copied to the sysroot, so // there's no need to rebuild it. - let download_rustc = run.builder.config.download_rustc; - run.all_krates("test").path("library").default_condition(!download_rustc) + let builder = run.builder; + run.all_krates("test") + .path("library") + .lazy_default_condition(Box::new(|| !builder.download_rustc())) } fn make_run(run: RunConfig<'_>) { @@ -66,7 +68,7 @@ impl Step for Std { // Don't recompile them. // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler, // so its artifacts can't be reused. - if builder.config.download_rustc && compiler.stage != 0 { + if builder.download_rustc() && compiler.stage != 0 { return; } @@ -551,7 +553,7 @@ impl Step for Rustc { // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler, // so its artifacts can't be reused. - if builder.config.download_rustc && compiler.stage != 0 { + if builder.download_rustc() && compiler.stage != 0 { // Copy the existing artifacts instead of rebuilding them. // NOTE: this path is only taken for tools linking to rustc-dev. builder.ensure(Sysroot { compiler }); @@ -995,7 +997,7 @@ impl Step for Sysroot { t!(fs::create_dir_all(&sysroot)); // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. - if builder.config.download_rustc && compiler.stage != 0 { + if builder.download_rustc() && compiler.stage != 0 { assert_eq!( builder.config.build, compiler.host, "Cross-compiling is not yet supported with `download-rustc`", @@ -1090,7 +1092,7 @@ impl Step for Assemble { let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build); // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. - if builder.config.download_rustc { + if builder.download_rustc() { builder.ensure(Sysroot { compiler: target_compiler }); return target_compiler; } diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 843d276cd7a..f51a6481e43 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -7,9 +7,11 @@ use std::cell::Cell; use std::cmp; use std::collections::{HashMap, HashSet}; use std::env; +use std::ffi::OsStr; use std::fmt; use std::fs; use std::path::{Path, PathBuf}; +use std::process::{exit, Command}; use std::str::FromStr; use crate::builder::{Builder, TaskPath}; @@ -17,7 +19,8 @@ use crate::cache::{Interned, INTERNER}; use crate::channel::GitInfo; pub use crate::flags::Subcommand; use crate::flags::{Color, Flags}; -use crate::util::{exe, t}; +use crate::util::{exe, output, program_out_of_date, t}; +use once_cell::sync::OnceCell; use serde::{Deserialize, Deserializer}; macro_rules! check_ci_llvm { @@ -81,7 +84,11 @@ pub struct Config { pub cmd: Subcommand, pub incremental: bool, pub dry_run: bool, - pub download_rustc: bool, + /// `None` if we shouldn't download CI compiler artifacts, or the commit to download if we should. + #[cfg(not(test))] + download_rustc_commit: Option, + #[cfg(test)] + pub download_rustc_commit: Option, pub deny_warnings: bool, pub backtrace_on_ice: bool, @@ -1080,7 +1087,8 @@ impl Config { config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config); config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use); config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate); - config.download_rustc = env::var("BOOTSTRAP_DOWNLOAD_RUSTC").as_deref() == Ok("1"); + config.download_rustc_commit = + download_ci_rustc_commit(rust.download_rustc, config.verbose > 0); } else { config.rust_profile_use = flags.rust_profile_use; config.rust_profile_generate = flags.rust_profile_generate; @@ -1192,7 +1200,7 @@ impl Config { let default = config.channel == "dev"; config.ignore_git = ignore_git.unwrap_or(default); - let download_rustc = config.download_rustc; + let download_rustc = config.download_rustc_commit.is_some(); // See https://github.com/rust-lang/compiler-team/issues/326 config.stage = match config.cmd { Subcommand::Check { .. } => flags.stage.or(build.check_stage).unwrap_or(0), @@ -1309,6 +1317,23 @@ impl Config { llvm_link_shared } + /// Return whether we will use a downloaded, pre-compiled version of rustc, or just build from source. + pub(crate) fn download_rustc(builder: &Builder<'_>) -> bool { + static DOWNLOAD_RUSTC: OnceCell = OnceCell::new(); + if builder.config.dry_run && DOWNLOAD_RUSTC.get().is_none() { + // avoid trying to actually download the commit + return false; + } + + *DOWNLOAD_RUSTC.get_or_init(|| match &builder.config.download_rustc_commit { + None => false, + Some(commit) => { + download_ci_rustc(builder, commit); + true + } + }) + } + pub fn verbose(&self) -> bool { self.verbose > 0 } @@ -1358,3 +1383,116 @@ fn threads_from_config(v: u32) -> u32 { n => n, } } + +/// Returns the commit to download, or `None` if we shouldn't download CI artifacts. +fn download_ci_rustc_commit(download_rustc: Option, verbose: bool) -> Option { + // If `download-rustc` is not set, default to rebuilding. + let if_unchanged = match download_rustc { + None | Some(StringOrBool::Bool(false)) => return None, + Some(StringOrBool::Bool(true)) => false, + Some(StringOrBool::String(s)) if s == "if-unchanged" => true, + Some(StringOrBool::String(other)) => { + panic!("unrecognized option for download-rustc: {}", other) + } + }; + + // Handle running from a directory other than the top level + let top_level = output(Command::new("git").args(&["rev-parse", "--show-toplevel"])); + let top_level = top_level.trim_end(); + let compiler = format!("{top_level}/compiler/"); + let library = format!("{top_level}/library/"); + + // Look for a version to compare to based on the current commit. + // Only commits merged by bors will have CI artifacts. + let merge_base = output(Command::new("git").args(&[ + "rev-list", + "--author=bors@rust-lang.org", + "-n1", + "--first-parent", + "HEAD", + ])); + let commit = merge_base.trim_end(); + if commit.is_empty() { + println!("error: could not find commit hash for downloading rustc"); + println!("help: maybe your repository history is too shallow?"); + println!("help: consider disabling `download-rustc`"); + println!("help: or fetch enough history to include one upstream commit"); + exit(1); + } + + // Warn if there were changes to the compiler or standard library since the ancestor commit. + let has_changes = !t!(Command::new("git") + .args(&["diff-index", "--quiet", &commit, "--", &compiler, &library]) + .status()) + .success(); + if has_changes { + if if_unchanged { + if verbose { + println!( + "warning: saw changes to compiler/ or library/ since {commit}; \ + ignoring `download-rustc`" + ); + } + return None; + } + println!( + "warning: `download-rustc` is enabled, but there are changes to \ + compiler/ or library/" + ); + } + + Some(commit.to_string()) +} + +fn download_ci_rustc(builder: &Builder<'_>, commit: &str) { + builder.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})")); + // FIXME: support downloading artifacts from the beta channel + const CHANNEL: &str = "nightly"; + let host = builder.config.build.triple; + let bin_root = builder.out.join(host).join("ci-rustc"); + let rustc_stamp = bin_root.join(".rustc-stamp"); + + if !bin_root.join("bin").join("rustc").exists() || program_out_of_date(&rustc_stamp, commit) { + if bin_root.exists() { + t!(fs::remove_dir_all(&bin_root)); + } + let filename = format!("rust-std-{CHANNEL}-{host}.tar.xz"); + let pattern = format!("rust-std-{host}"); + download_component(builder, filename, &pattern, commit); + let filename = format!("rustc-{CHANNEL}-{host}.tar.xz"); + download_component(builder, filename, "rustc", commit); + // download-rustc doesn't need its own cargo, it can just use beta's. + let filename = format!("rustc-dev-{CHANNEL}-{host}.tar.xz"); + download_component(builder, filename, "rustc-dev", commit); + + builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustc")); + builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc")); + let lib_dir = bin_root.join("lib"); + for lib in t!(fs::read_dir(lib_dir)) { + let lib = t!(lib); + if lib.path().extension() == Some(OsStr::new("so")) { + builder.fix_bin_or_dylib(&lib.path()); + } + } + t!(fs::write(rustc_stamp, commit)); + } +} + +/// Download a single component of a CI-built toolchain (not necessarily a published nightly). +// NOTE: intentionally takes an owned string to avoid downloading multiple times by accident +fn download_component(builder: &Builder<'_>, filename: String, prefix: &str, commit: &str) { + let cache_dst = builder.out.join("cache"); + let rustc_cache = cache_dst.join(commit); + if !rustc_cache.exists() { + t!(fs::create_dir_all(&rustc_cache)); + } + + let base = "https://ci-artifacts.rust-lang.org"; + let url = format!("rustc-builds/{commit}"); + let tarball = rustc_cache.join(&filename); + if !tarball.exists() { + builder.download_component(base, &format!("{url}/{filename}"), &tarball); + } + let bin_root = builder.out.join(builder.config.build.triple).join("ci-rustc"); + builder.unpack(&tarball, &bin_root, prefix) +} diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 5bb94f297c8..d7211b2c2af 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -182,7 +182,7 @@ fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) { builder.download_component(base, &format!("{}/{}", url, filename), &tarball); } let llvm_root = builder.config.ci_llvm_root(); - builder.unpack(&tarball, &llvm_root); + builder.unpack(&tarball, &llvm_root, "rust-dev"); } #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index bc851568f89..99e9e8794e3 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -2058,7 +2058,7 @@ impl Step for CrateRustdoc { let test_kind = self.test_kind; let target = self.host; - let compiler = if builder.config.download_rustc { + let compiler = if builder.download_rustc() { builder.compiler(builder.top_stage, target) } else { // Use the previous stage compiler to reuse the artifacts that are @@ -2127,7 +2127,7 @@ impl Step for CrateRustdoc { // with. // // Note that this set the host libdir for `download_rustc`, which uses a normal rust distribution. - let libdir = if builder.config.download_rustc { + let libdir = if builder.download_rustc() { builder.rustc_libdir(compiler) } else { builder.sysroot_libdir(compiler, target).to_path_buf() diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index 3b30e6de12a..e1b78904646 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -516,7 +516,7 @@ impl Step for Rustdoc { builder.ensure(compile::Rustc { compiler: build_compiler, target: target_compiler.host }); // NOTE: this implies that `download-rustc` is pretty useless when compiling with the stage0 // compiler, since you do just as much work. - if !builder.config.dry_run && builder.config.download_rustc && build_compiler.stage == 0 { + if !builder.config.dry_run && builder.download_rustc() && build_compiler.stage == 0 { println!( "warning: `download-rustc` does nothing when building stage1 tools; consider using `--stage 2` instead" ); From 23147273befa1312588d5ecc45d0316153125f04 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Wed, 4 May 2022 10:37:35 -0500 Subject: [PATCH 3/5] Remove FIXME about nixOS detection --- src/bootstrap/builder.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 94918cb318f..45d649e78c8 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -785,7 +785,9 @@ impl<'a> Builder<'a> { // If the user has asked binaries to be patched for Nix, then // don't check for NixOS or `/lib`, just continue to the patching. - // FIXME: shouldn't this take precedence over the `uname` check above? + // NOTE: this intentionally comes after the Linux check: + // - patchelf only works with ELF files, so no need to run it on Mac or Windows + // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc. if !self.config.patch_binaries_for_nix { // Use `/etc/os-release` instead of `/etc/NIXOS`. // The latter one does not exist on NixOS when using tmpfs as root. From 512d7bf874632c8ee133c0e6db382669344d3e48 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 8 May 2022 23:03:38 -0500 Subject: [PATCH 4/5] Add support for UTF-8 paths to downloads in builder.rs This is for a pre-existing FIXME, but it was easy enough to do. --- src/bootstrap/builder.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 45d649e78c8..b90a1d9d2a5 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -873,15 +873,15 @@ impl<'a> Builder<'a> { // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. let tempfile = self.tempdir().join(dest_path.file_name().unwrap()); // FIXME: support `do_verify` (only really needed for nightly rustfmt) - // FIXME: support non-utf8 paths? - self.download_with_retries(tempfile.to_str().unwrap(), &format!("{}/{}", base, url)); + self.download_with_retries(&tempfile, &format!("{}/{}", base, url)); t!(std::fs::rename(&tempfile, dest_path)); } - fn download_with_retries(&self, tempfile: &str, url: &str) { + fn download_with_retries(&self, tempfile: &Path, url: &str) { println!("downloading {}", url); // Try curl. If that fails and we are on windows, fallback to PowerShell. - if !self.check_run(Command::new("curl").args(&[ + let mut curl = Command::new("curl"); + curl.args(&[ "-#", "-y", "30", @@ -893,9 +893,10 @@ impl<'a> Builder<'a> { "3", "-Sf", "-o", - tempfile, - url, - ])) { + ]); + curl.arg(tempfile); + curl.arg(url); + if !self.check_run(&mut curl) { if self.build.build.contains("windows-msvc") { println!("Fallback to PowerShell"); for _ in 0..3 { @@ -905,7 +906,7 @@ impl<'a> Builder<'a> { "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", &format!( "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", - url, tempfile + url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"), ), ])) { return; From 00bb4df1fbfdcc9249f8fd5e009b0e18d30c96bf Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sat, 21 May 2022 21:13:48 -0500 Subject: [PATCH 5/5] Remove download-rustc handling from bootstrap.py --- src/bootstrap/bootstrap.py | 55 ++------------------------------------ 1 file changed, 2 insertions(+), 53 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index eee90ed2c1c..955edd94c78 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -444,9 +444,8 @@ class RustBuild(object): self.verbose = False self.git_version = None self.nix_deps_dir = None - self.rustc_commit = None - def download_toolchain(self, rustc_channel=None): + def download_toolchain(self): """Fetch the build system for Rust, written in Rust This method will build a cache directory, then it will fetch the @@ -456,8 +455,7 @@ class RustBuild(object): Each downloaded tarball is extracted, after that, the script will move all the content to the right place. """ - if rustc_channel is None: - rustc_channel = self.stage0_compiler.version + rustc_channel = self.stage0_compiler.version bin_root = self.bin_root() key = self.stage0_compiler.date @@ -628,51 +626,6 @@ class RustBuild(object): print("warning: failed to call patchelf:", reason) return - # If `download-rustc` is set, download the most recent commit with CI artifacts - def maybe_download_ci_toolchain(self): - # If `download-rustc` is not set, default to rebuilding. - download_rustc = self.get_toml("download-rustc", section="rust") - if download_rustc is None or download_rustc == "false": - return None - assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc - - # Handle running from a directory other than the top level - rev_parse = ["git", "rev-parse", "--show-toplevel"] - top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip() - compiler = "{}/compiler/".format(top_level) - library = "{}/library/".format(top_level) - - # Look for a version to compare to based on the current commit. - # Only commits merged by bors will have CI artifacts. - merge_base = [ - "git", "rev-list", "--author=bors@rust-lang.org", "-n1", - "--first-parent", "HEAD" - ] - commit = subprocess.check_output(merge_base, universal_newlines=True).strip() - if not commit: - print("error: could not find commit hash for downloading rustc") - print("help: maybe your repository history is too shallow?") - print("help: consider disabling `download-rustc`") - print("help: or fetch enough history to include one upstream commit") - exit(1) - - # Warn if there were changes to the compiler or standard library since the ancestor commit. - status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library]) - if status != 0: - if download_rustc == "if-unchanged": - if self.verbose: - print("warning: saw changes to compiler/ or library/ since {}; " \ - "ignoring `download-rustc`".format(commit)) - return None - print("warning: `download-rustc` is enabled, but there are changes to " \ - "compiler/ or library/") - - if self.verbose: - print("using downloaded stage2 artifacts from CI (commit {})".format(commit)) - self.rustc_commit = commit - # FIXME: support downloading artifacts from the beta channel - self.download_toolchain(False, "nightly") - def rustc_stamp(self): """Return the path for .rustc-stamp at the given stage @@ -1144,8 +1097,6 @@ def bootstrap(help_triggered): # Fetch/build the bootstrap build.download_toolchain() - # Download the master compiler if `download-rustc` is set - # build.maybe_download_ci_toolchain() sys.stdout.flush() build.ensure_vendored() build.build_bootstrap() @@ -1157,8 +1108,6 @@ def bootstrap(help_triggered): env = os.environ.copy() env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) env["BOOTSTRAP_PYTHON"] = sys.executable - if build.rustc_commit is not None: - env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1' run(args, env=env, verbose=build.verbose, is_bootstrap=True)