Rollup merge of #97828 - ferrocene:pa-config-artifacts, r=jyn514

Allow configuring where artifacts are downloaded from

Bootstrap has support for downloading prebuilt LLVM and rustc artifacts to speed up local builds, but that currently works only for users working on `rust-lang/rust`. Forks of the repository (for example Ferrocene) might have different URLs to download artifacts from, or might use a different email address on merge commits, breaking both LLVM and rustc artifact downloads.

This PR refactors bootstrap to load the download URLs and other constants from `src/stage0.json`, allowing downstream forks to tweak those values. It also future-proofs the download code to easily allow forks to add their own custom protocols (like `s3://`).

This PR is best reviewed commit-by-commit.

src/bootstrap/bootstrap.py

@@ -1043,7 +1043,7 @@ def bootstrap(help_triggered):
     build.checksums_sha256 = data["checksums_sha256"]
     build.stage0_compiler = Stage0Toolchain(data["compiler"])
 
-    build.set_dist_environment(data["dist_server"])
+    build.set_dist_environment(data["config"]["dist_server"])
 
     build.build = args.build or build.build_triple()
 

src/bootstrap/builder.rs

@@ -870,20 +870,23 @@ impl<'a> Builder<'a> {
         self.try_run(patchelf.arg(fname));
     }
 
-    pub(crate) fn download_component(
+    pub(crate) fn download_component(&self, url: &str, dest_path: &Path, help_on_error: &str) {
-        &self,
-        base: &str,
-        url: &str,
-        dest_path: &Path,
-        help_on_error: &str,
-    ) {
         // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
         let tempfile = self.tempdir().join(dest_path.file_name().unwrap());
-        self.download_with_retries(&tempfile, &format!("{}/{}", base, url), help_on_error);
+        // While bootstrap itself only supports http and https downloads, downstream forks might
+        // need to download components from other protocols. The match allows them adding more
+        // protocols without worrying about merge conficts if we change the HTTP implementation.
+        match url.split_once("://").map(|(proto, _)| proto) {
+            Some("http") | Some("https") => {
+                self.download_http_with_retries(&tempfile, url, help_on_error)
+            }
+            Some(other) => panic!("unsupported protocol {other} in {url}"),
+            None => panic!("no protocol in {url}"),
+        }
         t!(std::fs::rename(&tempfile, dest_path));
     }
 
-    fn download_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) {
+    fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) {
         println!("downloading {}", url);
         // Try curl. If that fails and we are on windows, fallback to PowerShell.
         let mut curl = Command::new("curl");

src/bootstrap/config.rs

@@ -20,7 +20,6 @@ use crate::channel::GitInfo;
 pub use crate::flags::Subcommand;
 use crate::flags::{Color, Flags};
 use crate::util::{exe, output, program_out_of_date, t};
-use crate::RustfmtMetadata;
 use once_cell::sync::OnceCell;
 use serde::{Deserialize, Deserializer};
 
@@ -73,6 +72,7 @@ pub struct Config {
     pub test_compare_mode: bool,
     pub color: Color,
     pub patch_binaries_for_nix: bool,
+    pub stage0_metadata: Stage0Metadata,
 
     pub on_fail: Option<String>,
     pub stage: u32,
@@ -212,6 +212,28 @@ pub struct Config {
     pub out: PathBuf,
 }
 
+#[derive(Default, Deserialize)]
+#[cfg_attr(test, derive(Clone))]
+pub struct Stage0Metadata {
+    pub config: Stage0Config,
+    pub checksums_sha256: HashMap<String, String>,
+    pub rustfmt: Option<RustfmtMetadata>,
+}
+#[derive(Default, Deserialize)]
+#[cfg_attr(test, derive(Clone))]
+pub struct Stage0Config {
+    pub dist_server: String,
+    pub artifacts_server: String,
+    pub artifacts_with_llvm_assertions_server: String,
+    pub git_merge_commit_email: String,
+}
+#[derive(Default, Deserialize)]
+#[cfg_attr(test, derive(Clone))]
+pub struct RustfmtMetadata {
+    pub date: String,
+    pub version: String,
+}
+
 #[derive(Clone, Debug)]
 pub enum RustfmtState {
     SystemToolchain(PathBuf),
@@ -776,6 +798,9 @@ impl Config {
         config.llvm_profile_use = flags.llvm_profile_use;
         config.llvm_profile_generate = flags.llvm_profile_generate;
 
+        let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json")));
+        config.stage0_metadata = t!(serde_json::from_slice::<Stage0Metadata>(&stage0_json));
+
         #[cfg(test)]
         let get_toml = |_| TomlConfig::default();
         #[cfg(not(test))]
@@ -1103,8 +1128,11 @@ impl Config {
             config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
             config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
             config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
-            config.download_rustc_commit =
+            config.download_rustc_commit = download_ci_rustc_commit(
-                download_ci_rustc_commit(rust.download_rustc, config.verbose > 0);
+                &config.stage0_metadata,
+                rust.download_rustc,
+                config.verbose > 0,
+            );
         } else {
             config.rust_profile_use = flags.rust_profile_use;
             config.rust_profile_generate = flags.rust_profile_generate;
@@ -1424,7 +1452,11 @@ fn threads_from_config(v: u32) -> u32 {
 }
 
 /// Returns the commit to download, or `None` if we shouldn't download CI artifacts.
-fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool) -> Option<String> {
+fn download_ci_rustc_commit(
+    stage0_metadata: &Stage0Metadata,
+    download_rustc: Option<StringOrBool>,
+    verbose: bool,
+) -> Option<String> {
     // If `download-rustc` is not set, default to rebuilding.
     let if_unchanged = match download_rustc {
         None | Some(StringOrBool::Bool(false)) => return None,
@@ -1443,13 +1475,12 @@ fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool)
 
     // Look for a version to compare to based on the current commit.
     // Only commits merged by bors will have CI artifacts.
-    let merge_base = output(Command::new("git").args(&[
+    let merge_base = output(
-        "rev-list",
+        Command::new("git")
-        "--author=bors@rust-lang.org",
+            .arg("rev-list")
-        "-n1",
+            .arg(format!("--author={}", stage0_metadata.config.git_merge_commit_email))
-        "--first-parent",
+            .args(&["-n1", "--first-parent", "HEAD"]),
-        "HEAD",
+    );
-    ]));
     let commit = merge_base.trim_end();
     if commit.is_empty() {
         println!("error: could not find commit hash for downloading rustc");
@@ -1484,7 +1515,7 @@ fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool)
 }
 
 fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> {
-    let RustfmtMetadata { date, version } = builder.stage0_metadata.rustfmt.as_ref()?;
+    let RustfmtMetadata { date, version } = builder.config.stage0_metadata.rustfmt.as_ref()?;
     let channel = format!("{version}-{date}");
 
     let host = builder.config.build;
@@ -1568,13 +1599,13 @@ fn download_component(
     let tarball = cache_dir.join(&filename);
     let (base_url, url, should_verify) = match mode {
         DownloadSource::CI => (
-            "https://ci-artifacts.rust-lang.org/rustc-builds".to_string(),
+            builder.config.stage0_metadata.config.artifacts_server.clone(),
             format!("{key}/{filename}"),
             false,
         ),
         DownloadSource::Dist => {
             let dist_server = env::var("RUSTUP_DIST_SERVER")
-                .unwrap_or(builder.stage0_metadata.dist_server.to_string());
+                .unwrap_or(builder.config.stage0_metadata.config.dist_server.to_string());
             // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json
             (dist_server, format!("dist/{key}/{filename}"), true)
         }
@@ -1590,7 +1621,7 @@ fn download_component(
             target at this time, see https://doc.rust-lang.org/nightly\
             /rustc/platform-support.html for more information."
         );
-        let sha256 = builder.stage0_metadata.checksums_sha256.get(&url).expect(&error);
+        let sha256 = builder.config.stage0_metadata.checksums_sha256.get(&url).expect(&error);
         if tarball.exists() {
             if builder.verify(&tarball, sha256) {
                 builder.unpack(&tarball, &bin_root, prefix);
@@ -1610,7 +1641,7 @@ fn download_component(
         None
     };
 
-    builder.download_component(&base_url, &url, &tarball, "");
+    builder.download_component(&format!("{base_url}/{url}"), &tarball, "");
     if let Some(sha256) = checksum {
         if !builder.verify(&tarball, sha256) {
             panic!("failed to verify {}", tarball.display());

src/bootstrap/lib.rs

@@ -118,7 +118,6 @@ use std::os::windows::fs::symlink_file;
 
 use filetime::FileTime;
 use once_cell::sync::OnceCell;
-use serde::Deserialize;
 
 use crate::builder::Kind;
 use crate::config::{LlvmLibunwind, TargetSelection};
@@ -294,8 +293,6 @@ pub struct Build {
     hosts: Vec<TargetSelection>,
     targets: Vec<TargetSelection>,
 
-    // Stage 0 (downloaded) compiler, lld and cargo or their local rust equivalents
-    stage0_metadata: Stage0Metadata,
     initial_rustc: PathBuf,
     initial_cargo: PathBuf,
     initial_lld: PathBuf,
@@ -322,18 +319,6 @@ pub struct Build {
     metrics: metrics::BuildMetrics,
 }
 
-#[derive(Deserialize)]
-struct Stage0Metadata {
-    dist_server: String,
-    checksums_sha256: HashMap<String, String>,
-    rustfmt: Option<RustfmtMetadata>,
-}
-#[derive(Deserialize)]
-struct RustfmtMetadata {
-    date: String,
-    version: String,
-}
-
 #[derive(Debug)]
 struct Crate {
     name: Interned<String>,
@@ -482,11 +467,7 @@ impl Build {
             bootstrap_out
         };
 
-        let stage0_json = t!(std::fs::read_to_string(&src.join("src").join("stage0.json")));
-        let stage0_metadata = t!(serde_json::from_str::<Stage0Metadata>(&stage0_json));
-
         let mut build = Build {
-            stage0_metadata,
             initial_rustc: config.initial_rustc.clone(),
             initial_cargo: config.initial_cargo.clone(),
             initial_lld,

src/bootstrap/native.rs

@@ -121,7 +121,7 @@ pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
     let mut rev_list = Command::new("git");
     rev_list.args(&[
         PathBuf::from("rev-list"),
-        "--author=bors@rust-lang.org".into(),
+        format!("--author={}", builder.config.stage0_metadata.config.git_merge_commit_email).into(),
         "-n1".into(),
         "--first-parent".into(),
         "HEAD".into(),
@@ -170,11 +170,10 @@ fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) {
     if !rustc_cache.exists() {
         t!(fs::create_dir_all(&rustc_cache));
     }
-    let base = "https://ci-artifacts.rust-lang.org";
+    let base = if llvm_assertions {
-    let url = if llvm_assertions {
+        &builder.config.stage0_metadata.config.artifacts_with_llvm_assertions_server
-        format!("rustc-builds-alt/{}", llvm_sha)
     } else {
-        format!("rustc-builds/{}", llvm_sha)
+        &builder.config.stage0_metadata.config.artifacts_server
     };
     let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple);
     let tarball = rustc_cache.join(&filename);
@@ -187,7 +186,11 @@ help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in
 [llvm]
 download-ci-llvm = false
 ";
-        builder.download_component(base, &format!("{}/{}", url, filename), &tarball, help_on_error);
+        builder.download_component(
+            &format!("{base}/{llvm_sha}/{filename}"),
+            &tarball,
+            help_on_error,
+        );
     }
     let llvm_root = builder.config.ci_llvm_root();
     builder.unpack(&tarball, &llvm_root, "rust-dev");

src/stage0.json

@@ -1,6 +1,20 @@
 {
-  "__comment": "Generated by `./x.py run src/tools/bump-stage0`. Run that command again to update the bootstrap compiler.",
+  "config": {
-  "dist_server": "https://static.rust-lang.org",
+    "dist_server": "https://static.rust-lang.org",
+    "artifacts_server": "https://ci-artifacts.rust-lang.org/rustc-builds",
+    "artifacts_with_llvm_assertions_server": "https://ci-artifacts.rust-lang.org/rustc-builds-alt",
+    "git_merge_commit_email": "bors@rust-lang.org"
+  },
+  "__comments": [
+    "The configuration above this comment is editable, and can be changed",
+    "by forks of the repository if they have alternate values.",
+    "",
+    "The section below is generated by `./x.py run src/tools/bump-stage0`,",
+    "run that command again to update the bootstrap compiler.",
+    "",
+    "All changes below this comment will be overridden the next time the",
+    "tool is executed."
+  ],
   "compiler": {
     "date": "2022-05-20",
     "version": "beta"

src/tools/bump-stage0/src/main.rs

@@ -4,11 +4,14 @@ use indexmap::IndexMap;
 use std::collections::HashMap;
 use std::convert::TryInto;
 
-const DIST_SERVER: &str = "https://static.rust-lang.org";
+const PATH: &str = "src/stage0.json";
 const COMPILER_COMPONENTS: &[&str] = &["rustc", "rust-std", "cargo"];
 const RUSTFMT_COMPONENTS: &[&str] = &["rustfmt-preview"];
 
 struct Tool {
+    config: Config,
+    comments: Vec<String>,
+
     channel: Channel,
     version: [u16; 3],
     checksums: IndexMap<String, String>,
@@ -32,18 +35,23 @@ impl Tool {
             .try_into()
             .map_err(|_| anyhow::anyhow!("failed to parse version"))?;
 
-        Ok(Self { channel, version, checksums: IndexMap::new() })
+        let existing: Stage0 = serde_json::from_slice(&std::fs::read(PATH)?)?;
+
+        Ok(Self {
+            channel,
+            version,
+            config: existing.config,
+            comments: existing.comments,
+            checksums: IndexMap::new(),
+        })
     }
 
     fn update_json(mut self) -> Result<(), Error> {
         std::fs::write(
-            "src/stage0.json",
+            PATH,
             format!(
                 "{}\n",
                 serde_json::to_string_pretty(&Stage0 {
-                    comment: "Generated by `./x.py run src/tools/bump-stage0`. \
-                              Run that command again to update the bootstrap compiler.",
-                    dist_server: DIST_SERVER.into(),
                     compiler: self.detect_compiler()?,
                     rustfmt: self.detect_rustfmt()?,
                     checksums_sha256: {
@@ -51,7 +59,9 @@ impl Tool {
                         // are added while filling the other struct fields just above this block.
                         self.checksums.sort_keys();
                         self.checksums
-                    }
+                    },
+                    config: self.config,
+                    comments: self.comments,
                 })?
             ),
         )?;
@@ -74,7 +84,7 @@ impl Tool {
             Channel::Nightly => "beta".to_string(),
         };
 
-        let manifest = fetch_manifest(&channel)?;
+        let manifest = fetch_manifest(&self.config, &channel)?;
         self.collect_checksums(&manifest, COMPILER_COMPONENTS)?;
         Ok(Stage0Toolchain {
             date: manifest.date,
@@ -100,13 +110,13 @@ impl Tool {
             return Ok(None);
         }
 
-        let manifest = fetch_manifest("nightly")?;
+        let manifest = fetch_manifest(&self.config, "nightly")?;
         self.collect_checksums(&manifest, RUSTFMT_COMPONENTS)?;
         Ok(Some(Stage0Toolchain { date: manifest.date, version: "nightly".into() }))
     }
 
     fn collect_checksums(&mut self, manifest: &Manifest, components: &[&str]) -> Result<(), Error> {
-        let prefix = format!("{}/", DIST_SERVER);
+        let prefix = format!("{}/", self.config.dist_server);
         for component in components {
             let pkg = manifest
                 .pkg
@@ -136,10 +146,10 @@ fn main() -> Result<(), Error> {
     Ok(())
 }
 
-fn fetch_manifest(channel: &str) -> Result<Manifest, Error> {
+fn fetch_manifest(config: &Config, channel: &str) -> Result<Manifest, Error> {
     Ok(toml::from_slice(&http_get(&format!(
         "{}/dist/channel-rust-{}.toml",
-        DIST_SERVER, channel
+        config.dist_server, channel
     ))?)?)
 }
 
@@ -166,35 +176,52 @@ enum Channel {
     Nightly,
 }
 
-#[derive(Debug, serde::Serialize)]
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
 struct Stage0 {
-    #[serde(rename = "__comment")]
+    config: Config,
-    comment: &'static str,
+    // Comments are explicitly below the config, do not move them above.
-    dist_server: String,
+    //
+    // Downstream forks of the compiler codebase can change the configuration values defined above,
+    // but doing so would risk merge conflicts whenever they import new changes that include a
+    // bootstrap compiler bump.
+    //
+    // To lessen the pain, a big block of comments is placed between the configuration and the
+    // auto-generated parts of the file, preventing git diffs of the config to include parts of the
+    // auto-generated content and vice versa. This should prevent merge conflicts.
+    #[serde(rename = "__comments")]
+    comments: Vec<String>,
     compiler: Stage0Toolchain,
     rustfmt: Option<Stage0Toolchain>,
     checksums_sha256: IndexMap<String, String>,
 }
 
-#[derive(Debug, serde::Serialize)]
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+struct Config {
+    dist_server: String,
+    artifacts_server: String,
+    artifacts_with_llvm_assertions_server: String,
+    git_merge_commit_email: String,
+}
+
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
 struct Stage0Toolchain {
     date: String,
     version: String,
 }
 
-#[derive(Debug, serde::Deserialize)]
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
 struct Manifest {
     date: String,
     pkg: HashMap<String, ManifestPackage>,
 }
 
-#[derive(Debug, serde::Deserialize)]
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
 struct ManifestPackage {
     version: String,
     target: HashMap<String, ManifestTargetPackage>,
 }
 
-#[derive(Debug, serde::Deserialize)]
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
 struct ManifestTargetPackage {
     url: Option<String>,
     hash: Option<String>,