Refactor Environment

.github/workflows/ci.yml

@@ -393,7 +393,7 @@ jobs:
           - name: dist-x86_64-msvc
             env:
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler"
-              SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths
+              SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
               DIST_REQUIRE_ALL_TOOLS: 1
             os: windows-2019-8core-32gb
           - name: dist-i686-msvc

Cargo.lock

@@ -858,14 +858,38 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "darling"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
+dependencies = [
+ "darling_core 0.14.4",
+ "darling_macro 0.14.4",
+]
+
 [[package]]
 name = "darling"
 version = "0.20.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e"
 dependencies = [
- "darling_core",
+ "darling_core 0.20.3",
- "darling_macro",
+ "darling_macro 0.20.3",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -882,13 +906,24 @@ dependencies = [
  "syn 2.0.29",
 ]
 
+[[package]]
+name = "darling_macro"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
+dependencies = [
+ "darling_core 0.14.4",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "darling_macro"
 version = "0.20.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
 dependencies = [
- "darling_core",
+ "darling_core 0.20.3",
  "quote",
  "syn 2.0.29",
 ]
@@ -919,6 +954,37 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "derive_builder"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8"
+dependencies = [
+ "derive_builder_macro",
+]
+
+[[package]]
+name = "derive_builder_core"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f"
+dependencies = [
+ "darling 0.14.4",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "derive_builder_macro"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
+dependencies = [
+ "derive_builder_core",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "derive_more"
 version = "0.99.17"
@@ -938,7 +1004,7 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4e8ef033054e131169b8f0f9a7af8f5533a9436fadf3c500ed547f730f07090d"
 dependencies = [
- "darling",
+ "darling 0.20.3",
  "proc-macro2",
  "quote",
  "syn 2.0.29",
@@ -2584,6 +2650,8 @@ dependencies = [
  "anyhow",
  "build_helper",
  "camino",
+ "clap",
+ "derive_builder",
  "env_logger 0.10.0",
  "fs_extra",
  "glob",

src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile

@@ -87,7 +87,7 @@ ENV RUST_CONFIGURE_ARGS \
       --set rust.lto=thin
 
 ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \
-    ./build/$HOSTS/stage0-tools-bin/opt-dist python3 ../x.py dist \
+    ./build/$HOSTS/stage0-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \
     --host $HOSTS --target $HOSTS \
     --include-default-paths \
     build-manifest bootstrap

src/ci/github-actions/ci.yml

@@ -624,7 +624,7 @@ jobs:
                 --target=x86_64-pc-windows-msvc
                 --enable-full-tools
                 --enable-profiler
-              SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths
+              SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
               DIST_REQUIRE_ALL_TOOLS: 1
             <<: *job-windows-8c
 

src/tools/opt-dist/Cargo.toml

@@ -21,3 +21,5 @@ serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 glob = "0.3"
 tempfile = "3.5"
+derive_builder = "0.12"
+clap = { version = "4", features = ["derive"] }

src/tools/opt-dist/src/environment.rs

@@ -0,0 +1,100 @@
+use camino::Utf8PathBuf;
+use derive_builder::Builder;
+
+#[derive(Builder)]
+pub struct Environment {
+    host_triple: String,
+    python_binary: String,
+    /// The rustc checkout, where the compiler source is located.
+    checkout_dir: Utf8PathBuf,
+    /// The main directory where the build occurs.
+    build_dir: Utf8PathBuf,
+    /// Directory where the optimization artifacts (PGO/BOLT profiles, etc.)
+    /// will be stored.
+    artifact_dir: Utf8PathBuf,
+    /// Path to the host LLVM used to compile LLVM in `src/llvm-project`.
+    host_llvm_dir: Utf8PathBuf,
+    /// List of test paths that should be skipped when testing the optimized artifacts.
+    skipped_tests: Vec<&'static str>,
+    use_bolt: bool,
+    shared_llvm: bool,
+}
+
+impl Environment {
+    pub fn host_triple(&self) -> &str {
+        &self.host_triple
+    }
+
+    pub fn python_binary(&self) -> &str {
+        &self.python_binary
+    }
+
+    pub fn checkout_path(&self) -> Utf8PathBuf {
+        self.checkout_dir.clone()
+    }
+
+    pub fn build_root(&self) -> Utf8PathBuf {
+        self.build_dir.clone()
+    }
+
+    pub fn build_artifacts(&self) -> Utf8PathBuf {
+        self.build_root().join("build").join(&self.host_triple)
+    }
+
+    pub fn artifact_dir(&self) -> Utf8PathBuf {
+        self.artifact_dir.clone()
+    }
+
+    pub fn cargo_stage_0(&self) -> Utf8PathBuf {
+        self.build_artifacts()
+            .join("stage0")
+            .join("bin")
+            .join(format!("cargo{}", executable_extension()))
+    }
+
+    pub fn rustc_stage_0(&self) -> Utf8PathBuf {
+        self.build_artifacts()
+            .join("stage0")
+            .join("bin")
+            .join(format!("rustc{}", executable_extension()))
+    }
+
+    pub fn rustc_stage_2(&self) -> Utf8PathBuf {
+        self.build_artifacts()
+            .join("stage2")
+            .join("bin")
+            .join(format!("rustc{}", executable_extension()))
+    }
+
+    /// Path to the built rustc-perf benchmark suite.
+    pub fn rustc_perf_dir(&self) -> Utf8PathBuf {
+        self.artifact_dir.join("rustc-perf")
+    }
+
+    pub fn host_llvm_dir(&self) -> Utf8PathBuf {
+        self.host_llvm_dir.clone()
+    }
+
+    pub fn use_bolt(&self) -> bool {
+        self.use_bolt
+    }
+
+    pub fn supports_shared_llvm(&self) -> bool {
+        self.shared_llvm
+    }
+
+    pub fn skipped_tests(&self) -> &[&'static str] {
+        &self.skipped_tests
+    }
+}
+
+/// What is the extension of binary executables on this platform?
+#[cfg(target_family = "unix")]
+pub fn executable_extension() -> &'static str {
+    ""
+}
+
+#[cfg(target_family = "windows")]
+pub fn executable_extension() -> &'static str {
+    ".exe"
+}

src/tools/opt-dist/src/environment/linux.rs

@@ -1,66 +0,0 @@
-use crate::environment::Environment;
-use crate::exec::cmd;
-use crate::utils::io::copy_directory;
-use camino::{Utf8Path, Utf8PathBuf};
-
-pub(super) struct LinuxEnvironment {
-    target_triple: String,
-}
-
-impl LinuxEnvironment {
-    pub fn new(target_triple: String) -> Self {
-        Self { target_triple }
-    }
-}
-
-impl Environment for LinuxEnvironment {
-    fn host_triple(&self) -> &str {
-        &self.target_triple
-    }
-
-    fn python_binary(&self) -> &'static str {
-        "python3"
-    }
-
-    fn checkout_path(&self) -> Utf8PathBuf {
-        Utf8PathBuf::from("/checkout")
-    }
-
-    fn host_llvm_dir(&self) -> Utf8PathBuf {
-        Utf8PathBuf::from("/rustroot")
-    }
-
-    fn opt_artifacts(&self) -> Utf8PathBuf {
-        Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts")
-    }
-
-    fn build_root(&self) -> Utf8PathBuf {
-        self.checkout_path().join("obj")
-    }
-
-    fn prepare_rustc_perf(&self) -> anyhow::Result<()> {
-        // /tmp/rustc-perf comes from the x64 dist Dockerfile
-        copy_directory(Utf8Path::new("/tmp/rustc-perf"), &self.rustc_perf_dir())?;
-        cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"])
-            .workdir(&self.rustc_perf_dir())
-            .env("RUSTC", &self.rustc_stage_0().into_string())
-            .env("RUSTC_BOOTSTRAP", "1")
-            .run()?;
-        Ok(())
-    }
-
-    fn supports_bolt(&self) -> bool {
-        true
-    }
-
-    fn supports_shared_llvm(&self) -> bool {
-        true
-    }
-
-    fn skipped_tests(&self) -> &'static [&'static str] {
-        &[
-            // Fails because of linker errors, as of June 2023.
-            "tests/ui/process/nofile-limit.rs",
-        ]
-    }
-}

src/tools/opt-dist/src/environment/mod.rs

@@ -1,83 +0,0 @@
-use camino::Utf8PathBuf;
-
-#[cfg(target_family = "unix")]
-mod linux;
-#[cfg(target_family = "windows")]
-mod windows;
-
-pub trait Environment {
-    fn host_triple(&self) -> &str;
-
-    fn python_binary(&self) -> &'static str;
-
-    /// The rustc checkout, where the compiler source is located.
-    fn checkout_path(&self) -> Utf8PathBuf;
-
-    /// Path to the host LLVM used to compile LLVM in `src/llvm-project`.
-    fn host_llvm_dir(&self) -> Utf8PathBuf;
-
-    /// Directory where the optimization artifacts (PGO/BOLT profiles, etc.)
-    /// will be stored.
-    fn opt_artifacts(&self) -> Utf8PathBuf;
-
-    /// The main directory where the build occurs.
-    fn build_root(&self) -> Utf8PathBuf;
-
-    fn build_artifacts(&self) -> Utf8PathBuf {
-        self.build_root().join("build").join(self.host_triple())
-    }
-
-    fn cargo_stage_0(&self) -> Utf8PathBuf {
-        self.build_artifacts()
-            .join("stage0")
-            .join("bin")
-            .join(format!("cargo{}", executable_extension()))
-    }
-
-    fn rustc_stage_0(&self) -> Utf8PathBuf {
-        self.build_artifacts()
-            .join("stage0")
-            .join("bin")
-            .join(format!("rustc{}", executable_extension()))
-    }
-
-    fn rustc_stage_2(&self) -> Utf8PathBuf {
-        self.build_artifacts()
-            .join("stage2")
-            .join("bin")
-            .join(format!("rustc{}", executable_extension()))
-    }
-
-    /// Path to the built rustc-perf benchmark suite.
-    fn rustc_perf_dir(&self) -> Utf8PathBuf {
-        self.opt_artifacts().join("rustc-perf")
-    }
-
-    /// Download and/or compile rustc-perf.
-    fn prepare_rustc_perf(&self) -> anyhow::Result<()>;
-
-    fn supports_bolt(&self) -> bool;
-
-    fn supports_shared_llvm(&self) -> bool;
-
-    /// List of test paths that should be skipped when testing the optimized artifacts.
-    fn skipped_tests(&self) -> &'static [&'static str];
-}
-
-pub fn create_environment(target_triple: String) -> Box<dyn Environment> {
-    #[cfg(target_family = "unix")]
-    return Box::new(linux::LinuxEnvironment::new(target_triple));
-    #[cfg(target_family = "windows")]
-    return Box::new(windows::WindowsEnvironment::new(target_triple));
-}
-
-/// What is the extension of binary executables on this platform?
-#[cfg(target_family = "unix")]
-pub fn executable_extension() -> &'static str {
-    ""
-}
-
-#[cfg(target_family = "windows")]
-pub fn executable_extension() -> &'static str {
-    ".exe"
-}

src/tools/opt-dist/src/environment/windows.rs

@@ -1,93 +0,0 @@
-use crate::environment::Environment;
-use crate::exec::cmd;
-use crate::utils::io::move_directory;
-use crate::utils::retry_action;
-use camino::Utf8PathBuf;
-use std::io::Cursor;
-use std::time::Duration;
-use zip::ZipArchive;
-
-pub(super) struct WindowsEnvironment {
-    checkout_dir: Utf8PathBuf,
-    target_triple: String,
-}
-
-impl WindowsEnvironment {
-    pub fn new(target_triple: String) -> Self {
-        Self { checkout_dir: std::env::current_dir().unwrap().try_into().unwrap(), target_triple }
-    }
-}
-
-impl Environment for WindowsEnvironment {
-    fn host_triple(&self) -> &str {
-        &self.target_triple
-    }
-
-    fn python_binary(&self) -> &'static str {
-        "python"
-    }
-
-    fn checkout_path(&self) -> Utf8PathBuf {
-        self.checkout_dir.clone()
-    }
-
-    fn host_llvm_dir(&self) -> Utf8PathBuf {
-        self.checkout_path().join("citools").join("clang-rust")
-    }
-
-    fn opt_artifacts(&self) -> Utf8PathBuf {
-        self.checkout_path().join("opt-artifacts")
-    }
-
-    fn build_root(&self) -> Utf8PathBuf {
-        self.checkout_path()
-    }
-
-    fn prepare_rustc_perf(&self) -> anyhow::Result<()> {
-        // FIXME: add some mechanism for synchronization of this commit SHA with
-        // Linux (which builds rustc-perf in a Dockerfile)
-        // rustc-perf version from 2023-05-30
-        const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1";
-
-        let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip");
-        let client = reqwest::blocking::Client::builder()
-            .timeout(Duration::from_secs(60 * 2))
-            .connect_timeout(Duration::from_secs(60 * 2))
-            .build()?;
-        let response = retry_action(
-            || Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()),
-            "Download rustc-perf archive",
-            5,
-        )?;
-
-        let mut archive = ZipArchive::new(Cursor::new(response))?;
-        archive.extract(self.rustc_perf_dir())?;
-        move_directory(
-            &self.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")),
-            &self.rustc_perf_dir(),
-        )?;
-
-        cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"])
-            .workdir(&self.rustc_perf_dir())
-            .env("RUSTC", &self.rustc_stage_0().into_string())
-            .env("RUSTC_BOOTSTRAP", "1")
-            .run()?;
-
-        Ok(())
-    }
-
-    fn supports_bolt(&self) -> bool {
-        false
-    }
-
-    fn supports_shared_llvm(&self) -> bool {
-        false
-    }
-
-    fn skipped_tests(&self) -> &'static [&'static str] {
-        &[
-            // Fails as of June 2023.
-            "tests\\codegen\\vec-shrink-panik.rs",
-        ]
-    }
-}

src/tools/opt-dist/src/exec.rs

@@ -96,7 +96,7 @@ pub struct Bootstrap {
 }
 
 impl Bootstrap {
-    pub fn build(env: &dyn Environment) -> Self {
+    pub fn build(env: &Environment) -> Self {
         let metrics_path = env.build_root().join("build").join("metrics.json");
         let cmd = cmd(&[
             env.python_binary(),
@@ -114,7 +114,7 @@ impl Bootstrap {
         Self { cmd, metrics_path }
     }
 
-    pub fn dist(env: &dyn Environment, dist_args: &[String]) -> Self {
+    pub fn dist(env: &Environment, dist_args: &[String]) -> Self {
         let metrics_path = env.build_root().join("build").join("metrics.json");
         let cmd = cmd(&dist_args.iter().map(|arg| arg.as_str()).collect::<Vec<_>>())
             .env("RUST_BACKTRACE", "full");

src/tools/opt-dist/src/main.rs

@@ -1,17 +1,22 @@
 use crate::bolt::{bolt_optimize, with_bolt_instrumented};
 use anyhow::Context;
+use camino::{Utf8Path, Utf8PathBuf};
+use clap::Parser;
 use log::LevelFilter;
+use std::io::Cursor;
+use std::time::Duration;
 use utils::io;
+use zip::ZipArchive;
 
-use crate::environment::{create_environment, Environment};
+use crate::environment::{Environment, EnvironmentBuilder};
-use crate::exec::Bootstrap;
+use crate::exec::{cmd, Bootstrap};
 use crate::tests::run_tests;
 use crate::timer::Timer;
 use crate::training::{gather_llvm_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles};
-use crate::utils::io::reset_directory;
+use crate::utils::io::{copy_directory, move_directory, reset_directory};
 use crate::utils::{
     clear_llvm_files, format_env_variables, print_binary_sizes, print_free_disk_space,
-    with_log_group,
+    retry_action, with_log_group,
 };
 
 mod bolt;
@@ -23,24 +28,104 @@ mod timer;
 mod training;
 mod utils;
 
+#[derive(clap::Parser, Debug)]
+struct Args {
+    #[clap(subcommand)]
+    env: EnvironmentCmd,
+}
+
+#[derive(clap::Parser, Clone, Debug)]
+struct SharedArgs {
+    // Arguments passed to `x` to perform the final (dist) build.
+    build_args: Vec<String>,
+}
+
+#[derive(clap::Parser, Clone, Debug)]
+enum EnvironmentCmd {
+    LinuxCi {
+        #[clap(flatten)]
+        shared: SharedArgs,
+    },
+    WindowsCi {
+        #[clap(flatten)]
+        shared: SharedArgs,
+    },
+}
+
 fn is_try_build() -> bool {
     std::env::var("DIST_TRY_BUILD").unwrap_or_else(|_| "0".to_string()) != "0"
 }
 
+fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec<String>)> {
+    let (env, args) = match args.env {
+        EnvironmentCmd::LinuxCi { shared } => {
+            let target_triple =
+                std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing");
+
+            let checkout_dir = Utf8PathBuf::from("/checkout");
+            let env = EnvironmentBuilder::default()
+                .host_triple(target_triple)
+                .python_binary("python3".to_string())
+                .checkout_dir(checkout_dir.clone())
+                .host_llvm_dir(Utf8PathBuf::from("/rustroot"))
+                .artifact_dir(Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts"))
+                .build_dir(checkout_dir.join("obj"))
+                .shared_llvm(true)
+                .use_bolt(true)
+                .skipped_tests(vec![
+                    // Fails because of linker errors, as of June 2023.
+                    "tests/ui/process/nofile-limit.rs",
+                ])
+                .build()?;
+            // /tmp/rustc-perf comes from the x64 dist Dockerfile
+            with_log_group("Building rustc-perf", || {
+                Ok::<(), anyhow::Error>(copy_rustc_perf(&env, Utf8Path::new("/tmp/rustc-perf"))?)
+            })?;
+
+            (env, shared.build_args)
+        }
+        EnvironmentCmd::WindowsCi { shared } => {
+            let target_triple =
+                std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing");
+
+            let checkout_dir: Utf8PathBuf = std::env::current_dir()?.try_into()?;
+            let env = EnvironmentBuilder::default()
+                .host_triple(target_triple)
+                .python_binary("python".to_string())
+                .checkout_dir(checkout_dir.clone())
+                .host_llvm_dir(checkout_dir.join("citools").join("clang-rust"))
+                .artifact_dir(checkout_dir.join("opt-artifacts"))
+                .build_dir(checkout_dir)
+                .shared_llvm(false)
+                .use_bolt(false)
+                .skipped_tests(vec![
+                    // Fails as of June 2023.
+                    "tests\\codegen\\vec-shrink-panik.rs",
+                ])
+                .build()?;
+
+            with_log_group("Building rustc-perf", || {
+                Ok::<(), anyhow::Error>(download_rustc_perf(&env)?)
+            })?;
+
+            (env, shared.build_args)
+        }
+    };
+    Ok((env, args))
+}
+
 fn execute_pipeline(
-    env: &dyn Environment,
+    env: &Environment,
     timer: &mut Timer,
     dist_args: Vec<String>,
 ) -> anyhow::Result<()> {
-    reset_directory(&env.opt_artifacts())?;
+    reset_directory(&env.artifact_dir())?;
-
-    with_log_group("Building rustc-perf", || env.prepare_rustc_perf())?;
 
     // Stage 1: Build PGO instrumented rustc
     // We use a normal build of LLVM, because gathering PGO profiles for LLVM and `rustc` at the
     // same time can cause issues, because the host and in-tree LLVM versions can diverge.
     let rustc_pgo_profile = timer.section("Stage 1 (Rustc PGO)", |stage| {
-        let rustc_profile_dir_root = env.opt_artifacts().join("rustc-pgo");
+        let rustc_profile_dir_root = env.artifact_dir().join("rustc-pgo");
 
         stage.section("Build PGO instrumented rustc and LLVM", |section| {
             let mut builder = Bootstrap::build(env).rustc_pgo_instrument(&rustc_profile_dir_root);
@@ -74,7 +159,7 @@ fn execute_pipeline(
         // Remove the previous, uninstrumented build of LLVM.
         clear_llvm_files(env)?;
 
-        let llvm_profile_dir_root = env.opt_artifacts().join("llvm-pgo");
+        let llvm_profile_dir_root = env.artifact_dir().join("llvm-pgo");
 
         stage.section("Build PGO instrumented LLVM", |section| {
             Bootstrap::build(env)
@@ -95,7 +180,7 @@ fn execute_pipeline(
         Ok(profile)
     })?;
 
-    let llvm_bolt_profile = if env.supports_bolt() {
+    let llvm_bolt_profile = if env.use_bolt() {
         // Stage 3: Build BOLT instrumented LLVM
         // We build a PGO optimized LLVM in this step, then instrument it with BOLT and gather BOLT profiles.
         // Note that we don't remove LLVM artifacts after this step, so that they are reused in the final dist build.
@@ -171,10 +256,9 @@ fn main() -> anyhow::Result<()> {
         .parse_default_env()
         .init();
 
-    let target_triple = std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing");
+    let args = Args::parse();
 
-    let mut build_args: Vec<String> = std::env::args().skip(1).collect();
+    println!("Running optimized build pipeline with args `{:?}`", args);
-    println!("Running optimized build pipeline with args `{}`", build_args.join(" "));
 
     with_log_group("Environment values", || {
         println!("Environment values\n{}", format_env_variables());
@@ -186,6 +270,8 @@ fn main() -> anyhow::Result<()> {
         }
     });
 
+    let (env, mut build_args) = create_environment(args).context("Cannot create environment")?;
+
     // Skip components that are not needed for try builds to speed them up
     if is_try_build() {
         log::info!("Skipping building of unimportant components for a try build");
@@ -204,14 +290,56 @@ fn main() -> anyhow::Result<()> {
     }
 
     let mut timer = Timer::new();
-    let env = create_environment(target_triple);
 
-    let result = execute_pipeline(env.as_ref(), &mut timer, build_args);
+    let result = execute_pipeline(&env, &mut timer, build_args);
     log::info!("Timer results\n{}", timer.format_stats());
 
     print_free_disk_space()?;
     result.context("Optimized build pipeline has failed")?;
-    print_binary_sizes(env.as_ref())?;
+    print_binary_sizes(&env)?;
 
     Ok(())
 }
+
+// Copy rustc-perf from the given path into the environment and build it.
+fn copy_rustc_perf(env: &Environment, dir: &Utf8Path) -> anyhow::Result<()> {
+    copy_directory(dir, &env.rustc_perf_dir())?;
+    build_rustc_perf(env)
+}
+
+// Download and build rustc-perf into the given environment.
+fn download_rustc_perf(env: &Environment) -> anyhow::Result<()> {
+    // FIXME: add some mechanism for synchronization of this commit SHA with
+    // Linux (which builds rustc-perf in a Dockerfile)
+    // rustc-perf version from 2023-05-30
+    const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1";
+
+    let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip");
+    let client = reqwest::blocking::Client::builder()
+        .timeout(Duration::from_secs(60 * 2))
+        .connect_timeout(Duration::from_secs(60 * 2))
+        .build()?;
+    let response = retry_action(
+        || Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()),
+        "Download rustc-perf archive",
+        5,
+    )?;
+
+    let mut archive = ZipArchive::new(Cursor::new(response))?;
+    archive.extract(env.rustc_perf_dir())?;
+    move_directory(
+        &env.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")),
+        &env.rustc_perf_dir(),
+    )?;
+
+    build_rustc_perf(env)
+}
+
+fn build_rustc_perf(env: &Environment) -> anyhow::Result<()> {
+    cmd(&[env.cargo_stage_0().as_str(), "build", "-p", "collector"])
+        .workdir(&env.rustc_perf_dir())
+        .env("RUSTC", &env.rustc_stage_0().into_string())
+        .env("RUSTC_BOOTSTRAP", "1")
+        .run()?;
+    Ok(())
+}

src/tools/opt-dist/src/tests.rs

@@ -5,7 +5,7 @@ use anyhow::Context;
 use camino::{Utf8Path, Utf8PathBuf};
 
 /// Run tests on optimized dist artifacts.
-pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> {
+pub fn run_tests(env: &Environment) -> anyhow::Result<()> {
     // After `dist` is executed, we extract its archived components into a sysroot directory,
     // and then use that extracted rustc as a stage0 compiler.
     // Then we run a subset of tests using that compiler, to have a basic smoke test which checks

src/tools/opt-dist/src/training.rs

@@ -30,7 +30,7 @@ const RUSTC_PGO_CRATES: &[&str] = &[
 const LLVM_BOLT_CRATES: &[&str] = LLVM_PGO_CRATES;
 
 fn init_compiler_benchmarks(
-    env: &dyn Environment,
+    env: &Environment,
     profiles: &[&str],
     scenarios: &[&str],
     crates: &[&str],
@@ -75,7 +75,7 @@ enum LlvmProfdata {
 }
 
 fn merge_llvm_profiles(
-    env: &dyn Environment,
+    env: &Environment,
     merged_path: &Utf8Path,
     profile_dir: &Utf8Path,
     profdata: LlvmProfdata,
@@ -116,7 +116,7 @@ fn log_profile_stats(
 pub struct LlvmPGOProfile(pub Utf8PathBuf);
 
 pub fn gather_llvm_profiles(
-    env: &dyn Environment,
+    env: &Environment,
     profile_root: &Utf8Path,
 ) -> anyhow::Result<LlvmPGOProfile> {
     log::info!("Running benchmarks with PGO instrumented LLVM");
@@ -127,7 +127,7 @@ pub fn gather_llvm_profiles(
             .context("Cannot gather LLVM PGO profiles")
     })?;
 
-    let merged_profile = env.opt_artifacts().join("llvm-pgo.profdata");
+    let merged_profile = env.artifact_dir().join("llvm-pgo.profdata");
     log::info!("Merging LLVM PGO profiles to {merged_profile}");
 
     merge_llvm_profiles(env, &merged_profile, profile_root, LlvmProfdata::Host)?;
@@ -143,7 +143,7 @@ pub fn gather_llvm_profiles(
 pub struct RustcPGOProfile(pub Utf8PathBuf);
 
 pub fn gather_rustc_profiles(
-    env: &dyn Environment,
+    env: &Environment,
     profile_root: &Utf8Path,
 ) -> anyhow::Result<RustcPGOProfile> {
     log::info!("Running benchmarks with PGO instrumented rustc");
@@ -163,7 +163,7 @@ pub fn gather_rustc_profiles(
             .context("Cannot gather rustc PGO profiles")
     })?;
 
-    let merged_profile = env.opt_artifacts().join("rustc-pgo.profdata");
+    let merged_profile = env.artifact_dir().join("rustc-pgo.profdata");
     log::info!("Merging Rustc PGO profiles to {merged_profile}");
 
     merge_llvm_profiles(env, &merged_profile, profile_root, LlvmProfdata::Target)?;
@@ -178,7 +178,7 @@ pub fn gather_rustc_profiles(
 
 pub struct LlvmBoltProfile(pub Utf8PathBuf);
 
-pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result<LlvmBoltProfile> {
+pub fn gather_llvm_bolt_profiles(env: &Environment) -> anyhow::Result<LlvmBoltProfile> {
     log::info!("Running benchmarks with BOLT instrumented LLVM");
 
     with_log_group("Running benchmarks", || {
@@ -187,7 +187,7 @@ pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result<LlvmBo
             .context("Cannot gather LLVM BOLT profiles")
     })?;
 
-    let merged_profile = env.opt_artifacts().join("llvm-bolt.profdata");
+    let merged_profile = env.artifact_dir().join("llvm-bolt.profdata");
     let profile_root = Utf8PathBuf::from("/tmp/prof.fdata");
     log::info!("Merging LLVM BOLT profiles to {merged_profile}");
 

src/tools/opt-dist/src/utils/mod.rs

@@ -26,7 +26,7 @@ pub fn print_free_disk_space() -> anyhow::Result<()> {
     Ok(())
 }
 
-pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> {
+pub fn print_binary_sizes(env: &Environment) -> anyhow::Result<()> {
     use std::fmt::Write;
 
     let root = env.build_artifacts().join("stage2");
@@ -48,7 +48,7 @@ pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> {
     Ok(())
 }
 
-pub fn clear_llvm_files(env: &dyn Environment) -> anyhow::Result<()> {
+pub fn clear_llvm_files(env: &Environment) -> anyhow::Result<()> {
     // Bootstrap currently doesn't support rebuilding LLVM when PGO options
     // change (or any other llvm-related options); so just clear out the relevant
     // directories ourselves.