From 074fb2c6b7c4c7fb1de7d303a37632c02dd10cef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 12 Sep 2023 18:02:45 +0200 Subject: [PATCH 1/8] Store target triple in environment --- src/tools/opt-dist/src/environment/linux.rs | 14 +++++++++++++- src/tools/opt-dist/src/environment/mod.rs | 10 ++++------ src/tools/opt-dist/src/environment/windows.rs | 9 +++++++-- src/tools/opt-dist/src/main.rs | 4 +++- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/tools/opt-dist/src/environment/linux.rs b/src/tools/opt-dist/src/environment/linux.rs index 58b7e6d2306..bd8dd037ae5 100644 --- a/src/tools/opt-dist/src/environment/linux.rs +++ b/src/tools/opt-dist/src/environment/linux.rs @@ -3,9 +3,21 @@ use crate::exec::cmd; use crate::utils::io::copy_directory; use camino::{Utf8Path, Utf8PathBuf}; -pub(super) struct LinuxEnvironment; +pub(super) struct LinuxEnvironment { + target_triple: String, +} + +impl LinuxEnvironment { + pub fn new(target_triple: String) -> Self { + Self { target_triple } + } +} impl Environment for LinuxEnvironment { + fn host_triple(&self) -> &str { + &self.target_triple + } + fn python_binary(&self) -> &'static str { "python3" } diff --git a/src/tools/opt-dist/src/environment/mod.rs b/src/tools/opt-dist/src/environment/mod.rs index a8650fad011..9f7f31d4791 100644 --- a/src/tools/opt-dist/src/environment/mod.rs +++ b/src/tools/opt-dist/src/environment/mod.rs @@ -6,9 +6,7 @@ mod linux; mod windows; pub trait Environment { - fn host_triple(&self) -> String { - std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing") - } + fn host_triple(&self) -> &str; fn python_binary(&self) -> &'static str; @@ -69,9 +67,9 @@ pub trait Environment { fn skipped_tests(&self) -> &'static [&'static str]; } -pub fn create_environment() -> Box { +pub fn create_environment(target_triple: String) -> Box { #[cfg(target_family = "unix")] - return Box::new(linux::LinuxEnvironment); + return Box::new(linux::LinuxEnvironment::new(target_triple)); #[cfg(target_family = "windows")] - return Box::new(windows::WindowsEnvironment::new()); + return Box::new(windows::WindowsEnvironment::new(target_triple)); } diff --git a/src/tools/opt-dist/src/environment/windows.rs b/src/tools/opt-dist/src/environment/windows.rs index 79399391798..90ddc01e893 100644 --- a/src/tools/opt-dist/src/environment/windows.rs +++ b/src/tools/opt-dist/src/environment/windows.rs @@ -9,15 +9,20 @@ use zip::ZipArchive; pub(super) struct WindowsEnvironment { checkout_dir: Utf8PathBuf, + target_triple: String, } impl WindowsEnvironment { - pub fn new() -> Self { - Self { checkout_dir: std::env::current_dir().unwrap().try_into().unwrap() } + pub fn new(target_triple: String) -> Self { + Self { checkout_dir: std::env::current_dir().unwrap().try_into().unwrap(), target_triple } } } impl Environment for WindowsEnvironment { + fn host_triple(&self) -> &str { + &self.target_triple + } + fn python_binary(&self) -> &'static str { "python" } diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs index 8ab19674d05..484ca5b3b86 100644 --- a/src/tools/opt-dist/src/main.rs +++ b/src/tools/opt-dist/src/main.rs @@ -171,6 +171,8 @@ fn main() -> anyhow::Result<()> { .parse_default_env() .init(); + let target_triple = std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing"); + let mut build_args: Vec = std::env::args().skip(1).collect(); println!("Running optimized build pipeline with args `{}`", build_args.join(" ")); @@ -202,7 +204,7 @@ fn main() -> anyhow::Result<()> { } let mut timer = Timer::new(); - let env = create_environment(); + let env = create_environment(target_triple); let result = execute_pipeline(env.as_ref(), &mut timer, build_args); log::info!("Timer results\n{}", timer.format_stats()); From 95500f494118b74c965a94acce3e92a803ee8e91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 12 Sep 2023 18:07:31 +0200 Subject: [PATCH 2/8] Make executable extension platform, rather than environment dependent --- src/tools/opt-dist/src/environment/linux.rs | 4 ---- src/tools/opt-dist/src/environment/mod.rs | 20 +++++++++++++------ src/tools/opt-dist/src/environment/windows.rs | 4 ---- src/tools/opt-dist/src/tests.rs | 8 ++++---- src/tools/opt-dist/src/training.rs | 4 ++-- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/tools/opt-dist/src/environment/linux.rs b/src/tools/opt-dist/src/environment/linux.rs index bd8dd037ae5..61f05137298 100644 --- a/src/tools/opt-dist/src/environment/linux.rs +++ b/src/tools/opt-dist/src/environment/linux.rs @@ -57,10 +57,6 @@ impl Environment for LinuxEnvironment { true } - fn executable_extension(&self) -> &'static str { - "" - } - fn skipped_tests(&self) -> &'static [&'static str] { &[ // Fails because of linker errors, as of June 2023. diff --git a/src/tools/opt-dist/src/environment/mod.rs b/src/tools/opt-dist/src/environment/mod.rs index 9f7f31d4791..271b41316ab 100644 --- a/src/tools/opt-dist/src/environment/mod.rs +++ b/src/tools/opt-dist/src/environment/mod.rs @@ -31,21 +31,21 @@ pub trait Environment { self.build_artifacts() .join("stage0") .join("bin") - .join(format!("cargo{}", self.executable_extension())) + .join(format!("cargo{}", executable_extension())) } fn rustc_stage_0(&self) -> Utf8PathBuf { self.build_artifacts() .join("stage0") .join("bin") - .join(format!("rustc{}", self.executable_extension())) + .join(format!("rustc{}", executable_extension())) } fn rustc_stage_2(&self) -> Utf8PathBuf { self.build_artifacts() .join("stage2") .join("bin") - .join(format!("rustc{}", self.executable_extension())) + .join(format!("rustc{}", executable_extension())) } /// Path to the built rustc-perf benchmark suite. @@ -60,9 +60,6 @@ pub trait Environment { fn supports_shared_llvm(&self) -> bool; - /// What is the extension of binary executables in this environment? - fn executable_extension(&self) -> &'static str; - /// List of test paths that should be skipped when testing the optimized artifacts. fn skipped_tests(&self) -> &'static [&'static str]; } @@ -73,3 +70,14 @@ pub fn create_environment(target_triple: String) -> Box { #[cfg(target_family = "windows")] return Box::new(windows::WindowsEnvironment::new(target_triple)); } + +/// What is the extension of binary executables on this platform? +#[cfg(target_family = "unix")] +pub fn executable_extension() -> &'static str { + "" +} + +#[cfg(target_family = "windows")] +pub fn executable_extension() -> &'static str { + ".exe" +} diff --git a/src/tools/opt-dist/src/environment/windows.rs b/src/tools/opt-dist/src/environment/windows.rs index 90ddc01e893..f705461ac81 100644 --- a/src/tools/opt-dist/src/environment/windows.rs +++ b/src/tools/opt-dist/src/environment/windows.rs @@ -84,10 +84,6 @@ impl Environment for WindowsEnvironment { false } - fn executable_extension(&self) -> &'static str { - ".exe" - } - fn skipped_tests(&self) -> &'static [&'static str] { &[ // Fails as of June 2023. diff --git a/src/tools/opt-dist/src/tests.rs b/src/tools/opt-dist/src/tests.rs index 3dd1a3223f5..e6b8df9c862 100644 --- a/src/tools/opt-dist/src/tests.rs +++ b/src/tools/opt-dist/src/tests.rs @@ -1,4 +1,4 @@ -use crate::environment::Environment; +use crate::environment::{executable_extension, Environment}; use crate::exec::cmd; use crate::utils::io::{copy_directory, find_file_in_dir, unpack_archive}; use anyhow::Context; @@ -45,9 +45,9 @@ pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> { &rustc_dir.join("lib").join("rustlib").join("src"), )?; - let rustc_path = rustc_dir.join("bin").join(format!("rustc{}", env.executable_extension())); + let rustc_path = rustc_dir.join("bin").join(format!("rustc{}", executable_extension())); assert!(rustc_path.is_file()); - let cargo_path = cargo_dir.join("bin").join(format!("cargo{}", env.executable_extension())); + let cargo_path = cargo_dir.join("bin").join(format!("cargo{}", executable_extension())); assert!(cargo_path.is_file()); // Specify path to a LLVM config so that LLVM is not rebuilt. @@ -56,7 +56,7 @@ pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> { .build_artifacts() .join("llvm") .join("bin") - .join(format!("llvm-config{}", env.executable_extension())); + .join(format!("llvm-config{}", executable_extension())); assert!(llvm_config.is_file()); let config_content = format!( diff --git a/src/tools/opt-dist/src/training.rs b/src/tools/opt-dist/src/training.rs index 59c73fbd695..950dd1127a9 100644 --- a/src/tools/opt-dist/src/training.rs +++ b/src/tools/opt-dist/src/training.rs @@ -1,4 +1,4 @@ -use crate::environment::Environment; +use crate::environment::{executable_extension, Environment}; use crate::exec::{cmd, CmdBuilder}; use crate::utils::io::{count_files, delete_directory}; use crate::utils::with_log_group; @@ -86,7 +86,7 @@ fn merge_llvm_profiles( .build_artifacts() .join("llvm") .join("build") - .join(format!("bin/llvm-profdata{}", env.executable_extension())), + .join(format!("bin/llvm-profdata{}", executable_extension())), }; cmd(&[llvm_profdata.as_str(), "merge", "-o", merged_path.as_str(), profile_dir.as_str()]) From f17047bc9046b3e1218200c6b649ee59ac6d0a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 12 Sep 2023 18:32:32 +0200 Subject: [PATCH 3/8] Refactor Environment --- .github/workflows/ci.yml | 2 +- Cargo.lock | 76 +++++++- .../host-x86_64/dist-x86_64-linux/Dockerfile | 2 +- src/ci/github-actions/ci.yml | 2 +- src/tools/opt-dist/Cargo.toml | 2 + src/tools/opt-dist/src/environment.rs | 100 +++++++++++ src/tools/opt-dist/src/environment/linux.rs | 66 ------- src/tools/opt-dist/src/environment/mod.rs | 83 --------- src/tools/opt-dist/src/environment/windows.rs | 93 ---------- src/tools/opt-dist/src/exec.rs | 4 +- src/tools/opt-dist/src/main.rs | 162 ++++++++++++++++-- src/tools/opt-dist/src/tests.rs | 2 +- src/tools/opt-dist/src/training.rs | 16 +- src/tools/opt-dist/src/utils/mod.rs | 4 +- 14 files changed, 335 insertions(+), 279 deletions(-) create mode 100644 src/tools/opt-dist/src/environment.rs delete mode 100644 src/tools/opt-dist/src/environment/linux.rs delete mode 100644 src/tools/opt-dist/src/environment/mod.rs delete mode 100644 src/tools/opt-dist/src/environment/windows.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3680136d89f..0c20e4dc535 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -393,7 +393,7 @@ jobs: - name: dist-x86_64-msvc env: RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 os: windows-2019-8core-32gb - name: dist-i686-msvc diff --git a/Cargo.lock b/Cargo.lock index 295a5b43a6f..51fce77150c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -858,14 +858,38 @@ dependencies = [ "winapi", ] +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + [[package]] name = "darling" version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.3", + "darling_macro 0.20.3", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", ] [[package]] @@ -882,13 +906,24 @@ dependencies = [ "syn 2.0.29", ] +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", +] + [[package]] name = "darling_macro" version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ - "darling_core", + "darling_core 0.20.3", "quote", "syn 2.0.29", ] @@ -919,6 +954,37 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_builder" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder_macro" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" +dependencies = [ + "derive_builder_core", + "syn 1.0.109", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -938,7 +1004,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e8ef033054e131169b8f0f9a7af8f5533a9436fadf3c500ed547f730f07090d" dependencies = [ - "darling", + "darling 0.20.3", "proc-macro2", "quote", "syn 2.0.29", @@ -2584,6 +2650,8 @@ dependencies = [ "anyhow", "build_helper", "camino", + "clap", + "derive_builder", "env_logger 0.10.0", "fs_extra", "glob", diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index f6954275ad4..6f1b2a6a638 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -87,7 +87,7 @@ ENV RUST_CONFIGURE_ARGS \ --set rust.lto=thin ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \ - ./build/$HOSTS/stage0-tools-bin/opt-dist python3 ../x.py dist \ + ./build/$HOSTS/stage0-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \ --host $HOSTS --target $HOSTS \ --include-default-paths \ build-manifest bootstrap diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 89b82d59d31..1712d65ece9 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -624,7 +624,7 @@ jobs: --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler - SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c diff --git a/src/tools/opt-dist/Cargo.toml b/src/tools/opt-dist/Cargo.toml index 3f7dba81c3a..f1c3dd6aa9b 100644 --- a/src/tools/opt-dist/Cargo.toml +++ b/src/tools/opt-dist/Cargo.toml @@ -21,3 +21,5 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" glob = "0.3" tempfile = "3.5" +derive_builder = "0.12" +clap = { version = "4", features = ["derive"] } diff --git a/src/tools/opt-dist/src/environment.rs b/src/tools/opt-dist/src/environment.rs new file mode 100644 index 00000000000..e730e610604 --- /dev/null +++ b/src/tools/opt-dist/src/environment.rs @@ -0,0 +1,100 @@ +use camino::Utf8PathBuf; +use derive_builder::Builder; + +#[derive(Builder)] +pub struct Environment { + host_triple: String, + python_binary: String, + /// The rustc checkout, where the compiler source is located. + checkout_dir: Utf8PathBuf, + /// The main directory where the build occurs. + build_dir: Utf8PathBuf, + /// Directory where the optimization artifacts (PGO/BOLT profiles, etc.) + /// will be stored. + artifact_dir: Utf8PathBuf, + /// Path to the host LLVM used to compile LLVM in `src/llvm-project`. + host_llvm_dir: Utf8PathBuf, + /// List of test paths that should be skipped when testing the optimized artifacts. + skipped_tests: Vec<&'static str>, + use_bolt: bool, + shared_llvm: bool, +} + +impl Environment { + pub fn host_triple(&self) -> &str { + &self.host_triple + } + + pub fn python_binary(&self) -> &str { + &self.python_binary + } + + pub fn checkout_path(&self) -> Utf8PathBuf { + self.checkout_dir.clone() + } + + pub fn build_root(&self) -> Utf8PathBuf { + self.build_dir.clone() + } + + pub fn build_artifacts(&self) -> Utf8PathBuf { + self.build_root().join("build").join(&self.host_triple) + } + + pub fn artifact_dir(&self) -> Utf8PathBuf { + self.artifact_dir.clone() + } + + pub fn cargo_stage_0(&self) -> Utf8PathBuf { + self.build_artifacts() + .join("stage0") + .join("bin") + .join(format!("cargo{}", executable_extension())) + } + + pub fn rustc_stage_0(&self) -> Utf8PathBuf { + self.build_artifacts() + .join("stage0") + .join("bin") + .join(format!("rustc{}", executable_extension())) + } + + pub fn rustc_stage_2(&self) -> Utf8PathBuf { + self.build_artifacts() + .join("stage2") + .join("bin") + .join(format!("rustc{}", executable_extension())) + } + + /// Path to the built rustc-perf benchmark suite. + pub fn rustc_perf_dir(&self) -> Utf8PathBuf { + self.artifact_dir.join("rustc-perf") + } + + pub fn host_llvm_dir(&self) -> Utf8PathBuf { + self.host_llvm_dir.clone() + } + + pub fn use_bolt(&self) -> bool { + self.use_bolt + } + + pub fn supports_shared_llvm(&self) -> bool { + self.shared_llvm + } + + pub fn skipped_tests(&self) -> &[&'static str] { + &self.skipped_tests + } +} + +/// What is the extension of binary executables on this platform? +#[cfg(target_family = "unix")] +pub fn executable_extension() -> &'static str { + "" +} + +#[cfg(target_family = "windows")] +pub fn executable_extension() -> &'static str { + ".exe" +} diff --git a/src/tools/opt-dist/src/environment/linux.rs b/src/tools/opt-dist/src/environment/linux.rs deleted file mode 100644 index 61f05137298..00000000000 --- a/src/tools/opt-dist/src/environment/linux.rs +++ /dev/null @@ -1,66 +0,0 @@ -use crate::environment::Environment; -use crate::exec::cmd; -use crate::utils::io::copy_directory; -use camino::{Utf8Path, Utf8PathBuf}; - -pub(super) struct LinuxEnvironment { - target_triple: String, -} - -impl LinuxEnvironment { - pub fn new(target_triple: String) -> Self { - Self { target_triple } - } -} - -impl Environment for LinuxEnvironment { - fn host_triple(&self) -> &str { - &self.target_triple - } - - fn python_binary(&self) -> &'static str { - "python3" - } - - fn checkout_path(&self) -> Utf8PathBuf { - Utf8PathBuf::from("/checkout") - } - - fn host_llvm_dir(&self) -> Utf8PathBuf { - Utf8PathBuf::from("/rustroot") - } - - fn opt_artifacts(&self) -> Utf8PathBuf { - Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts") - } - - fn build_root(&self) -> Utf8PathBuf { - self.checkout_path().join("obj") - } - - fn prepare_rustc_perf(&self) -> anyhow::Result<()> { - // /tmp/rustc-perf comes from the x64 dist Dockerfile - copy_directory(Utf8Path::new("/tmp/rustc-perf"), &self.rustc_perf_dir())?; - cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"]) - .workdir(&self.rustc_perf_dir()) - .env("RUSTC", &self.rustc_stage_0().into_string()) - .env("RUSTC_BOOTSTRAP", "1") - .run()?; - Ok(()) - } - - fn supports_bolt(&self) -> bool { - true - } - - fn supports_shared_llvm(&self) -> bool { - true - } - - fn skipped_tests(&self) -> &'static [&'static str] { - &[ - // Fails because of linker errors, as of June 2023. - "tests/ui/process/nofile-limit.rs", - ] - } -} diff --git a/src/tools/opt-dist/src/environment/mod.rs b/src/tools/opt-dist/src/environment/mod.rs deleted file mode 100644 index 271b41316ab..00000000000 --- a/src/tools/opt-dist/src/environment/mod.rs +++ /dev/null @@ -1,83 +0,0 @@ -use camino::Utf8PathBuf; - -#[cfg(target_family = "unix")] -mod linux; -#[cfg(target_family = "windows")] -mod windows; - -pub trait Environment { - fn host_triple(&self) -> &str; - - fn python_binary(&self) -> &'static str; - - /// The rustc checkout, where the compiler source is located. - fn checkout_path(&self) -> Utf8PathBuf; - - /// Path to the host LLVM used to compile LLVM in `src/llvm-project`. - fn host_llvm_dir(&self) -> Utf8PathBuf; - - /// Directory where the optimization artifacts (PGO/BOLT profiles, etc.) - /// will be stored. - fn opt_artifacts(&self) -> Utf8PathBuf; - - /// The main directory where the build occurs. - fn build_root(&self) -> Utf8PathBuf; - - fn build_artifacts(&self) -> Utf8PathBuf { - self.build_root().join("build").join(self.host_triple()) - } - - fn cargo_stage_0(&self) -> Utf8PathBuf { - self.build_artifacts() - .join("stage0") - .join("bin") - .join(format!("cargo{}", executable_extension())) - } - - fn rustc_stage_0(&self) -> Utf8PathBuf { - self.build_artifacts() - .join("stage0") - .join("bin") - .join(format!("rustc{}", executable_extension())) - } - - fn rustc_stage_2(&self) -> Utf8PathBuf { - self.build_artifacts() - .join("stage2") - .join("bin") - .join(format!("rustc{}", executable_extension())) - } - - /// Path to the built rustc-perf benchmark suite. - fn rustc_perf_dir(&self) -> Utf8PathBuf { - self.opt_artifacts().join("rustc-perf") - } - - /// Download and/or compile rustc-perf. - fn prepare_rustc_perf(&self) -> anyhow::Result<()>; - - fn supports_bolt(&self) -> bool; - - fn supports_shared_llvm(&self) -> bool; - - /// List of test paths that should be skipped when testing the optimized artifacts. - fn skipped_tests(&self) -> &'static [&'static str]; -} - -pub fn create_environment(target_triple: String) -> Box { - #[cfg(target_family = "unix")] - return Box::new(linux::LinuxEnvironment::new(target_triple)); - #[cfg(target_family = "windows")] - return Box::new(windows::WindowsEnvironment::new(target_triple)); -} - -/// What is the extension of binary executables on this platform? -#[cfg(target_family = "unix")] -pub fn executable_extension() -> &'static str { - "" -} - -#[cfg(target_family = "windows")] -pub fn executable_extension() -> &'static str { - ".exe" -} diff --git a/src/tools/opt-dist/src/environment/windows.rs b/src/tools/opt-dist/src/environment/windows.rs deleted file mode 100644 index f705461ac81..00000000000 --- a/src/tools/opt-dist/src/environment/windows.rs +++ /dev/null @@ -1,93 +0,0 @@ -use crate::environment::Environment; -use crate::exec::cmd; -use crate::utils::io::move_directory; -use crate::utils::retry_action; -use camino::Utf8PathBuf; -use std::io::Cursor; -use std::time::Duration; -use zip::ZipArchive; - -pub(super) struct WindowsEnvironment { - checkout_dir: Utf8PathBuf, - target_triple: String, -} - -impl WindowsEnvironment { - pub fn new(target_triple: String) -> Self { - Self { checkout_dir: std::env::current_dir().unwrap().try_into().unwrap(), target_triple } - } -} - -impl Environment for WindowsEnvironment { - fn host_triple(&self) -> &str { - &self.target_triple - } - - fn python_binary(&self) -> &'static str { - "python" - } - - fn checkout_path(&self) -> Utf8PathBuf { - self.checkout_dir.clone() - } - - fn host_llvm_dir(&self) -> Utf8PathBuf { - self.checkout_path().join("citools").join("clang-rust") - } - - fn opt_artifacts(&self) -> Utf8PathBuf { - self.checkout_path().join("opt-artifacts") - } - - fn build_root(&self) -> Utf8PathBuf { - self.checkout_path() - } - - fn prepare_rustc_perf(&self) -> anyhow::Result<()> { - // FIXME: add some mechanism for synchronization of this commit SHA with - // Linux (which builds rustc-perf in a Dockerfile) - // rustc-perf version from 2023-05-30 - const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1"; - - let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip"); - let client = reqwest::blocking::Client::builder() - .timeout(Duration::from_secs(60 * 2)) - .connect_timeout(Duration::from_secs(60 * 2)) - .build()?; - let response = retry_action( - || Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()), - "Download rustc-perf archive", - 5, - )?; - - let mut archive = ZipArchive::new(Cursor::new(response))?; - archive.extract(self.rustc_perf_dir())?; - move_directory( - &self.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")), - &self.rustc_perf_dir(), - )?; - - cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"]) - .workdir(&self.rustc_perf_dir()) - .env("RUSTC", &self.rustc_stage_0().into_string()) - .env("RUSTC_BOOTSTRAP", "1") - .run()?; - - Ok(()) - } - - fn supports_bolt(&self) -> bool { - false - } - - fn supports_shared_llvm(&self) -> bool { - false - } - - fn skipped_tests(&self) -> &'static [&'static str] { - &[ - // Fails as of June 2023. - "tests\\codegen\\vec-shrink-panik.rs", - ] - } -} diff --git a/src/tools/opt-dist/src/exec.rs b/src/tools/opt-dist/src/exec.rs index 4765dceb5dd..04e0184528a 100644 --- a/src/tools/opt-dist/src/exec.rs +++ b/src/tools/opt-dist/src/exec.rs @@ -96,7 +96,7 @@ pub struct Bootstrap { } impl Bootstrap { - pub fn build(env: &dyn Environment) -> Self { + pub fn build(env: &Environment) -> Self { let metrics_path = env.build_root().join("build").join("metrics.json"); let cmd = cmd(&[ env.python_binary(), @@ -114,7 +114,7 @@ impl Bootstrap { Self { cmd, metrics_path } } - pub fn dist(env: &dyn Environment, dist_args: &[String]) -> Self { + pub fn dist(env: &Environment, dist_args: &[String]) -> Self { let metrics_path = env.build_root().join("build").join("metrics.json"); let cmd = cmd(&dist_args.iter().map(|arg| arg.as_str()).collect::>()) .env("RUST_BACKTRACE", "full"); diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs index 484ca5b3b86..6a3bdd7cc75 100644 --- a/src/tools/opt-dist/src/main.rs +++ b/src/tools/opt-dist/src/main.rs @@ -1,17 +1,22 @@ use crate::bolt::{bolt_optimize, with_bolt_instrumented}; use anyhow::Context; +use camino::{Utf8Path, Utf8PathBuf}; +use clap::Parser; use log::LevelFilter; +use std::io::Cursor; +use std::time::Duration; use utils::io; +use zip::ZipArchive; -use crate::environment::{create_environment, Environment}; -use crate::exec::Bootstrap; +use crate::environment::{Environment, EnvironmentBuilder}; +use crate::exec::{cmd, Bootstrap}; use crate::tests::run_tests; use crate::timer::Timer; use crate::training::{gather_llvm_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles}; -use crate::utils::io::reset_directory; +use crate::utils::io::{copy_directory, move_directory, reset_directory}; use crate::utils::{ clear_llvm_files, format_env_variables, print_binary_sizes, print_free_disk_space, - with_log_group, + retry_action, with_log_group, }; mod bolt; @@ -23,24 +28,104 @@ mod timer; mod training; mod utils; +#[derive(clap::Parser, Debug)] +struct Args { + #[clap(subcommand)] + env: EnvironmentCmd, +} + +#[derive(clap::Parser, Clone, Debug)] +struct SharedArgs { + // Arguments passed to `x` to perform the final (dist) build. + build_args: Vec, +} + +#[derive(clap::Parser, Clone, Debug)] +enum EnvironmentCmd { + LinuxCi { + #[clap(flatten)] + shared: SharedArgs, + }, + WindowsCi { + #[clap(flatten)] + shared: SharedArgs, + }, +} + fn is_try_build() -> bool { std::env::var("DIST_TRY_BUILD").unwrap_or_else(|_| "0".to_string()) != "0" } +fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> { + let (env, args) = match args.env { + EnvironmentCmd::LinuxCi { shared } => { + let target_triple = + std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing"); + + let checkout_dir = Utf8PathBuf::from("/checkout"); + let env = EnvironmentBuilder::default() + .host_triple(target_triple) + .python_binary("python3".to_string()) + .checkout_dir(checkout_dir.clone()) + .host_llvm_dir(Utf8PathBuf::from("/rustroot")) + .artifact_dir(Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts")) + .build_dir(checkout_dir.join("obj")) + .shared_llvm(true) + .use_bolt(true) + .skipped_tests(vec![ + // Fails because of linker errors, as of June 2023. + "tests/ui/process/nofile-limit.rs", + ]) + .build()?; + // /tmp/rustc-perf comes from the x64 dist Dockerfile + with_log_group("Building rustc-perf", || { + Ok::<(), anyhow::Error>(copy_rustc_perf(&env, Utf8Path::new("/tmp/rustc-perf"))?) + })?; + + (env, shared.build_args) + } + EnvironmentCmd::WindowsCi { shared } => { + let target_triple = + std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing"); + + let checkout_dir: Utf8PathBuf = std::env::current_dir()?.try_into()?; + let env = EnvironmentBuilder::default() + .host_triple(target_triple) + .python_binary("python".to_string()) + .checkout_dir(checkout_dir.clone()) + .host_llvm_dir(checkout_dir.join("citools").join("clang-rust")) + .artifact_dir(checkout_dir.join("opt-artifacts")) + .build_dir(checkout_dir) + .shared_llvm(false) + .use_bolt(false) + .skipped_tests(vec![ + // Fails as of June 2023. + "tests\\codegen\\vec-shrink-panik.rs", + ]) + .build()?; + + with_log_group("Building rustc-perf", || { + Ok::<(), anyhow::Error>(download_rustc_perf(&env)?) + })?; + + (env, shared.build_args) + } + }; + Ok((env, args)) +} + fn execute_pipeline( - env: &dyn Environment, + env: &Environment, timer: &mut Timer, dist_args: Vec, ) -> anyhow::Result<()> { - reset_directory(&env.opt_artifacts())?; - - with_log_group("Building rustc-perf", || env.prepare_rustc_perf())?; + reset_directory(&env.artifact_dir())?; // Stage 1: Build PGO instrumented rustc // We use a normal build of LLVM, because gathering PGO profiles for LLVM and `rustc` at the // same time can cause issues, because the host and in-tree LLVM versions can diverge. let rustc_pgo_profile = timer.section("Stage 1 (Rustc PGO)", |stage| { - let rustc_profile_dir_root = env.opt_artifacts().join("rustc-pgo"); + let rustc_profile_dir_root = env.artifact_dir().join("rustc-pgo"); stage.section("Build PGO instrumented rustc and LLVM", |section| { let mut builder = Bootstrap::build(env).rustc_pgo_instrument(&rustc_profile_dir_root); @@ -74,7 +159,7 @@ fn execute_pipeline( // Remove the previous, uninstrumented build of LLVM. clear_llvm_files(env)?; - let llvm_profile_dir_root = env.opt_artifacts().join("llvm-pgo"); + let llvm_profile_dir_root = env.artifact_dir().join("llvm-pgo"); stage.section("Build PGO instrumented LLVM", |section| { Bootstrap::build(env) @@ -95,7 +180,7 @@ fn execute_pipeline( Ok(profile) })?; - let llvm_bolt_profile = if env.supports_bolt() { + let llvm_bolt_profile = if env.use_bolt() { // Stage 3: Build BOLT instrumented LLVM // We build a PGO optimized LLVM in this step, then instrument it with BOLT and gather BOLT profiles. // Note that we don't remove LLVM artifacts after this step, so that they are reused in the final dist build. @@ -171,10 +256,9 @@ fn main() -> anyhow::Result<()> { .parse_default_env() .init(); - let target_triple = std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing"); + let args = Args::parse(); - let mut build_args: Vec = std::env::args().skip(1).collect(); - println!("Running optimized build pipeline with args `{}`", build_args.join(" ")); + println!("Running optimized build pipeline with args `{:?}`", args); with_log_group("Environment values", || { println!("Environment values\n{}", format_env_variables()); @@ -186,6 +270,8 @@ fn main() -> anyhow::Result<()> { } }); + let (env, mut build_args) = create_environment(args).context("Cannot create environment")?; + // Skip components that are not needed for try builds to speed them up if is_try_build() { log::info!("Skipping building of unimportant components for a try build"); @@ -204,14 +290,56 @@ fn main() -> anyhow::Result<()> { } let mut timer = Timer::new(); - let env = create_environment(target_triple); - let result = execute_pipeline(env.as_ref(), &mut timer, build_args); + let result = execute_pipeline(&env, &mut timer, build_args); log::info!("Timer results\n{}", timer.format_stats()); print_free_disk_space()?; result.context("Optimized build pipeline has failed")?; - print_binary_sizes(env.as_ref())?; + print_binary_sizes(&env)?; Ok(()) } + +// Copy rustc-perf from the given path into the environment and build it. +fn copy_rustc_perf(env: &Environment, dir: &Utf8Path) -> anyhow::Result<()> { + copy_directory(dir, &env.rustc_perf_dir())?; + build_rustc_perf(env) +} + +// Download and build rustc-perf into the given environment. +fn download_rustc_perf(env: &Environment) -> anyhow::Result<()> { + // FIXME: add some mechanism for synchronization of this commit SHA with + // Linux (which builds rustc-perf in a Dockerfile) + // rustc-perf version from 2023-05-30 + const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1"; + + let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip"); + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(60 * 2)) + .connect_timeout(Duration::from_secs(60 * 2)) + .build()?; + let response = retry_action( + || Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()), + "Download rustc-perf archive", + 5, + )?; + + let mut archive = ZipArchive::new(Cursor::new(response))?; + archive.extract(env.rustc_perf_dir())?; + move_directory( + &env.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")), + &env.rustc_perf_dir(), + )?; + + build_rustc_perf(env) +} + +fn build_rustc_perf(env: &Environment) -> anyhow::Result<()> { + cmd(&[env.cargo_stage_0().as_str(), "build", "-p", "collector"]) + .workdir(&env.rustc_perf_dir()) + .env("RUSTC", &env.rustc_stage_0().into_string()) + .env("RUSTC_BOOTSTRAP", "1") + .run()?; + Ok(()) +} diff --git a/src/tools/opt-dist/src/tests.rs b/src/tools/opt-dist/src/tests.rs index e6b8df9c862..54f7185f88a 100644 --- a/src/tools/opt-dist/src/tests.rs +++ b/src/tools/opt-dist/src/tests.rs @@ -5,7 +5,7 @@ use anyhow::Context; use camino::{Utf8Path, Utf8PathBuf}; /// Run tests on optimized dist artifacts. -pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> { +pub fn run_tests(env: &Environment) -> anyhow::Result<()> { // After `dist` is executed, we extract its archived components into a sysroot directory, // and then use that extracted rustc as a stage0 compiler. // Then we run a subset of tests using that compiler, to have a basic smoke test which checks diff --git a/src/tools/opt-dist/src/training.rs b/src/tools/opt-dist/src/training.rs index 950dd1127a9..dd8aa532408 100644 --- a/src/tools/opt-dist/src/training.rs +++ b/src/tools/opt-dist/src/training.rs @@ -30,7 +30,7 @@ const RUSTC_PGO_CRATES: &[&str] = &[ const LLVM_BOLT_CRATES: &[&str] = LLVM_PGO_CRATES; fn init_compiler_benchmarks( - env: &dyn Environment, + env: &Environment, profiles: &[&str], scenarios: &[&str], crates: &[&str], @@ -75,7 +75,7 @@ enum LlvmProfdata { } fn merge_llvm_profiles( - env: &dyn Environment, + env: &Environment, merged_path: &Utf8Path, profile_dir: &Utf8Path, profdata: LlvmProfdata, @@ -116,7 +116,7 @@ fn log_profile_stats( pub struct LlvmPGOProfile(pub Utf8PathBuf); pub fn gather_llvm_profiles( - env: &dyn Environment, + env: &Environment, profile_root: &Utf8Path, ) -> anyhow::Result { log::info!("Running benchmarks with PGO instrumented LLVM"); @@ -127,7 +127,7 @@ pub fn gather_llvm_profiles( .context("Cannot gather LLVM PGO profiles") })?; - let merged_profile = env.opt_artifacts().join("llvm-pgo.profdata"); + let merged_profile = env.artifact_dir().join("llvm-pgo.profdata"); log::info!("Merging LLVM PGO profiles to {merged_profile}"); merge_llvm_profiles(env, &merged_profile, profile_root, LlvmProfdata::Host)?; @@ -143,7 +143,7 @@ pub fn gather_llvm_profiles( pub struct RustcPGOProfile(pub Utf8PathBuf); pub fn gather_rustc_profiles( - env: &dyn Environment, + env: &Environment, profile_root: &Utf8Path, ) -> anyhow::Result { log::info!("Running benchmarks with PGO instrumented rustc"); @@ -163,7 +163,7 @@ pub fn gather_rustc_profiles( .context("Cannot gather rustc PGO profiles") })?; - let merged_profile = env.opt_artifacts().join("rustc-pgo.profdata"); + let merged_profile = env.artifact_dir().join("rustc-pgo.profdata"); log::info!("Merging Rustc PGO profiles to {merged_profile}"); merge_llvm_profiles(env, &merged_profile, profile_root, LlvmProfdata::Target)?; @@ -178,7 +178,7 @@ pub fn gather_rustc_profiles( pub struct LlvmBoltProfile(pub Utf8PathBuf); -pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result { +pub fn gather_llvm_bolt_profiles(env: &Environment) -> anyhow::Result { log::info!("Running benchmarks with BOLT instrumented LLVM"); with_log_group("Running benchmarks", || { @@ -187,7 +187,7 @@ pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result anyhow::Result<()> { Ok(()) } -pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> { +pub fn print_binary_sizes(env: &Environment) -> anyhow::Result<()> { use std::fmt::Write; let root = env.build_artifacts().join("stage2"); @@ -48,7 +48,7 @@ pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> { Ok(()) } -pub fn clear_llvm_files(env: &dyn Environment) -> anyhow::Result<()> { +pub fn clear_llvm_files(env: &Environment) -> anyhow::Result<()> { // Bootstrap currently doesn't support rebuilding LLVM when PGO options // change (or any other llvm-related options); so just clear out the relevant // directories ourselves. From a2ed508f547e5538ba6bd882ef76687aad06f4db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 12 Sep 2023 19:32:46 +0200 Subject: [PATCH 4/8] Fix `reset_directory` function Before it was not deleting non-empty directories. --- src/tools/opt-dist/src/utils/io.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/opt-dist/src/utils/io.rs b/src/tools/opt-dist/src/utils/io.rs index 8bd516fa349..d24a1dc2d10 100644 --- a/src/tools/opt-dist/src/utils/io.rs +++ b/src/tools/opt-dist/src/utils/io.rs @@ -7,7 +7,7 @@ use std::path::Path; /// Delete and re-create the directory. pub fn reset_directory(path: &Utf8Path) -> anyhow::Result<()> { log::info!("Resetting directory {path}"); - let _ = std::fs::remove_dir(path); + let _ = std::fs::remove_dir_all(path); std::fs::create_dir_all(path)?; Ok(()) } From 11f9283da91ca4ab7fb634f3f586156f699007c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 12 Sep 2023 19:33:09 +0200 Subject: [PATCH 5/8] Add a Local environment to `opt-dist` This makes it easier to build a PGO/BOLT optimized `rustc` locally, outside of CI. --- src/tools/opt-dist/src/environment.rs | 7 +-- src/tools/opt-dist/src/main.rs | 75 ++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/src/tools/opt-dist/src/environment.rs b/src/tools/opt-dist/src/environment.rs index e730e610604..ed43f469c5e 100644 --- a/src/tools/opt-dist/src/environment.rs +++ b/src/tools/opt-dist/src/environment.rs @@ -7,7 +7,8 @@ pub struct Environment { python_binary: String, /// The rustc checkout, where the compiler source is located. checkout_dir: Utf8PathBuf, - /// The main directory where the build occurs. + /// The main directory where the build occurs. Stage0 rustc and cargo have to be available in + /// this directory before `opt-dist` is started. build_dir: Utf8PathBuf, /// Directory where the optimization artifacts (PGO/BOLT profiles, etc.) /// will be stored. @@ -15,7 +16,7 @@ pub struct Environment { /// Path to the host LLVM used to compile LLVM in `src/llvm-project`. host_llvm_dir: Utf8PathBuf, /// List of test paths that should be skipped when testing the optimized artifacts. - skipped_tests: Vec<&'static str>, + skipped_tests: Vec, use_bolt: bool, shared_llvm: bool, } @@ -83,7 +84,7 @@ impl Environment { self.shared_llvm } - pub fn skipped_tests(&self) -> &[&'static str] { + pub fn skipped_tests(&self) -> &[String] { &self.skipped_tests } } diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs index 6a3bdd7cc75..b5e123360f8 100644 --- a/src/tools/opt-dist/src/main.rs +++ b/src/tools/opt-dist/src/main.rs @@ -42,10 +42,51 @@ struct SharedArgs { #[derive(clap::Parser, Clone, Debug)] enum EnvironmentCmd { + /// Perform a custom local PGO/BOLT optimized build. + Local { + /// Target triple of the host. + #[arg(long)] + target_triple: String, + + /// Checkout directory of `rustc`. + #[arg(long)] + checkout_dir: Utf8PathBuf, + + /// Host LLVM installation directory. + #[arg(long)] + llvm_dir: Utf8PathBuf, + + /// Python binary to use in bootstrap invocations. + #[arg(long, default_value = "python3")] + python: String, + + /// Directory where artifacts (like PGO profiles or rustc-perf) of this workflow + /// will be stored. + #[arg(long, default_value = "opt-artifacts")] + artifact_dir: Utf8PathBuf, + + /// Is LLVM for `rustc` built in shared library mode? + #[arg(long, default_value_t = true)] + llvm_shared: bool, + + /// Should BOLT optimization be used? If yes, host LLVM must have BOLT binaries + /// (`llvm-bolt` and `merge-fdata`) available. + #[arg(long, default_value_t = false)] + use_bolt: bool, + + /// Tests that should be skipped when testing the optimized compiler. + #[arg(long)] + skipped_tests: Vec, + + #[clap(flatten)] + shared: SharedArgs, + }, + /// Perform an optimized build on Linux CI, from inside Docker. LinuxCi { #[clap(flatten)] shared: SharedArgs, }, + /// Perform an optimized build on Windows CI, directly inside Github Actions. WindowsCi { #[clap(flatten)] shared: SharedArgs, @@ -58,6 +99,34 @@ fn is_try_build() -> bool { fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> { let (env, args) = match args.env { + EnvironmentCmd::Local { + target_triple, + checkout_dir, + llvm_dir, + python, + artifact_dir, + llvm_shared, + use_bolt, + skipped_tests, + shared, + } => { + let env = EnvironmentBuilder::default() + .host_triple(target_triple) + .python_binary(python) + .checkout_dir(checkout_dir.clone()) + .host_llvm_dir(llvm_dir) + .artifact_dir(artifact_dir) + .build_dir(checkout_dir) + .shared_llvm(llvm_shared) + .use_bolt(use_bolt) + .skipped_tests(skipped_tests) + .build()?; + with_log_group("Building rustc-perf", || { + Ok::<(), anyhow::Error>(download_rustc_perf(&env)?) + })?; + + (env, shared.build_args) + } EnvironmentCmd::LinuxCi { shared } => { let target_triple = std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing"); @@ -74,7 +143,7 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> .use_bolt(true) .skipped_tests(vec![ // Fails because of linker errors, as of June 2023. - "tests/ui/process/nofile-limit.rs", + "tests/ui/process/nofile-limit.rs".to_string(), ]) .build()?; // /tmp/rustc-perf comes from the x64 dist Dockerfile @@ -100,7 +169,7 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> .use_bolt(false) .skipped_tests(vec![ // Fails as of June 2023. - "tests\\codegen\\vec-shrink-panik.rs", + "tests\\codegen\\vec-shrink-panik.rs".to_string(), ]) .build()?; @@ -309,6 +378,8 @@ fn copy_rustc_perf(env: &Environment, dir: &Utf8Path) -> anyhow::Result<()> { // Download and build rustc-perf into the given environment. fn download_rustc_perf(env: &Environment) -> anyhow::Result<()> { + reset_directory(&env.rustc_perf_dir())?; + // FIXME: add some mechanism for synchronization of this commit SHA with // Linux (which builds rustc-perf in a Dockerfile) // rustc-perf version from 2023-05-30 From 6c718b5b8a98ba756c69e8a2019131c6a514e64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 13 Sep 2023 18:11:34 +0200 Subject: [PATCH 6/8] Refactor rustc-perf building --- src/tools/opt-dist/src/environment.rs | 6 ++++++ src/tools/opt-dist/src/main.rs | 18 +++++++----------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/tools/opt-dist/src/environment.rs b/src/tools/opt-dist/src/environment.rs index ed43f469c5e..f7b5c176375 100644 --- a/src/tools/opt-dist/src/environment.rs +++ b/src/tools/opt-dist/src/environment.rs @@ -17,6 +17,8 @@ pub struct Environment { host_llvm_dir: Utf8PathBuf, /// List of test paths that should be skipped when testing the optimized artifacts. skipped_tests: Vec, + /// Directory containing a pre-built rustc-perf checkout. + prebuilt_rustc_perf: Option, use_bolt: bool, shared_llvm: bool, } @@ -67,6 +69,10 @@ impl Environment { .join(format!("rustc{}", executable_extension())) } + pub fn prebuilt_rustc_perf(&self) -> Option { + self.prebuilt_rustc_perf.clone() + } + /// Path to the built rustc-perf benchmark suite. pub fn rustc_perf_dir(&self) -> Utf8PathBuf { self.artifact_dir.join("rustc-perf") diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs index b5e123360f8..978e2dfa4e8 100644 --- a/src/tools/opt-dist/src/main.rs +++ b/src/tools/opt-dist/src/main.rs @@ -121,9 +121,6 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> .use_bolt(use_bolt) .skipped_tests(skipped_tests) .build()?; - with_log_group("Building rustc-perf", || { - Ok::<(), anyhow::Error>(download_rustc_perf(&env)?) - })?; (env, shared.build_args) } @@ -139,6 +136,8 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> .host_llvm_dir(Utf8PathBuf::from("/rustroot")) .artifact_dir(Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts")) .build_dir(checkout_dir.join("obj")) + // /tmp/rustc-perf comes from the x64 dist Dockerfile + .prebuilt_rustc_perf(Some(Utf8PathBuf::from("/tmp/rustc-perf"))) .shared_llvm(true) .use_bolt(true) .skipped_tests(vec![ @@ -146,10 +145,6 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> "tests/ui/process/nofile-limit.rs".to_string(), ]) .build()?; - // /tmp/rustc-perf comes from the x64 dist Dockerfile - with_log_group("Building rustc-perf", || { - Ok::<(), anyhow::Error>(copy_rustc_perf(&env, Utf8Path::new("/tmp/rustc-perf"))?) - })?; (env, shared.build_args) } @@ -173,10 +168,6 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> ]) .build()?; - with_log_group("Building rustc-perf", || { - Ok::<(), anyhow::Error>(download_rustc_perf(&env)?) - })?; - (env, shared.build_args) } }; @@ -190,6 +181,11 @@ fn execute_pipeline( ) -> anyhow::Result<()> { reset_directory(&env.artifact_dir())?; + with_log_group("Building rustc-perf", || match env.prebuilt_rustc_perf() { + Some(dir) => copy_rustc_perf(env, &dir), + None => download_rustc_perf(env), + })?; + // Stage 1: Build PGO instrumented rustc // We use a normal build of LLVM, because gathering PGO profiles for LLVM and `rustc` at the // same time can cause issues, because the host and in-tree LLVM versions can diverge. From f13b54546bb7aea9beee7d3f9178882a7f10f45c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 13 Sep 2023 18:12:41 +0200 Subject: [PATCH 7/8] Resolve clippy warnings --- src/tools/opt-dist/src/tests.rs | 6 +++--- src/tools/opt-dist/src/training.rs | 2 +- src/tools/opt-dist/src/utils/io.rs | 6 ++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/tools/opt-dist/src/tests.rs b/src/tools/opt-dist/src/tests.rs index 54f7185f88a..31aabca09f3 100644 --- a/src/tools/opt-dist/src/tests.rs +++ b/src/tools/opt-dist/src/tests.rs @@ -33,8 +33,8 @@ pub fn run_tests(env: &Environment) -> anyhow::Result<()> { // We need to manually copy libstd to the extracted rustc sysroot copy_directory( - &libstd_dir.join("lib").join("rustlib").join(&host_triple).join("lib"), - &rustc_dir.join("lib").join("rustlib").join(&host_triple).join("lib"), + &libstd_dir.join("lib").join("rustlib").join(host_triple).join("lib"), + &rustc_dir.join("lib").join("rustlib").join(host_triple).join("lib"), )?; // Extract sources - they aren't in the `rustc-nightly-{host}` tarball, so we need to manually copy libstd @@ -109,6 +109,6 @@ fn find_dist_version(directory: &Utf8Path) -> anyhow::Result { .unwrap() .to_string(); let (version, _) = - archive.strip_prefix("reproducible-artifacts-").unwrap().split_once("-").unwrap(); + archive.strip_prefix("reproducible-artifacts-").unwrap().split_once('-').unwrap(); Ok(version.to_string()) } diff --git a/src/tools/opt-dist/src/training.rs b/src/tools/opt-dist/src/training.rs index dd8aa532408..274f4cea0ab 100644 --- a/src/tools/opt-dist/src/training.rs +++ b/src/tools/opt-dist/src/training.rs @@ -192,7 +192,7 @@ pub fn gather_llvm_bolt_profiles(env: &Environment) -> anyhow::Result = - glob::glob(&format!("{profile_root}*"))?.into_iter().collect::, _>>()?; + glob::glob(&format!("{profile_root}*"))?.collect::, _>>()?; let mut merge_args = vec!["merge-fdata"]; merge_args.extend(profiles.iter().map(|p| p.to_str().unwrap())); diff --git a/src/tools/opt-dist/src/utils/io.rs b/src/tools/opt-dist/src/utils/io.rs index d24a1dc2d10..d6bd5cb85e4 100644 --- a/src/tools/opt-dist/src/utils/io.rs +++ b/src/tools/opt-dist/src/utils/io.rs @@ -63,7 +63,6 @@ pub fn get_files_from_dir( let path = format!("{dir}/*{}", suffix.unwrap_or("")); Ok(glob::glob(&path)? - .into_iter() .map(|p| p.map(|p| Utf8PathBuf::from_path_buf(p).unwrap())) .collect::, _>>()?) } @@ -74,9 +73,8 @@ pub fn find_file_in_dir( prefix: &str, suffix: &str, ) -> anyhow::Result { - let files = glob::glob(&format!("{directory}/{prefix}*{suffix}"))? - .into_iter() - .collect::, _>>()?; + let files = + glob::glob(&format!("{directory}/{prefix}*{suffix}"))?.collect::, _>>()?; match files.len() { 0 => Err(anyhow::anyhow!("No file with prefix {prefix} found in {directory}")), 1 => Ok(Utf8PathBuf::from_path_buf(files[0].clone()).unwrap()), From ee451f8faccf3050c76cdcd82543c917b40c7962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 18 Sep 2023 13:45:42 +0200 Subject: [PATCH 8/8] Fix build on Windows --- src/tools/opt-dist/src/environment.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tools/opt-dist/src/environment.rs b/src/tools/opt-dist/src/environment.rs index f7b5c176375..ff782a1687e 100644 --- a/src/tools/opt-dist/src/environment.rs +++ b/src/tools/opt-dist/src/environment.rs @@ -18,6 +18,7 @@ pub struct Environment { /// List of test paths that should be skipped when testing the optimized artifacts. skipped_tests: Vec, /// Directory containing a pre-built rustc-perf checkout. + #[builder(default)] prebuilt_rustc_perf: Option, use_bolt: bool, shared_llvm: bool,