nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-11-21 22:34:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Auto merge of #115795 - Kobzol:opt-dist-custom, r=Mark-Simulacrum

Browse Source 
Refactor `opt-dist` to simplify local building

This PR refactors the `opt-dist` tool to make it easier to invoke it locally, outside of CI, and thus simplify building PGO/BOLT optimized `rustc` builds e.g. for distro maintainers. It should also make it easier to run the PGO/BOLT workflow locally e.g. to profile performance or debug issues (looking at you, https://github.com/rust-lang/rust/pull/115554).
This commit is contained in:
bors 2023-09-18 14:26:40 +00:00
commit cbcf9a5368
15 changed files with 423 additions and 277 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/ci.yml vendored
View File

@ -393,7 +393,7 @@ jobs:
- name: dist-x86_64-msvc
env:
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler"
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
DIST_REQUIRE_ALL_TOOLS: 1
os: windows-2019-8core-32gb
- name: dist-i686-msvc

76
Cargo.lock
View File

@ -858,14 +858,38 @@ dependencies = [
"winapi",
]
[[package]]
name = "darling"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
dependencies = [
"darling_core 0.14.4",
"darling_macro 0.14.4",
]
[[package]]
name = "darling"
version = "0.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e"
dependencies = [
"darling_core",
"darling_macro",
"darling_core 0.20.3",
"darling_macro 0.20.3",
]
[[package]]
name = "darling_core"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn 1.0.109",
]
[[package]]
@ -882,13 +906,24 @@ dependencies = [
"syn 2.0.29",
]
[[package]]
name = "darling_macro"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
dependencies = [
"darling_core 0.14.4",
"quote",
"syn 1.0.109",
]
[[package]]
name = "darling_macro"
version = "0.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
dependencies = [
"darling_core",
"darling_core 0.20.3",
"quote",
"syn 2.0.29",
]
@ -919,6 +954,37 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "derive_builder"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8"
dependencies = [
"derive_builder_macro",
]
[[package]]
name = "derive_builder_core"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f"
dependencies = [
"darling 0.14.4",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "derive_builder_macro"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
dependencies = [
"derive_builder_core",
"syn 1.0.109",
]
[[package]]
name = "derive_more"
version = "0.99.17"
@ -938,7 +1004,7 @@ version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e8ef033054e131169b8f0f9a7af8f5533a9436fadf3c500ed547f730f07090d"
dependencies = [
"darling",
"darling 0.20.3",
"proc-macro2",
"quote",
"syn 2.0.29",
@ -2584,6 +2650,8 @@ dependencies = [
"anyhow",
"build_helper",
"camino",
"clap",
"derive_builder",
"env_logger 0.10.0",
"fs_extra",
"glob",

2
src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
View File

@ -87,7 +87,7 @@ ENV RUST_CONFIGURE_ARGS \
--set rust.lto=thin
ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \
./build/$HOSTS/stage0-tools-bin/opt-dist python3 ../x.py dist \
./build/$HOSTS/stage0-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \
--host $HOSTS --target $HOSTS \
--include-default-paths \
build-manifest bootstrap

2
src/ci/github-actions/ci.yml
View File

@ -624,7 +624,7 @@ jobs:
--target=x86_64-pc-windows-msvc
--enable-full-tools
--enable-profiler
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
DIST_REQUIRE_ALL_TOOLS: 1
<<: *job-windows-8c

2
src/tools/opt-dist/Cargo.toml
View File

@ -21,3 +21,5 @@ serde = { version = "1", features = ["derive"] }
serde_json = "1"
glob = "0.3"
tempfile = "3.5"
derive_builder = "0.12"
clap = { version = "4", features = ["derive"] }

108
src/tools/opt-dist/src/environment.rs Normal file
View File

@ -0,0 +1,108 @@
use camino::Utf8PathBuf;
use derive_builder::Builder;
#[derive(Builder)]
pub struct Environment {
host_triple: String,
python_binary: String,
/// The rustc checkout, where the compiler source is located.
checkout_dir: Utf8PathBuf,
/// The main directory where the build occurs. Stage0 rustc and cargo have to be available in
/// this directory before `opt-dist` is started.
build_dir: Utf8PathBuf,
/// Directory where the optimization artifacts (PGO/BOLT profiles, etc.)
/// will be stored.
artifact_dir: Utf8PathBuf,
/// Path to the host LLVM used to compile LLVM in `src/llvm-project`.
host_llvm_dir: Utf8PathBuf,
/// List of test paths that should be skipped when testing the optimized artifacts.
skipped_tests: Vec<String>,
/// Directory containing a pre-built rustc-perf checkout.
#[builder(default)]
prebuilt_rustc_perf: Option<Utf8PathBuf>,
use_bolt: bool,
shared_llvm: bool,
}
impl Environment {
pub fn host_triple(&self) -> &str {
&self.host_triple
}
pub fn python_binary(&self) -> &str {
&self.python_binary
}
pub fn checkout_path(&self) -> Utf8PathBuf {
self.checkout_dir.clone()
}
pub fn build_root(&self) -> Utf8PathBuf {
self.build_dir.clone()
}
pub fn build_artifacts(&self) -> Utf8PathBuf {
self.build_root().join("build").join(&self.host_triple)
}
pub fn artifact_dir(&self) -> Utf8PathBuf {
self.artifact_dir.clone()
}
pub fn cargo_stage_0(&self) -> Utf8PathBuf {
self.build_artifacts()
.join("stage0")
.join("bin")
.join(format!("cargo{}", executable_extension()))
}
pub fn rustc_stage_0(&self) -> Utf8PathBuf {
self.build_artifacts()
.join("stage0")
.join("bin")
.join(format!("rustc{}", executable_extension()))
}
pub fn rustc_stage_2(&self) -> Utf8PathBuf {
self.build_artifacts()
.join("stage2")
.join("bin")
.join(format!("rustc{}", executable_extension()))
}
pub fn prebuilt_rustc_perf(&self) -> Option<Utf8PathBuf> {
self.prebuilt_rustc_perf.clone()
}
/// Path to the built rustc-perf benchmark suite.
pub fn rustc_perf_dir(&self) -> Utf8PathBuf {
self.artifact_dir.join("rustc-perf")
}
pub fn host_llvm_dir(&self) -> Utf8PathBuf {
self.host_llvm_dir.clone()
}
pub fn use_bolt(&self) -> bool {
self.use_bolt
}
pub fn supports_shared_llvm(&self) -> bool {
self.shared_llvm
}
pub fn skipped_tests(&self) -> &[String] {
&self.skipped_tests
}
}
/// What is the extension of binary executables on this platform?
#[cfg(target_family = "unix")]
pub fn executable_extension() -> &'static str {
""
}
#[cfg(target_family = "windows")]
pub fn executable_extension() -> &'static str {
".exe"
}

58
src/tools/opt-dist/src/environment/linux.rs
View File

@ -1,58 +0,0 @@
use crate::environment::Environment;
use crate::exec::cmd;
use crate::utils::io::copy_directory;
use camino::{Utf8Path, Utf8PathBuf};
pub(super) struct LinuxEnvironment;
impl Environment for LinuxEnvironment {
fn python_binary(&self) -> &'static str {
"python3"
}
fn checkout_path(&self) -> Utf8PathBuf {
Utf8PathBuf::from("/checkout")
}
fn host_llvm_dir(&self) -> Utf8PathBuf {
Utf8PathBuf::from("/rustroot")
}
fn opt_artifacts(&self) -> Utf8PathBuf {
Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts")
}
fn build_root(&self) -> Utf8PathBuf {
self.checkout_path().join("obj")
}
fn prepare_rustc_perf(&self) -> anyhow::Result<()> {
// /tmp/rustc-perf comes from the x64 dist Dockerfile
copy_directory(Utf8Path::new("/tmp/rustc-perf"), &self.rustc_perf_dir())?;
cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"])
.workdir(&self.rustc_perf_dir())
.env("RUSTC", &self.rustc_stage_0().into_string())
.env("RUSTC_BOOTSTRAP", "1")
.run()?;
Ok(())
}
fn supports_bolt(&self) -> bool {
true
}
fn supports_shared_llvm(&self) -> bool {
true
}
fn executable_extension(&self) -> &'static str {
""
}
fn skipped_tests(&self) -> &'static [&'static str] {
&[
// Fails because of linker errors, as of June 2023.
"tests/ui/process/nofile-limit.rs",
]
}
}

77
src/tools/opt-dist/src/environment/mod.rs
View File

@ -1,77 +0,0 @@
use camino::Utf8PathBuf;
#[cfg(target_family = "unix")]
mod linux;
#[cfg(target_family = "windows")]
mod windows;
pub trait Environment {
fn host_triple(&self) -> String {
std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing")
}
fn python_binary(&self) -> &'static str;
/// The rustc checkout, where the compiler source is located.
fn checkout_path(&self) -> Utf8PathBuf;
/// Path to the host LLVM used to compile LLVM in `src/llvm-project`.
fn host_llvm_dir(&self) -> Utf8PathBuf;
/// Directory where the optimization artifacts (PGO/BOLT profiles, etc.)
/// will be stored.
fn opt_artifacts(&self) -> Utf8PathBuf;
/// The main directory where the build occurs.
fn build_root(&self) -> Utf8PathBuf;
fn build_artifacts(&self) -> Utf8PathBuf {
self.build_root().join("build").join(self.host_triple())
}
fn cargo_stage_0(&self) -> Utf8PathBuf {
self.build_artifacts()
.join("stage0")
.join("bin")
.join(format!("cargo{}", self.executable_extension()))
}
fn rustc_stage_0(&self) -> Utf8PathBuf {
self.build_artifacts()
.join("stage0")
.join("bin")
.join(format!("rustc{}", self.executable_extension()))
}
fn rustc_stage_2(&self) -> Utf8PathBuf {
self.build_artifacts()
.join("stage2")
.join("bin")
.join(format!("rustc{}", self.executable_extension()))
}
/// Path to the built rustc-perf benchmark suite.
fn rustc_perf_dir(&self) -> Utf8PathBuf {
self.opt_artifacts().join("rustc-perf")
}
/// Download and/or compile rustc-perf.
fn prepare_rustc_perf(&self) -> anyhow::Result<()>;
fn supports_bolt(&self) -> bool;
fn supports_shared_llvm(&self) -> bool;
/// What is the extension of binary executables in this environment?
fn executable_extension(&self) -> &'static str;
/// List of test paths that should be skipped when testing the optimized artifacts.
fn skipped_tests(&self) -> &'static [&'static str];
}
pub fn create_environment() -> Box<dyn Environment> {
#[cfg(target_family = "unix")]
return Box::new(linux::LinuxEnvironment);
#[cfg(target_family = "windows")]
return Box::new(windows::WindowsEnvironment::new());
}

92
src/tools/opt-dist/src/environment/windows.rs
View File

@ -1,92 +0,0 @@
use crate::environment::Environment;
use crate::exec::cmd;
use crate::utils::io::move_directory;
use crate::utils::retry_action;
use camino::Utf8PathBuf;
use std::io::Cursor;
use std::time::Duration;
use zip::ZipArchive;
pub(super) struct WindowsEnvironment {
checkout_dir: Utf8PathBuf,
}
impl WindowsEnvironment {
pub fn new() -> Self {
Self { checkout_dir: std::env::current_dir().unwrap().try_into().unwrap() }
}
}
impl Environment for WindowsEnvironment {
fn python_binary(&self) -> &'static str {
"python"
}
fn checkout_path(&self) -> Utf8PathBuf {
self.checkout_dir.clone()
}
fn host_llvm_dir(&self) -> Utf8PathBuf {
self.checkout_path().join("citools").join("clang-rust")
}
fn opt_artifacts(&self) -> Utf8PathBuf {
self.checkout_path().join("opt-artifacts")
}
fn build_root(&self) -> Utf8PathBuf {
self.checkout_path()
}
fn prepare_rustc_perf(&self) -> anyhow::Result<()> {
// FIXME: add some mechanism for synchronization of this commit SHA with
// Linux (which builds rustc-perf in a Dockerfile)
// rustc-perf version from 2023-05-30
const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1";
let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip");
let client = reqwest::blocking::Client::builder()
.timeout(Duration::from_secs(60 * 2))
.connect_timeout(Duration::from_secs(60 * 2))
.build()?;
let response = retry_action(
|| Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()),
"Download rustc-perf archive",
5,
)?;
let mut archive = ZipArchive::new(Cursor::new(response))?;
archive.extract(self.rustc_perf_dir())?;
move_directory(
&self.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")),
&self.rustc_perf_dir(),
)?;
cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"])
.workdir(&self.rustc_perf_dir())
.env("RUSTC", &self.rustc_stage_0().into_string())
.env("RUSTC_BOOTSTRAP", "1")
.run()?;
Ok(())
}
fn supports_bolt(&self) -> bool {
false
}
fn supports_shared_llvm(&self) -> bool {
false
}
fn executable_extension(&self) -> &'static str {
".exe"
}
fn skipped_tests(&self) -> &'static [&'static str] {
&[
// Fails as of June 2023.
"tests\\codegen\\vec-shrink-panik.rs",
]
}
}

4
src/tools/opt-dist/src/exec.rs
View File

@ -96,7 +96,7 @@ pub struct Bootstrap {
}
impl Bootstrap {
pub fn build(env: &dyn Environment) -> Self {
pub fn build(env: &Environment) -> Self {
let metrics_path = env.build_root().join("build").join("metrics.json");
let cmd = cmd(&[
env.python_binary(),
@ -114,7 +114,7 @@ impl Bootstrap {
Self { cmd, metrics_path }
}
pub fn dist(env: &dyn Environment, dist_args: &[String]) -> Self {
pub fn dist(env: &Environment, dist_args: &[String]) -> Self {
let metrics_path = env.build_root().join("build").join("metrics.json");
let cmd = cmd(&dist_args.iter().map(|arg| arg.as_str()).collect::<Vec<_>>())
.env("RUST_BACKTRACE", "full");

227
src/tools/opt-dist/src/main.rs
View File

@ -1,17 +1,22 @@
use crate::bolt::{bolt_optimize, with_bolt_instrumented};
use anyhow::Context;
use camino::{Utf8Path, Utf8PathBuf};
use clap::Parser;
use log::LevelFilter;
use std::io::Cursor;
use std::time::Duration;
use utils::io;
use zip::ZipArchive;
use crate::environment::{create_environment, Environment};
use crate::exec::Bootstrap;
use crate::environment::{Environment, EnvironmentBuilder};
use crate::exec::{cmd, Bootstrap};
use crate::tests::run_tests;
use crate::timer::Timer;
use crate::training::{gather_llvm_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles};
use crate::utils::io::reset_directory;
use crate::utils::io::{copy_directory, move_directory, reset_directory};
use crate::utils::{
clear_llvm_files, format_env_variables, print_binary_sizes, print_free_disk_space,
with_log_group,
retry_action, with_log_group,
};
mod bolt;
@ -23,24 +28,169 @@ mod timer;
mod training;
mod utils;
#[derive(clap::Parser, Debug)]
struct Args {
#[clap(subcommand)]
env: EnvironmentCmd,
}
#[derive(clap::Parser, Clone, Debug)]
struct SharedArgs {
// Arguments passed to `x` to perform the final (dist) build.
build_args: Vec<String>,
}
#[derive(clap::Parser, Clone, Debug)]
enum EnvironmentCmd {
/// Perform a custom local PGO/BOLT optimized build.
Local {
/// Target triple of the host.
#[arg(long)]
target_triple: String,
/// Checkout directory of `rustc`.
#[arg(long)]
checkout_dir: Utf8PathBuf,
/// Host LLVM installation directory.
#[arg(long)]
llvm_dir: Utf8PathBuf,
/// Python binary to use in bootstrap invocations.
#[arg(long, default_value = "python3")]
python: String,
/// Directory where artifacts (like PGO profiles or rustc-perf) of this workflow
/// will be stored.
#[arg(long, default_value = "opt-artifacts")]
artifact_dir: Utf8PathBuf,
/// Is LLVM for `rustc` built in shared library mode?
#[arg(long, default_value_t = true)]
llvm_shared: bool,
/// Should BOLT optimization be used? If yes, host LLVM must have BOLT binaries
/// (`llvm-bolt` and `merge-fdata`) available.
#[arg(long, default_value_t = false)]
use_bolt: bool,
/// Tests that should be skipped when testing the optimized compiler.
#[arg(long)]
skipped_tests: Vec<String>,
#[clap(flatten)]
shared: SharedArgs,
},
/// Perform an optimized build on Linux CI, from inside Docker.
LinuxCi {
#[clap(flatten)]
shared: SharedArgs,
},
/// Perform an optimized build on Windows CI, directly inside Github Actions.
WindowsCi {
#[clap(flatten)]
shared: SharedArgs,
},
}
fn is_try_build() -> bool {
std::env::var("DIST_TRY_BUILD").unwrap_or_else(|_| "0".to_string()) != "0"
}
fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec<String>)> {
let (env, args) = match args.env {
EnvironmentCmd::Local {
target_triple,
checkout_dir,
llvm_dir,
python,
artifact_dir,
llvm_shared,
use_bolt,
skipped_tests,
shared,
} => {
let env = EnvironmentBuilder::default()
.host_triple(target_triple)
.python_binary(python)
.checkout_dir(checkout_dir.clone())
.host_llvm_dir(llvm_dir)
.artifact_dir(artifact_dir)
.build_dir(checkout_dir)
.shared_llvm(llvm_shared)
.use_bolt(use_bolt)
.skipped_tests(skipped_tests)
.build()?;
(env, shared.build_args)
}
EnvironmentCmd::LinuxCi { shared } => {
let target_triple =
std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing");
let checkout_dir = Utf8PathBuf::from("/checkout");
let env = EnvironmentBuilder::default()
.host_triple(target_triple)
.python_binary("python3".to_string())
.checkout_dir(checkout_dir.clone())
.host_llvm_dir(Utf8PathBuf::from("/rustroot"))
.artifact_dir(Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts"))
.build_dir(checkout_dir.join("obj"))
// /tmp/rustc-perf comes from the x64 dist Dockerfile
.prebuilt_rustc_perf(Some(Utf8PathBuf::from("/tmp/rustc-perf")))
.shared_llvm(true)
.use_bolt(true)
.skipped_tests(vec![
// Fails because of linker errors, as of June 2023.
"tests/ui/process/nofile-limit.rs".to_string(),
])
.build()?;
(env, shared.build_args)
}
EnvironmentCmd::WindowsCi { shared } => {
let target_triple =
std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing");
let checkout_dir: Utf8PathBuf = std::env::current_dir()?.try_into()?;
let env = EnvironmentBuilder::default()
.host_triple(target_triple)
.python_binary("python".to_string())
.checkout_dir(checkout_dir.clone())
.host_llvm_dir(checkout_dir.join("citools").join("clang-rust"))
.artifact_dir(checkout_dir.join("opt-artifacts"))
.build_dir(checkout_dir)
.shared_llvm(false)
.use_bolt(false)
.skipped_tests(vec![
// Fails as of June 2023.
"tests\\codegen\\vec-shrink-panik.rs".to_string(),
])
.build()?;
(env, shared.build_args)
}
};
Ok((env, args))
}
fn execute_pipeline(
env: &dyn Environment,
env: &Environment,
timer: &mut Timer,
dist_args: Vec<String>,
) -> anyhow::Result<()> {
reset_directory(&env.opt_artifacts())?;
reset_directory(&env.artifact_dir())?;
with_log_group("Building rustc-perf", || env.prepare_rustc_perf())?;
with_log_group("Building rustc-perf", || match env.prebuilt_rustc_perf() {
Some(dir) => copy_rustc_perf(env, &dir),
None => download_rustc_perf(env),
})?;
// Stage 1: Build PGO instrumented rustc
// We use a normal build of LLVM, because gathering PGO profiles for LLVM and `rustc` at the
// same time can cause issues, because the host and in-tree LLVM versions can diverge.
let rustc_pgo_profile = timer.section("Stage 1 (Rustc PGO)", |stage| {
let rustc_profile_dir_root = env.opt_artifacts().join("rustc-pgo");
let rustc_profile_dir_root = env.artifact_dir().join("rustc-pgo");
stage.section("Build PGO instrumented rustc and LLVM", |section| {
let mut builder = Bootstrap::build(env).rustc_pgo_instrument(&rustc_profile_dir_root);
@ -74,7 +224,7 @@ fn execute_pipeline(
// Remove the previous, uninstrumented build of LLVM.
clear_llvm_files(env)?;
let llvm_profile_dir_root = env.opt_artifacts().join("llvm-pgo");
let llvm_profile_dir_root = env.artifact_dir().join("llvm-pgo");
stage.section("Build PGO instrumented LLVM", |section| {
Bootstrap::build(env)
@ -95,7 +245,7 @@ fn execute_pipeline(
Ok(profile)
})?;
let llvm_bolt_profile = if env.supports_bolt() {
let llvm_bolt_profile = if env.use_bolt() {
// Stage 3: Build BOLT instrumented LLVM
// We build a PGO optimized LLVM in this step, then instrument it with BOLT and gather BOLT profiles.
// Note that we don't remove LLVM artifacts after this step, so that they are reused in the final dist build.
@ -171,8 +321,9 @@ fn main() -> anyhow::Result<()> {
.parse_default_env()
.init();
let mut build_args: Vec<String> = std::env::args().skip(1).collect();
println!("Running optimized build pipeline with args `{}`", build_args.join(" "));
let args = Args::parse();
println!("Running optimized build pipeline with args `{:?}`", args);
with_log_group("Environment values", || {
println!("Environment values\n{}", format_env_variables());
@ -184,6 +335,8 @@ fn main() -> anyhow::Result<()> {
}
});
let (env, mut build_args) = create_environment(args).context("Cannot create environment")?;
// Skip components that are not needed for try builds to speed them up
if is_try_build() {
log::info!("Skipping building of unimportant components for a try build");
@ -202,14 +355,58 @@ fn main() -> anyhow::Result<()> {
}
let mut timer = Timer::new();
let env = create_environment();
let result = execute_pipeline(env.as_ref(), &mut timer, build_args);
let result = execute_pipeline(&env, &mut timer, build_args);
log::info!("Timer results\n{}", timer.format_stats());
print_free_disk_space()?;
result.context("Optimized build pipeline has failed")?;
print_binary_sizes(env.as_ref())?;
print_binary_sizes(&env)?;
Ok(())
}
// Copy rustc-perf from the given path into the environment and build it.
fn copy_rustc_perf(env: &Environment, dir: &Utf8Path) -> anyhow::Result<()> {
copy_directory(dir, &env.rustc_perf_dir())?;
build_rustc_perf(env)
}
// Download and build rustc-perf into the given environment.
fn download_rustc_perf(env: &Environment) -> anyhow::Result<()> {
reset_directory(&env.rustc_perf_dir())?;
// FIXME: add some mechanism for synchronization of this commit SHA with
// Linux (which builds rustc-perf in a Dockerfile)
// rustc-perf version from 2023-05-30
const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1";
let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip");
let client = reqwest::blocking::Client::builder()
.timeout(Duration::from_secs(60 * 2))
.connect_timeout(Duration::from_secs(60 * 2))
.build()?;
let response = retry_action(
|| Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()),
"Download rustc-perf archive",
5,
)?;
let mut archive = ZipArchive::new(Cursor::new(response))?;
archive.extract(env.rustc_perf_dir())?;
move_directory(
&env.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")),
&env.rustc_perf_dir(),
)?;
build_rustc_perf(env)
}
fn build_rustc_perf(env: &Environment) -> anyhow::Result<()> {
cmd(&[env.cargo_stage_0().as_str(), "build", "-p", "collector"])
.workdir(&env.rustc_perf_dir())
.env("RUSTC", &env.rustc_stage_0().into_string())
.env("RUSTC_BOOTSTRAP", "1")
.run()?;
Ok(())
}

16
src/tools/opt-dist/src/tests.rs
View File

@ -1,11 +1,11 @@
use crate::environment::Environment;
use crate::environment::{executable_extension, Environment};
use crate::exec::cmd;
use crate::utils::io::{copy_directory, find_file_in_dir, unpack_archive};
use anyhow::Context;
use camino::{Utf8Path, Utf8PathBuf};
/// Run tests on optimized dist artifacts.
pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> {
pub fn run_tests(env: &Environment) -> anyhow::Result<()> {
// After `dist` is executed, we extract its archived components into a sysroot directory,
// and then use that extracted rustc as a stage0 compiler.
// Then we run a subset of tests using that compiler, to have a basic smoke test which checks
@ -33,8 +33,8 @@ pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> {
// We need to manually copy libstd to the extracted rustc sysroot
copy_directory(
&libstd_dir.join("lib").join("rustlib").join(&host_triple).join("lib"),
&rustc_dir.join("lib").join("rustlib").join(&host_triple).join("lib"),
&libstd_dir.join("lib").join("rustlib").join(host_triple).join("lib"),
&rustc_dir.join("lib").join("rustlib").join(host_triple).join("lib"),
)?;
// Extract sources - they aren't in the `rustc-nightly-{host}` tarball, so we need to manually copy libstd
@ -45,9 +45,9 @@ pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> {
&rustc_dir.join("lib").join("rustlib").join("src"),
)?;
let rustc_path = rustc_dir.join("bin").join(format!("rustc{}", env.executable_extension()));
let rustc_path = rustc_dir.join("bin").join(format!("rustc{}", executable_extension()));
assert!(rustc_path.is_file());
let cargo_path = cargo_dir.join("bin").join(format!("cargo{}", env.executable_extension()));
let cargo_path = cargo_dir.join("bin").join(format!("cargo{}", executable_extension()));
assert!(cargo_path.is_file());
// Specify path to a LLVM config so that LLVM is not rebuilt.
@ -56,7 +56,7 @@ pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> {
.build_artifacts()
.join("llvm")
.join("bin")
.join(format!("llvm-config{}", env.executable_extension()));
.join(format!("llvm-config{}", executable_extension()));
assert!(llvm_config.is_file());
let config_content = format!(
@ -109,6 +109,6 @@ fn find_dist_version(directory: &Utf8Path) -> anyhow::Result<String> {
.unwrap()
.to_string();
let (version, _) =
archive.strip_prefix("reproducible-artifacts-").unwrap().split_once("-").unwrap();
archive.strip_prefix("reproducible-artifacts-").unwrap().split_once('-').unwrap();
Ok(version.to_string())
}

22
src/tools/opt-dist/src/training.rs
View File

@ -1,4 +1,4 @@
use crate::environment::Environment;
use crate::environment::{executable_extension, Environment};
use crate::exec::{cmd, CmdBuilder};
use crate::utils::io::{count_files, delete_directory};
use crate::utils::with_log_group;
@ -30,7 +30,7 @@ const RUSTC_PGO_CRATES: &[&str] = &[
const LLVM_BOLT_CRATES: &[&str] = LLVM_PGO_CRATES;
fn init_compiler_benchmarks(
env: &dyn Environment,
env: &Environment,
profiles: &[&str],
scenarios: &[&str],
crates: &[&str],
@ -75,7 +75,7 @@ enum LlvmProfdata {
}
fn merge_llvm_profiles(
env: &dyn Environment,
env: &Environment,
merged_path: &Utf8Path,
profile_dir: &Utf8Path,
profdata: LlvmProfdata,
@ -86,7 +86,7 @@ fn merge_llvm_profiles(
.build_artifacts()
.join("llvm")
.join("build")
.join(format!("bin/llvm-profdata{}", env.executable_extension())),
.join(format!("bin/llvm-profdata{}", executable_extension())),
};
cmd(&[llvm_profdata.as_str(), "merge", "-o", merged_path.as_str(), profile_dir.as_str()])
@ -116,7 +116,7 @@ fn log_profile_stats(
pub struct LlvmPGOProfile(pub Utf8PathBuf);
pub fn gather_llvm_profiles(
env: &dyn Environment,
env: &Environment,
profile_root: &Utf8Path,
) -> anyhow::Result<LlvmPGOProfile> {
log::info!("Running benchmarks with PGO instrumented LLVM");
@ -127,7 +127,7 @@ pub fn gather_llvm_profiles(
.context("Cannot gather LLVM PGO profiles")
})?;
let merged_profile = env.opt_artifacts().join("llvm-pgo.profdata");
let merged_profile = env.artifact_dir().join("llvm-pgo.profdata");
log::info!("Merging LLVM PGO profiles to {merged_profile}");
merge_llvm_profiles(env, &merged_profile, profile_root, LlvmProfdata::Host)?;
@ -143,7 +143,7 @@ pub fn gather_llvm_profiles(
pub struct RustcPGOProfile(pub Utf8PathBuf);
pub fn gather_rustc_profiles(
env: &dyn Environment,
env: &Environment,
profile_root: &Utf8Path,
) -> anyhow::Result<RustcPGOProfile> {
log::info!("Running benchmarks with PGO instrumented rustc");
@ -163,7 +163,7 @@ pub fn gather_rustc_profiles(
.context("Cannot gather rustc PGO profiles")
})?;
let merged_profile = env.opt_artifacts().join("rustc-pgo.profdata");
let merged_profile = env.artifact_dir().join("rustc-pgo.profdata");
log::info!("Merging Rustc PGO profiles to {merged_profile}");
merge_llvm_profiles(env, &merged_profile, profile_root, LlvmProfdata::Target)?;
@ -178,7 +178,7 @@ pub fn gather_rustc_profiles(
pub struct LlvmBoltProfile(pub Utf8PathBuf);
pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result<LlvmBoltProfile> {
pub fn gather_llvm_bolt_profiles(env: &Environment) -> anyhow::Result<LlvmBoltProfile> {
log::info!("Running benchmarks with BOLT instrumented LLVM");
with_log_group("Running benchmarks", || {
@ -187,12 +187,12 @@ pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result<LlvmBo
.context("Cannot gather LLVM BOLT profiles")
})?;
let merged_profile = env.opt_artifacts().join("llvm-bolt.profdata");
let merged_profile = env.artifact_dir().join("llvm-bolt.profdata");
let profile_root = Utf8PathBuf::from("/tmp/prof.fdata");
log::info!("Merging LLVM BOLT profiles to {merged_profile}");
let profiles: Vec<_> =
glob::glob(&format!("{profile_root}*"))?.into_iter().collect::<Result<Vec<_>, _>>()?;
glob::glob(&format!("{profile_root}*"))?.collect::<Result<Vec<_>, _>>()?;
let mut merge_args = vec!["merge-fdata"];
merge_args.extend(profiles.iter().map(|p| p.to_str().unwrap()));

8
src/tools/opt-dist/src/utils/io.rs
View File

@ -7,7 +7,7 @@ use std::path::Path;
/// Delete and re-create the directory.
pub fn reset_directory(path: &Utf8Path) -> anyhow::Result<()> {
log::info!("Resetting directory {path}");
let _ = std::fs::remove_dir(path);
let _ = std::fs::remove_dir_all(path);
std::fs::create_dir_all(path)?;
Ok(())
}
@ -63,7 +63,6 @@ pub fn get_files_from_dir(
let path = format!("{dir}/*{}", suffix.unwrap_or(""));
Ok(glob::glob(&path)?
.into_iter()
.map(|p| p.map(|p| Utf8PathBuf::from_path_buf(p).unwrap()))
.collect::<Result<Vec<_>, _>>()?)
}
@ -74,9 +73,8 @@ pub fn find_file_in_dir(
prefix: &str,
suffix: &str,
) -> anyhow::Result<Utf8PathBuf> {
let files = glob::glob(&format!("{directory}/{prefix}*{suffix}"))?
.into_iter()
.collect::<Result<Vec<_>, _>>()?;
let files =
glob::glob(&format!("{directory}/{prefix}*{suffix}"))?.collect::<Result<Vec<_>, _>>()?;
match files.len() {
0 => Err(anyhow::anyhow!("No file with prefix {prefix} found in {directory}")),
1 => Ok(Utf8PathBuf::from_path_buf(files[0].clone()).unwrap()),

4
src/tools/opt-dist/src/utils/mod.rs
View File

@ -26,7 +26,7 @@ pub fn print_free_disk_space() -> anyhow::Result<()> {
Ok(())
}
pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> {
pub fn print_binary_sizes(env: &Environment) -> anyhow::Result<()> {
use std::fmt::Write;
let root = env.build_artifacts().join("stage2");
@ -48,7 +48,7 @@ pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> {
Ok(())
}
pub fn clear_llvm_files(env: &dyn Environment) -> anyhow::Result<()> {
pub fn clear_llvm_files(env: &Environment) -> anyhow::Result<()> {
// Bootstrap currently doesn't support rebuilding LLVM when PGO options
// change (or any other llvm-related options); so just clear out the relevant
// directories ourselves.