mirror of
https://github.com/rust-lang/rust.git
synced 2024-11-22 06:44:35 +00:00
Auto merge of #115795 - Kobzol:opt-dist-custom, r=Mark-Simulacrum
Refactor `opt-dist` to simplify local building This PR refactors the `opt-dist` tool to make it easier to invoke it locally, outside of CI, and thus simplify building PGO/BOLT optimized `rustc` builds e.g. for distro maintainers. It should also make it easier to run the PGO/BOLT workflow locally e.g. to profile performance or debug issues (looking at you, https://github.com/rust-lang/rust/pull/115554).
This commit is contained in:
commit
cbcf9a5368
2
.github/workflows/ci.yml
vendored
2
.github/workflows/ci.yml
vendored
@ -393,7 +393,7 @@ jobs:
|
||||
- name: dist-x86_64-msvc
|
||||
env:
|
||||
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler"
|
||||
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths
|
||||
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
|
||||
DIST_REQUIRE_ALL_TOOLS: 1
|
||||
os: windows-2019-8core-32gb
|
||||
- name: dist-i686-msvc
|
||||
|
76
Cargo.lock
76
Cargo.lock
@ -858,14 +858,38 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
|
||||
dependencies = [
|
||||
"darling_core 0.14.4",
|
||||
"darling_macro 0.14.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.20.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"darling_macro",
|
||||
"darling_core 0.20.3",
|
||||
"darling_macro 0.20.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_core"
|
||||
version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
|
||||
dependencies = [
|
||||
"fnv",
|
||||
"ident_case",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"strsim",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -882,13 +906,24 @@ dependencies = [
|
||||
"syn 2.0.29",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_macro"
|
||||
version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
|
||||
dependencies = [
|
||||
"darling_core 0.14.4",
|
||||
"quote",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_macro"
|
||||
version = "0.20.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"darling_core 0.20.3",
|
||||
"quote",
|
||||
"syn 2.0.29",
|
||||
]
|
||||
@ -919,6 +954,37 @@ dependencies = [
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8"
|
||||
dependencies = [
|
||||
"derive_builder_macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder_core"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f"
|
||||
dependencies = [
|
||||
"darling 0.14.4",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder_macro"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
|
||||
dependencies = [
|
||||
"derive_builder_core",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_more"
|
||||
version = "0.99.17"
|
||||
@ -938,7 +1004,7 @@ version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e8ef033054e131169b8f0f9a7af8f5533a9436fadf3c500ed547f730f07090d"
|
||||
dependencies = [
|
||||
"darling",
|
||||
"darling 0.20.3",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.29",
|
||||
@ -2584,6 +2650,8 @@ dependencies = [
|
||||
"anyhow",
|
||||
"build_helper",
|
||||
"camino",
|
||||
"clap",
|
||||
"derive_builder",
|
||||
"env_logger 0.10.0",
|
||||
"fs_extra",
|
||||
"glob",
|
||||
|
@ -87,7 +87,7 @@ ENV RUST_CONFIGURE_ARGS \
|
||||
--set rust.lto=thin
|
||||
|
||||
ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \
|
||||
./build/$HOSTS/stage0-tools-bin/opt-dist python3 ../x.py dist \
|
||||
./build/$HOSTS/stage0-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \
|
||||
--host $HOSTS --target $HOSTS \
|
||||
--include-default-paths \
|
||||
build-manifest bootstrap
|
||||
|
@ -624,7 +624,7 @@ jobs:
|
||||
--target=x86_64-pc-windows-msvc
|
||||
--enable-full-tools
|
||||
--enable-profiler
|
||||
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths
|
||||
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
|
||||
DIST_REQUIRE_ALL_TOOLS: 1
|
||||
<<: *job-windows-8c
|
||||
|
||||
|
@ -21,3 +21,5 @@ serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
glob = "0.3"
|
||||
tempfile = "3.5"
|
||||
derive_builder = "0.12"
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
|
108
src/tools/opt-dist/src/environment.rs
Normal file
108
src/tools/opt-dist/src/environment.rs
Normal file
@ -0,0 +1,108 @@
|
||||
use camino::Utf8PathBuf;
|
||||
use derive_builder::Builder;
|
||||
|
||||
#[derive(Builder)]
|
||||
pub struct Environment {
|
||||
host_triple: String,
|
||||
python_binary: String,
|
||||
/// The rustc checkout, where the compiler source is located.
|
||||
checkout_dir: Utf8PathBuf,
|
||||
/// The main directory where the build occurs. Stage0 rustc and cargo have to be available in
|
||||
/// this directory before `opt-dist` is started.
|
||||
build_dir: Utf8PathBuf,
|
||||
/// Directory where the optimization artifacts (PGO/BOLT profiles, etc.)
|
||||
/// will be stored.
|
||||
artifact_dir: Utf8PathBuf,
|
||||
/// Path to the host LLVM used to compile LLVM in `src/llvm-project`.
|
||||
host_llvm_dir: Utf8PathBuf,
|
||||
/// List of test paths that should be skipped when testing the optimized artifacts.
|
||||
skipped_tests: Vec<String>,
|
||||
/// Directory containing a pre-built rustc-perf checkout.
|
||||
#[builder(default)]
|
||||
prebuilt_rustc_perf: Option<Utf8PathBuf>,
|
||||
use_bolt: bool,
|
||||
shared_llvm: bool,
|
||||
}
|
||||
|
||||
impl Environment {
|
||||
pub fn host_triple(&self) -> &str {
|
||||
&self.host_triple
|
||||
}
|
||||
|
||||
pub fn python_binary(&self) -> &str {
|
||||
&self.python_binary
|
||||
}
|
||||
|
||||
pub fn checkout_path(&self) -> Utf8PathBuf {
|
||||
self.checkout_dir.clone()
|
||||
}
|
||||
|
||||
pub fn build_root(&self) -> Utf8PathBuf {
|
||||
self.build_dir.clone()
|
||||
}
|
||||
|
||||
pub fn build_artifacts(&self) -> Utf8PathBuf {
|
||||
self.build_root().join("build").join(&self.host_triple)
|
||||
}
|
||||
|
||||
pub fn artifact_dir(&self) -> Utf8PathBuf {
|
||||
self.artifact_dir.clone()
|
||||
}
|
||||
|
||||
pub fn cargo_stage_0(&self) -> Utf8PathBuf {
|
||||
self.build_artifacts()
|
||||
.join("stage0")
|
||||
.join("bin")
|
||||
.join(format!("cargo{}", executable_extension()))
|
||||
}
|
||||
|
||||
pub fn rustc_stage_0(&self) -> Utf8PathBuf {
|
||||
self.build_artifacts()
|
||||
.join("stage0")
|
||||
.join("bin")
|
||||
.join(format!("rustc{}", executable_extension()))
|
||||
}
|
||||
|
||||
pub fn rustc_stage_2(&self) -> Utf8PathBuf {
|
||||
self.build_artifacts()
|
||||
.join("stage2")
|
||||
.join("bin")
|
||||
.join(format!("rustc{}", executable_extension()))
|
||||
}
|
||||
|
||||
pub fn prebuilt_rustc_perf(&self) -> Option<Utf8PathBuf> {
|
||||
self.prebuilt_rustc_perf.clone()
|
||||
}
|
||||
|
||||
/// Path to the built rustc-perf benchmark suite.
|
||||
pub fn rustc_perf_dir(&self) -> Utf8PathBuf {
|
||||
self.artifact_dir.join("rustc-perf")
|
||||
}
|
||||
|
||||
pub fn host_llvm_dir(&self) -> Utf8PathBuf {
|
||||
self.host_llvm_dir.clone()
|
||||
}
|
||||
|
||||
pub fn use_bolt(&self) -> bool {
|
||||
self.use_bolt
|
||||
}
|
||||
|
||||
pub fn supports_shared_llvm(&self) -> bool {
|
||||
self.shared_llvm
|
||||
}
|
||||
|
||||
pub fn skipped_tests(&self) -> &[String] {
|
||||
&self.skipped_tests
|
||||
}
|
||||
}
|
||||
|
||||
/// What is the extension of binary executables on this platform?
|
||||
#[cfg(target_family = "unix")]
|
||||
pub fn executable_extension() -> &'static str {
|
||||
""
|
||||
}
|
||||
|
||||
#[cfg(target_family = "windows")]
|
||||
pub fn executable_extension() -> &'static str {
|
||||
".exe"
|
||||
}
|
@ -1,58 +0,0 @@
|
||||
use crate::environment::Environment;
|
||||
use crate::exec::cmd;
|
||||
use crate::utils::io::copy_directory;
|
||||
use camino::{Utf8Path, Utf8PathBuf};
|
||||
|
||||
pub(super) struct LinuxEnvironment;
|
||||
|
||||
impl Environment for LinuxEnvironment {
|
||||
fn python_binary(&self) -> &'static str {
|
||||
"python3"
|
||||
}
|
||||
|
||||
fn checkout_path(&self) -> Utf8PathBuf {
|
||||
Utf8PathBuf::from("/checkout")
|
||||
}
|
||||
|
||||
fn host_llvm_dir(&self) -> Utf8PathBuf {
|
||||
Utf8PathBuf::from("/rustroot")
|
||||
}
|
||||
|
||||
fn opt_artifacts(&self) -> Utf8PathBuf {
|
||||
Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts")
|
||||
}
|
||||
|
||||
fn build_root(&self) -> Utf8PathBuf {
|
||||
self.checkout_path().join("obj")
|
||||
}
|
||||
|
||||
fn prepare_rustc_perf(&self) -> anyhow::Result<()> {
|
||||
// /tmp/rustc-perf comes from the x64 dist Dockerfile
|
||||
copy_directory(Utf8Path::new("/tmp/rustc-perf"), &self.rustc_perf_dir())?;
|
||||
cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"])
|
||||
.workdir(&self.rustc_perf_dir())
|
||||
.env("RUSTC", &self.rustc_stage_0().into_string())
|
||||
.env("RUSTC_BOOTSTRAP", "1")
|
||||
.run()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn supports_bolt(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn supports_shared_llvm(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn executable_extension(&self) -> &'static str {
|
||||
""
|
||||
}
|
||||
|
||||
fn skipped_tests(&self) -> &'static [&'static str] {
|
||||
&[
|
||||
// Fails because of linker errors, as of June 2023.
|
||||
"tests/ui/process/nofile-limit.rs",
|
||||
]
|
||||
}
|
||||
}
|
@ -1,77 +0,0 @@
|
||||
use camino::Utf8PathBuf;
|
||||
|
||||
#[cfg(target_family = "unix")]
|
||||
mod linux;
|
||||
#[cfg(target_family = "windows")]
|
||||
mod windows;
|
||||
|
||||
pub trait Environment {
|
||||
fn host_triple(&self) -> String {
|
||||
std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing")
|
||||
}
|
||||
|
||||
fn python_binary(&self) -> &'static str;
|
||||
|
||||
/// The rustc checkout, where the compiler source is located.
|
||||
fn checkout_path(&self) -> Utf8PathBuf;
|
||||
|
||||
/// Path to the host LLVM used to compile LLVM in `src/llvm-project`.
|
||||
fn host_llvm_dir(&self) -> Utf8PathBuf;
|
||||
|
||||
/// Directory where the optimization artifacts (PGO/BOLT profiles, etc.)
|
||||
/// will be stored.
|
||||
fn opt_artifacts(&self) -> Utf8PathBuf;
|
||||
|
||||
/// The main directory where the build occurs.
|
||||
fn build_root(&self) -> Utf8PathBuf;
|
||||
|
||||
fn build_artifacts(&self) -> Utf8PathBuf {
|
||||
self.build_root().join("build").join(self.host_triple())
|
||||
}
|
||||
|
||||
fn cargo_stage_0(&self) -> Utf8PathBuf {
|
||||
self.build_artifacts()
|
||||
.join("stage0")
|
||||
.join("bin")
|
||||
.join(format!("cargo{}", self.executable_extension()))
|
||||
}
|
||||
|
||||
fn rustc_stage_0(&self) -> Utf8PathBuf {
|
||||
self.build_artifacts()
|
||||
.join("stage0")
|
||||
.join("bin")
|
||||
.join(format!("rustc{}", self.executable_extension()))
|
||||
}
|
||||
|
||||
fn rustc_stage_2(&self) -> Utf8PathBuf {
|
||||
self.build_artifacts()
|
||||
.join("stage2")
|
||||
.join("bin")
|
||||
.join(format!("rustc{}", self.executable_extension()))
|
||||
}
|
||||
|
||||
/// Path to the built rustc-perf benchmark suite.
|
||||
fn rustc_perf_dir(&self) -> Utf8PathBuf {
|
||||
self.opt_artifacts().join("rustc-perf")
|
||||
}
|
||||
|
||||
/// Download and/or compile rustc-perf.
|
||||
fn prepare_rustc_perf(&self) -> anyhow::Result<()>;
|
||||
|
||||
fn supports_bolt(&self) -> bool;
|
||||
|
||||
fn supports_shared_llvm(&self) -> bool;
|
||||
|
||||
/// What is the extension of binary executables in this environment?
|
||||
fn executable_extension(&self) -> &'static str;
|
||||
|
||||
/// List of test paths that should be skipped when testing the optimized artifacts.
|
||||
fn skipped_tests(&self) -> &'static [&'static str];
|
||||
}
|
||||
|
||||
pub fn create_environment() -> Box<dyn Environment> {
|
||||
#[cfg(target_family = "unix")]
|
||||
return Box::new(linux::LinuxEnvironment);
|
||||
#[cfg(target_family = "windows")]
|
||||
return Box::new(windows::WindowsEnvironment::new());
|
||||
}
|
@ -1,92 +0,0 @@
|
||||
use crate::environment::Environment;
|
||||
use crate::exec::cmd;
|
||||
use crate::utils::io::move_directory;
|
||||
use crate::utils::retry_action;
|
||||
use camino::Utf8PathBuf;
|
||||
use std::io::Cursor;
|
||||
use std::time::Duration;
|
||||
use zip::ZipArchive;
|
||||
|
||||
pub(super) struct WindowsEnvironment {
|
||||
checkout_dir: Utf8PathBuf,
|
||||
}
|
||||
|
||||
impl WindowsEnvironment {
|
||||
pub fn new() -> Self {
|
||||
Self { checkout_dir: std::env::current_dir().unwrap().try_into().unwrap() }
|
||||
}
|
||||
}
|
||||
|
||||
impl Environment for WindowsEnvironment {
|
||||
fn python_binary(&self) -> &'static str {
|
||||
"python"
|
||||
}
|
||||
|
||||
fn checkout_path(&self) -> Utf8PathBuf {
|
||||
self.checkout_dir.clone()
|
||||
}
|
||||
|
||||
fn host_llvm_dir(&self) -> Utf8PathBuf {
|
||||
self.checkout_path().join("citools").join("clang-rust")
|
||||
}
|
||||
|
||||
fn opt_artifacts(&self) -> Utf8PathBuf {
|
||||
self.checkout_path().join("opt-artifacts")
|
||||
}
|
||||
|
||||
fn build_root(&self) -> Utf8PathBuf {
|
||||
self.checkout_path()
|
||||
}
|
||||
|
||||
fn prepare_rustc_perf(&self) -> anyhow::Result<()> {
|
||||
// FIXME: add some mechanism for synchronization of this commit SHA with
|
||||
// Linux (which builds rustc-perf in a Dockerfile)
|
||||
// rustc-perf version from 2023-05-30
|
||||
const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1";
|
||||
|
||||
let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip");
|
||||
let client = reqwest::blocking::Client::builder()
|
||||
.timeout(Duration::from_secs(60 * 2))
|
||||
.connect_timeout(Duration::from_secs(60 * 2))
|
||||
.build()?;
|
||||
let response = retry_action(
|
||||
|| Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()),
|
||||
"Download rustc-perf archive",
|
||||
5,
|
||||
)?;
|
||||
|
||||
let mut archive = ZipArchive::new(Cursor::new(response))?;
|
||||
archive.extract(self.rustc_perf_dir())?;
|
||||
move_directory(
|
||||
&self.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")),
|
||||
&self.rustc_perf_dir(),
|
||||
)?;
|
||||
|
||||
cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"])
|
||||
.workdir(&self.rustc_perf_dir())
|
||||
.env("RUSTC", &self.rustc_stage_0().into_string())
|
||||
.env("RUSTC_BOOTSTRAP", "1")
|
||||
.run()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn supports_bolt(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn supports_shared_llvm(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn executable_extension(&self) -> &'static str {
|
||||
".exe"
|
||||
}
|
||||
|
||||
fn skipped_tests(&self) -> &'static [&'static str] {
|
||||
&[
|
||||
// Fails as of June 2023.
|
||||
"tests\\codegen\\vec-shrink-panik.rs",
|
||||
]
|
||||
}
|
||||
}
|
@ -96,7 +96,7 @@ pub struct Bootstrap {
|
||||
}
|
||||
|
||||
impl Bootstrap {
|
||||
pub fn build(env: &dyn Environment) -> Self {
|
||||
pub fn build(env: &Environment) -> Self {
|
||||
let metrics_path = env.build_root().join("build").join("metrics.json");
|
||||
let cmd = cmd(&[
|
||||
env.python_binary(),
|
||||
@ -114,7 +114,7 @@ impl Bootstrap {
|
||||
Self { cmd, metrics_path }
|
||||
}
|
||||
|
||||
pub fn dist(env: &dyn Environment, dist_args: &[String]) -> Self {
|
||||
pub fn dist(env: &Environment, dist_args: &[String]) -> Self {
|
||||
let metrics_path = env.build_root().join("build").join("metrics.json");
|
||||
let cmd = cmd(&dist_args.iter().map(|arg| arg.as_str()).collect::<Vec<_>>())
|
||||
.env("RUST_BACKTRACE", "full");
|
||||
|
@ -1,17 +1,22 @@
|
||||
use crate::bolt::{bolt_optimize, with_bolt_instrumented};
|
||||
use anyhow::Context;
|
||||
use camino::{Utf8Path, Utf8PathBuf};
|
||||
use clap::Parser;
|
||||
use log::LevelFilter;
|
||||
use std::io::Cursor;
|
||||
use std::time::Duration;
|
||||
use utils::io;
|
||||
use zip::ZipArchive;
|
||||
|
||||
use crate::environment::{create_environment, Environment};
|
||||
use crate::exec::Bootstrap;
|
||||
use crate::environment::{Environment, EnvironmentBuilder};
|
||||
use crate::exec::{cmd, Bootstrap};
|
||||
use crate::tests::run_tests;
|
||||
use crate::timer::Timer;
|
||||
use crate::training::{gather_llvm_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles};
|
||||
use crate::utils::io::reset_directory;
|
||||
use crate::utils::io::{copy_directory, move_directory, reset_directory};
|
||||
use crate::utils::{
|
||||
clear_llvm_files, format_env_variables, print_binary_sizes, print_free_disk_space,
|
||||
with_log_group,
|
||||
retry_action, with_log_group,
|
||||
};
|
||||
|
||||
mod bolt;
|
||||
@ -23,24 +28,169 @@ mod timer;
|
||||
mod training;
|
||||
mod utils;
|
||||
|
||||
#[derive(clap::Parser, Debug)]
|
||||
struct Args {
|
||||
#[clap(subcommand)]
|
||||
env: EnvironmentCmd,
|
||||
}
|
||||
|
||||
#[derive(clap::Parser, Clone, Debug)]
|
||||
struct SharedArgs {
|
||||
// Arguments passed to `x` to perform the final (dist) build.
|
||||
build_args: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(clap::Parser, Clone, Debug)]
|
||||
enum EnvironmentCmd {
|
||||
/// Perform a custom local PGO/BOLT optimized build.
|
||||
Local {
|
||||
/// Target triple of the host.
|
||||
#[arg(long)]
|
||||
target_triple: String,
|
||||
|
||||
/// Checkout directory of `rustc`.
|
||||
#[arg(long)]
|
||||
checkout_dir: Utf8PathBuf,
|
||||
|
||||
/// Host LLVM installation directory.
|
||||
#[arg(long)]
|
||||
llvm_dir: Utf8PathBuf,
|
||||
|
||||
/// Python binary to use in bootstrap invocations.
|
||||
#[arg(long, default_value = "python3")]
|
||||
python: String,
|
||||
|
||||
/// Directory where artifacts (like PGO profiles or rustc-perf) of this workflow
|
||||
/// will be stored.
|
||||
#[arg(long, default_value = "opt-artifacts")]
|
||||
artifact_dir: Utf8PathBuf,
|
||||
|
||||
/// Is LLVM for `rustc` built in shared library mode?
|
||||
#[arg(long, default_value_t = true)]
|
||||
llvm_shared: bool,
|
||||
|
||||
/// Should BOLT optimization be used? If yes, host LLVM must have BOLT binaries
|
||||
/// (`llvm-bolt` and `merge-fdata`) available.
|
||||
#[arg(long, default_value_t = false)]
|
||||
use_bolt: bool,
|
||||
|
||||
/// Tests that should be skipped when testing the optimized compiler.
|
||||
#[arg(long)]
|
||||
skipped_tests: Vec<String>,
|
||||
|
||||
#[clap(flatten)]
|
||||
shared: SharedArgs,
|
||||
},
|
||||
/// Perform an optimized build on Linux CI, from inside Docker.
|
||||
LinuxCi {
|
||||
#[clap(flatten)]
|
||||
shared: SharedArgs,
|
||||
},
|
||||
/// Perform an optimized build on Windows CI, directly inside Github Actions.
|
||||
WindowsCi {
|
||||
#[clap(flatten)]
|
||||
shared: SharedArgs,
|
||||
},
|
||||
}
|
||||
|
||||
fn is_try_build() -> bool {
|
||||
std::env::var("DIST_TRY_BUILD").unwrap_or_else(|_| "0".to_string()) != "0"
|
||||
}
|
||||
|
||||
fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec<String>)> {
|
||||
let (env, args) = match args.env {
|
||||
EnvironmentCmd::Local {
|
||||
target_triple,
|
||||
checkout_dir,
|
||||
llvm_dir,
|
||||
python,
|
||||
artifact_dir,
|
||||
llvm_shared,
|
||||
use_bolt,
|
||||
skipped_tests,
|
||||
shared,
|
||||
} => {
|
||||
let env = EnvironmentBuilder::default()
|
||||
.host_triple(target_triple)
|
||||
.python_binary(python)
|
||||
.checkout_dir(checkout_dir.clone())
|
||||
.host_llvm_dir(llvm_dir)
|
||||
.artifact_dir(artifact_dir)
|
||||
.build_dir(checkout_dir)
|
||||
.shared_llvm(llvm_shared)
|
||||
.use_bolt(use_bolt)
|
||||
.skipped_tests(skipped_tests)
|
||||
.build()?;
|
||||
|
||||
(env, shared.build_args)
|
||||
}
|
||||
EnvironmentCmd::LinuxCi { shared } => {
|
||||
let target_triple =
|
||||
std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing");
|
||||
|
||||
let checkout_dir = Utf8PathBuf::from("/checkout");
|
||||
let env = EnvironmentBuilder::default()
|
||||
.host_triple(target_triple)
|
||||
.python_binary("python3".to_string())
|
||||
.checkout_dir(checkout_dir.clone())
|
||||
.host_llvm_dir(Utf8PathBuf::from("/rustroot"))
|
||||
.artifact_dir(Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts"))
|
||||
.build_dir(checkout_dir.join("obj"))
|
||||
// /tmp/rustc-perf comes from the x64 dist Dockerfile
|
||||
.prebuilt_rustc_perf(Some(Utf8PathBuf::from("/tmp/rustc-perf")))
|
||||
.shared_llvm(true)
|
||||
.use_bolt(true)
|
||||
.skipped_tests(vec![
|
||||
// Fails because of linker errors, as of June 2023.
|
||||
"tests/ui/process/nofile-limit.rs".to_string(),
|
||||
])
|
||||
.build()?;
|
||||
|
||||
(env, shared.build_args)
|
||||
}
|
||||
EnvironmentCmd::WindowsCi { shared } => {
|
||||
let target_triple =
|
||||
std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing");
|
||||
|
||||
let checkout_dir: Utf8PathBuf = std::env::current_dir()?.try_into()?;
|
||||
let env = EnvironmentBuilder::default()
|
||||
.host_triple(target_triple)
|
||||
.python_binary("python".to_string())
|
||||
.checkout_dir(checkout_dir.clone())
|
||||
.host_llvm_dir(checkout_dir.join("citools").join("clang-rust"))
|
||||
.artifact_dir(checkout_dir.join("opt-artifacts"))
|
||||
.build_dir(checkout_dir)
|
||||
.shared_llvm(false)
|
||||
.use_bolt(false)
|
||||
.skipped_tests(vec![
|
||||
// Fails as of June 2023.
|
||||
"tests\\codegen\\vec-shrink-panik.rs".to_string(),
|
||||
])
|
||||
.build()?;
|
||||
|
||||
(env, shared.build_args)
|
||||
}
|
||||
};
|
||||
Ok((env, args))
|
||||
}
|
||||
|
||||
fn execute_pipeline(
|
||||
env: &dyn Environment,
|
||||
env: &Environment,
|
||||
timer: &mut Timer,
|
||||
dist_args: Vec<String>,
|
||||
) -> anyhow::Result<()> {
|
||||
reset_directory(&env.opt_artifacts())?;
|
||||
reset_directory(&env.artifact_dir())?;
|
||||
|
||||
with_log_group("Building rustc-perf", || env.prepare_rustc_perf())?;
|
||||
with_log_group("Building rustc-perf", || match env.prebuilt_rustc_perf() {
|
||||
Some(dir) => copy_rustc_perf(env, &dir),
|
||||
None => download_rustc_perf(env),
|
||||
})?;
|
||||
|
||||
// Stage 1: Build PGO instrumented rustc
|
||||
// We use a normal build of LLVM, because gathering PGO profiles for LLVM and `rustc` at the
|
||||
// same time can cause issues, because the host and in-tree LLVM versions can diverge.
|
||||
let rustc_pgo_profile = timer.section("Stage 1 (Rustc PGO)", |stage| {
|
||||
let rustc_profile_dir_root = env.opt_artifacts().join("rustc-pgo");
|
||||
let rustc_profile_dir_root = env.artifact_dir().join("rustc-pgo");
|
||||
|
||||
stage.section("Build PGO instrumented rustc and LLVM", |section| {
|
||||
let mut builder = Bootstrap::build(env).rustc_pgo_instrument(&rustc_profile_dir_root);
|
||||
@ -74,7 +224,7 @@ fn execute_pipeline(
|
||||
// Remove the previous, uninstrumented build of LLVM.
|
||||
clear_llvm_files(env)?;
|
||||
|
||||
let llvm_profile_dir_root = env.opt_artifacts().join("llvm-pgo");
|
||||
let llvm_profile_dir_root = env.artifact_dir().join("llvm-pgo");
|
||||
|
||||
stage.section("Build PGO instrumented LLVM", |section| {
|
||||
Bootstrap::build(env)
|
||||
@ -95,7 +245,7 @@ fn execute_pipeline(
|
||||
Ok(profile)
|
||||
})?;
|
||||
|
||||
let llvm_bolt_profile = if env.supports_bolt() {
|
||||
let llvm_bolt_profile = if env.use_bolt() {
|
||||
// Stage 3: Build BOLT instrumented LLVM
|
||||
// We build a PGO optimized LLVM in this step, then instrument it with BOLT and gather BOLT profiles.
|
||||
// Note that we don't remove LLVM artifacts after this step, so that they are reused in the final dist build.
|
||||
@ -171,8 +321,9 @@ fn main() -> anyhow::Result<()> {
|
||||
.parse_default_env()
|
||||
.init();
|
||||
|
||||
let mut build_args: Vec<String> = std::env::args().skip(1).collect();
|
||||
println!("Running optimized build pipeline with args `{}`", build_args.join(" "));
|
||||
let args = Args::parse();
|
||||
|
||||
println!("Running optimized build pipeline with args `{:?}`", args);
|
||||
|
||||
with_log_group("Environment values", || {
|
||||
println!("Environment values\n{}", format_env_variables());
|
||||
@ -184,6 +335,8 @@ fn main() -> anyhow::Result<()> {
|
||||
}
|
||||
});
|
||||
|
||||
let (env, mut build_args) = create_environment(args).context("Cannot create environment")?;
|
||||
|
||||
// Skip components that are not needed for try builds to speed them up
|
||||
if is_try_build() {
|
||||
log::info!("Skipping building of unimportant components for a try build");
|
||||
@ -202,14 +355,58 @@ fn main() -> anyhow::Result<()> {
|
||||
}
|
||||
|
||||
let mut timer = Timer::new();
|
||||
let env = create_environment();
|
||||
|
||||
let result = execute_pipeline(env.as_ref(), &mut timer, build_args);
|
||||
let result = execute_pipeline(&env, &mut timer, build_args);
|
||||
log::info!("Timer results\n{}", timer.format_stats());
|
||||
|
||||
print_free_disk_space()?;
|
||||
result.context("Optimized build pipeline has failed")?;
|
||||
print_binary_sizes(env.as_ref())?;
|
||||
print_binary_sizes(&env)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Copy rustc-perf from the given path into the environment and build it.
|
||||
fn copy_rustc_perf(env: &Environment, dir: &Utf8Path) -> anyhow::Result<()> {
|
||||
copy_directory(dir, &env.rustc_perf_dir())?;
|
||||
build_rustc_perf(env)
|
||||
}
|
||||
|
||||
// Download and build rustc-perf into the given environment.
|
||||
fn download_rustc_perf(env: &Environment) -> anyhow::Result<()> {
|
||||
reset_directory(&env.rustc_perf_dir())?;
|
||||
|
||||
// FIXME: add some mechanism for synchronization of this commit SHA with
|
||||
// Linux (which builds rustc-perf in a Dockerfile)
|
||||
// rustc-perf version from 2023-05-30
|
||||
const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1";
|
||||
|
||||
let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip");
|
||||
let client = reqwest::blocking::Client::builder()
|
||||
.timeout(Duration::from_secs(60 * 2))
|
||||
.connect_timeout(Duration::from_secs(60 * 2))
|
||||
.build()?;
|
||||
let response = retry_action(
|
||||
|| Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()),
|
||||
"Download rustc-perf archive",
|
||||
5,
|
||||
)?;
|
||||
|
||||
let mut archive = ZipArchive::new(Cursor::new(response))?;
|
||||
archive.extract(env.rustc_perf_dir())?;
|
||||
move_directory(
|
||||
&env.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")),
|
||||
&env.rustc_perf_dir(),
|
||||
)?;
|
||||
|
||||
build_rustc_perf(env)
|
||||
}
|
||||
|
||||
fn build_rustc_perf(env: &Environment) -> anyhow::Result<()> {
|
||||
cmd(&[env.cargo_stage_0().as_str(), "build", "-p", "collector"])
|
||||
.workdir(&env.rustc_perf_dir())
|
||||
.env("RUSTC", &env.rustc_stage_0().into_string())
|
||||
.env("RUSTC_BOOTSTRAP", "1")
|
||||
.run()?;
|
||||
Ok(())
|
||||
}
|
||||
|
@ -1,11 +1,11 @@
|
||||
use crate::environment::Environment;
|
||||
use crate::environment::{executable_extension, Environment};
|
||||
use crate::exec::cmd;
|
||||
use crate::utils::io::{copy_directory, find_file_in_dir, unpack_archive};
|
||||
use anyhow::Context;
|
||||
use camino::{Utf8Path, Utf8PathBuf};
|
||||
|
||||
/// Run tests on optimized dist artifacts.
|
||||
pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> {
|
||||
pub fn run_tests(env: &Environment) -> anyhow::Result<()> {
|
||||
// After `dist` is executed, we extract its archived components into a sysroot directory,
|
||||
// and then use that extracted rustc as a stage0 compiler.
|
||||
// Then we run a subset of tests using that compiler, to have a basic smoke test which checks
|
||||
@ -33,8 +33,8 @@ pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> {
|
||||
|
||||
// We need to manually copy libstd to the extracted rustc sysroot
|
||||
copy_directory(
|
||||
&libstd_dir.join("lib").join("rustlib").join(&host_triple).join("lib"),
|
||||
&rustc_dir.join("lib").join("rustlib").join(&host_triple).join("lib"),
|
||||
&libstd_dir.join("lib").join("rustlib").join(host_triple).join("lib"),
|
||||
&rustc_dir.join("lib").join("rustlib").join(host_triple).join("lib"),
|
||||
)?;
|
||||
|
||||
// Extract sources - they aren't in the `rustc-nightly-{host}` tarball, so we need to manually copy libstd
|
||||
@ -45,9 +45,9 @@ pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> {
|
||||
&rustc_dir.join("lib").join("rustlib").join("src"),
|
||||
)?;
|
||||
|
||||
let rustc_path = rustc_dir.join("bin").join(format!("rustc{}", env.executable_extension()));
|
||||
let rustc_path = rustc_dir.join("bin").join(format!("rustc{}", executable_extension()));
|
||||
assert!(rustc_path.is_file());
|
||||
let cargo_path = cargo_dir.join("bin").join(format!("cargo{}", env.executable_extension()));
|
||||
let cargo_path = cargo_dir.join("bin").join(format!("cargo{}", executable_extension()));
|
||||
assert!(cargo_path.is_file());
|
||||
|
||||
// Specify path to a LLVM config so that LLVM is not rebuilt.
|
||||
@ -56,7 +56,7 @@ pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> {
|
||||
.build_artifacts()
|
||||
.join("llvm")
|
||||
.join("bin")
|
||||
.join(format!("llvm-config{}", env.executable_extension()));
|
||||
.join(format!("llvm-config{}", executable_extension()));
|
||||
assert!(llvm_config.is_file());
|
||||
|
||||
let config_content = format!(
|
||||
@ -109,6 +109,6 @@ fn find_dist_version(directory: &Utf8Path) -> anyhow::Result<String> {
|
||||
.unwrap()
|
||||
.to_string();
|
||||
let (version, _) =
|
||||
archive.strip_prefix("reproducible-artifacts-").unwrap().split_once("-").unwrap();
|
||||
archive.strip_prefix("reproducible-artifacts-").unwrap().split_once('-').unwrap();
|
||||
Ok(version.to_string())
|
||||
}
|
||||
|
@ -1,4 +1,4 @@
|
||||
use crate::environment::Environment;
|
||||
use crate::environment::{executable_extension, Environment};
|
||||
use crate::exec::{cmd, CmdBuilder};
|
||||
use crate::utils::io::{count_files, delete_directory};
|
||||
use crate::utils::with_log_group;
|
||||
@ -30,7 +30,7 @@ const RUSTC_PGO_CRATES: &[&str] = &[
|
||||
const LLVM_BOLT_CRATES: &[&str] = LLVM_PGO_CRATES;
|
||||
|
||||
fn init_compiler_benchmarks(
|
||||
env: &dyn Environment,
|
||||
env: &Environment,
|
||||
profiles: &[&str],
|
||||
scenarios: &[&str],
|
||||
crates: &[&str],
|
||||
@ -75,7 +75,7 @@ enum LlvmProfdata {
|
||||
}
|
||||
|
||||
fn merge_llvm_profiles(
|
||||
env: &dyn Environment,
|
||||
env: &Environment,
|
||||
merged_path: &Utf8Path,
|
||||
profile_dir: &Utf8Path,
|
||||
profdata: LlvmProfdata,
|
||||
@ -86,7 +86,7 @@ fn merge_llvm_profiles(
|
||||
.build_artifacts()
|
||||
.join("llvm")
|
||||
.join("build")
|
||||
.join(format!("bin/llvm-profdata{}", env.executable_extension())),
|
||||
.join(format!("bin/llvm-profdata{}", executable_extension())),
|
||||
};
|
||||
|
||||
cmd(&[llvm_profdata.as_str(), "merge", "-o", merged_path.as_str(), profile_dir.as_str()])
|
||||
@ -116,7 +116,7 @@ fn log_profile_stats(
|
||||
pub struct LlvmPGOProfile(pub Utf8PathBuf);
|
||||
|
||||
pub fn gather_llvm_profiles(
|
||||
env: &dyn Environment,
|
||||
env: &Environment,
|
||||
profile_root: &Utf8Path,
|
||||
) -> anyhow::Result<LlvmPGOProfile> {
|
||||
log::info!("Running benchmarks with PGO instrumented LLVM");
|
||||
@ -127,7 +127,7 @@ pub fn gather_llvm_profiles(
|
||||
.context("Cannot gather LLVM PGO profiles")
|
||||
})?;
|
||||
|
||||
let merged_profile = env.opt_artifacts().join("llvm-pgo.profdata");
|
||||
let merged_profile = env.artifact_dir().join("llvm-pgo.profdata");
|
||||
log::info!("Merging LLVM PGO profiles to {merged_profile}");
|
||||
|
||||
merge_llvm_profiles(env, &merged_profile, profile_root, LlvmProfdata::Host)?;
|
||||
@ -143,7 +143,7 @@ pub fn gather_llvm_profiles(
|
||||
pub struct RustcPGOProfile(pub Utf8PathBuf);
|
||||
|
||||
pub fn gather_rustc_profiles(
|
||||
env: &dyn Environment,
|
||||
env: &Environment,
|
||||
profile_root: &Utf8Path,
|
||||
) -> anyhow::Result<RustcPGOProfile> {
|
||||
log::info!("Running benchmarks with PGO instrumented rustc");
|
||||
@ -163,7 +163,7 @@ pub fn gather_rustc_profiles(
|
||||
.context("Cannot gather rustc PGO profiles")
|
||||
})?;
|
||||
|
||||
let merged_profile = env.opt_artifacts().join("rustc-pgo.profdata");
|
||||
let merged_profile = env.artifact_dir().join("rustc-pgo.profdata");
|
||||
log::info!("Merging Rustc PGO profiles to {merged_profile}");
|
||||
|
||||
merge_llvm_profiles(env, &merged_profile, profile_root, LlvmProfdata::Target)?;
|
||||
@ -178,7 +178,7 @@ pub fn gather_rustc_profiles(
|
||||
|
||||
pub struct LlvmBoltProfile(pub Utf8PathBuf);
|
||||
|
||||
pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result<LlvmBoltProfile> {
|
||||
pub fn gather_llvm_bolt_profiles(env: &Environment) -> anyhow::Result<LlvmBoltProfile> {
|
||||
log::info!("Running benchmarks with BOLT instrumented LLVM");
|
||||
|
||||
with_log_group("Running benchmarks", || {
|
||||
@ -187,12 +187,12 @@ pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result<LlvmBo
|
||||
.context("Cannot gather LLVM BOLT profiles")
|
||||
})?;
|
||||
|
||||
let merged_profile = env.opt_artifacts().join("llvm-bolt.profdata");
|
||||
let merged_profile = env.artifact_dir().join("llvm-bolt.profdata");
|
||||
let profile_root = Utf8PathBuf::from("/tmp/prof.fdata");
|
||||
log::info!("Merging LLVM BOLT profiles to {merged_profile}");
|
||||
|
||||
let profiles: Vec<_> =
|
||||
glob::glob(&format!("{profile_root}*"))?.into_iter().collect::<Result<Vec<_>, _>>()?;
|
||||
glob::glob(&format!("{profile_root}*"))?.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
let mut merge_args = vec!["merge-fdata"];
|
||||
merge_args.extend(profiles.iter().map(|p| p.to_str().unwrap()));
|
||||
|
@ -7,7 +7,7 @@ use std::path::Path;
|
||||
/// Delete and re-create the directory.
|
||||
pub fn reset_directory(path: &Utf8Path) -> anyhow::Result<()> {
|
||||
log::info!("Resetting directory {path}");
|
||||
let _ = std::fs::remove_dir(path);
|
||||
let _ = std::fs::remove_dir_all(path);
|
||||
std::fs::create_dir_all(path)?;
|
||||
Ok(())
|
||||
}
|
||||
@ -63,7 +63,6 @@ pub fn get_files_from_dir(
|
||||
let path = format!("{dir}/*{}", suffix.unwrap_or(""));
|
||||
|
||||
Ok(glob::glob(&path)?
|
||||
.into_iter()
|
||||
.map(|p| p.map(|p| Utf8PathBuf::from_path_buf(p).unwrap()))
|
||||
.collect::<Result<Vec<_>, _>>()?)
|
||||
}
|
||||
@ -74,9 +73,8 @@ pub fn find_file_in_dir(
|
||||
prefix: &str,
|
||||
suffix: &str,
|
||||
) -> anyhow::Result<Utf8PathBuf> {
|
||||
let files = glob::glob(&format!("{directory}/{prefix}*{suffix}"))?
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let files =
|
||||
glob::glob(&format!("{directory}/{prefix}*{suffix}"))?.collect::<Result<Vec<_>, _>>()?;
|
||||
match files.len() {
|
||||
0 => Err(anyhow::anyhow!("No file with prefix {prefix} found in {directory}")),
|
||||
1 => Ok(Utf8PathBuf::from_path_buf(files[0].clone()).unwrap()),
|
||||
|
@ -26,7 +26,7 @@ pub fn print_free_disk_space() -> anyhow::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> {
|
||||
pub fn print_binary_sizes(env: &Environment) -> anyhow::Result<()> {
|
||||
use std::fmt::Write;
|
||||
|
||||
let root = env.build_artifacts().join("stage2");
|
||||
@ -48,7 +48,7 @@ pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn clear_llvm_files(env: &dyn Environment) -> anyhow::Result<()> {
|
||||
pub fn clear_llvm_files(env: &Environment) -> anyhow::Result<()> {
|
||||
// Bootstrap currently doesn't support rebuilding LLVM when PGO options
|
||||
// change (or any other llvm-related options); so just clear out the relevant
|
||||
// directories ourselves.
|
||||
|
Loading…
Reference in New Issue
Block a user