Auto merge of #138039 - onur-ozkan:handle-forced-compiler-on-tools, r=jieyouxu

handle forced compiler and revert #137476

Fixes https://github.com/rust-lang/rust/issues/138004

I would appreciate it if we could measure CI pipelines with the current changes to see if this reduces recent CI overhead. cc `@rust-lang/infra`

try-job: dist-powerpc64le-linux
This commit is contained in:
bors 2025-03-06 17:20:24 +00:00
commit e6af292f91
5 changed files with 145 additions and 86 deletions

View File

@ -421,7 +421,11 @@ impl Step for Rustc {
if let Some(ra_proc_macro_srv) = builder.ensure_if_default(
tool::RustAnalyzerProcMacroSrv {
compiler: builder.compiler(compiler.stage, builder.config.build),
compiler: builder.compiler_for(
compiler.stage,
builder.config.build,
compiler.host,
),
target: compiler.host,
},
builder.kind,
@ -771,7 +775,11 @@ impl Step for Analysis {
// Find the actual compiler (handling the full bootstrap option) which
// produced the save-analysis data because that data isn't copied
// through the sysroot uplifting.
compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
compiler: run.builder.compiler_for(
run.builder.top_stage,
run.builder.config.build,
run.target,
),
target: run.target,
});
}
@ -1116,7 +1124,11 @@ impl Step for Cargo {
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(Cargo {
compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
compiler: run.builder.compiler_for(
run.builder.top_stage,
run.builder.config.build,
run.target,
),
target: run.target,
});
}
@ -1161,7 +1173,11 @@ impl Step for Rls {
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(Rls {
compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
compiler: run.builder.compiler_for(
run.builder.top_stage,
run.builder.config.build,
run.target,
),
target: run.target,
});
}
@ -1199,7 +1215,11 @@ impl Step for RustAnalyzer {
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(RustAnalyzer {
compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
compiler: run.builder.compiler_for(
run.builder.top_stage,
run.builder.config.build,
run.target,
),
target: run.target,
});
}
@ -1237,7 +1257,11 @@ impl Step for Clippy {
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(Clippy {
compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
compiler: run.builder.compiler_for(
run.builder.top_stage,
run.builder.config.build,
run.target,
),
target: run.target,
});
}
@ -1280,7 +1304,11 @@ impl Step for Miri {
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(Miri {
compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
compiler: run.builder.compiler_for(
run.builder.top_stage,
run.builder.config.build,
run.target,
),
target: run.target,
});
}
@ -1414,7 +1442,11 @@ impl Step for Rustfmt {
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(Rustfmt {
compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
compiler: run.builder.compiler_for(
run.builder.top_stage,
run.builder.config.build,
run.target,
),
target: run.target,
});
}
@ -1464,7 +1496,7 @@ impl Step for Extended {
fn run(self, builder: &Builder<'_>) {
let target = self.target;
let stage = self.stage;
let compiler = builder.compiler(self.stage, self.host);
let compiler = builder.compiler_for(self.stage, self.host, self.target);
builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
@ -2112,8 +2144,7 @@ pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection,
),
)]
pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
let dst_libdir =
sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
let dst_libdir = sysroot.join(builder.sysroot_libdir_relative(Compiler::new(1, target)));
// We do not need to copy LLVM files into the sysroot if it is not
// dynamically linked; it is already included into librustc_llvm
// statically.
@ -2228,7 +2259,11 @@ impl Step for LlvmBitcodeLinker {
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(LlvmBitcodeLinker {
compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
compiler: run.builder.compiler_for(
run.builder.top_stage,
run.builder.config.build,
run.target,
),
target: run.target,
});
}

View File

@ -322,6 +322,8 @@ pub(crate) fn get_tool_rustc_compiler(
if builder.download_rustc() && target_compiler.stage == 1 {
// We already have the stage 1 compiler, we don't need to cut the stage.
builder.compiler(target_compiler.stage, builder.config.build)
} else if target_compiler.is_forced_compiler() {
target_compiler
} else {
// Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise
// we'd have stageN/bin/rustc and stageN/bin/$rustc_tool be effectively different stage

View File

@ -1235,7 +1235,7 @@ impl<'a> Builder<'a> {
),
)]
pub fn compiler(&self, stage: u32, host: TargetSelection) -> Compiler {
self.ensure(compile::Assemble { target_compiler: Compiler { stage, host } })
self.ensure(compile::Assemble { target_compiler: Compiler::new(stage, host) })
}
/// Similar to `compiler`, except handles the full-bootstrap option to
@ -1273,7 +1273,7 @@ impl<'a> Builder<'a> {
target: TargetSelection,
) -> Compiler {
#![allow(clippy::let_and_return)]
let resolved_compiler = if self.build.force_use_stage2(stage) {
let mut resolved_compiler = if self.build.force_use_stage2(stage) {
trace!(target: "COMPILER_FOR", ?stage, "force_use_stage2");
self.compiler(2, self.config.build)
} else if self.build.force_use_stage1(stage, target) {
@ -1283,6 +1283,11 @@ impl<'a> Builder<'a> {
trace!(target: "COMPILER_FOR", ?stage, ?host, "no force, fallback to `compiler()`");
self.compiler(stage, host)
};
if stage != resolved_compiler.stage {
resolved_compiler.forced_compiler(true);
}
trace!(target: "COMPILER_FOR", ?resolved_compiler);
resolved_compiler
}

View File

@ -71,7 +71,7 @@ fn check_cli<const N: usize>(paths: [&str; N]) {
macro_rules! std {
($host:ident => $target:ident, stage = $stage:literal) => {
compile::Std::new(
Compiler { host: TargetSelection::from_user($host), stage: $stage },
Compiler::new($stage, TargetSelection::from_user($host)),
TargetSelection::from_user($target),
)
};
@ -84,7 +84,7 @@ macro_rules! doc_std {
macro_rules! rustc {
($host:ident => $target:ident, stage = $stage:literal) => {
compile::Rustc::new(
Compiler { host: TargetSelection::from_user($host), stage: $stage },
Compiler::new($stage, TargetSelection::from_user($host)),
TargetSelection::from_user($target),
)
};
@ -296,7 +296,7 @@ mod defaults {
first(cache.all::<tool::Rustdoc>()),
// Recall that rustdoc stages are off-by-one
// - this is the compiler it's _linked_ to, not built with.
&[tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }],
&[tool::Rustdoc { compiler: Compiler::new(1, a) }],
);
assert_eq!(
first(cache.all::<compile::Rustc>()),
@ -319,7 +319,7 @@ mod defaults {
first(cache.all::<tool::Rustdoc>()),
// This is the beta rustdoc.
// Add an assert here to make sure this is the only rustdoc built.
&[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } }],
&[tool::Rustdoc { compiler: Compiler::new(0, a) }],
);
assert!(cache.all::<compile::Rustc>().is_empty());
}
@ -352,16 +352,16 @@ mod defaults {
assert_eq!(
first(cache.all::<compile::Assemble>()),
&[
compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } },
compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } },
compile::Assemble { target_compiler: Compiler { host: b, stage: 1 } },
compile::Assemble { target_compiler: Compiler::new(0, a) },
compile::Assemble { target_compiler: Compiler::new(1, a) },
compile::Assemble { target_compiler: Compiler::new(1, b) },
]
);
assert_eq!(
first(cache.all::<tool::Rustdoc>()),
&[
tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } },
tool::Rustdoc { compiler: Compiler { host: b, stage: 1 } },
tool::Rustdoc { compiler: Compiler::new(1, a) },
tool::Rustdoc { compiler: Compiler::new(1, b) },
],
);
assert_eq!(
@ -386,14 +386,14 @@ mod defaults {
assert_eq!(first(cache.all::<doc::ErrorIndex>()), &[doc::ErrorIndex { target: a },]);
assert_eq!(
first(cache.all::<tool::ErrorIndex>()),
&[tool::ErrorIndex { compiler: Compiler { host: a, stage: 0 } }]
&[tool::ErrorIndex { compiler: Compiler::new(0, a) }]
);
// docs should be built with the beta compiler, not with the stage0 artifacts.
// recall that rustdoc is off-by-one: `stage` is the compiler rustdoc is _linked_ to,
// not the one it was built by.
assert_eq!(
first(cache.all::<tool::Rustdoc>()),
&[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } },]
&[tool::Rustdoc { compiler: Compiler::new(0, a) },]
);
}
}
@ -418,17 +418,17 @@ mod dist {
assert_eq!(first(cache.all::<dist::Mingw>()), &[dist::Mingw { host: a },]);
assert_eq!(
first(cache.all::<dist::Rustc>()),
&[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },]
&[dist::Rustc { compiler: Compiler::new(2, a) },]
);
assert_eq!(
first(cache.all::<dist::Std>()),
&[dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },]
&[dist::Std { compiler: Compiler::new(1, a), target: a },]
);
assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]);
// Make sure rustdoc is only built once.
assert_eq!(
first(cache.all::<tool::Rustdoc>()),
&[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },]
&[tool::Rustdoc { compiler: Compiler::new(2, a) },]
);
}
@ -450,13 +450,13 @@ mod dist {
);
assert_eq!(
first(cache.all::<dist::Rustc>()),
&[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },]
&[dist::Rustc { compiler: Compiler::new(2, a) },]
);
assert_eq!(
first(cache.all::<dist::Std>()),
&[
dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },
dist::Std { compiler: Compiler { host: a, stage: 2 }, target: b },
dist::Std { compiler: Compiler::new(1, a), target: a },
dist::Std { compiler: Compiler::new(2, a), target: b },
]
);
assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]);
@ -483,15 +483,15 @@ mod dist {
assert_eq!(
first(cache.all::<dist::Rustc>()),
&[
dist::Rustc { compiler: Compiler { host: a, stage: 2 } },
dist::Rustc { compiler: Compiler { host: b, stage: 2 } },
dist::Rustc { compiler: Compiler::new(2, a) },
dist::Rustc { compiler: Compiler::new(2, b) },
]
);
assert_eq!(
first(cache.all::<dist::Std>()),
&[
dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },
dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b },
dist::Std { compiler: Compiler::new(1, a), target: a },
dist::Std { compiler: Compiler::new(1, a), target: b },
]
);
assert_eq!(
@ -519,13 +519,12 @@ mod dist {
assert_eq!(
first(cache.all::<dist::Rustc>()),
&[dist::Rustc { compiler: Compiler { host: b, stage: 2 } },]
&[dist::Rustc { compiler: Compiler::new(2, b) },]
);
assert_eq!(
first(cache.all::<compile::Rustc>()),
&[
rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0),
rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 1),
rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 1),
]
);
@ -556,16 +555,16 @@ mod dist {
assert_eq!(
first(cache.all::<dist::Rustc>()),
&[
dist::Rustc { compiler: Compiler { host: a, stage: 2 } },
dist::Rustc { compiler: Compiler { host: b, stage: 2 } },
dist::Rustc { compiler: Compiler::new(2, a) },
dist::Rustc { compiler: Compiler::new(2, b) },
]
);
assert_eq!(
first(cache.all::<dist::Std>()),
&[
dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },
dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b },
dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c },
dist::Std { compiler: Compiler::new(1, a), target: a },
dist::Std { compiler: Compiler::new(1, a), target: b },
dist::Std { compiler: Compiler::new(2, a), target: c },
]
);
assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]);
@ -583,7 +582,7 @@ mod dist {
assert_eq!(first(cache.all::<dist::Mingw>()), &[dist::Mingw { host: c },]);
assert_eq!(
first(cache.all::<dist::Std>()),
&[dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c },]
&[dist::Std { compiler: Compiler::new(2, a), target: c },]
);
}
@ -608,15 +607,15 @@ mod dist {
assert_eq!(
first(cache.all::<dist::Rustc>()),
&[
dist::Rustc { compiler: Compiler { host: a, stage: 2 } },
dist::Rustc { compiler: Compiler { host: b, stage: 2 } },
dist::Rustc { compiler: Compiler::new(2, a) },
dist::Rustc { compiler: Compiler::new(2, b) },
]
);
assert_eq!(
first(cache.all::<dist::Std>()),
&[
dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },
dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b },
dist::Std { compiler: Compiler::new(1, a), target: a },
dist::Std { compiler: Compiler::new(1, a), target: b },
]
);
assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]);
@ -633,10 +632,10 @@ mod dist {
assert_eq!(
first(cache.all::<compile::Assemble>()),
&[
compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } },
compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } },
compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } },
compile::Assemble { target_compiler: Compiler { host: b, stage: 2 } },
compile::Assemble { target_compiler: Compiler::new(0, a) },
compile::Assemble { target_compiler: Compiler::new(1, a) },
compile::Assemble { target_compiler: Compiler::new(2, a) },
compile::Assemble { target_compiler: Compiler::new(2, b) },
]
);
}
@ -683,28 +682,16 @@ mod dist {
first(builder.cache.all::<compile::Assemble>()),
&[
compile::Assemble {
target_compiler: Compiler {
host: TargetSelection::from_user(TEST_TRIPLE_1),
stage: 0
}
target_compiler: Compiler::new(0, TargetSelection::from_user(TEST_TRIPLE_1),)
},
compile::Assemble {
target_compiler: Compiler {
host: TargetSelection::from_user(TEST_TRIPLE_1),
stage: 1
}
target_compiler: Compiler::new(1, TargetSelection::from_user(TEST_TRIPLE_1),)
},
compile::Assemble {
target_compiler: Compiler {
host: TargetSelection::from_user(TEST_TRIPLE_1),
stage: 2
}
target_compiler: Compiler::new(2, TargetSelection::from_user(TEST_TRIPLE_1),)
},
compile::Assemble {
target_compiler: Compiler {
host: TargetSelection::from_user(TEST_TRIPLE_2),
stage: 2
}
target_compiler: Compiler::new(2, TargetSelection::from_user(TEST_TRIPLE_2),)
},
]
);
@ -747,9 +734,9 @@ mod dist {
assert_eq!(
first(builder.cache.all::<compile::Assemble>()),
&[
compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } },
compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } },
compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } },
compile::Assemble { target_compiler: Compiler::new(0, a) },
compile::Assemble { target_compiler: Compiler::new(1, a) },
compile::Assemble { target_compiler: Compiler::new(2, a) },
]
);
assert_eq!(
@ -798,7 +785,7 @@ mod dist {
assert_eq!(
first(builder.cache.all::<test::Crate>()),
&[test::Crate {
compiler: Compiler { host, stage: 0 },
compiler: Compiler::new(0, host),
target: host,
mode: crate::Mode::Std,
crates: vec!["std".to_owned()],
@ -824,13 +811,13 @@ mod dist {
);
assert_eq!(
first(builder.cache.all::<tool::ErrorIndex>()),
&[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }]
&[tool::ErrorIndex { compiler: Compiler::new(1, a) }]
);
// This is actually stage 1, but Rustdoc::run swaps out the compiler with
// stage minus 1 if --stage is not 0. Very confusing!
assert_eq!(
first(builder.cache.all::<tool::Rustdoc>()),
&[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },]
&[tool::Rustdoc { compiler: Compiler::new(2, a) },]
);
}
@ -870,7 +857,7 @@ mod dist {
);
assert_eq!(
first(builder.cache.all::<tool::ErrorIndex>()),
&[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }]
&[tool::ErrorIndex { compiler: Compiler::new(1, a) }]
);
// Unfortunately rustdoc is built twice. Once from stage1 for compiletest
// (and other things), and once from stage0 for std crates. Ideally it
@ -886,9 +873,9 @@ mod dist {
assert_eq!(
first(builder.cache.all::<tool::Rustdoc>()),
&[
tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } },
tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } },
tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },
tool::Rustdoc { compiler: Compiler::new(0, a) },
tool::Rustdoc { compiler: Compiler::new(1, a) },
tool::Rustdoc { compiler: Compiler::new(2, a) },
]
);
}
@ -904,7 +891,7 @@ mod sysroot_target_dirs {
let build = Build::new(configure("build", &[TEST_TRIPLE_1], &[TEST_TRIPLE_1]));
let builder = Builder::new(&build);
let target_triple_1 = TargetSelection::from_user(TEST_TRIPLE_1);
let compiler = Compiler { stage: 1, host: target_triple_1 };
let compiler = Compiler::new(1, target_triple_1);
let target_triple_2 = TargetSelection::from_user(TEST_TRIPLE_2);
let actual = builder.sysroot_target_libdir(compiler, target_triple_2);
@ -924,7 +911,7 @@ mod sysroot_target_dirs {
let build = Build::new(configure("build", &[TEST_TRIPLE_1], &[TEST_TRIPLE_1]));
let builder = Builder::new(&build);
let target_triple_1 = TargetSelection::from_user(TEST_TRIPLE_1);
let compiler = Compiler { stage: 1, host: target_triple_1 };
let compiler = Compiler::new(1, target_triple_1);
let target_triple_2 = TargetSelection::from_user(TEST_TRIPLE_2);
let actual = builder.sysroot_target_bindir(compiler, target_triple_2);
@ -1128,13 +1115,13 @@ fn test_get_tool_rustc_compiler() {
let target_triple_1 = TargetSelection::from_user(TEST_TRIPLE_1);
let compiler = Compiler { stage: 2, host: target_triple_1 };
let expected = Compiler { stage: 1, host: target_triple_1 };
let compiler = Compiler::new(2, target_triple_1);
let expected = Compiler::new(1, target_triple_1);
let actual = tool::get_tool_rustc_compiler(&builder, compiler);
assert_eq!(expected, actual);
let compiler = Compiler { stage: 1, host: target_triple_1 };
let expected = Compiler { stage: 0, host: target_triple_1 };
let compiler = Compiler::new(1, target_triple_1);
let expected = Compiler::new(0, target_triple_1);
let actual = tool::get_tool_rustc_compiler(&builder, compiler);
assert_eq!(expected, actual);
@ -1143,8 +1130,8 @@ fn test_get_tool_rustc_compiler() {
let build = Build::new(config);
let builder = Builder::new(&build);
let compiler = Compiler { stage: 1, host: target_triple_1 };
let expected = Compiler { stage: 1, host: target_triple_1 };
let compiler = Compiler::new(1, target_triple_1);
let expected = Compiler::new(1, target_triple_1);
let actual = tool::get_tool_rustc_compiler(&builder, compiler);
assert_eq!(expected, actual);
}

View File

@ -92,10 +92,27 @@ const EXTRA_CHECK_CFGS: &[(Option<Mode>, &str, Option<&[&'static str]>)] = &[
/// Each compiler has a `stage` that it is associated with and a `host` that
/// corresponds to the platform the compiler runs on. This structure is used as
/// a parameter to many methods below.
#[derive(Eq, PartialOrd, Ord, PartialEq, Clone, Copy, Hash, Debug)]
#[derive(Eq, PartialOrd, Ord, Clone, Copy, Debug)]
pub struct Compiler {
stage: u32,
host: TargetSelection,
/// Indicates whether `compiler_for` was used to force a specific compiler stage.
/// This field is ignored in `Hash` and `PartialEq` implementations as only the `stage`
/// and `host` fields are relevant for those.
forced_compiler: bool,
}
impl std::hash::Hash for Compiler {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
self.stage.hash(state);
self.host.hash(state);
}
}
impl PartialEq for Compiler {
fn eq(&self, other: &Self) -> bool {
self.stage == other.stage && self.host == other.host
}
}
#[derive(PartialEq, Eq, Copy, Clone, Debug)]
@ -1963,6 +1980,14 @@ fn chmod(path: &Path, perms: u32) {
fn chmod(_path: &Path, _perms: u32) {}
impl Compiler {
pub fn new(stage: u32, host: TargetSelection) -> Self {
Self { stage, host, forced_compiler: false }
}
pub fn forced_compiler(&mut self, forced_compiler: bool) {
self.forced_compiler = forced_compiler;
}
pub fn with_stage(mut self, stage: u32) -> Compiler {
self.stage = stage;
self
@ -1972,6 +1997,11 @@ impl Compiler {
pub fn is_snapshot(&self, build: &Build) -> bool {
self.stage == 0 && self.host == build.build
}
/// Indicates whether `compiler_for` was used to force a specific compiler stage.
pub fn is_forced_compiler(&self) -> bool {
self.forced_compiler
}
}
fn envify(s: &str) -> String {