Auto merge of #122709 - onur-ozkan:use-precompiled-rustc-by-default, r=Mark-Simulacrum

use precompiled rustc for non-dist builders

Makes non-dist builders to use precompiled CI rustc by default if they are available for the target triple.

As we are going to make `rust.download-rustc=if-unchanged` default option with https://github.com/rust-lang/rust/pull/119899, we need to make sure `if-unchanged` logic never breaks and works as expected.

As an addition, this will significantly improve the build times on CI when there's no change on the compiler.

blocker for #119899

try-job: x86_64-gnu-nopt
try-job: aarch64-apple

src/bootstrap/src/core/builder.rs

@@ -414,6 +414,15 @@ impl StepDescription {
             .map(|desc| (desc.should_run)(ShouldRun::new(builder, desc.kind)))
             .collect::<Vec<_>>();
 
+        if builder.download_rustc() && (builder.kind == Kind::Dist || builder.kind == Kind::Install)
+        {
+            eprintln!(
+                "ERROR: '{}' subcommand is incompatible with `rust.download-rustc`.",
+                builder.kind.as_str()
+            );
+            crate::exit!(1);
+        }
+
         // sanity checks on rules
         for (desc, should_run) in v.iter().zip(&should_runs) {
             assert!(

src/bootstrap/src/core/builder/tests.rs

@@ -1,5 +1,7 @@
 use std::thread;
 
+use build_helper::git::get_closest_merge_commit;
+
 use super::*;
 use crate::Flags;
 use crate::core::build_steps::doc::DocumentationFormat;
@@ -212,6 +214,52 @@ fn alias_and_path_for_library() {
     assert_eq!(first(cache.all::<doc::Std>()), &[doc_std!(A => A, stage = 0)]);
 }
 
+#[test]
+fn ci_rustc_if_unchanged_logic() {
+    let config = Config::parse_inner(
+        Flags::parse(&[
+            "build".to_owned(),
+            "--dry-run".to_owned(),
+            "--set=rust.download-rustc='if-unchanged'".to_owned(),
+        ]),
+        |&_| Ok(Default::default()),
+    );
+
+    if config.rust_info.is_from_tarball() {
+        return;
+    }
+
+    let build = Build::new(config.clone());
+    let builder = Builder::new(&build);
+
+    if config.out.exists() {
+        fs::remove_dir_all(&config.out).unwrap();
+    }
+
+    builder.run_step_descriptions(&Builder::get_step_descriptions(config.cmd.kind()), &[]);
+
+    let compiler_path = build.src.join("compiler");
+    let library_path = build.src.join("compiler");
+
+    let commit =
+        get_closest_merge_commit(Some(&builder.config.src), &builder.config.git_config(), &[
+            compiler_path.clone(),
+            library_path.clone(),
+        ])
+        .unwrap();
+
+    let has_changes = !helpers::git(Some(&builder.src))
+        .args(["diff-index", "--quiet", &commit])
+        .arg("--")
+        .args([compiler_path, library_path])
+        .as_command_mut()
+        .status()
+        .unwrap()
+        .success();
+
+    assert!(has_changes == config.download_rustc_commit.is_none());
+}
+
 mod defaults {
     use pretty_assertions::assert_eq;
 

src/bootstrap/src/core/config/config.rs

@@ -13,6 +13,7 @@ use std::str::FromStr;
 use std::sync::OnceLock;
 use std::{cmp, env, fs};
 
+use build_helper::ci::CiEnv;
 use build_helper::exit;
 use build_helper::git::{GitConfig, get_closest_merge_commit, output_result};
 use serde::{Deserialize, Deserializer};
@@ -22,6 +23,7 @@ use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX;
 use crate::core::build_steps::llvm;
 pub use crate::core::config::flags::Subcommand;
 use crate::core::config::flags::{Color, Flags, Warnings};
+use crate::core::download::is_download_ci_available;
 use crate::utils::cache::{INTERNER, Interned};
 use crate::utils::channel::{self, GitInfo};
 use crate::utils::helpers::{self, exe, output, t};
@@ -1627,9 +1629,11 @@ impl Config {
             config.mandir = mandir.map(PathBuf::from);
         }
 
+        config.llvm_assertions =
+            toml.llvm.as_ref().map_or(false, |llvm| llvm.assertions.unwrap_or(false));
+
         // Store off these values as options because if they're not provided
         // we'll infer default values for them later
-        let mut llvm_assertions = None;
         let mut llvm_tests = None;
         let mut llvm_enzyme = None;
         let mut llvm_plugins = None;
@@ -1712,7 +1716,8 @@ impl Config {
             is_user_configured_rust_channel = channel.is_some();
             set(&mut config.channel, channel.clone());
 
-            config.download_rustc_commit = config.download_ci_rustc_commit(download_rustc);
+            config.download_rustc_commit =
+                config.download_ci_rustc_commit(download_rustc, config.llvm_assertions);
 
             debug = debug_toml;
             debug_assertions = debug_assertions_toml;
@@ -1848,7 +1853,7 @@ impl Config {
                 optimize: optimize_toml,
                 thin_lto,
                 release_debuginfo,
-                assertions,
+                assertions: _,
                 tests,
                 enzyme,
                 plugins,
@@ -1882,7 +1887,6 @@ impl Config {
                 Some(StringOrBool::Bool(false)) | None => {}
             }
             set(&mut config.ninja_in_file, ninja);
-            llvm_assertions = assertions;
             llvm_tests = tests;
             llvm_enzyme = enzyme;
             llvm_plugins = plugins;
@@ -1911,8 +1915,8 @@ impl Config {
             config.llvm_enable_warnings = enable_warnings.unwrap_or(false);
             config.llvm_build_config = build_config.clone().unwrap_or(Default::default());
 
-            let asserts = llvm_assertions.unwrap_or(false);
+            config.llvm_from_ci =
-            config.llvm_from_ci = config.parse_download_ci_llvm(download_ci_llvm, asserts);
+                config.parse_download_ci_llvm(download_ci_llvm, config.llvm_assertions);
 
             if config.llvm_from_ci {
                 let warn = |option: &str| {
@@ -2080,7 +2084,6 @@ impl Config {
         // Now that we've reached the end of our configuration, infer the
         // default values for all options that we haven't otherwise stored yet.
 
-        config.llvm_assertions = llvm_assertions.unwrap_or(false);
         config.llvm_tests = llvm_tests.unwrap_or(false);
         config.llvm_enzyme = llvm_enzyme.unwrap_or(false);
         config.llvm_plugins = llvm_plugins.unwrap_or(false);
@@ -2419,8 +2422,9 @@ impl Config {
                             ci_config_toml,
                         );
 
-                        let disable_ci_rustc_if_incompatible =
+                        // Primarily used by CI runners to avoid handling download-rustc incompatible
-                            env::var_os("DISABLE_CI_RUSTC_IF_INCOMPATIBLE")
+                        // options one by one on shell scripts.
+                        let disable_ci_rustc_if_incompatible = env::var_os("DISABLE_CI_RUSTC_IF_INCOMPATIBLE")
                             .is_some_and(|s| s == "1" || s == "true");
 
                         if disable_ci_rustc_if_incompatible && res.is_err() {
@@ -2711,7 +2715,15 @@ impl Config {
     }
 
     /// Returns the commit to download, or `None` if we shouldn't download CI artifacts.
-    fn download_ci_rustc_commit(&self, download_rustc: Option<StringOrBool>) -> Option<String> {
+    fn download_ci_rustc_commit(
+        &self,
+        download_rustc: Option<StringOrBool>,
+        llvm_assertions: bool,
+    ) -> Option<String> {
+        if !is_download_ci_available(&self.build.triple, llvm_assertions) {
+            return None;
+        }
+
         // If `download-rustc` is not set, default to rebuilding.
         let if_unchanged = match download_rustc {
             None | Some(StringOrBool::Bool(false)) => return None,
@@ -2724,7 +2736,11 @@ impl Config {
 
         // Look for a version to compare to based on the current commit.
         // Only commits merged by bors will have CI artifacts.
-        let commit = get_closest_merge_commit(Some(&self.src), &self.git_config(), &[]).unwrap();
+        let commit = get_closest_merge_commit(Some(&self.src), &self.git_config(), &[
+            self.src.join("compiler"),
+            self.src.join("library"),
+        ])
+        .unwrap();
         if commit.is_empty() {
             println!("ERROR: could not find commit hash for downloading rustc");
             println!("HELP: maybe your repository history is too shallow?");
@@ -2733,6 +2749,19 @@ impl Config {
             crate::exit!(1);
         }
 
+        if CiEnv::is_ci() && {
+            let head_sha =
+                output(helpers::git(Some(&self.src)).arg("rev-parse").arg("HEAD").as_command_mut());
+            let head_sha = head_sha.trim();
+            commit == head_sha
+        } {
+            eprintln!("CI rustc commit matches with HEAD and we are in CI.");
+            eprintln!(
+                "`rustc.download-ci` functionality will be skipped as artifacts are not available."
+            );
+            return None;
+        }
+
         // Warn if there were changes to the compiler or standard library since the ancestor commit.
         let has_changes = !t!(helpers::git(Some(&self.src))
             .args(["diff-index", "--quiet", &commit])

src/bootstrap/src/core/config/tests.rs

@@ -12,7 +12,7 @@ use super::{ChangeIdWrapper, Config};
 use crate::core::build_steps::clippy::get_clippy_rules_in_order;
 use crate::core::config::{LldMode, Target, TargetSelection, TomlConfig};
 
-fn parse(config: &str) -> Config {
+pub(crate) fn parse(config: &str) -> Config {
     Config::parse_inner(
         Flags::parse(&["check".to_string(), "--config=/does/not/exist".to_string()]),
         |&_| toml::from_str(&config),

src/bootstrap/src/core/download.rs

@@ -832,3 +832,43 @@ fn path_is_dylib(path: &Path) -> bool {
     // The .so is not necessarily the extension, it might be libLLVM.so.18.1
     path.to_str().map_or(false, |path| path.contains(".so"))
 }
+
+/// Checks whether the CI rustc is available for the given target triple.
+pub(crate) fn is_download_ci_available(target_triple: &str, llvm_assertions: bool) -> bool {
+    // All tier 1 targets and tier 2 targets with host tools.
+    const SUPPORTED_PLATFORMS: &[&str] = &[
+        "aarch64-apple-darwin",
+        "aarch64-pc-windows-msvc",
+        "aarch64-unknown-linux-gnu",
+        "aarch64-unknown-linux-musl",
+        "arm-unknown-linux-gnueabi",
+        "arm-unknown-linux-gnueabihf",
+        "armv7-unknown-linux-gnueabihf",
+        "i686-pc-windows-gnu",
+        "i686-pc-windows-msvc",
+        "i686-unknown-linux-gnu",
+        "loongarch64-unknown-linux-gnu",
+        "powerpc-unknown-linux-gnu",
+        "powerpc64-unknown-linux-gnu",
+        "powerpc64le-unknown-linux-gnu",
+        "riscv64gc-unknown-linux-gnu",
+        "s390x-unknown-linux-gnu",
+        "x86_64-apple-darwin",
+        "x86_64-pc-windows-gnu",
+        "x86_64-pc-windows-msvc",
+        "x86_64-unknown-freebsd",
+        "x86_64-unknown-illumos",
+        "x86_64-unknown-linux-gnu",
+        "x86_64-unknown-linux-musl",
+        "x86_64-unknown-netbsd",
+    ];
+
+    const SUPPORTED_PLATFORMS_WITH_ASSERTIONS: &[&str] =
+        &["x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"];
+
+    if llvm_assertions {
+        SUPPORTED_PLATFORMS_WITH_ASSERTIONS.contains(&target_triple)
+    } else {
+        SUPPORTED_PLATFORMS.contains(&target_triple)
+    }
+}

src/ci/docker/host-x86_64/mingw-check/Dockerfile

@@ -46,7 +46,20 @@ ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1
 # Check library crates on all tier 1 targets.
 # We disable optimized compiler built-ins because that requires a C toolchain for the target.
 # We also skip the x86_64-unknown-linux-gnu target as it is well-tested by other jobs.
-ENV SCRIPT python3 ../x.py check --stage 0 --set build.optimized-compiler-builtins=false core alloc std --target=aarch64-unknown-linux-gnu,i686-pc-windows-msvc,i686-unknown-linux-gnu,x86_64-apple-darwin,x86_64-pc-windows-gnu,x86_64-pc-windows-msvc && \
+ENV SCRIPT \
+           # `core::builder::tests::ci_rustc_if_unchanged_logic` bootstrap test covers the `rust.download-rustc=if-unchanged` logic.
+           # Here we are adding a dummy commit on compiler and running that test to ensure when there is a change on the compiler,
+           # we never download ci rustc with `rust.download-rustc=if-unchanged` option.
+           echo \"\" >> ../compiler/rustc/src/main.rs && \
+           git config --global user.email \"dummy@dummy.com\" && \
+           git config --global user.name \"dummy\" && \
+           git add ../compiler/rustc/src/main.rs && \
+           git commit -m \"test commit for rust.download-rustc=if-unchanged logic\" && \
+           DISABLE_CI_RUSTC_IF_INCOMPATIBLE=0 python3 ../x.py test bootstrap -- core::builder::tests::ci_rustc_if_unchanged_logic && \
+           # Revert the dummy commit
+           git reset --hard HEAD~1 && \
+
+           python3 ../x.py check --stage 0 --set build.optimized-compiler-builtins=false core alloc std --target=aarch64-unknown-linux-gnu,i686-pc-windows-msvc,i686-unknown-linux-gnu,x86_64-apple-darwin,x86_64-pc-windows-gnu,x86_64-pc-windows-msvc && \
            /scripts/check-default-config-profiles.sh && \
            python3 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \
            python3 ../x.py clippy bootstrap -Dwarnings && \

src/ci/docker/host-x86_64/x86_64-fuchsia/Dockerfile

@@ -58,6 +58,9 @@ RUN mkdir -p $RUST_INSTALL_DIR/etc
 # Fuchsia only supports LLVM.
 ENV CODEGEN_BACKENDS llvm
 
+# download-rustc is not allowed for `x install`
+ENV NO_DOWNLOAD_CI_RUSTC 1
+
 ENV RUST_CONFIGURE_ARGS \
   --prefix=$RUST_INSTALL_DIR \
   --sysconfdir=etc \
@@ -70,6 +73,7 @@ ENV RUST_CONFIGURE_ARGS \
   --set target.x86_64-unknown-fuchsia.ar=/usr/local/bin/llvm-ar \
   --set target.x86_64-unknown-fuchsia.ranlib=/usr/local/bin/llvm-ranlib \
   --set target.x86_64-unknown-fuchsia.linker=/usr/local/bin/ld.lld
+
 ENV SCRIPT \
     python3 ../x.py install --target $TARGETS compiler/rustc library/std clippy && \
     bash ../src/ci/docker/host-x86_64/x86_64-fuchsia/build-fuchsia.sh

src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile

@@ -84,6 +84,7 @@ ENV RUST_CONFIGURE_ARGS \
   --enable-new-symbol-mangling
 
 ENV HOST_TARGET x86_64-unknown-linux-gnu
+ENV FORCE_CI_RUSTC 1
 
 COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/
 COPY host-x86_64/dist-x86_64-linux/build-gccjit.sh /scripts/

src/ci/docker/run.sh

@@ -343,6 +343,7 @@ docker \
   --env PR_CI_JOB \
   --env OBJDIR_ON_HOST="$objdir" \
   --env CODEGEN_BACKENDS \
+  --env DISABLE_CI_RUSTC_IF_INCOMPATIBLE="$DISABLE_CI_RUSTC_IF_INCOMPATIBLE" \
   --init \
   --rm \
   rust-ci \

src/ci/github-actions/jobs.yml

@@ -85,6 +85,9 @@ envs:
 # it in each job definition.
 pr:
   - image: mingw-check
+    env:
+      # We are adding (temporarily) a dummy commit on the compiler
+      READ_ONLY_SRC: "0"
     <<: *job-linux-4c
   - image: mingw-check-tidy
     continue_on_error: true
@@ -207,6 +210,8 @@ auto:
     <<: *job-linux-8c
 
   - image: mingw-check
+    env:
+      READ_ONLY_SRC: 0
     <<: *job-linux-4c
 
   - image: test-various

src/ci/run.sh

@@ -52,6 +52,13 @@ if [ "$CI" != "" ]; then
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set change-id=99999999"
 fi
 
+# If runner uses an incompatible option and `FORCE_CI_RUSTC` is not defined,
+# switch to in-tree rustc.
+if [ "$FORCE_CI_RUSTC" == "" ]; then
+    echo "debug: `DISABLE_CI_RUSTC_IF_INCOMPATIBLE` configured."
+    DISABLE_CI_RUSTC_IF_INCOMPATIBLE=1
+fi
+
 if ! isCI || isCiBranch auto || isCiBranch beta || isCiBranch try || isCiBranch try-perf || \
   isCiBranch automation/bors/try; then
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests"
@@ -169,10 +176,16 @@ else
   if [ "$NO_DOWNLOAD_CI_LLVM" = "" ]; then
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.download-ci-llvm=if-unchanged"
   else
+    # CI rustc requires CI LLVM to be enabled (see https://github.com/rust-lang/rust/issues/123586).
+    NO_DOWNLOAD_CI_RUSTC=1
     # When building for CI we want to use the static C++ Standard library
     # included with LLVM, since a dynamic libstdcpp may not be available.
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.static-libstdcpp"
   fi
+
+  if [ "$NO_DOWNLOAD_CI_RUSTC" = "" ]; then
+    RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.download-rustc=if-unchanged"
+  fi
 fi
 
 if [ "$ENABLE_GCC_CODEGEN" = "1" ]; then