Rollup merge of #138051 - Kobzol:download-ci-gcc, r=onur-ozkan

Add support for downloading GCC from CI

This PR adds a new bootstrap config section called `gcc` and implements a single config `download-ci-gcc`. Its behavior is similar to `download-ci-llvm`. Since https://github.com/rust-lang/rust/pull/137667, we distribute a CI component that contains the prebuilt `libgccjit.so` library on x64 Linux. With `download-ci-gcc`, this component is downloaded from CI to avoid building GCC locally.

This is an MVP of this functionality, designed for local usage. This PR does not enable this functionality on the LLVM 18 PR CI job which builds `cg_gcc`, and does not implement more complex detection logic. It simply uses `false` (build locally) or `true` (download from CI if you're on the right target, if CI download fails, then bootstrap fails).

The original LLVM CI download functionality has a lot of features and complexity, which we don't need for GCC (yet). I don't like how the LLVM CI stuff is threaded through multiple parts of bootstrap, so with GCC I would like to take a more centralized approach, where the `build::Gcc` step handles download from CI internally. This means that:
- For the rest of bootstrap, it should be transparent whether GCC was built locally or downloaded from CI.
- GCC is not downloaded eagerly unless you actually requested GCC (either you requested `x build gcc` or you asked to build/test the GCC backend).

This approach will require some modifications once we extend this feature, but so far I like this approach much more than putting this stuff into `Config[::parse]`, which already does a ton of stuff that it arguably shouldn't (but it's super difficult to extract its logic out).

This PR is an alternative to https://github.com/rust-lang/rust/pull/130749, which did a more 1:1 copy of the `download-ci-llvm` logic.

r? ``@onur-ozkan``

config.example.toml

@@ -163,6 +163,16 @@
 # Custom CMake defines to set when building LLVM.
 #build-config = {}
 
+# =============================================================================
+# Tweaking how GCC is compiled
+# =============================================================================
+[gcc]
+# Download GCC from CI instead of building it locally.
+# Note that this will attempt to download GCC even if there are local
+# modifications to the `src/gcc` submodule.
+# Currently, this is only supported for the `x86_64-unknown-linux-gnu` target.
+# download-ci-gcc = false
+
 # =============================================================================
 # General build configuration options
 # =============================================================================

src/bootstrap/download-ci-gcc-stamp

@@ -0,0 +1,4 @@
+Change this file to make users of the `download-ci-gcc` configuration download
+a new version of GCC from CI, even if the GCC submodule hasn’t changed.
+
+Last change is for: https://github.com/rust-lang/rust/pull/138051

src/bootstrap/src/core/build_steps/gcc.rs

@@ -14,13 +14,67 @@ use std::sync::OnceLock;
 
 use build_helper::ci::CiEnv;
 
-use crate::Kind;
-use crate::core::builder::{Builder, Cargo, RunConfig, ShouldRun, Step};
+use crate::core::builder::{Builder, Cargo, Kind, RunConfig, ShouldRun, Step};
 use crate::core::config::TargetSelection;
 use crate::utils::build_stamp::{BuildStamp, generate_smart_stamp_hash};
 use crate::utils::exec::command;
 use crate::utils::helpers::{self, t};
 
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+pub struct Gcc {
+    pub target: TargetSelection,
+}
+
+#[derive(Clone)]
+pub struct GccOutput {
+    pub libgccjit: PathBuf,
+}
+
+impl Step for Gcc {
+    type Output = GccOutput;
+
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/gcc").alias("gcc")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Gcc { target: run.target });
+    }
+
+    /// Compile GCC (specifically `libgccjit`) for `target`.
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
+        let target = self.target;
+
+        // If GCC has already been built, we avoid building it again.
+        let metadata = match get_gcc_build_status(builder, target) {
+            GccBuildStatus::AlreadyBuilt(path) => return GccOutput { libgccjit: path },
+            GccBuildStatus::ShouldBuild(m) => m,
+        };
+
+        let _guard = builder.msg_unstaged(Kind::Build, "GCC", target);
+        t!(metadata.stamp.remove());
+        let _time = helpers::timeit(builder);
+
+        let libgccjit_path = libgccjit_built_path(&metadata.install_dir);
+        if builder.config.dry_run() {
+            return GccOutput { libgccjit: libgccjit_path };
+        }
+
+        build_gcc(&metadata, builder, target);
+
+        let lib_alias = metadata.install_dir.join("lib/libgccjit.so.0");
+        if !lib_alias.exists() {
+            t!(builder.symlink_file(&libgccjit_path, lib_alias));
+        }
+
+        t!(metadata.stamp.write());
+
+        GccOutput { libgccjit: libgccjit_path }
+    }
+}
+
 pub struct Meta {
     stamp: BuildStamp,
     out_dir: PathBuf,
@@ -34,17 +88,45 @@ pub enum GccBuildStatus {
     ShouldBuild(Meta),
 }
 
-/// This returns whether we've already previously built GCC.
+/// Tries to download GCC from CI if it is enabled and GCC artifacts
+/// are available for the given target.
+/// Returns a path to the libgccjit.so file.
+#[cfg(not(test))]
+fn try_download_gcc(builder: &Builder<'_>, target: TargetSelection) -> Option<PathBuf> {
+    // Try to download GCC from CI if configured and available
+    if !matches!(builder.config.gcc_ci_mode, crate::core::config::GccCiMode::DownloadFromCi) {
+        return None;
+    }
+    if target != "x86_64-unknown-linux-gnu" {
+        eprintln!("GCC CI download is only available for the `x86_64-unknown-linux-gnu` target");
+        return None;
+    }
+    let sha =
+        detect_gcc_sha(&builder.config, builder.config.rust_info.is_managed_git_subrepository());
+    let root = ci_gcc_root(&builder.config);
+    let gcc_stamp = BuildStamp::new(&root).with_prefix("gcc").add_stamp(&sha);
+    if !gcc_stamp.is_up_to_date() && !builder.config.dry_run() {
+        builder.config.download_ci_gcc(&sha, &root);
+        t!(gcc_stamp.write());
+    }
+    // FIXME: put libgccjit.so into a lib directory in dist::Gcc
+    Some(root.join("libgccjit.so"))
+}
+
+#[cfg(test)]
+fn try_download_gcc(_builder: &Builder<'_>, _target: TargetSelection) -> Option<PathBuf> {
+    None
+}
+
+/// This returns information about whether GCC should be built or if it's already built.
+/// It transparently handles downloading GCC from CI if needed.
 ///
 /// It's used to avoid busting caches during x.py check -- if we've already built
 /// GCC, it's fine for us to not try to avoid doing so.
-pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> GccBuildStatus {
-    // Initialize the gcc submodule if not initialized already.
-    builder.config.update_submodule("src/gcc");
-
-    let root = builder.src.join("src/gcc");
-    let out_dir = builder.gcc_out(target).join("build");
-    let install_dir = builder.gcc_out(target).join("install");
+pub fn get_gcc_build_status(builder: &Builder<'_>, target: TargetSelection) -> GccBuildStatus {
+    if let Some(path) = try_download_gcc(builder, target) {
+        return GccBuildStatus::AlreadyBuilt(path);
+    }
 
     static STAMP_HASH_MEMO: OnceLock<String> = OnceLock::new();
     let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| {
@@ -55,6 +137,13 @@ pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> Gc
         )
     });
 
+    // Initialize the gcc submodule if not initialized already.
+    builder.config.update_submodule("src/gcc");
+
+    let root = builder.src.join("src/gcc");
+    let out_dir = builder.gcc_out(target).join("build");
+    let install_dir = builder.gcc_out(target).join("install");
+
     let stamp = BuildStamp::new(&out_dir).with_prefix("gcc").add_stamp(smart_stamp_hash);
 
     if stamp.is_up_to_date() {
@@ -87,125 +176,72 @@ fn libgccjit_built_path(install_dir: &Path) -> PathBuf {
     install_dir.join("lib/libgccjit.so")
 }
 
-#[derive(Debug, Clone, Hash, PartialEq, Eq)]
-pub struct Gcc {
-    pub target: TargetSelection,
-}
+fn build_gcc(metadata: &Meta, builder: &Builder<'_>, target: TargetSelection) {
+    let Meta { stamp: _, out_dir, install_dir, root } = metadata;
 
-#[derive(Clone)]
-pub struct GccOutput {
-    pub libgccjit: PathBuf,
-}
+    t!(fs::create_dir_all(out_dir));
+    t!(fs::create_dir_all(install_dir));
 
-impl Step for Gcc {
-    type Output = GccOutput;
-
-    const ONLY_HOSTS: bool = true;
-
-    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
-        run.path("src/gcc").alias("gcc")
-    }
-
-    fn make_run(run: RunConfig<'_>) {
-        run.builder.ensure(Gcc { target: run.target });
-    }
-
-    /// Compile GCC (specifically `libgccjit`) for `target`.
-    fn run(self, builder: &Builder<'_>) -> Self::Output {
-        let target = self.target;
-
-        // If GCC has already been built, we avoid building it again.
-        let Meta { stamp, out_dir, install_dir, root } = match prebuilt_gcc_config(builder, target)
-        {
-            GccBuildStatus::AlreadyBuilt(path) => return GccOutput { libgccjit: path },
-            GccBuildStatus::ShouldBuild(m) => m,
-        };
-
-        let _guard = builder.msg_unstaged(Kind::Build, "GCC", target);
-        t!(stamp.remove());
-        let _time = helpers::timeit(builder);
-        t!(fs::create_dir_all(&out_dir));
-        t!(fs::create_dir_all(&install_dir));
-
-        let libgccjit_path = libgccjit_built_path(&install_dir);
-        if builder.config.dry_run() {
-            return GccOutput { libgccjit: libgccjit_path };
+    // GCC creates files (e.g. symlinks to the downloaded dependencies)
+    // in the source directory, which does not work with our CI setup, where we mount
+    // source directories as read-only on Linux.
+    // Therefore, as a part of the build in CI, we first copy the whole source directory
+    // to the build directory, and perform the build from there.
+    let src_dir = if CiEnv::is_ci() {
+        let src_dir = builder.gcc_out(target).join("src");
+        if src_dir.exists() {
+            builder.remove_dir(&src_dir);
         }
+        builder.create_dir(&src_dir);
+        builder.cp_link_r(root, &src_dir);
+        src_dir
+    } else {
+        root.clone()
+    };
 
-        // GCC creates files (e.g. symlinks to the downloaded dependencies)
-        // in the source directory, which does not work with our CI setup, where we mount
-        // source directories as read-only on Linux.
-        // Therefore, as a part of the build in CI, we first copy the whole source directory
-        // to the build directory, and perform the build from there.
-        let src_dir = if CiEnv::is_ci() {
-            let src_dir = builder.gcc_out(target).join("src");
-            if src_dir.exists() {
-                builder.remove_dir(&src_dir);
-            }
-            builder.create_dir(&src_dir);
-            builder.cp_link_r(&root, &src_dir);
-            src_dir
-        } else {
-            root
-        };
+    command(src_dir.join("contrib/download_prerequisites")).current_dir(&src_dir).run(builder);
+    let mut configure_cmd = command(src_dir.join("configure"));
+    configure_cmd
+        .current_dir(out_dir)
+        // On CI, we compile GCC with Clang.
+        // The -Wno-everything flag is needed to make GCC compile with Clang 19.
+        // `-g -O2` are the default flags that are otherwise used by Make.
+        // FIXME(kobzol): change the flags once we have [gcc] configuration in config.toml.
+        .env("CXXFLAGS", "-Wno-everything -g -O2")
+        .env("CFLAGS", "-Wno-everything -g -O2")
+        .arg("--enable-host-shared")
+        .arg("--enable-languages=jit")
+        .arg("--enable-checking=release")
+        .arg("--disable-bootstrap")
+        .arg("--disable-multilib")
+        .arg(format!("--prefix={}", install_dir.display()));
+    let cc = builder.build.cc(target).display().to_string();
+    let cc = builder
+        .build
+        .config
+        .ccache
+        .as_ref()
+        .map_or_else(|| cc.clone(), |ccache| format!("{ccache} {cc}"));
+    configure_cmd.env("CC", cc);
 
-        command(src_dir.join("contrib/download_prerequisites")).current_dir(&src_dir).run(builder);
-        let mut configure_cmd = command(src_dir.join("configure"));
-        configure_cmd
-            .current_dir(&out_dir)
-            // On CI, we compile GCC with Clang.
-            // The -Wno-everything flag is needed to make GCC compile with Clang 19.
-            // `-g -O2` are the default flags that are otherwise used by Make.
-            // FIXME(kobzol): change the flags once we have [gcc] configuration in config.toml.
-            .env("CXXFLAGS", "-Wno-everything -g -O2")
-            .env("CFLAGS", "-Wno-everything -g -O2")
-            .arg("--enable-host-shared")
-            .arg("--enable-languages=jit")
-            .arg("--enable-checking=release")
-            .arg("--disable-bootstrap")
-            .arg("--disable-multilib")
-            .arg(format!("--prefix={}", install_dir.display()));
-        let cc = builder.build.cc(target).display().to_string();
-        let cc = builder
+    if let Ok(ref cxx) = builder.build.cxx(target) {
+        let cxx = cxx.display().to_string();
+        let cxx = builder
             .build
             .config
             .ccache
             .as_ref()
-            .map_or_else(|| cc.clone(), |ccache| format!("{ccache} {cc}"));
-        configure_cmd.env("CC", cc);
-
-        if let Ok(ref cxx) = builder.build.cxx(target) {
-            let cxx = cxx.display().to_string();
-            let cxx = builder
-                .build
-                .config
-                .ccache
-                .as_ref()
-                .map_or_else(|| cxx.clone(), |ccache| format!("{ccache} {cxx}"));
-            configure_cmd.env("CXX", cxx);
-        }
-        configure_cmd.run(builder);
-
-        command("make")
-            .current_dir(&out_dir)
-            .arg("--silent")
-            .arg(format!("-j{}", builder.jobs()))
-            .run_capture_stdout(builder);
-        command("make")
-            .current_dir(&out_dir)
-            .arg("--silent")
-            .arg("install")
-            .run_capture_stdout(builder);
-
-        let lib_alias = install_dir.join("lib/libgccjit.so.0");
-        if !lib_alias.exists() {
-            t!(builder.symlink_file(&libgccjit_path, lib_alias));
-        }
-
-        t!(stamp.write());
-
-        GccOutput { libgccjit: libgccjit_path }
+            .map_or_else(|| cxx.clone(), |ccache| format!("{ccache} {cxx}"));
+        configure_cmd.env("CXX", cxx);
     }
+    configure_cmd.run(builder);
+
+    command("make")
+        .current_dir(out_dir)
+        .arg("--silent")
+        .arg(format!("-j{}", builder.jobs()))
+        .run_capture_stdout(builder);
+    command("make").current_dir(out_dir).arg("--silent").arg("install").run_capture_stdout(builder);
 }
 
 /// Configures a Cargo invocation so that it can build the GCC codegen backend.
@@ -213,3 +249,38 @@ pub fn add_cg_gcc_cargo_flags(cargo: &mut Cargo, gcc: &GccOutput) {
     // Add the path to libgccjit.so to the linker search paths.
     cargo.rustflag(&format!("-L{}", gcc.libgccjit.parent().unwrap().to_str().unwrap()));
 }
+
+/// The absolute path to the downloaded GCC artifacts.
+#[cfg(not(test))]
+fn ci_gcc_root(config: &crate::Config) -> PathBuf {
+    config.out.join(config.build).join("ci-gcc")
+}
+
+/// This retrieves the GCC sha we *want* to use, according to git history.
+#[cfg(not(test))]
+fn detect_gcc_sha(config: &crate::Config, is_git: bool) -> String {
+    use build_helper::git::get_closest_merge_commit;
+
+    let gcc_sha = if is_git {
+        get_closest_merge_commit(
+            Some(&config.src),
+            &config.git_config(),
+            &[config.src.join("src/gcc"), config.src.join("src/bootstrap/download-ci-gcc-stamp")],
+        )
+        .unwrap()
+    } else if let Some(info) = crate::utils::channel::read_commit_info_file(&config.src) {
+        info.sha.trim().to_owned()
+    } else {
+        "".to_owned()
+    };
+
+    if gcc_sha.is_empty() {
+        eprintln!("error: could not find commit hash for downloading GCC");
+        eprintln!("HELP: maybe your repository history is too shallow?");
+        eprintln!("HELP: consider disabling `download-ci-gcc`");
+        eprintln!("HELP: or fetch enough history to include one upstream commit");
+        panic!();
+    }
+
+    gcc_sha
+}

src/bootstrap/src/core/config/config.rs

@@ -171,6 +171,17 @@ impl LldMode {
     }
 }
 
+/// Determines how will GCC be provided.
+#[derive(Default, Clone)]
+pub enum GccCiMode {
+    /// Build GCC from the local `src/gcc` submodule.
+    #[default]
+    BuildLocally,
+    /// Try to download GCC from CI.
+    /// If it is not available on CI, it will be built locally instead.
+    DownloadFromCi,
+}
+
 /// Global configuration for the entire build and/or bootstrap.
 ///
 /// This structure is parsed from `config.toml`, and some of the fields are inferred from `git` or build-time parameters.
@@ -283,6 +294,9 @@ pub struct Config {
     pub llvm_ldflags: Option<String>,
     pub llvm_use_libcxx: bool,
 
+    // gcc codegen options
+    pub gcc_ci_mode: GccCiMode,
+
     // rust codegen options
     pub rust_optimize: RustOptimize,
     pub rust_codegen_units: Option<u32>,
@@ -676,6 +690,7 @@ pub(crate) struct TomlConfig {
     build: Option<Build>,
     install: Option<Install>,
     llvm: Option<Llvm>,
+    gcc: Option<Gcc>,
     rust: Option<Rust>,
     target: Option<HashMap<String, TomlTarget>>,
     dist: Option<Dist>,
@@ -710,7 +725,7 @@ trait Merge {
 impl Merge for TomlConfig {
     fn merge(
         &mut self,
-        TomlConfig { build, install, llvm, rust, dist, target, profile, change_id }: Self,
+        TomlConfig { build, install, llvm, gcc, rust, dist, target, profile, change_id }: Self,
         replace: ReplaceOpt,
     ) {
         fn do_merge<T: Merge>(x: &mut Option<T>, y: Option<T>, replace: ReplaceOpt) {
@@ -729,6 +744,7 @@ impl Merge for TomlConfig {
         do_merge(&mut self.build, build, replace);
         do_merge(&mut self.install, install, replace);
         do_merge(&mut self.llvm, llvm, replace);
+        do_merge(&mut self.gcc, gcc, replace);
         do_merge(&mut self.rust, rust, replace);
         do_merge(&mut self.dist, dist, replace);
 
@@ -995,6 +1011,13 @@ define_config! {
     }
 }
 
+define_config! {
+    /// TOML representation of how the GCC build is configured.
+    struct Gcc {
+        download_ci_gcc: Option<bool> = "download-ci-gcc",
+    }
+}
+
 define_config! {
     struct Dist {
         sign_folder: Option<String> = "sign-folder",
@@ -2136,6 +2159,16 @@ impl Config {
             config.llvm_from_ci = config.parse_download_ci_llvm(None, false);
         }
 
+        if let Some(gcc) = toml.gcc {
+            config.gcc_ci_mode = match gcc.download_ci_gcc {
+                Some(value) => match value {
+                    true => GccCiMode::DownloadFromCi,
+                    false => GccCiMode::BuildLocally,
+                },
+                None => GccCiMode::default(),
+            };
+        }
+
         if let Some(t) = toml.target {
             for (triple, cfg) in t {
                 let mut target = Target::from_triple(&triple);

src/bootstrap/src/core/download.rs

@@ -826,6 +826,34 @@ download-rustc = false
         let llvm_root = self.ci_llvm_root();
         self.unpack(&tarball, &llvm_root, "rust-dev");
     }
+
+    pub fn download_ci_gcc(&self, gcc_sha: &str, root_dir: &Path) {
+        let cache_prefix = format!("gcc-{gcc_sha}");
+        let cache_dst =
+            self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache"));
+
+        let gcc_cache = cache_dst.join(cache_prefix);
+        if !gcc_cache.exists() {
+            t!(fs::create_dir_all(&gcc_cache));
+        }
+        let base = &self.stage0_metadata.config.artifacts_server;
+        let filename = format!("gcc-nightly-{}.tar.xz", self.build.triple);
+        let tarball = gcc_cache.join(&filename);
+        if !tarball.exists() {
+            let help_on_error = "ERROR: failed to download gcc from ci
+
+    HELP: There could be two reasons behind this:
+        1) The host triple is not supported for `download-ci-gcc`.
+        2) Old builds get deleted after a certain time.
+    HELP: In either case, disable `download-ci-gcc` in your config.toml:
+
+    [gcc]
+    download-ci-gcc = false
+    ";
+            self.download_file(&format!("{base}/{gcc_sha}/{filename}"), &tarball, help_on_error);
+        }
+        self.unpack(&tarball, root_dir, "gcc");
+    }
 }
 
 fn path_is_dylib(path: &Path) -> bool {

src/bootstrap/src/utils/change_tracker.rs

@@ -370,4 +370,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[
         severity: ChangeSeverity::Info,
         summary: "The rust.description option has moved to build.description and rust.description is now deprecated.",
     },
+    ChangeInfo {
+        change_id: 138051,
+        severity: ChangeSeverity::Info,
+        summary: "There is now a new `gcc` config section that can be used to download GCC from CI using `gcc.download-ci-gcc = true`",
+    },
 ];

triagebot.toml

@@ -958,6 +958,9 @@ If appropriate, please update `CONFIG_CHANGE_HISTORY` in `src/bootstrap/src/util
 [mentions."src/bootstrap/src/core/build_steps/llvm.rs"]
 message = "This PR changes how LLVM is built. Consider updating src/bootstrap/download-ci-llvm-stamp."
 
+[mentions."src/bootstrap/src/core/build_steps/gcc.rs"]
+message = "This PR changes how GCC is built. Consider updating src/bootstrap/download-ci-gcc-stamp."
+
 [mentions."tests/crashes"]
 message = "This PR changes a file inside `tests/crashes`. If a crash was fixed, please move into the corresponding `ui` subdir and add 'Fixes #<issueNr>' to the PR description to autoclose the issue upon merge."