Auto merge of #100557 - dawnofmidnight:tarball-commit-info, r=Mark-Simulacrum

fix: use git-commit-info for version information

Fixes #33286.
Fixes #86587.

This PR changes the current `git-commit-hash` file that `./x.py` dist puts in the `rustc-{version}-src.tar.{x,g}z` to contain the hash, the short hash, and the commit date from which the tarball was created, assuming git was available when it was. It uses this for reading the version so that rustc has all the appropriate metadata.

# Testing

Testing this is kind of a pain. I did it with something like
```sh
./x.py dist # ensure that `ignore-git` is `false` in config.toml
cp ./build/dist/rustc-1.65.0-dev-src.tar.gz ../rustc-1.65.0-dev-src.tar.gz
cd .. && tar -xzf rustc-1.65.0-dev-src && cd rustc-1.65.0-dev-src
./x.py build
```

Then, the output of  `rustc -vV` with the stage1 compiler should have the `commit-hash` and `commit-date` fields filled, rather than be `unknown`. To be completely sure, you can use `rustc --sysroot` with the stdlib that the original `./x.py dist` made, which will require that the metadata matches.

config.toml.example

@@ -525,6 +525,12 @@ changelog-seen = 2
 # A descriptive string to be appended to `rustc --version` output, which is
 # also used in places like debuginfo `DW_AT_producer`. This may be useful for
 # supplementary build information, like distro-specific package versions.
+#
+# The Rust compiler will differentiate between versions of itself, including
+# based on this string, which means that if you wish to be compatible with
+# upstream Rust you need to set this to "". However, note that if you are not
+# actually compatible -- for example if you've backported patches that change
+# behavior -- this may lead to miscompilations or other bugs.
 #description = <none> (string)
 
 # The root location of the musl installation directory. The library directory

src/bootstrap/channel.rs

@@ -5,10 +5,12 @@
 //! `package_vers`, and otherwise indicating to the compiler what it should
 //! print out as part of its version information.
 
+use std::fs;
 use std::path::Path;
 use std::process::Command;
 
 use crate::util::output;
+use crate::util::t;
 use crate::Build;
 
 pub enum GitInfo {
@@ -18,19 +20,25 @@ pub enum GitInfo {
     /// If the info should be used (`ignore_git` is false), this will be
     /// `Some`, otherwise it will be `None`.
     Present(Option<Info>),
+    /// This is not a git repostory, but the info can be fetched from the
+    /// `git-commit-info` file.
+    RecordedForTarball(Info),
 }
 
 pub struct Info {
-    commit_date: String,
-    sha: String,
-    short_sha: String,
+    pub commit_date: String,
+    pub sha: String,
+    pub short_sha: String,
 }
 
 impl GitInfo {
     pub fn new(ignore_git: bool, dir: &Path) -> GitInfo {
         // See if this even begins to look like a git dir
         if !dir.join(".git").exists() {
-            return GitInfo::Absent;
+            match read_commit_info_file(dir) {
+                Some(info) => return GitInfo::RecordedForTarball(info),
+                None => return GitInfo::Absent,
+            }
         }
 
         // Make sure git commands work
@@ -65,10 +73,11 @@ impl GitInfo {
         }))
     }
 
-    fn info(&self) -> Option<&Info> {
+    pub fn info(&self) -> Option<&Info> {
         match self {
-            GitInfo::Present(info) => info.as_ref(),
             GitInfo::Absent => None,
+            GitInfo::Present(info) => info.as_ref(),
+            GitInfo::RecordedForTarball(info) => Some(info),
         }
     }
 
@@ -96,10 +105,48 @@ impl GitInfo {
         version
     }
 
-    pub fn is_git(&self) -> bool {
+    /// Returns whether this directory has a `.git` directory which should be managed by bootstrap.
+    pub fn is_managed_git_subrepository(&self) -> bool {
         match self {
-            GitInfo::Absent => false,
+            GitInfo::Absent | GitInfo::RecordedForTarball(_) => false,
             GitInfo::Present(_) => true,
         }
     }
+
+    /// Returns whether this is being built from a tarball.
+    pub fn is_from_tarball(&self) -> bool {
+        match self {
+            GitInfo::Absent | GitInfo::Present(_) => false,
+            GitInfo::RecordedForTarball(_) => true,
+        }
+    }
+}
+
+/// Read the commit information from the `git-commit-info` file given the
+/// project root.
+pub fn read_commit_info_file(root: &Path) -> Option<Info> {
+    if let Ok(contents) = fs::read_to_string(root.join("git-commit-info")) {
+        let mut lines = contents.lines();
+        let sha = lines.next();
+        let short_sha = lines.next();
+        let commit_date = lines.next();
+        let info = match (commit_date, sha, short_sha) {
+            (Some(commit_date), Some(sha), Some(short_sha)) => Info {
+                commit_date: commit_date.to_owned(),
+                sha: sha.to_owned(),
+                short_sha: short_sha.to_owned(),
+            },
+            _ => panic!("the `git-comit-info` file is malformed"),
+        };
+        Some(info)
+    } else {
+        None
+    }
+}
+
+/// Write the commit information to the `git-commit-info` file given the project
+/// root.
+pub fn write_commit_info_file(root: &Path, info: &Info) {
+    let commit_info = format!("{}\n{}\n{}\n", info.sha, info.short_sha, info.commit_date);
+    t!(fs::write(root.join("git-commit-info"), &commit_info));
 }

src/bootstrap/config.rs

@@ -1334,11 +1334,22 @@ impl Config {
         git
     }
 
-    pub(crate) fn artifact_channel(&self, commit: &str) -> String {
-        let mut channel = self.git();
-        channel.arg("show").arg(format!("{}:src/ci/channel", commit));
-        let channel = output(&mut channel);
-        channel.trim().to_owned()
+    pub(crate) fn artifact_channel(&self, builder: &Builder<'_>, commit: &str) -> String {
+        if builder.rust_info.is_managed_git_subrepository() {
+            let mut channel = self.git();
+            channel.arg("show").arg(format!("{}:src/ci/channel", commit));
+            let channel = output(&mut channel);
+            channel.trim().to_owned()
+        } else if let Ok(channel) = fs::read_to_string(builder.src.join("src/ci/channel")) {
+            channel.trim().to_owned()
+        } else {
+            let src = builder.src.display();
+            eprintln!("error: failed to determine artifact channel");
+            eprintln!(
+                "help: either use git or ensure that {src}/src/ci/channel contains the name of the channel to use"
+            );
+            panic!();
+        }
     }
 
     /// Try to find the relative path of `bindir`, otherwise return it in full.
@@ -1475,7 +1486,7 @@ impl Config {
     }
 
     pub fn submodules(&self, rust_info: &GitInfo) -> bool {
-        self.submodules.unwrap_or(rust_info.is_git())
+        self.submodules.unwrap_or(rust_info.is_managed_git_subrepository())
     }
 }
 
@@ -1580,7 +1591,7 @@ fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> {
 
 fn download_ci_rustc(builder: &Builder<'_>, commit: &str) {
     builder.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})"));
-    let channel = builder.config.artifact_channel(commit);
+    let channel = builder.config.artifact_channel(builder, commit);
     let host = builder.config.build.triple;
     let bin_root = builder.out.join(host).join("ci-rustc");
     let rustc_stamp = bin_root.join(".rustc-stamp");

src/bootstrap/dist.rs

@@ -16,6 +16,7 @@ use std::process::Command;
 
 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
 use crate::cache::{Interned, INTERNER};
+use crate::channel;
 use crate::compile;
 use crate::config::TargetSelection;
 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
@@ -918,12 +919,12 @@ impl Step for PlainSourceTarball {
 
         // Create the version file
         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
-        if let Some(sha) = builder.rust_sha() {
-            builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
+        if let Some(info) = builder.rust_info.info() {
+            channel::write_commit_info_file(&plain_dst_src, info);
         }
 
         // If we're building from git sources, we need to vendor a complete distribution.
-        if builder.rust_info.is_git() {
+        if builder.rust_info.is_managed_git_subrepository() {
             // Ensure we have the submodules checked out.
             builder.update_submodule(Path::new("src/tools/rust-analyzer"));
 

src/bootstrap/lib.rs

@@ -400,7 +400,7 @@ impl Build {
     /// line and the filesystem `config`.
     ///
     /// By default all build output will be placed in the current directory.
-    pub fn new(config: Config) -> Build {
+    pub fn new(mut config: Config) -> Build {
         let src = config.src.clone();
         let out = config.out.clone();
 
@@ -474,6 +474,10 @@ impl Build {
             )
         }
 
+        if rust_info.is_from_tarball() && config.description.is_none() {
+            config.description = Some("built from a source tarball".to_owned());
+        }
+
         let mut build = Build {
             initial_rustc: config.initial_rustc.clone(),
             initial_cargo: config.initial_cargo.clone(),
@@ -573,7 +577,9 @@ impl Build {
 
         // NOTE: The check for the empty directory is here because when running x.py the first time,
         // the submodule won't be checked out. Check it out now so we can build it.
-        if !channel::GitInfo::new(false, &absolute_path).is_git() && !dir_is_empty(&absolute_path) {
+        if !channel::GitInfo::new(false, &absolute_path).is_managed_git_subrepository()
+            && !dir_is_empty(&absolute_path)
+        {
             return;
         }
 
@@ -644,7 +650,7 @@ impl Build {
             // Sample output: `submodule.src/rust-installer.path src/tools/rust-installer`
             let submodule = Path::new(line.splitn(2, ' ').nth(1).unwrap());
             // Don't update the submodule unless it's already been cloned.
-            if channel::GitInfo::new(false, submodule).is_git() {
+            if channel::GitInfo::new(false, submodule).is_managed_git_subrepository() {
                 self.update_submodule(submodule);
             }
         }
@@ -1260,7 +1266,7 @@ impl Build {
         match &self.config.channel[..] {
             "stable" => num.to_string(),
             "beta" => {
-                if self.rust_info.is_git() && !self.config.ignore_git {
+                if self.rust_info.is_managed_git_subrepository() && !self.config.ignore_git {
                     format!("{}-beta.{}", num, self.beta_prerelease_version())
                 } else {
                     format!("{}-beta", num)

src/bootstrap/native.rs

@@ -17,6 +17,7 @@ use std::path::{Path, PathBuf};
 use std::process::Command;
 
 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
+use crate::channel;
 use crate::config::TargetSelection;
 use crate::util::get_clang_cl_resource_dir;
 use crate::util::{self, exe, output, program_out_of_date, t, up_to_date};
@@ -115,24 +116,29 @@ pub fn prebuilt_llvm_config(
 }
 
 /// This retrieves the LLVM sha we *want* to use, according to git history.
-pub(crate) fn detect_llvm_sha(config: &crate::config::Config) -> String {
-    let mut rev_list = config.git();
-    rev_list.args(&[
-        PathBuf::from("rev-list"),
-        format!("--author={}", config.stage0_metadata.config.git_merge_commit_email).into(),
-        "-n1".into(),
-        "--first-parent".into(),
-        "HEAD".into(),
-        "--".into(),
-        config.src.join("src/llvm-project"),
-        config.src.join("src/bootstrap/download-ci-llvm-stamp"),
-        // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
-        config.src.join("src/version"),
-    ]);
-    let llvm_sha = output(&mut rev_list);
-    let llvm_sha = llvm_sha.trim();
+pub(crate) fn detect_llvm_sha(config: &crate::config::Config, is_git: bool) -> String {
+    let llvm_sha = if is_git {
+        let mut rev_list = config.git();
+        rev_list.args(&[
+            PathBuf::from("rev-list"),
+            format!("--author={}", config.stage0_metadata.config.git_merge_commit_email).into(),
+            "-n1".into(),
+            "--first-parent".into(),
+            "HEAD".into(),
+            "--".into(),
+            config.src.join("src/llvm-project"),
+            config.src.join("src/bootstrap/download-ci-llvm-stamp"),
+            // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
+            config.src.join("src/version"),
+        ]);
+        output(&mut rev_list).trim().to_owned()
+    } else if let Some(info) = channel::read_commit_info_file(&config.src) {
+        info.sha.trim().to_owned()
+    } else {
+        "".to_owned()
+    };
 
-    if llvm_sha == "" {
+    if &llvm_sha == "" {
         eprintln!("error: could not find commit hash for downloading LLVM");
         eprintln!("help: maybe your repository history is too shallow?");
         eprintln!("help: consider disabling `download-ci-llvm`");
@@ -140,7 +146,7 @@ pub(crate) fn detect_llvm_sha(config: &crate::config::Config) -> String {
         panic!();
     }
 
-    llvm_sha.to_owned()
+    llvm_sha
 }
 
 /// Returns whether the CI-found LLVM is currently usable.
@@ -194,7 +200,9 @@ pub(crate) fn is_ci_llvm_available(config: &crate::config::Config, asserts: bool
     }
 
     if crate::util::CiEnv::is_ci() {
-        let llvm_sha = detect_llvm_sha(config);
+        // We assume we have access to git, so it's okay to unconditionally pass
+        // `true` here.
+        let llvm_sha = detect_llvm_sha(config, true);
         let head_sha = output(config.git().arg("rev-parse").arg("HEAD"));
         let head_sha = head_sha.trim();
         if llvm_sha == head_sha {
@@ -215,7 +223,7 @@ pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
     }
     let llvm_root = config.ci_llvm_root();
     let llvm_stamp = llvm_root.join(".llvm-stamp");
-    let llvm_sha = detect_llvm_sha(&config);
+    let llvm_sha = detect_llvm_sha(&config, builder.rust_info.is_managed_git_subrepository());
     let key = format!("{}{}", llvm_sha, config.llvm_assertions);
     if program_out_of_date(&llvm_stamp, &key) && !config.dry_run {
         download_ci_llvm(builder, &llvm_sha);
@@ -260,7 +268,7 @@ fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) {
     } else {
         &builder.config.stage0_metadata.config.artifacts_server
     };
-    let channel = builder.config.artifact_channel(llvm_sha);
+    let channel = builder.config.artifact_channel(builder, llvm_sha);
     let filename = format!("rust-dev-{}-{}.tar.xz", channel, builder.build.build.triple);
     let tarball = rustc_cache.join(&filename);
     if !tarball.exists() {

src/bootstrap/sanity.rs

@@ -74,7 +74,7 @@ pub fn check(build: &mut Build) {
     let mut cmd_finder = Finder::new();
     // If we've got a git directory we're gonna need git to update
     // submodules and learn about various other aspects.
-    if build.rust_info.is_git() {
+    if build.rust_info.is_managed_git_subrepository() {
         cmd_finder.must_have("git");
     }
 

src/bootstrap/tarball.rs

@@ -4,6 +4,7 @@ use std::{
 };
 
 use crate::builder::Builder;
+use crate::channel;
 use crate::util::t;
 
 #[derive(Copy, Clone)]
@@ -297,8 +298,8 @@ impl<'a> Tarball<'a> {
     fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut Command)) -> GeneratedTarball {
         t!(std::fs::create_dir_all(&self.overlay_dir));
         self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder));
-        if let Some(sha) = self.builder.rust_sha() {
-            self.builder.create(&self.overlay_dir.join("git-commit-hash"), &sha);
+        if let Some(info) = self.builder.rust_info.info() {
+            channel::write_commit_info_file(&self.overlay_dir, info);
         }
         for file in self.overlay.legal_and_readme() {
             self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644);