Auto merge of #102565 - jyn514:refactor-build-manifest, r=Mark-Simulacrum

Refactor build-manifest to minimize the number of changes needed to add a new component

- Add all components to `PkgType`
- Automate functionality wherever possible, so functions often don't have to be manually edited
- Where that's not possible, use exhaustive matches on `PkgType` instead of adding individual strings.
- Add documentation for how to add a component. Improve the existing documentation for how to test changes.

I tested locally that this generates an identical manifest before and after my change, as follows:
```sh
git checkout d44e14225a
cargo +nightly run --manifest-path src/tools/build-manifest/Cargo.toml build/dist build/manifest-before 1970-01-01 http://example.com nightly
git checkout refactor-build-manifest
cargo +nightly run --manifest-path src/tools/build-manifest/Cargo.toml build/dist build/manifest-before 1970-01-01 http://example.com nightly
sort -u build/manifest-before/channel-rust-nightly.toml | diff - <(sort -u build/manifest-after/channel-rust-nightly.toml)
```
I then verified by hand that the differences before sorting are inconsequential (mostly targets being slightly reordered).

The only change in behavior is that `llvm-tools` is now properly renamed to `llvm-tools-preview`:
```
; sort -u build/manifest-before/channel-rust-nightly.toml | diff - <(sort -u build/manifest-after/channel-rust-nightly.toml)
784a785
> [renames.llvm-tools]
894a896
> to = "llvm-tools-preview"
```

This is based on https://github.com/rust-lang/rust/pull/102241 and should not be merged before.

src/tools/build-manifest/README.md

@@ -1,7 +1,16 @@
 # build-manifest
 
-This tool generates the manifests uploaded to static.rust-lang.org and used by
-rustup. The tool is invoked by the bootstrap tool.
+This tool generates the manifests uploaded to static.rust-lang.org and used by rustup.
+You can see a full list of all manifests at <https://static.rust-lang.org/manifests.txt>.
+This listing is updated by <https://github.com/rust-lang/generate-manifest-list> every 7 days.
+
+This gets called by `promote-release` <https://github.com/rust-lang/promote-release> via `x.py dist hash-and-sign`.
+
+## Adding a new component
+
+1. Add a new `Step` to `dist.rs`. This should usually be named after the filename of the uploaded tarball. See https://github.com/rust-lang/rust/pull/101799/files#diff-2c56335faa24486df09ba392d8900c57e2fac4633e1f7038469bcf9ed3feb871 for an example.
+    a. If appropriate, call `tarball.is_preview(true)` for the component.
+2. Add a new `PkgType` to build-manifest. Fix all the compile errors as appropriate.
 
 ## Testing changes locally
 
@@ -9,19 +18,16 @@ In order to test the changes locally you need to have a valid dist directory
 available locally. If you don't want to build all the compiler, you can easily
 create one from the nightly artifacts with:
 
-```
-#!/bin/bash
-for cmpn in rust rustc rust-std rust-docs cargo; do
-    wget https://static.rust-lang.org/dist/${cmpn}-nightly-x86_64-unknown-linux-gnu.tar.gz
+```sh
+for component in rust rustc rust-std rust-docs cargo; do
+    wget -P build/dist https://static.rust-lang.org/dist/${component}-nightly-x86_64-unknown-linux-gnu.tar.gz
 done
 ```
 
-Then, you can generate the manifest and all the packages from `path/to/dist` to
-`path/to/output` with:
+Then, you can generate the manifest and all the packages from `build/dist` to
+`build/manifest` with:
 
+```sh
+mkdir -p build/manifest
+cargo +nightly run --release -p build-manifest build/dist build/manifest 1970-01-01 http://example.com nightly
 ```
-$ cargo +nightly run path/to/dist path/to/output 1970-01-01 http://example.com CHANNEL
-```
-
-Remember to replace `CHANNEL` with the channel you produced dist artifacts of
-and `VERSION` with the current Rust version.

src/tools/build-manifest/src/main.rs

@@ -1,8 +1,4 @@
-//! Build a dist manifest, hash and sign everything.
-//! This gets called by `promote-release`
-//! (https://github.com/rust-lang/rust-central-station/tree/master/promote-release)
-//! via `x.py dist hash-and-sign`; the cmdline arguments are set up
-//! by rustbuild (in `src/bootstrap/dist.rs`).
+#![doc = include_str!("../README.md")]
 
 mod checksum;
 mod manifest;
@@ -184,7 +180,7 @@ static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin"
 
 static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
 
-static NIGHTLY_ONLY_COMPONENTS: &[&str] = &["miri-preview", "rust-docs-json-preview"];
+static NIGHTLY_ONLY_COMPONENTS: &[PkgType] = &[PkgType::Miri, PkgType::JsonDocs];
 
 macro_rules! t {
     ($e:expr) => {
@@ -287,28 +283,9 @@ impl Builder {
     }
 
     fn add_packages_to(&mut self, manifest: &mut Manifest) {
-        macro_rules! package {
-            ($name:expr, $targets:expr) => {
-                self.package($name, &mut manifest.pkg, $targets, &[])
-            };
+        for pkg in PkgType::all() {
+            self.package(pkg, &mut manifest.pkg);
         }
-        package!("rustc", HOSTS);
-        package!("rustc-dev", HOSTS);
-        package!("reproducible-artifacts", HOSTS);
-        package!("rustc-docs", HOSTS);
-        package!("cargo", HOSTS);
-        package!("rust-mingw", MINGW);
-        package!("rust-std", TARGETS);
-        self.package("rust-docs", &mut manifest.pkg, HOSTS, DOCS_FALLBACK);
-        self.package("rust-docs-json-preview", &mut manifest.pkg, HOSTS, DOCS_FALLBACK);
-        package!("rust-src", &["*"]);
-        package!("rls-preview", HOSTS);
-        package!("rust-analyzer-preview", HOSTS);
-        package!("clippy-preview", HOSTS);
-        package!("miri-preview", HOSTS);
-        package!("rustfmt-preview", HOSTS);
-        package!("rust-analysis", TARGETS);
-        package!("llvm-tools-preview", TARGETS);
     }
 
     fn add_artifacts_to(&mut self, manifest: &mut Manifest) {
@@ -333,44 +310,28 @@ impl Builder {
     }
 
     fn add_profiles_to(&mut self, manifest: &mut Manifest) {
-        let mut profile = |name, pkgs| self.profile(name, &mut manifest.profiles, pkgs);
-        profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]);
-        profile(
-            "default",
-            &[
-                "rustc",
-                "cargo",
-                "rust-std",
-                "rust-mingw",
-                "rust-docs",
-                "rustfmt-preview",
-                "clippy-preview",
-            ],
-        );
-        profile(
-            "complete",
-            &[
-                "rustc",
-                "cargo",
-                "rust-std",
-                "rust-mingw",
-                "rust-docs",
-                "rustfmt-preview",
-                "clippy-preview",
-                "rls-preview",
-                "rust-analyzer-preview",
-                "rust-src",
-                "llvm-tools-preview",
-                "rust-analysis",
-                "miri-preview",
-            ],
-        );
+        use PkgType::*;
+
+        let mut profile = |name, pkgs: &_| self.profile(name, &mut manifest.profiles, pkgs);
+
+        // Use a Vec here to make sure we don't exclude any components in an earlier profile.
+        let minimal = vec![Rustc, Cargo, RustStd, RustMingw];
+        profile("minimal", &minimal);
+
+        let mut default = minimal;
+        default.extend([HtmlDocs, Rustfmt, Clippy]);
+        profile("default", &default);
+
+        // NOTE: this profile is effectively deprecated; do not add new components to it.
+        let mut complete = default;
+        complete.extend([Rls, RustAnalyzer, RustSrc, LlvmTools, RustAnalysis, Miri]);
+        profile("complete", &complete);
 
         // The compiler libraries are not stable for end users, and they're also huge, so we only
         // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible
         // for users to install the additional component manually, if needed.
         if self.versions.channel() == "nightly" {
-            self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);
+            self.extend_profile("complete", &mut manifest.profiles, &[RustcDev]);
             // Do not include the rustc-docs component for now, as it causes
             // conflicts with the rust-docs component when installed. See
             // #75833.
@@ -382,12 +343,11 @@ impl Builder {
         let mut rename = |from: &str, to: &str| {
             manifest.renames.insert(from.to_owned(), Rename { to: to.to_owned() })
         };
-        rename("rls", "rls-preview");
-        rename("rustfmt", "rustfmt-preview");
-        rename("clippy", "clippy-preview");
-        rename("miri", "miri-preview");
-        rename("rust-docs-json", "rust-docs-json-preview");
-        rename("rust-analyzer", "rust-analyzer-preview");
+        for pkg in PkgType::all() {
+            if pkg.is_preview() {
+                rename(pkg.tarball_component_name(), &pkg.manifest_component_name());
+            }
+        }
     }
 
     fn rust_package(&mut self, manifest: &Manifest) -> Package {
@@ -419,43 +379,53 @@ impl Builder {
         let mut components = Vec::new();
         let mut extensions = Vec::new();
 
-        let host_component = |pkg| Component::from_str(pkg, host);
+        let host_component = |pkg: &_| Component::from_pkg(pkg, host);
 
-        // rustc/rust-std/cargo/docs are all required,
-        // and so is rust-mingw if it's available for the target.
-        components.extend(vec![
-            host_component("rustc"),
-            host_component("rust-std"),
-            host_component("cargo"),
-            host_component("rust-docs"),
-        ]);
-        if host.contains("pc-windows-gnu") {
-            components.push(host_component("rust-mingw"));
+        for pkg in PkgType::all() {
+            match pkg {
+                // rustc/rust-std/cargo/docs are all required
+                PkgType::Rustc | PkgType::Cargo | PkgType::HtmlDocs => {
+                    components.push(host_component(pkg));
+                }
+                PkgType::RustStd => {
+                    components.push(host_component(pkg));
+                    extensions.extend(
+                        TARGETS
+                            .iter()
+                            .filter(|&&target| target != host)
+                            .map(|target| Component::from_pkg(pkg, target)),
+                    );
+                }
+                // so is rust-mingw if it's available for the target
+                PkgType::RustMingw => {
+                    if host.contains("pc-windows-gnu") {
+                        components.push(host_component(pkg));
+                    }
+                }
+                // Tools are always present in the manifest,
+                // but might be marked as unavailable if they weren't built.
+                PkgType::Clippy
+                | PkgType::Miri
+                | PkgType::Rls
+                | PkgType::RustAnalyzer
+                | PkgType::Rustfmt
+                | PkgType::LlvmTools
+                | PkgType::RustAnalysis
+                | PkgType::JsonDocs => {
+                    extensions.push(host_component(pkg));
+                }
+                PkgType::RustcDev | PkgType::RustcDocs => {
+                    extensions.extend(HOSTS.iter().map(|target| Component::from_pkg(pkg, target)));
+                }
+                PkgType::RustSrc => {
+                    extensions.push(Component::from_pkg(pkg, "*"));
+                }
+                PkgType::Rust => {}
+                // NOTE: this is intentional, these artifacts aren't intended to be used with rustup
+                PkgType::ReproducibleArtifacts => {}
+            }
         }
 
-        // Tools are always present in the manifest,
-        // but might be marked as unavailable if they weren't built.
-        extensions.extend(vec![
-            host_component("clippy-preview"),
-            host_component("miri-preview"),
-            host_component("rls-preview"),
-            host_component("rust-analyzer-preview"),
-            host_component("rustfmt-preview"),
-            host_component("llvm-tools-preview"),
-            host_component("rust-analysis"),
-            host_component("rust-docs-json-preview"),
-        ]);
-
-        extensions.extend(
-            TARGETS
-                .iter()
-                .filter(|&&target| target != host)
-                .map(|target| Component::from_str("rust-std", target)),
-        );
-        extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-dev", target)));
-        extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-docs", target)));
-        extensions.push(Component::from_str("rust-src", "*"));
-
         // If the components/extensions don't actually exist for this
         // particular host/target combination then nix it entirely from our
         // lists.
@@ -481,43 +451,44 @@ impl Builder {
         &mut self,
         profile_name: &str,
         dst: &mut BTreeMap<String, Vec<String>>,
-        pkgs: &[&str],
+        pkgs: &[PkgType],
     ) {
-        dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect());
+        dst.insert(
+            profile_name.to_owned(),
+            pkgs.iter().map(|s| s.manifest_component_name()).collect(),
+        );
     }
 
     fn extend_profile(
         &mut self,
         profile_name: &str,
         dst: &mut BTreeMap<String, Vec<String>>,
-        pkgs: &[&str],
+        pkgs: &[PkgType],
     ) {
         dst.get_mut(profile_name)
             .expect("existing profile")
-            .extend(pkgs.iter().map(|s| (*s).to_owned()));
+            .extend(pkgs.iter().map(|s| s.manifest_component_name()));
     }
 
-    fn package(
-        &mut self,
-        pkgname: &str,
-        dst: &mut BTreeMap<String, Package>,
-        targets: &[&str],
-        fallback: &[(&str, &str)],
-    ) {
-        let version_info = self
-            .versions
-            .version(&PkgType::from_component(pkgname))
-            .expect("failed to load package version");
+    fn package(&mut self, pkg: &PkgType, dst: &mut BTreeMap<String, Package>) {
+        if *pkg == PkgType::Rust {
+            // This is handled specially by `rust_package` later.
+            // Order is important, so don't call `rust_package` here.
+            return;
+        }
+
+        let fallback = if pkg.use_docs_fallback() { DOCS_FALLBACK } else { &[] };
+        let version_info = self.versions.version(&pkg).expect("failed to load package version");
         let mut is_present = version_info.present;
 
         // Never ship nightly-only components for other trains.
-        if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) {
+        if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkg) {
             is_present = false; // Pretend the component is entirely missing.
         }
 
         macro_rules! tarball_name {
             ($target_name:expr) => {
-                self.versions.tarball_name(&PkgType::from_component(pkgname), $target_name).unwrap()
+                self.versions.tarball_name(pkg, $target_name).unwrap()
             };
         }
         let mut target_from_compressed_tar = |target_name| {
@@ -546,7 +517,8 @@ impl Builder {
             Target::unavailable()
         };
 
-        let targets = targets
+        let targets = pkg
+            .targets()
             .iter()
             .map(|name| {
                 let target = if is_present {
@@ -561,7 +533,7 @@ impl Builder {
             .collect();
 
         dst.insert(
-            pkgname.to_string(),
+            pkg.manifest_component_name(),
             Package {
                 version: version_info.version.unwrap_or_default(),
                 git_commit_hash: version_info.git_commit,

src/tools/build-manifest/src/manifest.rs

@@ -1,3 +1,4 @@
+use crate::versions::PkgType;
 use crate::Builder;
 use serde::{Serialize, Serializer};
 use std::collections::BTreeMap;
@@ -116,8 +117,8 @@ pub(crate) struct Component {
 }
 
 impl Component {
-    pub(crate) fn from_str(pkg: &str, target: &str) -> Self {
-        Self { pkg: pkg.to_string(), target: target.to_string() }
+    pub(crate) fn from_pkg(pkg: &PkgType, target: &str) -> Self {
+        Self { pkg: pkg.manifest_component_name(), target: target.to_string() }
     }
 }
 

src/tools/build-manifest/src/versions.rs

@@ -8,55 +8,63 @@ use tar::Archive;
 
 const DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu";
 
-#[derive(Debug, Hash, Eq, PartialEq, Clone)]
-pub(crate) enum PkgType {
-    Rust,
-    RustSrc,
-    Rustc,
-    Cargo,
-    Rls,
-    RustAnalyzer,
-    Clippy,
-    Rustfmt,
-    LlvmTools,
-    Miri,
-    JsonDocs,
-    Other(String),
+macro_rules! pkg_type {
+    ( $($variant:ident = $component:literal $(; preview = true $(@$is_preview:tt)? )? ),+ $(,)? ) => {
+        #[derive(Debug, Hash, Eq, PartialEq, Clone)]
+        pub(crate) enum PkgType {
+            $($variant,)+
+        }
+
+        impl PkgType {
+            pub(crate) fn is_preview(&self) -> bool {
+                match self {
+                    $( $( $($is_preview)? PkgType::$variant => true, )? )+
+                    _ => false,
+                }
+            }
+
+            /// First part of the tarball name.
+            pub(crate) fn tarball_component_name(&self) -> &str {
+                match self {
+                    $( PkgType::$variant => $component,)+
+                }
+            }
+
+            pub(crate) fn all() -> &'static [PkgType] {
+                &[ $(PkgType::$variant),+ ]
+            }
+        }
+    }
+}
+
+pkg_type! {
+    Rust = "rust",
+    RustSrc = "rust-src",
+    Rustc = "rustc",
+    RustcDev = "rustc-dev",
+    RustcDocs = "rustc-docs",
+    ReproducibleArtifacts = "reproducible-artifacts",
+    RustMingw = "rust-mingw",
+    RustStd = "rust-std",
+    Cargo = "cargo",
+    HtmlDocs = "rust-docs",
+    RustAnalysis = "rust-analysis",
+    Rls = "rls"; preview = true,
+    RustAnalyzer = "rust-analyzer"; preview = true,
+    Clippy = "clippy"; preview = true,
+    Rustfmt = "rustfmt"; preview = true,
+    LlvmTools = "llvm-tools"; preview = true,
+    Miri = "miri"; preview = true,
+    JsonDocs = "rust-docs-json"; preview = true,
 }
 
 impl PkgType {
-    pub(crate) fn from_component(component: &str) -> Self {
-        match component {
-            "rust" => PkgType::Rust,
-            "rust-src" => PkgType::RustSrc,
-            "rustc" => PkgType::Rustc,
-            "cargo" => PkgType::Cargo,
-            "rls" | "rls-preview" => PkgType::Rls,
-            "rust-analyzer" | "rust-analyzer-preview" => PkgType::RustAnalyzer,
-            "clippy" | "clippy-preview" => PkgType::Clippy,
-            "rustfmt" | "rustfmt-preview" => PkgType::Rustfmt,
-            "llvm-tools" | "llvm-tools-preview" => PkgType::LlvmTools,
-            "miri" | "miri-preview" => PkgType::Miri,
-            "rust-docs-json" | "rust-docs-json-preview" => PkgType::JsonDocs,
-            other => PkgType::Other(other.into()),
-        }
-    }
-
-    /// First part of the tarball name.
-    fn tarball_component_name(&self) -> &str {
-        match self {
-            PkgType::Rust => "rust",
-            PkgType::RustSrc => "rust-src",
-            PkgType::Rustc => "rustc",
-            PkgType::Cargo => "cargo",
-            PkgType::Rls => "rls",
-            PkgType::RustAnalyzer => "rust-analyzer",
-            PkgType::Clippy => "clippy",
-            PkgType::Rustfmt => "rustfmt",
-            PkgType::LlvmTools => "llvm-tools",
-            PkgType::Miri => "miri",
-            PkgType::JsonDocs => "rust-docs-json",
-            PkgType::Other(component) => component,
+    /// Component name in the manifest. In particular, this includes the `-preview` suffix where appropriate.
+    pub(crate) fn manifest_component_name(&self) -> String {
+        if self.is_preview() {
+            format!("{}-preview", self.tarball_component_name())
+        } else {
+            self.tarball_component_name().to_string()
         }
     }
 
@@ -73,10 +81,42 @@ impl PkgType {
             PkgType::Miri => false,
 
             PkgType::Rust => true,
+            PkgType::RustStd => true,
             PkgType::RustSrc => true,
             PkgType::Rustc => true,
             PkgType::JsonDocs => true,
-            PkgType::Other(_) => true,
+            PkgType::HtmlDocs => true,
+            PkgType::RustcDev => true,
+            PkgType::RustcDocs => true,
+            PkgType::ReproducibleArtifacts => true,
+            PkgType::RustMingw => true,
+            PkgType::RustAnalysis => true,
+        }
+    }
+
+    pub(crate) fn targets(&self) -> &[&str] {
+        use crate::{HOSTS, MINGW, TARGETS};
+        use PkgType::*;
+
+        match self {
+            Rust => HOSTS, // doesn't matter in practice, but return something to avoid panicking
+            Rustc => HOSTS,
+            RustcDev => HOSTS,
+            ReproducibleArtifacts => HOSTS,
+            RustcDocs => HOSTS,
+            Cargo => HOSTS,
+            RustMingw => MINGW,
+            RustStd => TARGETS,
+            HtmlDocs => HOSTS,
+            JsonDocs => HOSTS,
+            RustSrc => &["*"],
+            Rls => HOSTS,
+            RustAnalyzer => HOSTS,
+            Clippy => HOSTS,
+            Miri => HOSTS,
+            Rustfmt => HOSTS,
+            RustAnalysis => TARGETS,
+            LlvmTools => TARGETS,
         }
     }
 
@@ -84,6 +124,14 @@ impl PkgType {
     fn target_independent(&self) -> bool {
         *self == PkgType::RustSrc
     }
+
+    /// Whether to package these target-specific docs for another similar target.
+    pub(crate) fn use_docs_fallback(&self) -> bool {
+        match self {
+            PkgType::JsonDocs | PkgType::HtmlDocs => true,
+            _ => false,
+        }
+    }
 }
 
 #[derive(Debug, Default, Clone)]