From 0917b2123fff6cf5357b5edd5db6d675313ab8bd Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Fri, 18 Sep 2020 12:44:25 +0200
Subject: [PATCH 1/5] build-manifest: move PkgType into the versions module
---
src/tools/build-manifest/src/main.rs | 35 +++---------------------
src/tools/build-manifest/src/versions.rs | 27 ++++++++++++++++++
2 files changed, 31 insertions(+), 31 deletions(-)
create mode 100644 src/tools/build-manifest/src/versions.rs
diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs
index ff9ee67763b..7a1878f749e 100644
--- a/src/tools/build-manifest/src/main.rs
+++ b/src/tools/build-manifest/src/main.rs
@@ -4,8 +4,10 @@
//! via `x.py dist hash-and-sign`; the cmdline arguments are set up
//! by rustbuild (in `src/bootstrap/dist.rs`).
-use serde::Serialize;
+mod versions;
+use crate::versions::PkgType;
+use serde::Serialize;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::env;
@@ -336,35 +338,6 @@ fn main() {
.build();
}
-enum PkgType {
- RustSrc,
- Cargo,
- Rls,
- RustAnalyzer,
- Clippy,
- Rustfmt,
- LlvmTools,
- Miri,
- Other,
-}
-
-impl PkgType {
- fn from_component(component: &str) -> Self {
- use PkgType::*;
- match component {
- "rust-src" => RustSrc,
- "cargo" => Cargo,
- "rls" | "rls-preview" => Rls,
- "rust-analyzer" | "rust-analyzer-preview" => RustAnalyzer,
- "clippy" | "clippy-preview" => Clippy,
- "rustfmt" | "rustfmt-preview" => Rustfmt,
- "llvm-tools" | "llvm-tools-preview" => LlvmTools,
- "miri" | "miri-preview" => Miri,
- _ => Other,
- }
- }
-}
-
impl Builder {
fn build(&mut self) {
self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
@@ -702,7 +675,7 @@ impl Builder {
Rustfmt => format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target),
LlvmTools => format!("llvm-tools-{}-{}.tar.gz", self.llvm_tools_release, target),
Miri => format!("miri-{}-{}.tar.gz", self.miri_release, target),
- Other => format!("{}-{}-{}.tar.gz", component, self.rust_release, target),
+ Other(_) => format!("{}-{}-{}.tar.gz", component, self.rust_release, target),
}
}
diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs
new file mode 100644
index 00000000000..044c04914b1
--- /dev/null
+++ b/src/tools/build-manifest/src/versions.rs
@@ -0,0 +1,27 @@
+pub(crate) enum PkgType {
+ RustSrc,
+ Cargo,
+ Rls,
+ RustAnalyzer,
+ Clippy,
+ Rustfmt,
+ LlvmTools,
+ Miri,
+ Other(String),
+}
+
+impl PkgType {
+ pub(crate) fn from_component(component: &str) -> Self {
+ match component {
+ "rust-src" => PkgType::RustSrc,
+ "cargo" => PkgType::Cargo,
+ "rls" | "rls-preview" => PkgType::Rls,
+ "rust-analyzer" | "rust-analyzer-preview" => PkgType::RustAnalyzer,
+ "clippy" | "clippy-preview" => PkgType::Clippy,
+ "rustfmt" | "rustfmt-preview" => PkgType::Rustfmt,
+ "llvm-tools" | "llvm-tools-preview" => PkgType::LlvmTools,
+ "miri" | "miri-preview" => PkgType::Miri,
+ other => PkgType::Other(other.into()),
+ }
+ }
+}
From 3bddfea7e23341f788526f8b303b9a27038511e3 Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Fri, 18 Sep 2020 14:40:01 +0200
Subject: [PATCH 2/5] build-manifest: stop receiving release numbers from
bootstrap
---
Cargo.lock | 1 +
src/bootstrap/dist.rs | 10 +-
src/tools/build-manifest/Cargo.toml | 1 +
src/tools/build-manifest/README.md | 9 +-
src/tools/build-manifest/src/main.rs | 77 +++++----------
src/tools/build-manifest/src/versions.rs | 114 +++++++++++++++++++++++
6 files changed, 146 insertions(+), 66 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index d3f777bc663..5a2dd23a7ae 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -230,6 +230,7 @@ dependencies = [
name = "build-manifest"
version = "0.1.0"
dependencies = [
+ "anyhow",
"serde",
"serde_json",
"toml",
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
index f25ad50c9b7..991f8d4eea3 100644
--- a/src/bootstrap/dist.rs
+++ b/src/bootstrap/dist.rs
@@ -2356,15 +2356,9 @@ impl Step for HashSign {
cmd.arg(sign);
cmd.arg(distdir(builder));
cmd.arg(today.trim());
- cmd.arg(builder.rust_package_vers());
cmd.arg(addr);
- cmd.arg(builder.package_vers(&builder.release_num("cargo")));
- cmd.arg(builder.package_vers(&builder.release_num("rls")));
- cmd.arg(builder.package_vers(&builder.release_num("rust-analyzer/crates/rust-analyzer")));
- cmd.arg(builder.package_vers(&builder.release_num("clippy")));
- cmd.arg(builder.package_vers(&builder.release_num("miri")));
- cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
- cmd.arg(builder.llvm_tools_package_vers());
+ cmd.arg(&builder.config.channel);
+ cmd.arg(&builder.src);
builder.create_dir(&distdir(builder));
diff --git a/src/tools/build-manifest/Cargo.toml b/src/tools/build-manifest/Cargo.toml
index 0bbbabd2998..2da224a54dc 100644
--- a/src/tools/build-manifest/Cargo.toml
+++ b/src/tools/build-manifest/Cargo.toml
@@ -8,3 +8,4 @@ edition = "2018"
toml = "0.5"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
+anyhow = "1.0.32"
diff --git a/src/tools/build-manifest/README.md b/src/tools/build-manifest/README.md
index a80f36d4969..4d7d9f7da18 100644
--- a/src/tools/build-manifest/README.md
+++ b/src/tools/build-manifest/README.md
@@ -21,10 +21,9 @@ Then, you can generate the manifest and all the packages from `path/to/dist` to
```
$ BUILD_MANIFEST_DISABLE_SIGNING=1 cargo +nightly run \
- path/to/dist path/to/output 1970-01-01 \
- nightly nightly nightly nightly nightly nightly nightly nightly \
- http://example.com
+ path/to/dist path/to/output 1970-01-01 http://example.com \
+ CHANNEL path/to/rust/repo
```
-In the future, if the tool complains about missing arguments just add more
-`nightly`s in the middle.
+Remember to replace `CHANNEL` with the channel you produced dist artifacts of
+and `path/to/rust/repo` with the path to your checkout of the Rust repository.
diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs
index 7a1878f749e..c694948bac0 100644
--- a/src/tools/build-manifest/src/main.rs
+++ b/src/tools/build-manifest/src/main.rs
@@ -6,7 +6,7 @@
mod versions;
-use crate::versions::PkgType;
+use crate::versions::{PkgType, Versions};
use serde::Serialize;
use std::collections::BTreeMap;
use std::collections::HashMap;
@@ -227,14 +227,7 @@ macro_rules! t {
}
struct Builder {
- rust_release: String,
- cargo_release: String,
- rls_release: String,
- rust_analyzer_release: String,
- clippy_release: String,
- rustfmt_release: String,
- llvm_tools_release: String,
- miri_release: String,
+ versions: Versions,
input: PathBuf,
output: PathBuf,
@@ -281,15 +274,9 @@ fn main() {
let input = PathBuf::from(args.next().unwrap());
let output = PathBuf::from(args.next().unwrap());
let date = args.next().unwrap();
- let rust_release = args.next().unwrap();
let s3_address = args.next().unwrap();
- let cargo_release = args.next().unwrap();
- let rls_release = args.next().unwrap();
- let rust_analyzer_release = args.next().unwrap();
- let clippy_release = args.next().unwrap();
- let miri_release = args.next().unwrap();
- let rustfmt_release = args.next().unwrap();
- let llvm_tools_release = args.next().unwrap();
+ let channel = args.next().unwrap();
+ let monorepo_path = args.next().unwrap();
// Do not ask for a passphrase while manually testing
let mut passphrase = String::new();
@@ -299,14 +286,7 @@ fn main() {
}
Builder {
- rust_release,
- cargo_release,
- rls_release,
- rust_analyzer_release,
- clippy_release,
- rustfmt_release,
- llvm_tools_release,
- miri_release,
+ versions: Versions::new(&channel, Path::new(&monorepo_path)).unwrap(),
input,
output,
@@ -363,10 +343,11 @@ impl Builder {
self.check_toolstate();
self.digest_and_sign();
let manifest = self.build_manifest();
- self.write_channel_files(&self.rust_release, &manifest);
- if self.rust_release != "beta" && self.rust_release != "nightly" {
- self.write_channel_files("stable", &manifest);
+ let rust_version = self.versions.package_version(&PkgType::Rust).unwrap();
+ self.write_channel_files(self.versions.channel(), &manifest);
+ if self.versions.channel() != rust_version {
+ self.write_channel_files(&rust_version, &manifest);
}
}
@@ -473,7 +454,7 @@ impl Builder {
// The compiler libraries are not stable for end users, and they're also huge, so we only
// `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible
// for users to install the additional component manually, if needed.
- if self.rust_release == "nightly" {
+ if self.versions.channel() == "nightly" {
self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);
self.extend_profile("complete", &mut manifest.profiles, &["rustc-docs"]);
}
@@ -511,7 +492,7 @@ impl Builder {
}
fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option<Target> {
- let filename = self.filename("rust", host);
+ let filename = self.versions.tarball_name(&PkgType::Rust, host).unwrap();
let digest = self.digests.remove(&filename)?;
let xz_filename = filename.replace(".tar.gz", ".tar.xz");
let xz_digest = self.digests.remove(&xz_filename);
@@ -610,7 +591,7 @@ impl Builder {
.unwrap_or_default(); // `is_present` defaults to `false` here.
// Never ship nightly-only components for other trains.
- if self.rust_release != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) {
+ if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) {
is_present = false; // Pretend the component is entirely missing.
}
@@ -619,7 +600,10 @@ impl Builder {
.map(|name| {
if is_present {
// The component generally exists, but it might still be missing for this target.
- let filename = self.filename(pkgname, name);
+ let filename = self
+ .versions
+ .tarball_name(&PkgType::from_component(pkgname), name)
+ .unwrap();
let digest = match self.digests.remove(&filename) {
Some(digest) => digest,
// This component does not exist for this target -- skip it.
@@ -662,23 +646,6 @@ impl Builder {
format!("{}/{}/{}", self.s3_address, self.date, filename)
}
- fn filename(&self, component: &str, target: &str) -> String {
- use PkgType::*;
- match PkgType::from_component(component) {
- RustSrc => format!("rust-src-{}.tar.gz", self.rust_release),
- Cargo => format!("cargo-{}-{}.tar.gz", self.cargo_release, target),
- Rls => format!("rls-{}-{}.tar.gz", self.rls_release, target),
- RustAnalyzer => {
- format!("rust-analyzer-{}-{}.tar.gz", self.rust_analyzer_release, target)
- }
- Clippy => format!("clippy-{}-{}.tar.gz", self.clippy_release, target),
- Rustfmt => format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target),
- LlvmTools => format!("llvm-tools-{}-{}.tar.gz", self.llvm_tools_release, target),
- Miri => format!("miri-{}-{}.tar.gz", self.miri_release, target),
- Other(_) => format!("{}-{}-{}.tar.gz", component, self.rust_release, target),
- }
- }
-
fn cached_version(&self, component: &str) -> &Option<String> {
use PkgType::*;
match PkgType::from_component(component) {
@@ -707,20 +674,24 @@ impl Builder {
}
}
- fn version(&self, component: &str, target: &str) -> Option<String> {
+ fn version(&mut self, component: &str, target: &str) -> Option<String> {
self.untar(component, target, |filename| format!("{}/version", filename))
}
- fn git_commit_hash(&self, component: &str, target: &str) -> Option<String> {
+ fn git_commit_hash(&mut self, component: &str, target: &str) -> Option<String> {
self.untar(component, target, |filename| format!("{}/git-commit-hash", filename))
}
- fn untar<F>(&self, component: &str, target: &str, dir: F) -> Option<String>
+ fn untar<F>(&mut self, component: &str, target: &str, dir: F) -> Option<String>
where
F: FnOnce(String) -> String,
{
+ let filename = self
+ .versions
+ .tarball_name(&PkgType::from_component(component), target)
+ .expect("failed to retrieve the tarball path");
+
let mut cmd = Command::new("tar");
- let filename = self.filename(component, target);
cmd.arg("xf")
.arg(self.input.join(&filename))
.arg(dir(filename.replace(".tar.gz", "")))
diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs
index 044c04914b1..151cfa0d9c6 100644
--- a/src/tools/build-manifest/src/versions.rs
+++ b/src/tools/build-manifest/src/versions.rs
@@ -1,4 +1,10 @@
+use anyhow::{Context, Error};
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+
+#[derive(Debug, Hash, Eq, PartialEq, Clone)]
pub(crate) enum PkgType {
+ Rust,
RustSrc,
Cargo,
Rls,
@@ -13,6 +19,7 @@ pub(crate) enum PkgType {
impl PkgType {
pub(crate) fn from_component(component: &str) -> Self {
match component {
+ "rust" => PkgType::Rust,
"rust-src" => PkgType::RustSrc,
"cargo" => PkgType::Cargo,
"rls" | "rls-preview" => PkgType::Rls,
@@ -24,4 +31,111 @@ impl PkgType {
other => PkgType::Other(other.into()),
}
}
+
+ fn rust_monorepo_path(&self) -> Option<&'static str> {
+ match self {
+ PkgType::Cargo => Some("src/tools/cargo"),
+ PkgType::Rls => Some("src/tools/rls"),
+ PkgType::RustAnalyzer => Some("src/tools/rust-analyzer/crates/rust-analyzer"),
+ PkgType::Clippy => Some("src/tools/clippy"),
+ PkgType::Rustfmt => Some("src/tools/rustfmt"),
+ PkgType::Miri => Some("src/tools/miri"),
+ PkgType::Rust => None,
+ PkgType::RustSrc => None,
+ PkgType::LlvmTools => None,
+ PkgType::Other(_) => None,
+ }
+ }
+
+ fn tarball_component_name(&self) -> &str {
+ match self {
+ PkgType::Rust => "rust",
+ PkgType::RustSrc => "rust-src",
+ PkgType::Cargo => "cargo",
+ PkgType::Rls => "rls",
+ PkgType::RustAnalyzer => "rust-analyzer",
+ PkgType::Clippy => "clippy",
+ PkgType::Rustfmt => "rustfmt",
+ PkgType::LlvmTools => "llvm-tools",
+ PkgType::Miri => "miri",
+ PkgType::Other(component) => component,
+ }
+ }
+}
+
+pub(crate) struct Versions {
+ channel: String,
+ rustc_version: String,
+ monorepo_root: PathBuf,
+ package_versions: HashMap<PkgType, String>,
+}
+
+impl Versions {
+ pub(crate) fn new(channel: &str, monorepo_root: &Path) -> Result<Self, Error> {
+ Ok(Self {
+ channel: channel.into(),
+ rustc_version: std::fs::read_to_string(monorepo_root.join("src").join("version"))
+ .context("failed to read the rustc version from src/version")?
+ .trim()
+ .to_string(),
+ monorepo_root: monorepo_root.into(),
+ package_versions: HashMap::new(),
+ })
+ }
+
+ pub(crate) fn channel(&self) -> &str {
+ &self.channel
+ }
+
+ pub(crate) fn tarball_name(
+ &mut self,
+ package: &PkgType,
+ target: &str,
+ ) -> Result<String, Error> {
+ Ok(format!(
+ "{}-{}-{}.tar.gz",
+ package.tarball_component_name(),
+ self.package_version(package).with_context(|| format!(
+ "failed to get the package version for component {:?}",
+ package,
+ ))?,
+ target
+ ))
+ }
+
+ pub(crate) fn package_version(&mut self, package: &PkgType) -> Result<String, Error> {
+ match self.package_versions.get(package) {
+ Some(release) => Ok(release.clone()),
+ None => {
+ let version = match package.rust_monorepo_path() {
+ Some(path) => {
+ let path = self.monorepo_root.join(path).join("Cargo.toml");
+ let cargo_toml: CargoToml = toml::from_slice(&std::fs::read(path)?)?;
+ cargo_toml.package.version
+ }
+ None => self.rustc_version.clone(),
+ };
+
+ let release = match self.channel.as_str() {
+ "stable" => version,
+ "beta" => "beta".into(),
+ "nightly" => "nightly".into(),
+ _ => format!("{}-dev", version),
+ };
+
+ self.package_versions.insert(package.clone(), release.clone());
+ Ok(release)
+ }
+ }
+ }
+}
+
+#[derive(serde::Deserialize)]
+struct CargoToml {
+ package: CargoTomlPackage,
+}
+
+#[derive(serde::Deserialize)]
+struct CargoTomlPackage {
+ version: String,
}
From 89ffab76b7a17355f8fa0c82b06e6fb7ec48f09c Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Thu, 24 Sep 2020 14:13:40 +0200
Subject: [PATCH 3/5] build-manifest: refactor detecting package versions
---
Cargo.lock | 2 +
src/tools/build-manifest/Cargo.toml | 2 +
src/tools/build-manifest/src/main.rs | 143 ++---------------------
src/tools/build-manifest/src/versions.rs | 103 +++++++++++++++-
4 files changed, 118 insertions(+), 132 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 5a2dd23a7ae..f18db340aa6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -231,8 +231,10 @@ name = "build-manifest"
version = "0.1.0"
dependencies = [
"anyhow",
+ "flate2",
"serde",
"serde_json",
+ "tar",
"toml",
]
diff --git a/src/tools/build-manifest/Cargo.toml b/src/tools/build-manifest/Cargo.toml
index 2da224a54dc..4f89c31936d 100644
--- a/src/tools/build-manifest/Cargo.toml
+++ b/src/tools/build-manifest/Cargo.toml
@@ -9,3 +9,5 @@ toml = "0.5"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
anyhow = "1.0.32"
+flate2 = "1.0.16"
+tar = "0.4.29"
diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs
index c694948bac0..d8813f95be4 100644
--- a/src/tools/build-manifest/src/main.rs
+++ b/src/tools/build-manifest/src/main.rs
@@ -236,24 +236,6 @@ struct Builder {
s3_address: String,
date: String,
- rust_version: Option<String>,
- cargo_version: Option<String>,
- rls_version: Option<String>,
- rust_analyzer_version: Option<String>,
- clippy_version: Option<String>,
- rustfmt_version: Option<String>,
- llvm_tools_version: Option<String>,
- miri_version: Option<String>,
-
- rust_git_commit_hash: Option<String>,
- cargo_git_commit_hash: Option<String>,
- rls_git_commit_hash: Option<String>,
- rust_analyzer_git_commit_hash: Option<String>,
- clippy_git_commit_hash: Option<String>,
- rustfmt_git_commit_hash: Option<String>,
- llvm_tools_git_commit_hash: Option<String>,
- miri_git_commit_hash: Option<String>,
-
should_sign: bool,
}
@@ -286,7 +268,7 @@ fn main() {
}
Builder {
- versions: Versions::new(&channel, Path::new(&monorepo_path)).unwrap(),
+ versions: Versions::new(&channel, &input, Path::new(&monorepo_path)).unwrap(),
input,
output,
@@ -295,24 +277,6 @@ fn main() {
s3_address,
date,
- rust_version: None,
- cargo_version: None,
- rls_version: None,
- rust_analyzer_version: None,
- clippy_version: None,
- rustfmt_version: None,
- llvm_tools_version: None,
- miri_version: None,
-
- rust_git_commit_hash: None,
- cargo_git_commit_hash: None,
- rls_git_commit_hash: None,
- rust_analyzer_git_commit_hash: None,
- clippy_git_commit_hash: None,
- rustfmt_git_commit_hash: None,
- llvm_tools_git_commit_hash: None,
- miri_git_commit_hash: None,
-
should_sign,
}
.build();
@@ -320,26 +284,6 @@ fn main() {
impl Builder {
fn build(&mut self) {
- self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
- self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");
- self.rls_version = self.version("rls", "x86_64-unknown-linux-gnu");
- self.rust_analyzer_version = self.version("rust-analyzer", "x86_64-unknown-linux-gnu");
- self.clippy_version = self.version("clippy", "x86_64-unknown-linux-gnu");
- self.rustfmt_version = self.version("rustfmt", "x86_64-unknown-linux-gnu");
- self.llvm_tools_version = self.version("llvm-tools", "x86_64-unknown-linux-gnu");
- self.miri_version = self.version("miri", "x86_64-unknown-linux-gnu");
-
- self.rust_git_commit_hash = self.git_commit_hash("rust", "x86_64-unknown-linux-gnu");
- self.cargo_git_commit_hash = self.git_commit_hash("cargo", "x86_64-unknown-linux-gnu");
- self.rls_git_commit_hash = self.git_commit_hash("rls", "x86_64-unknown-linux-gnu");
- self.rust_analyzer_git_commit_hash =
- self.git_commit_hash("rust-analyzer", "x86_64-unknown-linux-gnu");
- self.clippy_git_commit_hash = self.git_commit_hash("clippy", "x86_64-unknown-linux-gnu");
- self.rustfmt_git_commit_hash = self.git_commit_hash("rustfmt", "x86_64-unknown-linux-gnu");
- self.llvm_tools_git_commit_hash =
- self.git_commit_hash("llvm-tools", "x86_64-unknown-linux-gnu");
- self.miri_git_commit_hash = self.git_commit_hash("miri", "x86_64-unknown-linux-gnu");
-
self.check_toolstate();
self.digest_and_sign();
let manifest = self.build_manifest();
@@ -368,8 +312,7 @@ impl Builder {
// Mark some tools as missing based on toolstate.
if toolstates.get("miri").map(|s| &*s as &str) != Some("test-pass") {
println!("Miri tests are not passing, removing component");
- self.miri_version = None;
- self.miri_git_commit_hash = None;
+ self.versions.disable_version(&PkgType::Miri);
}
}
@@ -471,13 +414,10 @@ impl Builder {
}
fn rust_package(&mut self, manifest: &Manifest) -> Package {
+ let version_info = self.versions.version(&PkgType::Rust).expect("missing Rust tarball");
let mut pkg = Package {
- version: self
- .cached_version("rust")
- .as_ref()
- .expect("Couldn't find Rust version")
- .clone(),
- git_commit_hash: self.cached_git_commit_hash("rust").clone(),
+ version: version_info.version.expect("missing Rust version"),
+ git_commit_hash: version_info.git_commit,
target: BTreeMap::new(),
};
for host in HOSTS {
@@ -583,12 +523,11 @@ impl Builder {
}
fn package(&mut self, pkgname: &str, dst: &mut BTreeMap<String, Package>, targets: &[&str]) {
- let (version, mut is_present) = self
- .cached_version(pkgname)
- .as_ref()
- .cloned()
- .map(|version| (version, true))
- .unwrap_or_default(); // `is_present` defaults to `false` here.
+ let version_info = self
+ .versions
+ .version(&PkgType::from_component(pkgname))
+ .expect("failed to load package version");
+ let mut is_present = version_info.present;
// Never ship nightly-only components for other trains.
if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) {
@@ -635,8 +574,8 @@ impl Builder {
dst.insert(
pkgname.to_string(),
Package {
- version,
- git_commit_hash: self.cached_git_commit_hash(pkgname).clone(),
+ version: version_info.version.unwrap_or_default(),
+ git_commit_hash: version_info.git_commit,
target: targets,
},
);
@@ -646,64 +585,6 @@ impl Builder {
format!("{}/{}/{}", self.s3_address, self.date, filename)
}
- fn cached_version(&self, component: &str) -> &Option<String> {
- use PkgType::*;
- match PkgType::from_component(component) {
- Cargo => &self.cargo_version,
- Rls => &self.rls_version,
- RustAnalyzer => &self.rust_analyzer_version,
- Clippy => &self.clippy_version,
- Rustfmt => &self.rustfmt_version,
- LlvmTools => &self.llvm_tools_version,
- Miri => &self.miri_version,
- _ => &self.rust_version,
- }
- }
-
- fn cached_git_commit_hash(&self, component: &str) -> &Option<String> {
- use PkgType::*;
- match PkgType::from_component(component) {
- Cargo => &self.cargo_git_commit_hash,
- Rls => &self.rls_git_commit_hash,
- RustAnalyzer => &self.rust_analyzer_git_commit_hash,
- Clippy => &self.clippy_git_commit_hash,
- Rustfmt => &self.rustfmt_git_commit_hash,
- LlvmTools => &self.llvm_tools_git_commit_hash,
- Miri => &self.miri_git_commit_hash,
- _ => &self.rust_git_commit_hash,
- }
- }
-
- fn version(&mut self, component: &str, target: &str) -> Option<String> {
- self.untar(component, target, |filename| format!("{}/version", filename))
- }
-
- fn git_commit_hash(&mut self, component: &str, target: &str) -> Option<String> {
- self.untar(component, target, |filename| format!("{}/git-commit-hash", filename))
- }
-
- fn untar<F>(&mut self, component: &str, target: &str, dir: F) -> Option<String>
- where
- F: FnOnce(String) -> String,
- {
- let filename = self
- .versions
- .tarball_name(&PkgType::from_component(component), target)
- .expect("failed to retrieve the tarball path");
-
- let mut cmd = Command::new("tar");
- cmd.arg("xf")
- .arg(self.input.join(&filename))
- .arg(dir(filename.replace(".tar.gz", "")))
- .arg("-O");
- let output = t!(cmd.output());
- if output.status.success() {
- Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
- } else {
- None
- }
- }
-
fn hash(&self, path: &Path) -> String {
let sha = t!(Command::new("shasum")
.arg("-a")
diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs
index 151cfa0d9c6..3b5caf1cd7b 100644
--- a/src/tools/build-manifest/src/versions.rs
+++ b/src/tools/build-manifest/src/versions.rs
@@ -1,6 +1,12 @@
use anyhow::{Context, Error};
+use flate2::read::GzDecoder;
use std::collections::HashMap;
+use std::fs::File;
+use std::io::Read;
use std::path::{Path, PathBuf};
+use tar::Archive;
+
+const DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu";
#[derive(Debug, Hash, Eq, PartialEq, Clone)]
pub(crate) enum PkgType {
@@ -61,17 +67,46 @@ impl PkgType {
PkgType::Other(component) => component,
}
}
+
+ fn should_use_rust_version(&self) -> bool {
+ match self {
+ PkgType::Cargo => false,
+ PkgType::Rls => false,
+ PkgType::RustAnalyzer => false,
+ PkgType::Clippy => false,
+ PkgType::Rustfmt => false,
+ PkgType::LlvmTools => false,
+ PkgType::Miri => false,
+
+ PkgType::Rust => true,
+ PkgType::RustSrc => true,
+ PkgType::Other(_) => true,
+ }
+ }
+}
+
+#[derive(Debug, Default, Clone)]
+pub(crate) struct VersionInfo {
+ pub(crate) version: Option<String>,
+ pub(crate) git_commit: Option<String>,
+ pub(crate) present: bool,
}
pub(crate) struct Versions {
channel: String,
rustc_version: String,
monorepo_root: PathBuf,
+ dist_path: PathBuf,
package_versions: HashMap<PkgType, String>,
+ versions: HashMap<PkgType, VersionInfo>,
}
impl Versions {
- pub(crate) fn new(channel: &str, monorepo_root: &Path) -> Result<Self, Error> {
+ pub(crate) fn new(
+ channel: &str,
+ dist_path: &Path,
+ monorepo_root: &Path,
+ ) -> Result<Self, Error> {
Ok(Self {
channel: channel.into(),
rustc_version: std::fs::read_to_string(monorepo_root.join("src").join("version"))
@@ -79,7 +114,9 @@ impl Versions {
.trim()
.to_string(),
monorepo_root: monorepo_root.into(),
+ dist_path: dist_path.into(),
package_versions: HashMap::new(),
+ versions: HashMap::new(),
})
}
@@ -87,6 +124,70 @@ impl Versions {
&self.channel
}
+ pub(crate) fn version(&mut self, mut package: &PkgType) -> Result<VersionInfo, Error> {
+ if package.should_use_rust_version() {
+ package = &PkgType::Rust;
+ }
+
+ match self.versions.get(package) {
+ Some(version) => Ok(version.clone()),
+ None => {
+ let version_info = self.load_version_from_tarball(package)?;
+ self.versions.insert(package.clone(), version_info.clone());
+ Ok(version_info)
+ }
+ }
+ }
+
+ fn load_version_from_tarball(&mut self, package: &PkgType) -> Result<VersionInfo, Error> {
+ let tarball_name = self.tarball_name(package, DEFAULT_TARGET)?;
+ let tarball = self.dist_path.join(tarball_name);
+
+ let file = match File::open(&tarball) {
+ Ok(file) => file,
+ Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
+ // Missing tarballs do not return an error, but return empty data.
+ return Ok(VersionInfo::default());
+ }
+ Err(err) => return Err(err.into()),
+ };
+ let mut tar = Archive::new(GzDecoder::new(file));
+
+ let mut version = None;
+ let mut git_commit = None;
+ for entry in tar.entries()? {
+ let mut entry = entry?;
+
+ let dest;
+ match entry.path()?.components().nth(1).and_then(|c| c.as_os_str().to_str()) {
+ Some("version") => dest = &mut version,
+ Some("git-commit-hash") => dest = &mut git_commit,
+ _ => continue,
+ }
+ let mut buf = String::new();
+ entry.read_to_string(&mut buf)?;
+ *dest = Some(buf);
+
+ // Short circuit to avoid reading the whole tar file if not necessary.
+ if version.is_some() && git_commit.is_some() {
+ break;
+ }
+ }
+
+ Ok(VersionInfo { version, git_commit, present: true })
+ }
+
+ pub(crate) fn disable_version(&mut self, package: &PkgType) {
+ match self.versions.get_mut(package) {
+ Some(version) => {
+ *version = VersionInfo::default();
+ }
+ None => {
+ self.versions.insert(package.clone(), VersionInfo::default());
+ }
+ }
+ }
+
pub(crate) fn tarball_name(
&mut self,
package: &PkgType,
From 73d9c24b3edf0f87a6e1c78ca56349f94ced5788 Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Thu, 24 Sep 2020 15:23:35 +0200
Subject: [PATCH 4/5] build-manifest: add documentation on the PkgType methods
---
src/tools/build-manifest/src/versions.rs | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs
index 3b5caf1cd7b..9f37b22c494 100644
--- a/src/tools/build-manifest/src/versions.rs
+++ b/src/tools/build-manifest/src/versions.rs
@@ -38,6 +38,8 @@ impl PkgType {
}
}
+ /// The directory containing the `Cargo.toml` of this component inside the monorepo, to
+ /// retrieve the source code version. If `None` is returned Rust's version will be used.
fn rust_monorepo_path(&self) -> Option<&'static str> {
match self {
PkgType::Cargo => Some("src/tools/cargo"),
@@ -53,6 +55,7 @@ impl PkgType {
}
}
+ /// First part of the tarball name.
fn tarball_component_name(&self) -> &str {
match self {
PkgType::Rust => "rust",
@@ -68,6 +71,8 @@ impl PkgType {
}
}
+ /// Whether this package has the same version as Rust itself, or has its own `version` and
+ /// `git-commit-hash` files inside the tarball.
fn should_use_rust_version(&self) -> bool {
match self {
PkgType::Cargo => false,
From 0749ad02d0b8429e2a3a50158994268c0b0af044 Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Fri, 25 Sep 2020 15:36:16 +0200
Subject: [PATCH 5/5] build-manifest: handle rust-src being target-independent
---
src/tools/build-manifest/src/versions.rs | 23 ++++++++++++++---------
1 file changed, 14 insertions(+), 9 deletions(-)
diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs
index 9f37b22c494..d949dff7279 100644
--- a/src/tools/build-manifest/src/versions.rs
+++ b/src/tools/build-manifest/src/versions.rs
@@ -88,6 +88,11 @@ impl PkgType {
PkgType::Other(_) => true,
}
}
+
+ /// Whether this package is target-independent or not.
+ fn target_independent(&self) -> bool {
+ *self == PkgType::RustSrc
+ }
}
#[derive(Debug, Default, Clone)]
@@ -198,15 +203,15 @@ impl Versions {
package: &PkgType,
target: &str,
) -> Result<String, Error> {
- Ok(format!(
- "{}-{}-{}.tar.gz",
- package.tarball_component_name(),
- self.package_version(package).with_context(|| format!(
- "failed to get the package version for component {:?}",
- package,
- ))?,
- target
- ))
+ let component_name = package.tarball_component_name();
+ let version = self.package_version(package).with_context(|| {
+ format!("failed to get the package version for component {:?}", package,)
+ })?;
+ if package.target_independent() {
+ Ok(format!("{}-{}.tar.gz", component_name, version))
+ } else {
+ Ok(format!("{}-{}-{}.tar.gz", component_name, version, target))
+ }
}
pub(crate) fn package_version(&mut self, package: &PkgType) -> Result<String, Error> {