Rollup merge of #39284 - alexcrichton:manifesting, r=brson

rustbuild: Add manifest generation in-tree

This commit adds a new tool, `build-manifest`, which is used to generate a
distribution manifest of all produced artifacts. This tool is intended to
replace the `build-rust-manifest.py` script that's currently located on the
buildmaster. The intention is that we'll have a builder which periodically:

* Downloads all artifacts for a commit
* Runs `./x.py dist hash-and-sign`. This will generate `sha256` and `asc` files
  as well as TOML manifests.
* Upload all generated hashes and manifests to the directory the artifacts came
  from.
* Upload *all* artifacts (tarballs and hashes and manifests) to an archived
  location.
* If necessary, upload all artifacts to the main location.

This script is intended to just be the second step here where orchestrating
uploads and such will all happen externally from the build system itself.

cc #38531

src/Cargo.lock

@@ -50,6 +50,14 @@ dependencies = [
  "toml 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "build-manifest"
+version = "0.1.0"
+dependencies = [
+ "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "toml 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "build_helper"
 version = "0.1.0"

src/Cargo.toml

@@ -10,6 +10,7 @@ members = [
   "tools/linkchecker",
   "tools/rustbook",
   "tools/tidy",
+  "tools/build-manifest",
 ]
 
 # Curiously, compiletest will segfault if compiled with opt-level=3 on 64-bit

src/bootstrap/config.rs

@@ -78,6 +78,11 @@ pub struct Config {
     pub cargo: Option<PathBuf>,
     pub local_rebuild: bool,
 
+    // dist misc
+    pub dist_sign_folder: Option<PathBuf>,
+    pub dist_upload_addr: Option<String>,
+    pub dist_gpg_password_file: Option<PathBuf>,
+
     // libstd features
     pub debug_jemalloc: bool,
     pub use_jemalloc: bool,
@@ -123,6 +128,7 @@ struct TomlConfig {
     llvm: Option<Llvm>,
     rust: Option<Rust>,
     target: Option<HashMap<String, TomlTarget>>,
+    dist: Option<Dist>,
 }
 
 /// TOML representation of various global build decisions.
@@ -166,6 +172,13 @@ struct Llvm {
     targets: Option<String>,
 }
 
+#[derive(RustcDecodable, Default, Clone)]
+struct Dist {
+    sign_folder: Option<String>,
+    gpg_password_file: Option<String>,
+    upload_addr: Option<String>,
+}
+
 #[derive(RustcDecodable)]
 enum StringOrBool {
     String(String),
@@ -352,6 +365,12 @@ impl Config {
             }
         }
 
+        if let Some(ref t) = toml.dist {
+            config.dist_sign_folder = t.sign_folder.clone().map(PathBuf::from);
+            config.dist_gpg_password_file = t.gpg_password_file.clone().map(PathBuf::from);
+            config.dist_upload_addr = t.upload_addr.clone();
+        }
+
         return config
     }
 

src/bootstrap/config.toml.example

@@ -242,3 +242,33 @@
 # that this option only makes sense for MUSL targets that produce statically
 # linked binaries
 #musl-root = "..."
+
+# =============================================================================
+# Distribution options
+#
+# These options are related to distribution, mostly for the Rust project itself.
+# You probably won't need to concern yourself with any of these options
+# =============================================================================
+[dist]
+
+# This is the folder of artifacts that the build system will sign. All files in
+# this directory will be signed with the default gpg key using the system `gpg`
+# binary. The `asc` and `sha256` files will all be output into the standard dist
+# output folder (currently `build/dist`)
+#
+# This folder should be populated ahead of time before the build system is
+# invoked.
+#sign-folder = "path/to/folder/to/sign"
+
+# This is a file which contains the password of the default gpg key. This will
+# be passed to `gpg` down the road when signing all files in `sign-folder`
+# above. This should be stored in plaintext.
+#gpg-password-file = "path/to/gpg/password"
+
+# The remote address that all artifacts will eventually be uploaded to. The
+# build system generates manifests which will point to these urls, and for the
+# manifests to be correct they'll have to have the right URLs encoded.
+#
+# Note that this address should not contain a trailing slash as file names will
+# be appended to it.
+#upload-addr = "https://example.com/folder"

src/bootstrap/dist.rs

@@ -22,7 +22,7 @@ use std::env;
 use std::fs::{self, File};
 use std::io::{Read, Write};
 use std::path::{PathBuf, Path};
-use std::process::Command;
+use std::process::{Command, Stdio};
 
 use build_helper::output;
 
@@ -876,3 +876,34 @@ fn add_env(build: &Build, cmd: &mut Command, target: &str) {
        cmd.env("CFG_PLATFORM", "x86");
     }
 }
+
+pub fn hash_and_sign(build: &Build) {
+    let compiler = Compiler::new(0, &build.config.build);
+    let mut cmd = build.tool_cmd(&compiler, "build-manifest");
+    let sign = build.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
+        panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
+    });
+    let addr = build.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
+        panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
+    });
+    let file = build.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
+        panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
+    });
+    let mut pass = String::new();
+    t!(t!(File::open(&file)).read_to_string(&mut pass));
+
+    let today = output(Command::new("date").arg("+%Y-%m-%d"));
+
+    cmd.arg(sign);
+    cmd.arg(distdir(build));
+    cmd.arg(today.trim());
+    cmd.arg(package_vers(build));
+    cmd.arg(addr);
+
+    t!(fs::create_dir_all(distdir(build)));
+
+    let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
+    t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
+    let status = t!(child.wait());
+    assert!(status.success());
+}

src/bootstrap/step.rs

@@ -513,6 +513,9 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
     rules.build("tool-compiletest", "src/tools/compiletest")
          .dep(|s| s.name("libtest"))
          .run(move |s| compile::tool(build, s.stage, s.target, "compiletest"));
+    rules.build("tool-build-manifest", "src/tools/build-manifest")
+         .dep(|s| s.name("libstd"))
+         .run(move |s| compile::tool(build, s.stage, s.target, "build-manifest"));
 
     // ========================================================================
     // Documentation targets
@@ -633,6 +636,13 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .dep(|d| d.name("dist-cargo"))
          .run(move |s| dist::extended(build, s.stage, s.target));
 
+    rules.dist("dist-sign", "hash-and-sign")
+         .host(true)
+         .only_build(true)
+         .only_host_build(true)
+         .dep(move |s| s.name("tool-build-manifest").target(&build.config.build).stage(0))
+         .run(move |_| dist::hash_and_sign(build));
+
     rules.verify();
     return rules;
 }

src/tools/build-manifest/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "build-manifest"
+version = "0.1.0"
+authors = ["Alex Crichton <alex@alexcrichton.com>"]
+
+[dependencies]
+toml = "0.1"
+rustc-serialize = "0.3"

src/tools/build-manifest/src/main.rs

@@ -0,0 +1,404 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+extern crate toml;
+extern crate rustc_serialize;
+
+use std::collections::HashMap;
+use std::env;
+use std::fs::File;
+use std::io::{self, Read, Write};
+use std::path::{PathBuf, Path};
+use std::process::{Command, Stdio};
+
+static HOSTS: &'static [&'static str] = &[
+    "aarch64-unknown-linux-gnu",
+    "arm-unknown-linux-gnueabi",
+    "arm-unknown-linux-gnueabihf",
+    "armv7-unknown-linux-gnueabihf",
+    "i686-apple-darwin",
+    "i686-pc-windows-gnu",
+    "i686-pc-windows-msvc",
+    "i686-unknown-linux-gnu",
+    "mips-unknown-linux-gnu",
+    "mips64-unknown-linux-gnuabi64",
+    "mips64el-unknown-linux-gnuabi64",
+    "mipsel-unknown-linux-gnu",
+    "powerpc-unknown-linux-gnu",
+    "powerpc64-unknown-linux-gnu",
+    "powerpc64le-unknown-linux-gnu",
+    "s390x-unknown-linux-gnu",
+    "x86_64-apple-darwin",
+    "x86_64-pc-windows-gnu",
+    "x86_64-pc-windows-msvc",
+    "x86_64-unknown-freebsd",
+    "x86_64-unknown-linux-gnu",
+    "x86_64-unknown-netbsd",
+];
+
+static TARGETS: &'static [&'static str] = &[
+    "aarch64-apple-ios",
+    "aarch64-linux-android",
+    "aarch64-unknown-linux-gnu",
+    "arm-linux-androideabi",
+    "arm-unknown-linux-gnueabi",
+    "arm-unknown-linux-gnueabihf",
+    "arm-unknown-linux-musleabi",
+    "arm-unknown-linux-musleabihf",
+    "armv7-apple-ios",
+    "armv7-linux-androideabi",
+    "armv7-unknown-linux-gnueabihf",
+    "armv7-unknown-linux-musleabihf",
+    "armv7s-apple-ios",
+    "asmjs-unknown-emscripten",
+    "i386-apple-ios",
+    "i586-pc-windows-msvc",
+    "i586-unknown-linux-gnu",
+    "i686-apple-darwin",
+    "i686-linux-android",
+    "i686-pc-windows-gnu",
+    "i686-pc-windows-msvc",
+    "i686-unknown-freebsd",
+    "i686-unknown-linux-gnu",
+    "i686-unknown-linux-musl",
+    "mips-unknown-linux-gnu",
+    "mips-unknown-linux-musl",
+    "mips64-unknown-linux-gnuabi64",
+    "mips64el-unknown-linux-gnuabi64",
+    "mipsel-unknown-linux-gnu",
+    "mipsel-unknown-linux-musl",
+    "powerpc-unknown-linux-gnu",
+    "powerpc64-unknown-linux-gnu",
+    "powerpc64le-unknown-linux-gnu",
+    "s390x-unknown-linux-gnu",
+    "wasm32-unknown-emscripten",
+    "x86_64-apple-darwin",
+    "x86_64-apple-ios",
+    "x86_64-pc-windows-gnu",
+    "x86_64-pc-windows-msvc",
+    "x86_64-rumprun-netbsd",
+    "x86_64-unknown-freebsd",
+    "x86_64-unknown-linux-gnu",
+    "x86_64-unknown-linux-musl",
+    "x86_64-unknown-netbsd",
+];
+
+static MINGW: &'static [&'static str] = &[
+    "i686-pc-windows-gnu",
+    "x86_64-pc-windows-gnu",
+];
+
+#[derive(RustcEncodable)]
+struct Manifest {
+    manifest_version: String,
+    date: String,
+    pkg: HashMap<String, Package>,
+}
+
+#[derive(RustcEncodable)]
+struct Package {
+    version: String,
+    target: HashMap<String, Target>,
+}
+
+#[derive(RustcEncodable)]
+struct Target {
+    available: bool,
+    url: Option<String>,
+    hash: Option<String>,
+    components: Option<Vec<Component>>,
+    extensions: Option<Vec<Component>>,
+}
+
+#[derive(RustcEncodable)]
+struct Component {
+    pkg: String,
+    target: String,
+}
+
+macro_rules! t {
+    ($e:expr) => (match $e {
+        Ok(e) => e,
+        Err(e) => panic!("{} failed with {}", stringify!($e), e),
+    })
+}
+
+struct Builder {
+    channel: String,
+    input: PathBuf,
+    output: PathBuf,
+    gpg_passphrase: String,
+    digests: HashMap<String, String>,
+    s3_address: String,
+    date: String,
+    rust_version: String,
+    cargo_version: String,
+}
+
+fn main() {
+    let mut args = env::args().skip(1);
+    let input = PathBuf::from(args.next().unwrap());
+    let output = PathBuf::from(args.next().unwrap());
+    let date = args.next().unwrap();
+    let channel = args.next().unwrap();
+    let s3_address = args.next().unwrap();
+    let mut passphrase = String::new();
+    t!(io::stdin().read_to_string(&mut passphrase));
+
+    Builder {
+        channel: channel,
+        input: input,
+        output: output,
+        gpg_passphrase: passphrase,
+        digests: HashMap::new(),
+        s3_address: s3_address,
+        date: date,
+        rust_version: String::new(),
+        cargo_version: String::new(),
+    }.build();
+}
+
+impl Builder {
+    fn build(&mut self) {
+        self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
+        self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");
+
+        self.digest_and_sign();
+        let manifest = self.build_manifest();
+        let manifest = toml::encode(&manifest).to_string();
+
+        let filename = format!("channel-rust-{}.toml", self.channel);
+        self.write_manifest(&manifest, &filename);
+
+        if self.channel != "beta" && self.channel != "nightly" {
+            self.write_manifest(&manifest, "channel-rust-stable.toml");
+        }
+    }
+
+    fn digest_and_sign(&mut self) {
+        for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
+            let filename = file.file_name().unwrap().to_str().unwrap();
+            let digest = self.hash(&file);
+            self.sign(&file);
+            assert!(self.digests.insert(filename.to_string(), digest).is_none());
+        }
+    }
+
+    fn build_manifest(&mut self) -> Manifest {
+        let mut manifest = Manifest {
+            manifest_version: "2".to_string(),
+            date: self.date.to_string(),
+            pkg: HashMap::new(),
+        };
+
+        self.package("rustc", &mut manifest.pkg, HOSTS);
+        self.package("cargo", &mut manifest.pkg, HOSTS);
+        self.package("rust-mingw", &mut manifest.pkg, MINGW);
+        self.package("rust-std", &mut manifest.pkg, TARGETS);
+        self.package("rust-docs", &mut manifest.pkg, TARGETS);
+        self.package("rust-src", &mut manifest.pkg, &["*"]);
+
+        let mut pkg = Package {
+            version: self.cached_version("rust").to_string(),
+            target: HashMap::new(),
+        };
+        for host in HOSTS {
+            let filename = self.filename("rust", host);
+            let digest = match self.digests.remove(&filename) {
+                Some(digest) => digest,
+                None => {
+                    pkg.target.insert(host.to_string(), Target {
+                        available: false,
+                        url: None,
+                        hash: None,
+                        components: None,
+                        extensions: None,
+                    });
+                    continue
+                }
+            };
+            let mut components = Vec::new();
+            let mut extensions = Vec::new();
+
+            // rustc/rust-std/cargo are all required, and so is rust-mingw if it's
+            // available for the target.
+            components.extend(vec![
+                Component { pkg: "rustc".to_string(), target: host.to_string() },
+                Component { pkg: "rust-std".to_string(), target: host.to_string() },
+                Component { pkg: "cargo".to_string(), target: host.to_string() },
+            ]);
+            if host.contains("pc-windows-gnu") {
+                components.push(Component {
+                    pkg: "rust-mingw".to_string(),
+                    target: host.to_string(),
+                });
+            }
+
+            // Docs, other standard libraries, and the source package are all
+            // optional.
+            extensions.push(Component {
+                pkg: "rust-docs".to_string(),
+                target: host.to_string(),
+            });
+            for target in TARGETS {
+                if target != host {
+                    extensions.push(Component {
+                        pkg: "rust-std".to_string(),
+                        target: target.to_string(),
+                    });
+                }
+            }
+            extensions.push(Component {
+                pkg: "rust-src".to_string(),
+                target: "*".to_string(),
+            });
+
+            pkg.target.insert(host.to_string(), Target {
+                available: true,
+                url: Some(self.url("rust", host)),
+                hash: Some(to_hex(digest.as_ref())),
+                components: Some(components),
+                extensions: Some(extensions),
+            });
+        }
+        manifest.pkg.insert("rust".to_string(), pkg);
+
+        return manifest
+    }
+
+    fn package(&mut self,
+               pkgname: &str,
+               dst: &mut HashMap<String, Package>,
+               targets: &[&str]) {
+        let targets = targets.iter().map(|name| {
+            let filename = self.filename(pkgname, name);
+            let digest = match self.digests.remove(&filename) {
+                Some(digest) => digest,
+                None => {
+                    return (name.to_string(), Target {
+                        available: false,
+                        url: None,
+                        hash: None,
+                        components: None,
+                        extensions: None,
+                    })
+                }
+            };
+
+            (name.to_string(), Target {
+                available: true,
+                url: Some(self.url(pkgname, name)),
+                hash: Some(digest),
+                components: None,
+                extensions: None,
+            })
+        }).collect();
+
+        dst.insert(pkgname.to_string(), Package {
+            version: self.cached_version(pkgname).to_string(),
+            target: targets,
+        });
+    }
+
+    fn url(&self, component: &str, target: &str) -> String {
+        format!("{}/{}/{}",
+                self.s3_address,
+                self.date,
+                self.filename(component, target))
+    }
+
+    fn filename(&self, component: &str, target: &str) -> String {
+        if component == "rust-src" {
+            format!("rust-src-{}.tar.gz", self.channel)
+        } else {
+            format!("{}-{}-{}.tar.gz", component, self.channel, target)
+        }
+    }
+
+    fn cached_version(&self, component: &str) -> &str {
+        if component == "cargo" {
+            &self.cargo_version
+        } else {
+            &self.rust_version
+        }
+    }
+
+    fn version(&self, component: &str, target: &str) -> String {
+        let mut cmd = Command::new("tar");
+        let filename = self.filename(component, target);
+        cmd.arg("xf")
+           .arg(self.input.join(&filename))
+           .arg(format!("{}/version", filename.replace(".tar.gz", "")))
+           .arg("-O");
+        let version = t!(cmd.output());
+        if !version.status.success() {
+            panic!("failed to learn version:\n\n{:?}\n\n{}\n\n{}",
+                   cmd,
+                   String::from_utf8_lossy(&version.stdout),
+                   String::from_utf8_lossy(&version.stderr));
+        }
+        String::from_utf8_lossy(&version.stdout).trim().to_string()
+    }
+
+    fn hash(&self, path: &Path) -> String {
+        let sha = t!(Command::new("shasum")
+                        .arg("-a").arg("256")
+                        .arg(path)
+                        .output());
+        assert!(sha.status.success());
+
+        let filename = path.file_name().unwrap().to_str().unwrap();
+        let sha256 = self.output.join(format!("{}.sha256", filename));
+        t!(t!(File::create(&sha256)).write_all(&sha.stdout));
+
+        let stdout = String::from_utf8_lossy(&sha.stdout);
+        stdout.split_whitespace().next().unwrap().to_string()
+    }
+
+    fn sign(&self, path: &Path) {
+        let filename = path.file_name().unwrap().to_str().unwrap();
+        let asc = self.output.join(format!("{}.asc", filename));
+        println!("signing: {:?}", path);
+        let mut cmd = Command::new("gpg");
+        cmd.arg("--no-tty")
+            .arg("--yes")
+            .arg("--passphrase-fd").arg("0")
+            .arg("--armor")
+            .arg("--output").arg(&asc)
+            .arg("--detach-sign").arg(path)
+            .stdin(Stdio::piped());
+        let mut child = t!(cmd.spawn());
+        t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes()));
+        assert!(t!(child.wait()).success());
+    }
+
+    fn write_manifest(&self, manifest: &str, name: &str) {
+        let dst = self.output.join(name);
+        t!(t!(File::create(&dst)).write_all(manifest.as_bytes()));
+        self.hash(&dst);
+        self.sign(&dst);
+    }
+}
+
+fn to_hex(digest: &[u8]) -> String {
+    let mut ret = String::new();
+    for byte in digest {
+        ret.push(hex((byte & 0xf0) >> 4));
+        ret.push(hex(byte & 0xf));
+    }
+    return ret;
+
+    fn hex(b: u8) -> char {
+        match b {
+            0...9 => (b'0' + b) as char,
+            _ => (b'a' + b - 10) as char,
+        }
+    }
+}