Auto merge of #38708 - alexcrichton:add-distcheck, r=brson

Gate on distcheck on Travis

This commit adds a new entry to the Travis matrix to gate on distcheck, the illustrious test process that has historically taken *8 hours* to complete and also breaks all the time on nightly. By adding it to Travis we should hope to never see nightly breakage (like https://github.com/rust-lang/rust/issues/38690) because of this ever again!

"But wait, surely we can't wait 8 hours for all PRs!" you might be thinking, and you are indeed correct. The distcheck added here is much more optimized for speed than the old buildbot instances for a number of reasons:

* We're not building *two host compilers* beforehand. The current distcheck bot does a cross for i686 Linux and x86_64 Linux before it actually runs distcheck, building 6 compilers and LLVM twice. None of this is done in parallel as well (e.g. `-j1`). Not doing any of this work will be a huge win!
* We're using sccache to compile LLVM, so it should be much faster. Distcheck on the bots didn't cache LLVM well and rebuilt it every time.

All in all, this version of "distcheck" should be exactly like other matrix entries that run tests except that it's a *little* slower to start as it has to create the source tarball then rebuild the build system in the distcheck dir. Overall this should be well under the 2 hours that Android is currently taking anyway.

Closes https://github.com/rust-lang/rust/issues/38691

.travis.yml

@@ -26,6 +26,7 @@ matrix:
     - env: IMAGE=x86_64-gnu-make
     - env: IMAGE=x86_64-gnu-llvm-3.7 ALLOW_PR=1 RUST_BACKTRACE=1
     - env: IMAGE=x86_64-musl
+    - env: IMAGE=x86_64-gnu-distcheck
 
     # OSX builders
     - env: >

src/bootstrap/check.rs

@@ -562,6 +562,7 @@ pub fn distcheck(build: &Build) {
        .current_dir(&dir);
     build.run(&mut cmd);
     build.run(Command::new("./configure")
+                     .args(&build.config.configure_args)
                      .current_dir(&dir));
     build.run(Command::new(build_helper::make(&build.config.build))
                      .arg("check")

src/bootstrap/config.rs

@@ -94,6 +94,7 @@ pub struct Config {
     pub nodejs: Option<PathBuf>,
     pub gdb: Option<PathBuf>,
     pub python: Option<PathBuf>,
+    pub configure_args: Vec<String>,
 }
 
 /// Per-target configuration stored in the global configuration structure.
@@ -519,6 +520,11 @@ impl Config {
                 "CFG_ENABLE_SCCACHE" if value == "1" => {
                     self.ccache = Some("sccache".to_string());
                 }
+                "CFG_CONFIGURE_ARGS" if value.len() > 0 => {
+                    self.configure_args = value.split_whitespace()
+                                               .map(|s| s.to_string())
+                                               .collect();
+                }
                 _ => {}
             }
         }

src/bootstrap/dist.rs

@@ -397,6 +397,13 @@ pub fn rust_src(build: &Build, host: &str) {
             }
         }
 
+        // If we're inside the vendor directory then we need to preserve
+        // everything as Cargo's vendoring support tracks all checksums and we
+        // want to be sure we don't accidentally leave out a file.
+        if spath.contains("vendor") {
+            return true
+        }
+
         let excludes = [
             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",

src/bootstrap/util.rs

@@ -76,9 +76,9 @@ pub fn cp_r(src: &Path, dst: &Path) {
 /// Copies the `src` directory recursively to `dst`. Both are assumed to exist
 /// when this function is called. Unwanted files or directories can be skipped
 /// by returning `false` from the filter function.
-pub fn cp_filtered<F: Fn(&Path) -> bool>(src: &Path, dst: &Path, filter: &F) {
+pub fn cp_filtered(src: &Path, dst: &Path, filter: &Fn(&Path) -> bool) {
     // Inner function does the actual work
-    fn recurse<F: Fn(&Path) -> bool>(src: &Path, dst: &Path, relative: &Path, filter: &F) {
+    fn recurse(src: &Path, dst: &Path, relative: &Path, filter: &Fn(&Path) -> bool) {
         for f in t!(fs::read_dir(src)) {
             let f = t!(f);
             let path = f.path();

src/ci/docker/x86_64-gnu-distcheck/Dockerfile

@@ -0,0 +1,26 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  g++ \
+  make \
+  file \
+  curl \
+  ca-certificates \
+  python2.7 \
+  git \
+  cmake \
+  sudo \
+  gdb \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
+
+RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
+    dpkg -i dumb-init_*.deb && \
+    rm dumb-init_*.deb
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu
+ENV XPY_RUN test distcheck

src/ci/run.sh

@@ -43,10 +43,14 @@ else
     ncpus=$(nproc)
 fi
 
-make -j $ncpus tidy
-make -j $ncpus
-if [ ! -z "$XPY_CHECK" ]; then
-  exec python2.7 $SRC/x.py $XPY_CHECK
+if [ ! -z "$XPY_RUN" ]; then
+  exec python2.7 $SRC/x.py $XPY_RUN
 else
-  exec make $RUST_CHECK_TARGET -j $ncpus
+  make -j $ncpus tidy
+  make -j $ncpus
+  if [ ! -z "$XPY_CHECK" ]; then
+    exec python2.7 $SRC/x.py $XPY_CHECK
+  else
+    exec make $RUST_CHECK_TARGET -j $ncpus
+  fi
 fi