Add Azure Pipelines

.azure-pipelines/auto.fanout.yml

@@ -0,0 +1,521 @@
+#
+# Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows.
+# 
+# Notes:
+# To get sccache working, I (John Erickson) had to build it with the Azure feature enabled
+# and with openssl statically-linked for Linux.
+# Here's the build (with a backpointer to source) of where the bits came from:
+# https://dev.azure.com/johnterickson/rust-lang/_build/results?buildId=275
+
+name: fanout 
+pr: none
+trigger:
+- auto
+
+variables:
+- group: caching
+
+jobs:
+# WINDOWS JOBS
+- job: Windows
+  timeoutInMinutes: 600
+  variables:
+    TAR_COMMAND: tar
+  pool:
+    vmImage: 'vs2017-win2016'
+  steps:
+  - template: steps/windows.yml
+  strategy:
+    matrix:
+      # 32/64 bit MSVC tests
+      x86_64-msvc-1:
+        MSYS_BITS: 64
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+        SCRIPT: make ci-subset-1
+      x86_64-msvc-2:
+        MSYS_BITS: 64
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+        SCRIPT: make ci-subset-2
+
+- job: Windows_build  
+  timeoutInMinutes: 600
+  variables:
+    TAR_COMMAND: tar
+  pool:
+    vmImage: 'vs2017-win2016'
+  steps:
+  - template: steps/windows.yml
+  - script: |
+      echo on
+      echo "##vso[task.setvariable variable=TAR_PATH]%TEMP%\drop.tar"
+  - bash: |
+      set -x
+      $TAR_COMMAND --dereference --exclude=.git --exclude=citools $SUBMODULES_EXCLUDES -cf $TEMP/drop.tar .
+      $TAR_COMMAND --append --file=$TEMP/drop.tar citools
+  - task: PublishPipelineArtifact@0
+    inputs:
+      artifactName: $(System.JobDisplayName)
+      targetPath: $(TAR_PATH)
+  strategy:
+    matrix:
+      # 32/64 bit MSVC tests
+      x86_64-msvc-build:
+        MSYS_BITS: 64
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+        SCRIPT: make ci-test-prepare
+
+- job: Windows_test
+  dependsOn: Windows_build
+  timeoutInMinutes: 600
+  variables:
+    TAR_COMMAND: tar
+    SKIP_PREP: 1
+    KEEP_STAGE: '--keep-stage 0 --keep-stage 1'
+    SKIP_LLVM_BUILD: 1
+  pool:
+    vmImage: 'vs2017-win2016'
+  steps:
+  - checkout: self
+    fetchDepth: 2
+  - script: |
+      set MSYS_PATH=%CD%\citools\msys64
+      set PATH=%MSYS_PATH%\usr\bin;%PATH%
+      where rev
+      rev --help
+      where make
+      echo ##vso[task.prependpath]%MSYS_PATH%\usr\bin
+    displayName: Install msys2
+
+  # # If we need to download a custom MinGW, do so here and set the path
+  # # appropriately.
+  # #
+  # # Note that this *also* means that we're not using what is typically
+  # # /mingw32/bin/python2.7.exe, which is a "correct" python interpreter where
+  # # /usr/bin/python2.7.exe is not. To ensure we use the right interpreter we
+  # # move `C:\Python27` ahead in PATH and then also make sure the `python2.7.exe`
+  # # file exists in there (which it doesn't by default).
+  # - script: |
+  #     powershell -Command "iwr -outf %MINGW_ARCHIVE% %MINGW_URL%/%MINGW_ARCHIVE%"
+  #     7z x -y %MINGW_ARCHIVE% > nul
+  #     echo ##vso[task.prependpath]%CD%\%MINGW_DIR%\bin
+  #   condition: and(succeeded(), ne(variables['MINGW_URL'],''))
+  #   displayName: Download custom MinGW
+
+  # If we're compiling for MSVC then we, like most other distribution builders,
+  # switch to clang as the compiler. This'll allow us eventually to enable LTO
+  # amongst LLVM and rustc. Note that we only do this on MSVC as I don't think
+  # clang has an output mode compatible with MinGW that we need. If it does we
+  # should switch to clang for MinGW as well!
+  #
+  # Note that the LLVM installer is an NSIS installer
+  #
+  # Original downloaded here came from
+  # http://releases.llvm.org/7.0.0/LLVM-7.0.0-win64.exe
+  - script: |
+      # powershell -Command "iwr -outf LLVM-7.0.0-win64.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/LLVM-7.0.0-win64.exe"
+      # .\LLVM-7.0.0-win64.exe /S /NCRC /D=C:\clang-rust
+      set CLANG_DIR=%CD%\citools\clang-rust
+      set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --set llvm.clang-cl=%CLANG_DIR%\bin\clang-cl.exe
+      echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS%
+    condition: and(succeeded(), eq(variables['MINGW_URL'],''))
+    displayName: Download clang
+
+  # # Here we do a pretty heinous thing which is to mangle the MinGW installation
+  # # we just had above. Currently, as of this writing, we're using MinGW-w64
+  # # builds of gcc, and that's currently at 6.3.0. We use 6.3.0 as it appears to
+  # # be the first version which contains a fix for #40546, builds randomly
+  # # failing during LLVM due to ar.exe/ranlib.exe failures.
+  # #
+  # # Unfortunately, though, 6.3.0 *also* is the first version of MinGW-w64 builds
+  # # to contain a regression in gdb (#40184). As a result if we were to use the
+  # # gdb provided (7.11.1) then we would fail all debuginfo tests.
+  # #
+  # # In order to fix spurious failures (pretty high priority) we use 6.3.0. To
+  # # avoid disabling gdb tests we download an *old* version of gdb, specifically
+  # # that found inside the 6.2.0 distribution. We then overwrite the 6.3.0 gdb
+  # # with the 6.2.0 gdb to get tests passing.
+  # #
+  # # Note that we don't literally overwrite the gdb.exe binary because it appears
+  # # to just use gdborig.exe, so that's the binary we deal with instead.
+  # - script: |
+  #     echo ON
+  #     powershell -Command "iwr -outf 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_URL%/2017-04-20-%MSYS_BITS%bit-gdborig.exe"
+  #     mv 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_DIR%\bin\gdborig.exe
+  #   condition: and(succeeded(), ne(variables['MINGW_URL'],''))
+  #   displayName: Override with 6.3.0 gdb with 6.2.0 gdb
+
+  # Otherwise pull in the MinGW installed on appveyor
+  - script: |
+      echo Find mingw
+      set PATH | findstr /i msys
+      set PATH | findstr /i mingw
+      echo ##vso[task.prependpath]C:\msys64\mingw%MSYS_BITS%\bin
+    condition: and(succeeded(), eq(variables['MINGW_URL'],''))
+    displayName: Add MinGW to path
+
+  - script: |
+      copy C:\Python27amd64\python.exe C:\Python27amd64\python2.7.exe
+      echo ##vso[task.prependpath]C:\Python27amd64
+    displayName: Prefer the "native" Python as LLVM has trouble building with MSYS sometimes
+
+  - script: |
+      REM md sccache
+      REM powershell -Command "iwr -outf sccache\sccache.exe https://rustlangtools.blob.core.windows.net/public/stable-x86_64-pc-windows-msvc.sccache.exe"
+      echo ##vso[task.prependpath]%CD%\sccache
+    displayName: Download and install sccache
+
+  # Note that this is originally from the github releases patch of Ninja
+  - script: |
+      REM md ninja
+      REM powershell -Command "iwr -outf 2017-03-15-ninja-win.zip https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-15-ninja-win.zip"
+      REM 7z x -oninja 2017-03-15-ninja-win.zip
+      REM del 2017-03-15-ninja-win.zip
+      set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --enable-ninja
+      echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS%
+      echo ##vso[task.prependpath]%CD%\ninja
+    displayName: Download and install ninja
+  
+  - script: |
+      IF NOT "%DISABLE_DISK_SPACE_HACK%"=="1" (
+        IF NOT EXIST build (
+          mkdir c:\MORE_SPACE
+          mklink /J build c:\MORE_SPACE
+        )
+      )
+  - script: |
+      echo on
+      echo ##vso[task.setvariable variable=TAR_FOLDER_PATH]%TEMP%
+  - template: steps/show-environment-variables.yml
+  - task: DownloadPipelineArtifact@1
+    inputs:
+      artifactName: $(ARTIFACT_NAME)
+      downloadPath: $(TAR_FOLDER_PATH)
+  # - task: DownloadPipelineArtifact@1
+  #   inputs:
+  #     artifactName: $(ARTIFACT_NAME)
+  #     downloadPath: $(TAR_FOLDER_PATH)
+  #     buildType: specific
+  #     project: rust
+  #     pipeline: 8
+  #     buildVersionToDownload: specific
+  #     buildId: 200
+  #     # https://dev.azure.com/rust-lang-azure/rust/_build/results?buildId=145
+
+  # - template: steps/macos.yml 
+  - bash: $TAR_COMMAND -tvf $TEMP/drop.tar
+  - bash: $TAR_COMMAND --touch -xf $TEMP/drop.tar
+  - bash: rm $TEMP/drop.tar
+  - script: |
+      REM mkdir handle
+      REM powershell -Command "iwr -outf 2017-05-15-Handle.zip https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-05-15-Handle.zip"
+      REM 7z x -ohandle 2017-05-15-Handle.zip
+      REM del 2017-05-15-Handle.zip
+      set PATH=%PATH%;%CD%\handle
+      handle.exe -accepteula -help
+      echo ##vso[task.setvariable variable=PATH]%PATH%
+    displayName: Help debug handle issues
+  - script: |
+      REM echo force the specific VS version https://github.com/johnterickson/rust/issues/8
+      IF "%VCVARS_BAT%" NEQ "" ( 
+        CALL "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\%VCVARS_BAT%"
+      )
+
+      where sccache
+      where rev
+      set | findstr /v SCCACHE_AZURE_CONNECTION_STRING
+
+      if not exist D:\cache\rustsrc\NUL mkdir D:\cache\rustsrc
+      
+      sh src/ci/run.sh
+    env:      
+      CI: true
+      CI_JOB_NAME: $(System.JobDisplayName)
+      SRC: .
+      NO_CCACHE: 1
+
+      # explicitly decrypt secret variables
+      # see https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch
+      SCCACHE_AZURE_CONNECTION_STRING: $(SCCACHE_AZURE_CONNECTION_STRING_SECRET)
+      DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY: $(DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY_SECRET)
+    displayName: Run script
+  
+  strategy:
+    matrix:
+      # 32/64 bit MSVC tests
+      x86_64-msvc-A:
+        MSYS_BITS: 64
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+        ARTIFACT_NAME: x86_64-msvc-build
+        SCRIPT: make ci-resume-subset-A
+      x86_64-msvc-B:
+        MSYS_BITS: 64
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+        ARTIFACT_NAME: x86_64-msvc-build
+        SCRIPT: make ci-resume-subset-B
+      x86_64-msvc-C:
+        MSYS_BITS: 64
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+        ARTIFACT_NAME: x86_64-msvc-build
+        SCRIPT: make ci-resume-subset-C
+      x86_64-msvc-D:
+        MSYS_BITS: 64
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+        ARTIFACT_NAME: x86_64-msvc-build
+        SCRIPT: make ci-resume-subset-D
+      x86_64-msvc-E:
+        MSYS_BITS: 64
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+        ARTIFACT_NAME: x86_64-msvc-build
+        SCRIPT: make ci-resume-subset-E
+      x86_64-msvc-F:
+        MSYS_BITS: 64
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+        ARTIFACT_NAME: x86_64-msvc-build
+        SCRIPT: make ci-resume-subset-F
+      x86_64-msvc-G:
+        MSYS_BITS: 64
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+        ARTIFACT_NAME: x86_64-msvc-build
+        SCRIPT: make ci-resume-subset-G
+      x86_64-msvc-H:
+        MSYS_BITS: 64
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+        ARTIFACT_NAME: x86_64-msvc-build
+        SCRIPT: make ci-resume-subset-H
+
+# macOS JOBS
+- job: macOS
+  timeoutInMinutes: 600
+  variables:
+    TAR_COMMAND: gtar
+  pool:
+    vmImage: macos-10.13
+  steps:
+  - checkout: self
+    fetchDepth: 2
+  - template: steps/macos.yml  
+  strategy:
+    matrix:
+      x86_64-apple:
+        RUST_CHECK_TARGET: check
+        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
+        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+        MACOSX_DEPLOYMENT_TARGET: 10.8
+        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
+        NO_LLVM_ASSERTIONS: 1
+        NO_DEBUG_ASSERTIONS: 1
+
+- job: macOS_build
+  timeoutInMinutes: 600
+  variables:
+    TAR_COMMAND: gtar
+  pool:
+    vmImage: macos-10.13
+  steps:
+  - checkout: self
+    fetchDepth: 2
+  - template: steps/macos.yml
+  - script: |
+      set -x
+
+      git submodule
+      export SUBMODULES_EXCLUDES=$(git submodule | grep -Eow 'src/[^ ]+' | sed 's/\(.*\)/--exclude=\1\/\.git/')
+      echo $SUBMODULES_EXCLUDES
+
+      $TAR_COMMAND --exclude=.git $SUBMODULES_EXCLUDES -cf /tmp/drop.tar .
+  - task: PublishPipelineArtifact@0
+    inputs:
+      artifactName: $(System.JobDisplayName)
+      targetPath: '/tmp/drop.tar'
+
+  strategy:
+    matrix:
+      # macOS builders. These are placed near the beginning because they are very
+      # slow to run.
+
+      # OSX builders running tests, these run the full test suite.
+      # NO_DEBUG_ASSERTIONS=1 to make them go faster, but also do have some
+      # runners that run `//ignore-debug` tests.
+      #
+      # Note that the compiler is compiled to target 10.8 here because the Xcode
+      # version that we're using, 8.2, cannot compile LLVM for OSX 10.7.
+      x86_64-apple-build:
+        RUST_CHECK_TARGET: check
+        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
+        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+        MACOSX_DEPLOYMENT_TARGET: 10.8
+        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
+        NO_LLVM_ASSERTIONS: 1
+        NO_DEBUG_ASSERTIONS: 1
+        SCRIPT: make ci-test-prepare
+
+- job: macOS_test
+  dependsOn: macOS_build
+  timeoutInMinutes: 600
+  variables:
+    TAR_COMMAND: gtar
+    SKIP_PREP: 1
+    KEEP_STAGE: '--keep-stage 0 --keep-stage 1'
+    SKIP_LLVM_BUILD: 1
+  pool:
+    vmImage: macos-10.13
+  steps:
+  - checkout: self
+    fetchDepth: 2
+  - bash: brew install gnu-tar
+    displayName: install a tar that works well
+
+  - template: steps/show-disk-usage.yml
+
+  - task: DownloadPipelineArtifact@1
+    inputs:
+      artifactName: $(ARTIFACT_NAME)
+      downloadPath: $(System.DefaultWorkingDirectory)
+  # - task: DownloadPipelineArtifact@1
+  #   inputs:
+  #     artifactName: $(ARTIFACT_NAME)
+  #     downloadPath: $(System.DefaultWorkingDirectory)
+  #     buildType: specific
+  #     project: rust
+  #     pipeline: 8
+  #     buildVersionToDownload: specific
+  #     buildId: 200
+
+  # - template: steps/macos.yml 
+  - template: steps/show-disk-usage.yml
+  - bash: $TAR_COMMAND -tvf ./drop.tar
+  - bash: $TAR_COMMAND -xf ./drop.tar
+  - bash: rm ./drop.tar
+  - template: steps/show-disk-usage.yml
+  - bash: |
+      export PATH=$PATH:$HOME/.local/bin:$HOME/Library/Python/2.7/bin/
+      mkdir -p $HOME/rustsrc    
+      echo "##vso[task.setvariable variable=PATH;]$PATH"
+
+      curl -fo /usr/local/bin/sccache https://rustlangtools.blob.core.windows.net/public/stable-x86_64-apple-darwin.sccache
+      chmod +x /usr/local/bin/sccache
+
+      curl -fo /usr/local/bin/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-apple-darwin
+      chmod +x /usr/local/bin/stamp
+      
+      export CC=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang
+      echo "##vso[task.setvariable variable=CC]$CC"
+      
+      export CXX=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang++
+      echo "##vso[task.setvariable variable=CXX]$CXX"
+      
+      echo "##vso[task.setvariable variable=AR]ar"
+    displayName: Prep
+
+  - bash: brew install gnu-tar
+    displayName: install a tar that works well
+
+  - bash: |
+      brew update
+      brew install xz
+      brew install swig
+    condition: and(succeeded(), eq(variables['RUST_CHECK_TARGET'],'dist'))
+    displayName: Install xz and swigw
+
+  - bash: |
+      export RUN_SCRIPT="src/ci/run.sh"
+      echo "##vso[task.setvariable variable=RUN_SCRIPT]$RUN_SCRIPT"
+    displayName: Prepare run script (init and run)
+
+  - template: steps/show-environment-variables.yml
+
+  - template: steps/verify-publish-toolstate.yml
+
+  - template: steps/run-script.yml
+
+  strategy:
+    matrix:
+      # macOS builders. These are placed near the beginning because they are very
+      # slow to run.
+
+      # OSX builders running tests, these run the full test suite.
+      # NO_DEBUG_ASSERTIONS=1 to make them go faster, but also do have some
+      # runners that run `//ignore-debug` tests.
+      #
+      # Note that the compiler is compiled to target 10.8 here because the Xcode
+      # version that we're using, 8.2, cannot compile LLVM for OSX 10.7.
+      x86_64-apple-A:
+        RUST_CHECK_TARGET: check
+        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
+        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+        MACOSX_DEPLOYMENT_TARGET: 10.8
+        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
+        NO_LLVM_ASSERTIONS: 1
+        NO_DEBUG_ASSERTIONS: 1
+        ARTIFACT_NAME: x86_64-apple-build
+        SCRIPT: make ci-resume-subset-A
+      x86_64-apple-B:
+        RUST_CHECK_TARGET: check
+        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
+        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+        MACOSX_DEPLOYMENT_TARGET: 10.8
+        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
+        NO_LLVM_ASSERTIONS: 1
+        NO_DEBUG_ASSERTIONS: 1
+        ARTIFACT_NAME: x86_64-apple-build
+        SCRIPT: make ci-resume-subset-B
+      x86_64-apple-C:
+        RUST_CHECK_TARGET: check
+        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
+        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+        MACOSX_DEPLOYMENT_TARGET: 10.8
+        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
+        NO_LLVM_ASSERTIONS: 1
+        NO_DEBUG_ASSERTIONS: 1
+        ARTIFACT_NAME: x86_64-apple-build
+        SCRIPT: make ci-resume-subset-C
+      x86_64-apple-D:
+        RUST_CHECK_TARGET: check
+        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
+        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+        MACOSX_DEPLOYMENT_TARGET: 10.8
+        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
+        NO_LLVM_ASSERTIONS: 1
+        NO_DEBUG_ASSERTIONS: 1
+        ARTIFACT_NAME: x86_64-apple-build
+        SCRIPT: make ci-resume-subset-D
+      x86_64-apple-E:
+        RUST_CHECK_TARGET: check
+        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
+        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+        MACOSX_DEPLOYMENT_TARGET: 10.8
+        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
+        NO_LLVM_ASSERTIONS: 1
+        NO_DEBUG_ASSERTIONS: 1
+        ARTIFACT_NAME: x86_64-apple-build
+        SCRIPT: make ci-resume-subset-E
+      x86_64-apple-F:
+        RUST_CHECK_TARGET: check
+        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
+        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+        MACOSX_DEPLOYMENT_TARGET: 10.8
+        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
+        NO_LLVM_ASSERTIONS: 1
+        NO_DEBUG_ASSERTIONS: 1
+        ARTIFACT_NAME: x86_64-apple-build
+        SCRIPT: make ci-resume-subset-F
+      x86_64-apple-G:
+        RUST_CHECK_TARGET: check
+        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
+        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+        MACOSX_DEPLOYMENT_TARGET: 10.8
+        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
+        NO_LLVM_ASSERTIONS: 1
+        NO_DEBUG_ASSERTIONS: 1
+        ARTIFACT_NAME: x86_64-apple-build
+        SCRIPT: make ci-resume-subset-G
+      x86_64-apple-H:
+        RUST_CHECK_TARGET: check
+        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
+        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+        MACOSX_DEPLOYMENT_TARGET: 10.8
+        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
+        NO_LLVM_ASSERTIONS: 1
+        NO_DEBUG_ASSERTIONS: 1
+        ARTIFACT_NAME: x86_64-apple-build
+        SCRIPT: make ci-resume-subset-H

.azure-pipelines/auto.yml

@@ -0,0 +1,361 @@
+#
+# Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows.
+# 
+# Notes:
+# To get sccache working, I (John Erickson) had to build it with the Azure feature enabled
+# and with openssl statically-linked for Linux.
+# Here's the build (with a backpointer to source) of where the bits came from:
+# https://dev.azure.com/johnterickson/rust-lang/_build/results?buildId=275
+
+pr: none
+# trigger:
+# - auto
+
+variables:
+- group: caching
+
+jobs:
+- job: Linux
+  timeoutInMinutes: 600
+  pool:
+    vmImage: ubuntu-16.04
+  steps:
+  - template: steps/linux.yml
+  strategy:
+    matrix:
+      x86_64-gnu-llvm-6.0:
+        IMAGE: x86_64-gnu-llvm-6.0
+        RUST_BACKTRACE: 1
+
+      dist-x86_64-linux:
+        IMAGE: dist-x86_64-linux
+        DEPLOY: 1
+
+      # "alternate" deployments, these are "nightlies" but have LLVM assertions
+      # turned on, they're deployed to a different location primarily for
+      # additional testing.
+      dist-x86_64-linux-alt:
+        IMAGE: dist-x86_64-linux
+        DEPLOY_ALT: 1
+
+      # Linux builders, remaining docker images        
+      arm-android:
+        IMAGE: arm-android
+
+      armhf-gnu:
+        IMAGE: armhf-gnu
+
+      dist-various-1:
+        IMAGE: dist-various-1
+        DEPLOY: 1
+
+      dist-various-2:
+        IMAGE: dist-various-2
+        DEPLOY: 1
+
+      dist-aarch64-linux:
+        IMAGE: dist-aarch64-linux
+        DEPLOY: 1
+
+      dist-android:
+        IMAGE: dist-android
+        DEPLOY: 1
+
+      dist-arm-linux:
+        IMAGE: dist-arm-linux
+        DEPLOY: 1
+
+      dist-armhf-linux:
+        IMAGE: dist-armhf-linux
+        DEPLOY: 1
+
+      dist-armv7-linux:
+        IMAGE: dist-armv7-linux
+        DEPLOY: 1
+
+      dist-i586-gnu-i586-i686-musl:
+        IMAGE: dist-i586-gnu-i586-i686-musl
+        DEPLOY: 1
+
+      dist-i686-freebsd:
+        IMAGE: dist-i686-freebsd
+        DEPLOY: 1
+
+      dist-i686-linux:
+        IMAGE: dist-i686-linux
+        DEPLOY: 1
+
+      dist-mips-linux:
+        IMAGE: dist-mips-linux
+        DEPLOY: 1
+
+      dist-mips64-linux:
+        IMAGE: dist-mips64-linux
+        DEPLOY: 1
+
+      dist-mips64el-linux:
+        IMAGE: dist-mips64el-linux
+        DEPLOY: 1
+
+      dist-mipsel-linux:
+        IMAGE: dist-mipsel-linux
+        DEPLOY: 1
+
+      dist-powerpc-linux:
+        IMAGE: dist-powerpc-linux
+        DEPLOY: 1
+
+      dist-powerpc64-linux:
+        IMAGE: dist-powerpc64-linux
+        DEPLOY: 1
+
+      dist-powerpc64le-linux:
+        IMAGE: dist-powerpc64le-linux
+        DEPLOY: 1
+
+      dist-s390x-linux:
+        IMAGE: dist-s390x-linux
+        DEPLOY: 1
+
+      dist-x86_64-freebsd:
+        IMAGE: dist-x86_64-freebsd
+        DEPLOY: 1
+
+      dist-x86_64-musl:
+        IMAGE: dist-x86_64-musl
+        DEPLOY: 1
+
+      dist-x86_64-netbsd:
+        IMAGE: dist-x86_64-netbsd
+        DEPLOY: 1
+
+      asmjs:
+        IMAGE: asmjs
+      i686-gnu-1:
+        IMAGE: i686-gnu-1
+      i686-gnu-2:
+        IMAGE: i686-gnu-2
+      i686-gnu-nopt:
+        IMAGE: i686-gnu-nopt
+      test-various:
+        IMAGE: test-various
+      x86_64-gnu:
+        IMAGE: x86_64-gnu
+      x86_64-gnu-full-bootstrap:
+        IMAGE: x86_64-gnu-full-bootstrap
+      x86_64-gnu-aux:
+        IMAGE: x86_64-gnu-aux
+      x86_64-gnu-tools:
+        IMAGE: x86_64-gnu-tools
+      # FIXME if: branch = auto OR (type = pull_request AND commit_message =~ /(?i:^update.*\b(rls|rustfmt|clippy|miri|cargo)\b)/)
+      x86_64-gnu-debug:
+        IMAGE: x86_64-gnu-debug
+      x86_64-gnu-nopt:
+        IMAGE: x86_64-gnu-nopt
+      x86_64-gnu-distcheck:
+        IMAGE: x86_64-gnu-distcheck
+      mingw-check:
+        IMAGE: mingw-check
+
+- job: macOS
+  timeoutInMinutes: 600
+  pool:
+    vmImage: macos-10.13
+  steps:
+  - checkout: self
+    fetchDepth: 2
+  - template: steps/macos.yml  
+  strategy:
+    matrix:
+      dist-x86_64-apple-alt:
+        RUST_CHECK_TARGET: dist
+        RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc
+        DEPLOY_ALT: 1
+        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+        MACOSX_DEPLOYMENT_TARGET: 10.7
+        NO_LLVM_ASSERTIONS: 1
+        NO_DEBUG_ASSERTIONS: 1
+
+      # macOS builders. These are placed near the beginning because they are very
+      # slow to run.
+
+      # OSX builders running tests, these run the full test suite.
+      # NO_DEBUG_ASSERTIONS=1 to make them go faster, but also do have some
+      # runners that run `//ignore-debug` tests.
+      #
+      # Note that the compiler is compiled to target 10.8 here because the Xcode
+      # version that we're using, 8.2, cannot compile LLVM for OSX 10.7.
+      x86_64-apple:
+        RUST_CHECK_TARGET: check
+        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
+        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+        MACOSX_DEPLOYMENT_TARGET: 10.8
+        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
+        NO_LLVM_ASSERTIONS: 1
+        NO_DEBUG_ASSERTIONS: 1
+
+      i686-apple:
+        RUST_CHECK_TARGET: check 
+        RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc
+        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+        MACOSX_DEPLOYMENT_TARGET: 10.8
+        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
+        NO_LLVM_ASSERTIONS: 1
+        NO_DEBUG_ASSERTIONS: 1
+
+      dist-i686-apple:
+        RUST_CHECK_TARGET: dist
+        RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc
+        DEPLOY: 1
+        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+        MACOSX_DEPLOYMENT_TARGET: 10.7
+        NO_LLVM_ASSERTIONS: 1
+        NO_DEBUG_ASSERTIONS: 1
+        DIST_REQUIRE_ALL_TOOLS: 1
+
+      dist-x86_64-apple:
+        RUST_CHECK_TARGET: dist
+        RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --enable-lldb --set rust.jemalloc
+        DEPLOY: 1
+        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+        MACOSX_DEPLOYMENT_TARGET: 10.7
+        NO_LLVM_ASSERTIONS: 1
+        NO_DEBUG_ASSERTIONS: 1
+        DIST_REQUIRE_ALL_TOOLS: 1
+
+- job: Windows
+  timeoutInMinutes: 600
+  pool:
+    vmImage: 'vs2017-win2016'
+  steps:
+  - template: steps/windows.yml
+  strategy:
+    matrix:
+      # 32/64 bit MSVC tests
+      x86_64-msvc-1:
+        MSYS_BITS: 64
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+        SCRIPT: make ci-subset-1
+        # FIXME(#59637)
+        NO_DEBUG_ASSERTIONS: 1
+        NO_LLVM_ASSERTIONS: 1
+      x86_64-msvc-2:
+        MSYS_BITS: 64
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+        SCRIPT: make ci-subset-2
+      i686-msvc-1:
+        MSYS_BITS: 32
+        RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
+        SCRIPT: make ci-subset-1
+      i686-msvc-2:
+        MSYS_BITS: 32
+        RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
+        SCRIPT: make ci-subset-2
+      # MSVC aux tests
+      x86_64-msvc-aux:
+        MSYS_BITS: 64
+        RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
+      x86_64-msvc-cargo:
+        MSYS_BITS: 64
+        SCRIPT: python x.py test src/tools/cargotest src/tools/cargo
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
+        VCVARS_BAT: vcvars64.bat
+      # MSVC tools tests
+      x86_64-msvc-tools:
+        MSYS_BITS: 64
+        SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri
+
+      # 32/64-bit MinGW builds.
+      #
+      # We are using MinGW with posix threads since LLVM does not compile with
+      # the win32 threads version due to missing support for C++'s std::thread.
+      #
+      # Instead of relying on the MinGW version installed on appveryor we download
+      # and install one ourselves so we won't be surprised by changes to appveyor's
+      # build image.
+      #
+      # Finally, note that the downloads below are all in the `rust-lang-ci` S3
+      # bucket, but they cleraly didn't originate there! The downloads originally
+      # came from the mingw-w64 SourceForge download site. Unfortunately
+      # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
+      i686-mingw-1:
+        MSYS_BITS: 32
+        RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
+        SCRIPT: make ci-subset-1
+        MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
+        MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
+        MINGW_DIR: mingw32
+        # FIXME(#59637)
+        NO_DEBUG_ASSERTIONS: 1
+        NO_LLVM_ASSERTIONS: 1
+      i686-mingw-2:
+        MSYS_BITS: 32
+        RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
+        SCRIPT: make ci-subset-2
+        MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
+        MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
+        MINGW_DIR: mingw32
+      x86_64-mingw-1:
+        MSYS_BITS: 64
+        SCRIPT: make ci-subset-1
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
+        MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
+        MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
+        MINGW_DIR: mingw64
+        # FIXME(#59637)
+        NO_DEBUG_ASSERTIONS: 1
+        NO_LLVM_ASSERTIONS: 1
+      x86_64-mingw-2:
+        MSYS_BITS: 64
+        SCRIPT: make ci-subset-2
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
+        MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
+        MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
+        MINGW_DIR: mingw64
+
+      # 32/64 bit MSVC and GNU deployment
+      dist-x86_64-msvc:
+        RUST_CONFIGURE_ARGS: >
+          --build=x86_64-pc-windows-msvc
+          --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc
+          --enable-full-tools
+          --enable-profiler
+        SCRIPT: python x.py dist
+        DIST_REQUIRE_ALL_TOOLS: 1
+        DEPLOY: 1
+      dist-i686-msvc:
+        RUST_CONFIGURE_ARGS: >
+          --build=i686-pc-windows-msvc
+          --target=i586-pc-windows-msvc
+          --enable-full-tools
+          --enable-profiler
+        SCRIPT: python x.py dist
+        DIST_REQUIRE_ALL_TOOLS: 1
+        DEPLOY: 1
+      dist-i686-mingw:
+        MSYS_BITS: 32
+        RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools
+        SCRIPT: python x.py dist
+        MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
+        MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
+        MINGW_DIR: mingw32
+        DIST_REQUIRE_ALL_TOOLS: 1
+        DEPLOY: 1
+      dist-x86_64-mingw:
+        MSYS_BITS: 64
+        SCRIPT: python x.py dist
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools
+        MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
+        MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
+        MINGW_DIR: mingw64
+        DIST_REQUIRE_ALL_TOOLS: 1
+        DEPLOY: 1
+
+      # "alternate" deployment, see .travis.yml for more info
+      dist-x86_64-msvc-alt:
+        MSYS_BITS: 64
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
+        SCRIPT: python x.py dist
+        DEPLOY_ALT: 1

.azure-pipelines/master.yml

@@ -0,0 +1,22 @@
+#
+# Azure Pipelines job to publish toolstate. Only triggers on pushes to master.
+#
+
+pr: none
+trigger:
+- master
+
+pool:
+  vmImage: ubuntu-16.04
+
+steps: 
+- checkout: self
+  fetchDepth: 2
+
+- script: |
+    export MESSAGE_FILE=$(mktemp -t msg.XXXXXX)
+    . src/ci/docker/x86_64-gnu-tools/repo.sh
+    commit_toolstate_change "$MESSAGE_FILE" "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "$MESSAGE_FILE" "$TOOLSTATE_REPO_ACCESS_TOKEN"
+  displayName: Publish toolstate  
+  env:
+    TOOLSTATE_REPO_ACCESS_TOKEN: $(TOOLSTATE_REPO_ACCESS_TOKEN_SECRET)

.azure-pipelines/pr.yml

@@ -0,0 +1,26 @@
+#
+# Azure Pipelines pull request build for Rust
+# 
+
+trigger: none
+pr:
+- master # FIXME: really just want any branch, but want an explicit "pr" property set so it's clear
+
+variables:
+- group: caching-pr
+
+jobs:
+- job: Linux
+  timeoutInMinutes: 600
+  pool:
+    vmImage: ubuntu-16.04
+  steps:
+    - template: steps/linux.yml
+  strategy:
+    matrix:
+      x86_64-gnu-llvm-6.0:
+        RUST_BACKTRACE: 1
+      
+      x86_64-gnu-tools: {}
+      # if: branch = auto OR (type = pull_request AND commit_message =~ /(?i:^update.*\b(rls|rustfmt|clippy|miri|cargo)\b)/)
+      mingw-check: {}

.azure-pipelines/steps/linux.yml

@@ -0,0 +1,42 @@
+steps:
+- checkout: self
+  fetchDepth: 2
+
+- template: show-environment-variables.yml
+- template: show-disk-usage.yml
+
+- bash: |
+    sudo apt install gdb
+        
+    curl -fo $HOME/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-unknown-linux-musl
+    chmod +x $HOME/stamp
+    
+    export PATH=$PATH:$HOME/.local/bin:$HOME/Library/Python/2.7/bin/:$HOME
+    echo "##vso[task.setvariable variable=PATH;]$PATH"
+
+    mkdir -p $HOME/rustsrc
+  displayName:  Prep
+
+# FIXME(#46924): these two commands are required to enable IPv6,
+# they shouldn't exist, please revert once more official solutions appeared.
+# see https://github.com/travis-ci/travis-ci/issues/8891#issuecomment-353403729
+- bash: |
+    echo '{"ipv6":true,"fixed-cidr-v6":"fd9a:8454:6789:13f7::/64"}' | sudo tee /etc/docker/daemon.json
+    sudo service docker restart
+  displayName: Enable IPv6  
+
+# FIXME: using the job display name feels brittle. Is there a better variable that represents the current matrix key?
+- bash: |
+    export RUN_SCRIPT="$BUILD_SOURCESDIRECTORY/src/ci/init_repo.sh . $HOME/rustsrc && src/ci/docker/run.sh $IMAGE"
+    echo "##vso[task.setvariable variable=IMAGE]$IMAGE"
+    echo "##vso[task.setvariable variable=RUN_SCRIPT]$RUN_SCRIPT"
+  displayName: Prepare run script
+
+- template: show-environment-variables.yml
+
+- bash: sudo sh -c 'echo "/checkout/obj/cores/core.%p.%E" > /proc/sys/kernel/core_pattern'
+  displayName: Enable core dump
+
+- template: verify-publish-toolstate.yml
+
+- template: run-script.yml

.azure-pipelines/steps/macos.yml

@@ -0,0 +1,47 @@
+steps:
+- template: show-disk-usage.yml
+
+- bash: |
+    export PATH=$PATH:$HOME/.local/bin:$HOME/Library/Python/2.7/bin/
+    mkdir -p $HOME/rustsrc    
+    echo "##vso[task.setvariable variable=PATH;]$PATH"
+
+    curl -fo /usr/local/bin/sccache https://rustlangtools.blob.core.windows.net/public/stable-x86_64-apple-darwin.sccache
+    chmod +x /usr/local/bin/sccache
+
+    curl -fo /usr/local/bin/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-apple-darwin
+    chmod +x /usr/local/bin/stamp
+    
+    export CC=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang
+    echo "##vso[task.setvariable variable=CC]$CC"
+    
+    export CXX=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang++
+    echo "##vso[task.setvariable variable=CXX]$CXX"
+    
+    echo "##vso[task.setvariable variable=AR]ar"
+  displayName: Prep
+
+- bash: brew install gnu-tar
+  displayName: install a tar that works well
+
+- bash: |
+    curl -f http://releases.llvm.org/7.0.0/clang+llvm-7.0.0-x86_64-apple-darwin.tar.xz | tar xJf - 
+  displayName: Download clang
+  
+- bash: |
+    brew update
+    brew install xz
+    brew install swig
+  condition: and(succeeded(), eq(variables['RUST_CHECK_TARGET'],'dist'))
+  displayName: Install xz and swigw
+
+- bash: |
+    export RUN_SCRIPT="$BUILD_SOURCESDIRECTORY/src/ci/init_repo.sh . $HOME/rustsrc && src/ci/run.sh"
+    echo "##vso[task.setvariable variable=RUN_SCRIPT]$RUN_SCRIPT"
+  displayName: Prepare run script (init and run)
+
+- template: show-environment-variables.yml
+
+- template: verify-publish-toolstate.yml
+
+- template: run-script.yml

.azure-pipelines/steps/run-script.yml

@@ -0,0 +1,24 @@
+steps:
+# Log time information from this machine and an external machine for insight into possible
+# clock drift. Timezones don't matter since relative deltas give all the necessary info.
+- bash: |
+    date && (curl -fs --head https://google.com | grep ^Date: | sed 's/Date: //g' || true)
+  displayName: Log time information (before)
+
+- bash: |    
+    which sccache
+    stamp sh -x -c "$RUN_SCRIPT"
+  env:
+    CI: true
+    CI_JOB_NAME: $(IMAGE)
+    SRC: .
+
+    # Explicitly decrypt secret variables
+    # See https://docs.microsoft.com/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#secret-variables
+    SCCACHE_AZURE_CONNECTION_STRING: $(SCCACHE_AZURE_CONNECTION_STRING_SECRET)
+    DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY: $(DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY_SECRET)
+  displayName: Run script
+
+- bash: |
+    date && (curl -fs --head https://google.com | grep ^Date: | sed 's/Date: //g' || true)
+  displayName: Log time information (after)

.azure-pipelines/steps/show-disk-usage.yml

@@ -0,0 +1,5 @@
+steps:
+- bash: |
+    df -h
+    du . | sort -nr | head -n100
+  displayName: Show disk usage

.azure-pipelines/steps/show-environment-variables.yml

@@ -0,0 +1,3 @@
+steps:
+- bash: printenv | sort
+  displayName: Show environment variables

.azure-pipelines/steps/verify-publish-toolstate.yml

@@ -0,0 +1,9 @@
+steps:
+- bash: |
+    git clone --depth=1 https://github.com/rust-lang-nursery/rust-toolstate.git
+    cd rust-toolstate
+    python2.7 "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "" ""
+    cd ..
+    rm -rf rust-toolstate
+  condition: and(succeeded(), eq(variables['IMAGE'], 'mingw-check'))
+  displayName: Verify the publish_toolstate script works

.azure-pipelines/steps/windows.yml

@@ -0,0 +1,160 @@
+steps:
+- checkout: self
+  fetchDepth: 2
+
+- bash: |
+    set -x
+    git submodule
+    export SUBMODULES_EXCLUDES=$(git submodule | grep -Eow 'src/[^ ]+' | sed 's/\(.*\)/--exclude=\1\/\.git/')
+    echo "##vso[task.setvariable variable=SUBMODULES_EXCLUDES;]$SUBMODULES_EXCLUDES"
+    
+- script: |
+    REM echo hack as drive D is too small
+    IF NOT "%DISABLE_DISK_SPACE_HACK%"=="1" (
+      mkdir c:\MORE_SPACE
+      mklink /J build c:\MORE_SPACE
+    )
+
+- script: |
+    set MSYS_PATH=%CD%\citools\msys64
+    choco install msys2 --params="/InstallDir:%MSYS_PATH% /NoPath" -y
+    set PATH=%MSYS_PATH%\usr\bin;%PATH%
+    pacman -S --noconfirm --needed base-devel ca-certificates make diffutils tar 
+    IF "%MINGW_URL%"=="" (
+      IF "%MSYS_BITS%"=="32" pacman -S --noconfirm --needed mingw-w64-i686-toolchain mingw-w64-i686-cmake mingw-w64-i686-gcc mingw-w64-i686-python2
+      IF "%MSYS_BITS%"=="64" pacman -S --noconfirm --needed mingw-w64-x86_64-toolchain mingw-w64-x86_64-cmake mingw-w64-x86_64-gcc mingw-w64-x86_64-python2
+    )
+    where rev
+    rev --help
+    where make
+
+    echo ##vso[task.setvariable variable=MSYS_PATH]%MSYS_PATH%
+    echo ##vso[task.prependpath]%MSYS_PATH%\usr\bin
+  displayName: Install msys2
+
+# If we need to download a custom MinGW, do so here and set the path
+# appropriately.
+#
+# Note that this *also* means that we're not using what is typically
+# /mingw32/bin/python2.7.exe, which is a "correct" python interpreter where
+# /usr/bin/python2.7.exe is not. To ensure we use the right interpreter we
+# move `C:\Python27` ahead in PATH and then also make sure the `python2.7.exe`
+# file exists in there (which it doesn't by default).
+- script: |
+    powershell -Command "iwr -outf %MINGW_ARCHIVE% %MINGW_URL%/%MINGW_ARCHIVE%"
+    7z x -y %MINGW_ARCHIVE% > nul
+    echo ##vso[task.prependpath]%CD%\%MINGW_DIR%\bin
+  condition: and(succeeded(), ne(variables['MINGW_URL'],''))
+  displayName: Download custom MinGW
+
+# If we're compiling for MSVC then we, like most other distribution builders,
+# switch to clang as the compiler. This'll allow us eventually to enable LTO
+# amongst LLVM and rustc. Note that we only do this on MSVC as I don't think
+# clang has an output mode compatible with MinGW that we need. If it does we
+# should switch to clang for MinGW as well!
+#
+# Note that the LLVM installer is an NSIS installer
+#
+# Original downloaded here came from
+# http://releases.llvm.org/7.0.0/LLVM-7.0.0-win64.exe
+- script: |
+    powershell -Command "iwr -outf %TEMP%\LLVM-7.0.0-win64.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/LLVM-7.0.0-win64.exe"
+    set CLANG_DIR=%CD%\citools\clang-rust
+    %TEMP%\LLVM-7.0.0-win64.exe /S /NCRC /D=%CLANG_DIR%
+    set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --set llvm.clang-cl=%CLANG_DIR%\bin\clang-cl.exe
+    echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS%
+  condition: and(succeeded(), eq(variables['MINGW_URL'],''))
+  displayName: Download clang
+
+# Here we do a pretty heinous thing which is to mangle the MinGW installation
+# we just had above. Currently, as of this writing, we're using MinGW-w64
+# builds of gcc, and that's currently at 6.3.0. We use 6.3.0 as it appears to
+# be the first version which contains a fix for #40546, builds randomly
+# failing during LLVM due to ar.exe/ranlib.exe failures.
+#
+# Unfortunately, though, 6.3.0 *also* is the first version of MinGW-w64 builds
+# to contain a regression in gdb (#40184). As a result if we were to use the
+# gdb provided (7.11.1) then we would fail all debuginfo tests.
+#
+# In order to fix spurious failures (pretty high priority) we use 6.3.0. To
+# avoid disabling gdb tests we download an *old* version of gdb, specifically
+# that found inside the 6.2.0 distribution. We then overwrite the 6.3.0 gdb
+# with the 6.2.0 gdb to get tests passing.
+#
+# Note that we don't literally overwrite the gdb.exe binary because it appears
+# to just use gdborig.exe, so that's the binary we deal with instead.
+- script: |
+    echo ON
+    powershell -Command "iwr -outf 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_URL%/2017-04-20-%MSYS_BITS%bit-gdborig.exe"
+    mv 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_DIR%\bin\gdborig.exe
+  condition: and(succeeded(), ne(variables['MINGW_URL'],''))
+  displayName: Override with 6.3.0 gdb with 6.2.0 gdb
+
+# Otherwise pull in the MinGW installed on appveyor
+- script: |
+    echo Find mingw
+    set PATH | findstr /i msys
+    set PATH | findstr /i mingw
+    echo ##vso[task.prependpath]%MSYS_PATH%\mingw%MSYS_BITS%\bin
+  condition: and(succeeded(), eq(variables['MINGW_URL'],''))
+  displayName: Add MinGW to path
+
+- script: |
+    copy C:\Python27amd64\python.exe C:\Python27amd64\python2.7.exe
+    echo ##vso[task.prependpath]C:\Python27amd64
+  displayName: Prefer the "native" Python as LLVM has trouble building with MSYS sometimes
+
+- script: |
+    md sccache
+    powershell -Command "iwr -outf sccache\sccache.exe https://rustlangtools.blob.core.windows.net/public/stable-x86_64-pc-windows-msvc.sccache.exe"
+    echo ##vso[task.prependpath]%CD%\sccache
+  displayName: Download and install sccache
+
+# Note that this is originally from the github releases patch of Ninja
+- script: |
+    md ninja
+    powershell -Command "iwr -outf 2017-03-15-ninja-win.zip https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-15-ninja-win.zip"
+    7z x -oninja 2017-03-15-ninja-win.zip
+    del 2017-03-15-ninja-win.zip
+    set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --enable-ninja
+    echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS%
+    echo ##vso[task.prependpath]%CD%\ninja
+  displayName: Download and install ninja
+
+- script: |
+    mkdir handle
+    powershell -Command "iwr -outf 2017-05-15-Handle.zip https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-05-15-Handle.zip"
+    7z x -ohandle 2017-05-15-Handle.zip
+    del 2017-05-15-Handle.zip
+    set PATH=%PATH%;%CD%\handle
+    handle.exe -accepteula -help
+    echo ##vso[task.setvariable variable=PATH]%PATH%
+  displayName: Help debug handle issues
+
+- template: show-environment-variables.yml
+
+- script: |
+    REM echo force the specific VS version https://github.com/johnterickson/rust/issues/8
+    IF "%VCVARS_BAT%" NEQ "" ( 
+      CALL "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\%VCVARS_BAT%"
+    )
+
+    where sccache
+    where rev
+    set | findstr /v SCCACHE_AZURE_CONNECTION_STRING
+
+    if not exist D:\cache\rustsrc\NUL mkdir D:\cache\rustsrc
+    
+    sh src/ci/init_repo.sh . /d/cache/rustsrc
+    sh src/ci/run.sh
+  env:      
+    CI: true
+    CI_JOB_NAME: $(System.JobDisplayName)
+    SRC: .
+    NO_CCACHE: 1
+
+    # explicitly decrypt secret variables
+    # see https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch
+    SCCACHE_AZURE_CONNECTION_STRING: $(SCCACHE_AZURE_CONNECTION_STRING_SECRET)
+    DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY: $(DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY_SECRET)
+  displayName: Run script

.azure-pipelines/test.yml

@@ -0,0 +1,131 @@
+#
+# Azure Pipelines test build for Rust on Linux, macOS, and Windows.
+# 
+
+pr: none
+trigger: none
+
+variables:
+- group: caching
+
+jobs:
+- job: Linux
+  timeoutInMinutes: 600
+  pool:
+    vmImage: ubuntu-16.04
+  steps:
+  - template: steps/linux.yml
+  strategy:
+    matrix:
+      x86_64-gnu-llvm-6.0:
+        RUST_BACKTRACE: 1
+
+      dist-x86_64-linux:
+        DEPLOY: 1
+
+      # "alternate" deployments, these are "nightlies" but have LLVM assertions
+      # turned on, they're deployed to a different location primarily for
+      # additional testing.
+      dist-x86_64-linux-alt:
+        IMAGE: dist-x86_64-linux
+        DEPLOY_ALT: 1
+
+      dist-various-1:
+        DEPLOY: 1
+
+      dist-android:
+        DEPLOY: 1
+
+      dist-x86_64-freebsd:
+        DEPLOY: 1
+
+      i686-gnu-nopt: {}
+      test-various: {}      
+      x86_64-gnu-tools: {}
+      # FIXME if: branch = auto OR (type = pull_request AND commit_message =~ /(?i:^update.*\b(rls|rustfmt|clippy|miri|cargo)\b)/)
+      x86_64-gnu-nopt: {}
+      x86_64-gnu-distcheck: {}
+      mingw-check: {}
+
+- job: macOS
+  timeoutInMinutes: 600
+  pool:
+    vmImage: macos-10.13
+  steps:
+  - template: steps/macos.yml  
+  strategy:
+    matrix:
+      dist-x86_64-apple-alt:
+        RUST_CHECK_TARGET: dist
+        RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc
+        DEPLOY_ALT: 1
+        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+        MACOSX_DEPLOYMENT_TARGET: 10.7
+        NO_LLVM_ASSERTIONS: 1
+        NO_DEBUG_ASSERTIONS: 1
+
+      # macOS builders. These are placed near the beginning because they are very
+      # slow to run.
+
+      # OSX builders running tests, these run the full test suite.
+      # NO_DEBUG_ASSERTIONS=1 to make them go faster, but also do have some
+      # runners that run `//ignore-debug` tests.
+      #
+      # Note that the compiler is compiled to target 10.8 here because the Xcode
+      # version that we're using, 8.2, cannot compile LLVM for OSX 10.7.
+      x86_64-apple:
+        RUST_CHECK_TARGET: check
+        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
+        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+        MACOSX_DEPLOYMENT_TARGET: 10.8
+        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
+        NO_LLVM_ASSERTIONS: 1
+        NO_DEBUG_ASSERTIONS: 1
+
+- job: Windows
+  timeoutInMinutes: 600
+  pool:
+    vmImage: 'vs2017-win2016'
+  steps:
+  - template: steps/windows.yml
+  strategy:
+    matrix:
+      x86_64-msvc-tools:
+        MSYS_BITS: 64
+        SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri
+
+      # 32/64-bit MinGW builds.
+      #
+      # We are using MinGW with posix threads since LLVM does not compile with
+      # the win32 threads version due to missing support for C++'s std::thread.
+      #
+      # Instead of relying on the MinGW version installed on appveryor we download
+      # and install one ourselves so we won't be surprised by changes to appveyor's
+      # build image.
+      #
+      # Finally, note that the downloads below are all in the `rust-lang-ci` S3
+      # bucket, but they cleraly didn't originate there! The downloads originally
+      # came from the mingw-w64 SourceForge download site. Unfortunately
+      # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
+      i686-mingw-1:
+        MSYS_BITS: 32
+        RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
+        SCRIPT: make ci-subset-1
+        MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
+        MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
+        MINGW_DIR: mingw32
+      x86_64-mingw:
+        MSYS_BITS: 64
+        SCRIPT: python x.py test
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
+        MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
+        MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
+        MINGW_DIR: mingw64
+
+      # "alternate" deployment, see .travis.yml for more info
+      dist-x86_64-msvc-alt:
+        MSYS_BITS: 64
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
+        SCRIPT: python x.py dist
+        DEPLOY_ALT: 1

.azure-pipelines/try.yml

@@ -0,0 +1,29 @@
+#
+# Azure Pipelines "try" branch build for Rust
+# 
+
+pr: none
+trigger:
+- try
+
+variables:
+- group: caching
+
+jobs:
+- job: Linux
+  timeoutInMinutes: 300
+  pool:
+    vmImage: ubuntu-16.04
+  strategy:
+    matrix:
+      dist-x86_64-linux:
+        DEPLOY: 1
+
+      # "alternate" deployments, these are "nightlies" but have LLVM assertions
+      # turned on, they're deployed to a different location primarily for
+      # additional testing.
+      dist-x86_64-linux-alt:
+        IMAGE: dist-x86_64-linux
+        DEPLOY_ALT: 1
+  steps:
+  - template: steps/linux.yml

README.md

@@ -1,3 +1,5 @@
+[![Build Status](https://dev.azure.com/rust-lang-azure/rust/_apis/build/status/johnterickson.rust?branchName=master)](https://dev.azure.com/rust-lang-azure/rust/_build/latest?definitionId=6&branchName=master)
+
 # The Rust Programming Language
 
 This is the main source code repository for [Rust]. It contains the compiler,

appveyor.yml

@@ -20,11 +20,11 @@ environment:
   - CI_JOB_NAME: i686-msvc-1
     MSYS_BITS: 32
     RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
-    SCRIPT: make appveyor-subset-1
+    SCRIPT: make ci-subset-1
   - CI_JOB_NAME: i686-msvc-2
     MSYS_BITS: 32
     RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
-    SCRIPT: make appveyor-subset-2
+    SCRIPT: make ci-subset-2
 
   # MSVC aux tests
   - CI_JOB_NAME: x86_64-msvc-aux
@@ -58,7 +58,7 @@ environment:
   - CI_JOB_NAME: i686-mingw-1
     MSYS_BITS: 32
     RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
-    SCRIPT: make appveyor-subset-1
+    SCRIPT: make ci-subset-1
     MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
     MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
     MINGW_DIR: mingw32
@@ -68,7 +68,7 @@ environment:
   - CI_JOB_NAME: i686-mingw-2
     MSYS_BITS: 32
     RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
-    SCRIPT: make appveyor-subset-2
+    SCRIPT: make ci-subset-2
     MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
     MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
     MINGW_DIR: mingw32

src/bootstrap/README.md

@@ -39,8 +39,7 @@ The script accepts commands, flags, and arguments to determine what to do:
   ```
 
   If files are dirty that would normally be rebuilt from stage 0, that can be
-  overridden using `--keep-stage 0`. Using `--keep-stage n` will skip all steps
+  overridden using `--keep-stage 0`.
-  that belong to stage n or earlier:
 
   ```
   # keep old build products for stage 0 and build stage 1

src/bootstrap/mk/Makefile.in

@@ -14,9 +14,10 @@ endif
 
 BOOTSTRAP := $(CFG_PYTHON) $(CFG_SRC_DIR)src/bootstrap/bootstrap.py
 
-all:
+build:
 	$(Q)$(BOOTSTRAP) build $(BOOTSTRAP_ARGS)
-	$(Q)$(BOOTSTRAP) doc $(BOOTSTRAP_ARGS)
+
+all: build doc
 
 help:
 	$(Q)echo 'Welcome to the rustbuild build system!'
@@ -80,10 +81,75 @@ TESTS_IN_2 := \
 	src/test/run-pass-fulldeps \
 	src/tools/linkchecker
 
-appveyor-subset-1:
+MIN_TEST := \
+	src/test/debuginfo
+
+ci-subset-1:
 	$(Q)$(BOOTSTRAP) test $(TESTS_IN_2:%=--exclude %)
-appveyor-subset-2:
+ci-subset-2:
 	$(Q)$(BOOTSTRAP) test $(TESTS_IN_2)
 
+ci-test-prepare: 
+	$(Q)$(BOOTSTRAP) test $(MIN_TEST)
+
+ci-resume-subset-1:
+	$(Q)$(BOOTSTRAP) test $(TESTS_IN_2:%=--exclude %)
+ci-resume-subset-2:
+	$(Q)$(BOOTSTRAP) test $(TESTS_IN_2)
+
+TESTS_IN_B := \
+	src/tools/linkchecker
+
+TESTS_IN_C := \
+	src/test/run-pass \
+	src/test/run-pass-fulldeps
+
+TESTS_IN_D := \
+	src/test/compile-fail \
+	src/test/rustdoc \
+	src/test/pretty
+
+TESTS_IN_E := \
+	src/test/ui 
+
+TESTS_IN_F := \
+	src/test/run-fail \
+	src/liballoc \
+	src/libcore
+
+TESTS_IN_G := \
+	src/tools/rustdoc \
+	src/test/rustdoc-js-std \
+	src/test/run-make-fulldeps \
+	src/libstd
+
+TESTS_IN_H := \
+	src/librustc_driver
+
+ci-resume-subset-A:
+	$(Q)$(BOOTSTRAP) test \
+		$(KEEP_STAGE) \
+		$(TESTS_IN_B:%=--exclude %) \
+		$(TESTS_IN_C:%=--exclude %) \
+		$(TESTS_IN_D:%=--exclude %) \
+		$(TESTS_IN_E:%=--exclude %) \
+		$(TESTS_IN_F:%=--exclude %) \
+		$(TESTS_IN_G:%=--exclude %) \
+		$(TESTS_IN_H:%=--exclude %) \
+		$(MIN_TEST:%=--exclude %) 
+ci-resume-subset-B:
+	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_B)
+ci-resume-subset-C:
+	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_C)
+ci-resume-subset-D:
+	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_D)
+ci-resume-subset-E:
+	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_E)
+ci-resume-subset-F:
+	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_F)
+ci-resume-subset-G:
+	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_G)
+ci-resume-subset-H:
+	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_H)
 
 .PHONY: dist

src/bootstrap/native.rs

@@ -81,6 +81,13 @@ impl Step for Llvm {
             (info, "src/llvm-project/llvm", builder.llvm_out(target), dir.join("bin"))
         };
 
+        let build_llvm_config = llvm_config_ret_dir
+            .join(exe("llvm-config", &*builder.config.build));
+
+        if env::var_os("SKIP_LLVM_BUILD").is_some() {
+            return build_llvm_config
+        }
+
         if !llvm_info.is_git() {
             println!(
                 "git could not determine the LLVM submodule commit hash. \
@@ -88,8 +95,6 @@ impl Step for Llvm {
             );
         }
 
-        let build_llvm_config = llvm_config_ret_dir
-            .join(exe("llvm-config", &*builder.config.build));
         let done_stamp = out_dir.join("llvm-finished-building");
 
         if let Some(llvm_commit) = llvm_info.sha() {

src/bootstrap/util.rs

@@ -326,6 +326,8 @@ pub enum CiEnv {
     Travis,
     /// The AppVeyor environment, for Windows builds.
     AppVeyor,
+    /// The Azure Pipelines environment, for Linux (including Docker), Window, and macOS builds.
+    AzurePipelines,
 }
 
 impl CiEnv {
@@ -335,6 +337,8 @@ impl CiEnv {
             CiEnv::Travis
         } else if env::var("APPVEYOR").ok().map_or(false, |e| &*e == "True") {
             CiEnv::AppVeyor
+        } else if env::var("TF_BUILD").ok().map_or(false, |e| &*e == "True") {
+            CiEnv::AzurePipelines
         } else {
             CiEnv::None
         }

src/ci/docker/i686-gnu-1/Dockerfile

@@ -18,10 +18,5 @@ COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
 ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu
-# Exclude some tests that are unlikely to be platform specific, to speed up
+ENV SCRIPT make ci-subset-1
-# this slow job.
+
-ENV SCRIPT python2.7 ../x.py test \
-  --exclude src/bootstrap \
-  --exclude src/test/rustdoc-js \
-  --exclude src/tools/error_index_generator \
-  --exclude src/tools/linkchecker

src/ci/docker/i686-gnu-2/Dockerfile

@@ -0,0 +1,21 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  g++-multilib \
+  make \
+  file \
+  curl \
+  ca-certificates \
+  python2.7 \
+  git \
+  cmake \
+  sudo \
+  gdb \
+  xz-utils
+
+
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu
+ENV SCRIPT make ci-subset-2

src/ci/docker/run.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+# ignore-tidy-linelength
+
 set -e
 
 export MSYS_NO_PATHCONV=1
@@ -40,9 +42,24 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
       docker --version >> $hash_key
       cksum=$(sha512sum $hash_key | \
         awk '{print $1}')
+
+      if [ "$DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT" != "" ]; then
+        # install azcopy
+        echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod/ xenial main" > azure.list
+        sudo cp ./azure.list /etc/apt/sources.list.d/
+        sudo apt-key adv --keyserver packages.microsoft.com --recv-keys EB3E94ADBE1229CF
+        sudo apt-get update
+        sudo apt-get install azcopy
+
+        url="https://$DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT.blob.core.windows.net/$DOCKER_LAYER_CACHE_AZURE_STORAGE_CONTAINER/$cksum"
+        upload="azcopy --quiet --destination $url --dest-key $DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY"
+      else
         s3url="s3://$SCCACHE_BUCKET/docker/$cksum"
         url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum"
-      echo "Attempting to download $s3url"
+        upload="aws s3 cp - $s3url"
+      fi
+
+      echo "Attempting to download $url"
       rm -f /tmp/rustci_docker_cache
       set +e
       retry curl -y 30 -Y 10 --connect-timeout 30 -f -L -C - -o /tmp/rustci_docker_cache "$url"
@@ -65,17 +82,17 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
       -f "$dockerfile" \
       "$context"
 
-    if [ "$s3url" != "" ]; then
+    if [ "$upload" != "" ]; then
       digest=$(docker inspect rust-ci --format '{{.Id}}')
       echo "Built container $digest"
       if ! grep -q "$digest" <(echo "$loaded_images"); then
-        echo "Uploading finished image to $s3url"
+        echo "Uploading finished image to $url"
         set +e
         docker history -q rust-ci | \
           grep -v missing | \
           xargs docker save | \
           gzip | \
-          aws s3 cp - $s3url
+          $upload
         set -e
       else
         echo "Looks like docker image is the same as before, not uploading"
@@ -87,8 +104,8 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
       echo "$digest" >>"$info"
     fi
 elif [ -f "$docker_dir/disabled/$image/Dockerfile" ]; then
-    if [ -n "$TRAVIS_OS_NAME" ]; then
+    if isCI; then
-        echo Cannot run disabled images on travis!
+        echo Cannot run disabled images on CI!
         exit 1
     fi
     # retry messes with the pipe from tar to docker. Not needed on non-travis
@@ -117,6 +134,9 @@ if [ "$SCCACHE_BUCKET" != "" ]; then
     args="$args --env SCCACHE_REGION"
     args="$args --env AWS_ACCESS_KEY_ID"
     args="$args --env AWS_SECRET_ACCESS_KEY"
+elif [ "$SCCACHE_AZURE_CONNECTION_STRING" != "" ]; then
+    args="$args --env SCCACHE_AZURE_CONNECTION_STRING"
+    args="$args --env SCCACHE_AZURE_BLOB_CONTAINER"
 else
     mkdir -p $HOME/.cache/sccache
     args="$args --env SCCACHE_DIR=/sccache --volume $HOME/.cache/sccache:/sccache"
@@ -140,8 +160,11 @@ exec docker \
   --env DEPLOY \
   --env DEPLOY_ALT \
   --env LOCAL_USER_ID=`id -u` \
+  --env CI \
   --env TRAVIS \
   --env TRAVIS_BRANCH \
+  --env TF_BUILD \
+  --env BUILD_SOURCEBRANCHNAME \
   --env TOOLSTATE_REPO_ACCESS_TOKEN \
   --env CI_JOB_NAME="${CI_JOB_NAME-$IMAGE}" \
   --volume "$HOME/.cargo:/cargo" \

src/ci/docker/scripts/sccache.sh

@@ -1,6 +1,7 @@
 set -ex
 
+echo Use sccache built with Azure Storage support
 curl -fo /usr/local/bin/sccache \
-  https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-unknown-linux-musl
+  https://rustlangtools.blob.core.windows.net/public/stable-x86_64-unknown-linux-musl.sccache
 
 chmod +x /usr/local/bin/sccache

src/ci/docker/x86_64-gnu-tools/checktools.sh

@@ -6,7 +6,7 @@ X_PY="$1"
 TOOLSTATE_FILE="$(realpath $2)"
 OS="$3"
 COMMIT="$(git rev-parse HEAD)"
-CHANGED_FILES="$(git diff --name-status HEAD HEAD^)"
+CHANGED_FILES="$(git diff --name-status HEAD HEAD^ || echo)"
 SIX_WEEK_CYCLE="$(( ($(date +%s) / 86400 - 20) % 42 ))"
 # ^ Number of days after the last promotion of beta.
 #   Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens.

src/ci/run.sh

@@ -23,7 +23,9 @@ fi
 ci_dir=`cd $(dirname $0) && pwd`
 source "$ci_dir/shared.sh"
 
-if [ "$TRAVIS" != "true" ] || [ "$TRAVIS_BRANCH" == "auto" ]; then
+branch_name=$(getCIBranch)
+
+if [ ! isCI ] || [ "$branch_name" = "auto" ]; then
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests"
 fi
 
@@ -44,7 +46,7 @@ fi
 # FIXME: need a scheme for changing this `nightly` value to `beta` and `stable`
 #        either automatically or manually.
 export RUST_RELEASE_CHANNEL=nightly
-if [ "$DEPLOY$DEPLOY_ALT" != "" ]; then
+if [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --release-channel=$RUST_RELEASE_CHANNEL"
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-static-stdcpp"
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.remap-debuginfo"
@@ -88,28 +90,30 @@ if [ "$RUN_CHECK_WITH_PARALLEL_QUERIES" != "" ]; then
   rm -rf build
 fi
 
-travis_fold start configure
+if [ "$SKIP_PREP" != "1" ]; then
-travis_time_start
+  travis_fold start configure
-$SRC/configure $RUST_CONFIGURE_ARGS
+  travis_time_start
-travis_fold end configure
+  $SRC/configure $RUST_CONFIGURE_ARGS
-travis_time_finish
+  travis_fold end configure
+  travis_time_finish
 
-travis_fold start make-prepare
+  travis_fold start make-prepare
-travis_time_start
+  travis_time_start
-retry make prepare
+  retry make prepare
-travis_fold end make-prepare
+  travis_fold end make-prepare
-travis_time_finish
+  travis_time_finish
 
-travis_fold start check-bootstrap
+  travis_fold start check-bootstrap
-travis_time_start
+  travis_time_start
-make check-bootstrap
+  make check-bootstrap
-travis_fold end check-bootstrap
+  travis_fold end check-bootstrap
-travis_time_finish
+  travis_time_finish
+fi
 
 # Display the CPU and memory information. This helps us know why the CI timing
 # is fluctuating.
 travis_fold start log-system-info
-if [ "$TRAVIS_OS_NAME" = "osx" ]; then
+if isOSX; then
     system_profiler SPHardwareDataType || true
     sysctl hw || true
     ncpus=$(sysctl -n hw.ncpu)
@@ -138,3 +142,5 @@ else
   do_make all
   do_make "$RUST_CHECK_TARGET"
 fi
+
+sccache --show-stats

src/ci/shared.sh

@@ -24,6 +24,22 @@ function retry {
   done
 }
 
+function isCI {
+  [ "$CI" = "true" ] || [ "$TRAVIS" = "true" ] || [ "$TF_BUILD" = "True" ]
+}
+
+function isOSX {
+  [ "$TRAVIS_OS_NAME" = "osx" ] || [ "$AGENT_OS" = "Darwin" ]
+}
+
+function getCIBranch {
+  if [ "$TRAVIS" = "true" ]; then
+    echo "$TRAVIS_BRANCH"
+  else
+    echo "$BUILD_SOURCEBRANCHNAME"
+  fi;
+}
+
 if ! declare -F travis_fold; then
   if [ "${TRAVIS-false}" = 'true' ]; then
     # This is a trimmed down copy of

src/test/run-pass/fds-are-cloexec.rs

@@ -3,6 +3,7 @@
 // ignore-cloudabi no processes
 // ignore-emscripten no processes
 // ignore-haiku
+// ignore-macos
 
 #![feature(rustc_private)]