Upload the toolstate to the remote repository.

.travis.yml

@@ -179,6 +179,17 @@ matrix:
     - env: IMAGE=x86_64-gnu-incremental
       if: branch = auto
 
+    - stage: publish toolstate
+      if: branch = master AND type = push
+      before_install: []
+      install: []
+      cache: false
+      sudo: false
+      script:
+        MESSAGE_FILE=$(mktemp -t msg.XXXXXX);
+        . src/ci/docker/x86_64-gnu-tools/repo.sh;
+        commit_toolstate_change "$MESSAGE_FILE" "$TRAVIS_BUILD_DIR/src/ci/docker/x86_64-gnu-tools/publish.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "$MESSAGE_FILE"
+
 env:
   global:
     - SCCACHE_BUCKET=rust-lang-ci-sccache2
@@ -186,6 +197,8 @@ env:
     - AWS_ACCESS_KEY_ID=AKIAJAMV3QAMMA6AXHFQ
     # AWS_SECRET_ACCESS_KEY=...
     - secure: "j96XxTVOSUf4s4r4htIxn/fvIa5DWbMgLqWl7r8z2QfgUwscmkMXAwXuFNc7s7bGTpV/+CgDiMFFM6BAFLGKutytIF6oA02s9b+usQYnM0th7YQ2AIgm9GtMTJCJp4AoyfFmh8F2faUICBZlfVLUJ34udHEe35vOklix+0k4WDo="
+    # TOOLSTATE_REPO_ACCESS_TOKEN=...
+    - secure: "cFh8thThqEJLC98XKI5pfqflUzOlxsYPRW20AWRaYOOgYHPTiGWypTXiPbGSKaeAXTZoOA+DpQtEmefc0U6lt9dHc7a/MIaK6isFurjlnKYiLOeTruzyu1z7PWCeZ/jKXsU2RK/88DBtlNwfMdaMIeuKj14IVfpepPPL71ETbuk="
 
 before_install:
   - zcat $HOME/docker/rust-ci.tar.gz | docker load || true

appveyor.yml

@@ -5,6 +5,8 @@ environment:
   AWS_SECRET_ACCESS_KEY:
     secure: 7Y+JiquYedOAgnUU26uL0DPzrxmTtR+qIwG6rNKSuWDffqU3vVZxbGXim9QpTO80
   SCCACHE_DIGEST: f808afabb4a4eb1d7112bcb3fa6be03b61e93412890c88e177c667eb37f46353d7ec294e559b16f9f4b5e894f2185fe7670a0df15fd064889ecbd80f0c34166c
+  TOOLSTATE_REPO_ACCESS_TOKEN:
+    secure: PTZiSxJMVUZ0VnMR5i13E4OagbXfglj7pcskDQiKufVrDm13mLoI0vDJAEM35+bY
 
   # By default schannel checks revocation of certificates unlike some other SSL
   # backends, but we've historically had problems on CI where a revocation
@@ -27,8 +29,8 @@ environment:
 
   # MSVC tools tests
   - MSYS_BITS: 64
-    SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py toolstates.json
+    SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows
-    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=toolstates.json --enable-test-miri
+    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri
 
   # 32/64-bit MinGW builds.
   #

src/ci/docker/run.sh

@@ -99,6 +99,7 @@ exec docker \
   --env LOCAL_USER_ID=`id -u` \
   --env TRAVIS \
   --env TRAVIS_BRANCH \
+  --env TOOLSTATE_REPO_ACCESS_TOKEN \
   --volume "$HOME/.cargo:/cargo" \
   --volume "$HOME/rustsrc:$HOME/rustsrc" \
   --init \

src/ci/docker/x86_64-gnu-tools/Dockerfile

@@ -18,9 +18,10 @@ COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
 COPY x86_64-gnu-tools/checktools.sh /tmp/
+COPY x86_64-gnu-tools/repo.sh /tmp/
 
 ENV RUST_CONFIGURE_ARGS \
   --build=x86_64-unknown-linux-gnu \
   --enable-test-miri \
   --save-toolstates=/tmp/toolstates.json
-ENV SCRIPT /tmp/checktools.sh ../x.py /tmp/toolstates.json
+ENV SCRIPT /tmp/checktools.sh ../x.py /tmp/toolstates.json linux

src/ci/docker/x86_64-gnu-tools/checktools.sh

@@ -13,7 +13,9 @@
 set -eu
 
 X_PY="$1"
-TOOLSTATE_FILE="$2"
+TOOLSTATE_FILE="$(realpath $2)"
+OS="$3"
+COMMIT="$(git rev-parse HEAD)"
 
 touch "$TOOLSTATE_FILE"
 
@@ -23,17 +25,22 @@ python2.7 "$X_PY" test --no-fail-fast \
     src/tools/rustfmt \
     src/tools/miri \
     src/tools/clippy
-TEST_RESULT=$?
 set -e
 
-# FIXME: Upload this file to the repository.
 cat "$TOOLSTATE_FILE"
 
-# FIXME: After we can properly inform dev-tool maintainers about failure,
-#        comment out the `exit 0` below.
 if [ "$RUST_RELEASE_CHANNEL" = nightly ]; then
-    # exit 0
+    . "$(dirname $0)/repo.sh"
-    true
+    MESSAGE_FILE=$(mktemp -t msg.XXXXXX)
+    echo "($OS CI update)" > "$MESSAGE_FILE"
+    commit_toolstate_change "$MESSAGE_FILE" \
+        sed -i "1 a\\
+$COMMIT\t$(cat "$TOOLSTATE_FILE")
+" "history/$OS.tsv"
+    rm -f "$MESSAGE_FILE"
+    exit 0
 fi
 
-exit $TEST_RESULT
+if grep -q 'Broken\|Compiling' "$TOOLSTATE_FILE"; then
+    exit 4
+fi

src/ci/docker/x86_64-gnu-tools/publish.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import sys
+import re
+import json
+import copy
+import datetime
+import collections
+
+# List of people to ping when the status of a tool changed.
+MAINTAINERS = {
+    'miri': '@oli-obk @RalfJung @eddyb',
+    'clippy-driver': '@Manishearth @llogiq @mcarton @oli-obk',
+    'rls': '@nrc',
+    'rustfmt': '@nrc',
+}
+
+
+def read_current_status(current_commit, path):
+    '''Reads build status of `current_commit` from content of `history/*.tsv`
+    '''
+    with open(path, 'rU') as f:
+        for line in f:
+            (commit, status) = line.split('\t', 1)
+            if commit == current_commit:
+                return json.loads(status)
+    return {}
+
+
+def update_latest(current_commit, relevant_pr_number, current_datetime):
+    '''Updates `_data/latest.json` to match build result of the given commit.
+    '''
+    with open('_data/latest.json', 'rb+') as f:
+        latest = json.load(f, object_pairs_hook=collections.OrderedDict)
+
+        current_status = {
+            os: read_current_status(current_commit, 'history/' + os + '.tsv')
+            for os in ['windows', 'linux']
+        }
+
+        slug = 'rust-lang/rust'
+        message = '📣 Toolstate changed by {}!\n\nTested on commit {}@{}.\n\n' \
+            .format(relevant_pr_number, slug, current_commit)
+        anything_changed = False
+        for status in latest:
+            tool = status['tool']
+            changed = False
+
+            for os, s in current_status.items():
+                old = status[os]
+                new = s.get(tool, old)
+                status[os] = new
+                if new > old:
+                    changed = True
+                    message += '🎉 {} on {}: {} → {}.\n' \
+                        .format(tool, os, old, new)
+                elif new < old:
+                    changed = True
+                    message += '💔 {} on {}: {} → {} (cc {}).\n' \
+                        .format(tool, os, old, new, MAINTAINERS[tool])
+
+            if changed:
+                status['commit'] = current_commit
+                status['datetime'] = current_datetime
+                anything_changed = True
+
+        if not anything_changed:
+            return ''
+
+        f.seek(0)
+        f.truncate(0)
+        json.dump(latest, f, indent=4, separators=(',', ': '))
+        return message
+
+
+if __name__ == '__main__':
+    cur_commit = sys.argv[1]
+    cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
+    cur_commit_msg = sys.argv[2]
+    save_message_to_path = sys.argv[3]
+
+    relevant_pr_match = re.search('#[0-9]+', cur_commit_msg)
+    if relevant_pr_match:
+        relevant_pr_number = 'rust-lang/rust' + relevant_pr_match.group(0)
+    else:
+        relevant_pr_number = '<unknown PR>'
+
+    message = update_latest(cur_commit, relevant_pr_number, cur_datetime)
+    if message:
+        print(message)
+        with open(save_message_to_path, 'w') as f:
+            f.write(message)
+    else:
+        print('<Nothing changed>')

src/ci/docker/x86_64-gnu-tools/repo.sh

@@ -0,0 +1,90 @@
+#!/bin/sh
+
+# Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+# This file provides the function `commit_toolstate_change` for pushing a change
+# to the `rust-toolstate` repository.
+#
+# The function relies on a GitHub bot user, which should have a Personal access
+# token defined in the environment variable $TOOLSTATE_REPO_ACCESS_TOKEN. If for
+# some reason you need to change the token, please update `.travis.yml` and
+# `appveyor.yml`:
+#
+#   1. Generate a new Personal access token:
+#
+#       * Login to the bot account, and go to Settings -> Developer settings ->
+#           Personal access tokens
+#       * Click "Generate new token"
+#       * Enable the "public_repo" permission, then click "Generate token"
+#       * Copy the generated token (should be a 40-digit hexadecimal number).
+#           Save it somewhere secure, as the token would be gone once you leave
+#           the page.
+#
+#   2. Encrypt the token for Travis CI
+#
+#       * Install the `travis` tool locally (`gem install travis`).
+#       * Encrypt the token:
+#           ```
+#           travis -r rust-lang/rust encrypt \
+#                   TOOLSTATE_REPO_ACCESS_TOKEN=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+#           ```
+#       * Copy output to replace the existing one in `.travis.yml`.
+#       * Details of this step can be found in
+#           <https://docs.travis-ci.com/user/encryption-keys/>
+#
+#   3. Encrypt the token for AppVeyor
+#
+#       * Login to AppVeyor using your main account, and login as the rust-lang
+#           organization.
+#       * Open the ["Encrypt data" tool](https://ci.appveyor.com/tools/encrypt)
+#       * Paste the 40-digit token into the "Value to encrypt" box, then click
+#           "Encrypt"
+#       * Copy the output to replace the existing one in `appveyor.yml`.
+#       * Details of this step can be found in
+#           <https://www.appveyor.com/docs/how-to/git-push/>
+#
+#   4. Replace the email address below if the bot account identity is changed
+#
+#       * See <https://help.github.com/articles/about-commit-email-addresses/>
+#           if a private email by GitHub is wanted.
+
+commit_toolstate_change() {
+    OLDFLAGS="$-"
+    set -eu
+
+    git config --global user.email '34210020+rust-toolstate-update@users.noreply.github.com'
+    git config --global user.name 'Rust Toolstate Update'
+    git config --global credential.helper store
+    printf 'https://%s:x-oauth-basic@github.com\n' "$TOOLSTATE_REPO_ACCESS_TOKEN" \
+        > "$HOME/.git-credentials"
+    git clone --depth=1 https://github.com/rust-lang-nursery/rust-toolstate.git
+
+    cd rust-toolstate
+    FAILURE=1
+    MESSAGE_FILE="$1"
+    shift
+    for RETRY_COUNT in 1 2 3 4 5; do
+        "$@"
+        # `git commit` failing means nothing to commit.
+        FAILURE=0
+        git commit -a -F "$MESSAGE_FILE" || break
+        # On failure randomly sleep for 0 to 3 seconds as a crude way to introduce jittering.
+        git push origin master && break || sleep $(LC_ALL=C tr -cd 0-3 < /dev/urandom | head -c 1)
+        FAILURE=1
+        git fetch origin master
+        git reset --hard origin/master
+    done
+    cd ..
+
+    set +eu
+    set "-$OLDFLAGS"
+    return $FAILURE
+}