Auto merge of #124366 - Kobzol:remove-yaml-expansion, r=pietroalbini

CI: remove `expand-yaml-anchors`

This PR unifies all CI outcome jobs in a single job, and then removes the `expand-yaml-anchors` tool, since it is no longer needed after this change.

I have tested try builds for both situations with the new `outcome` job (note that these two workflow runs use a different step structure in the outcome job, I have simplified it since):
- [Success](https://github.com/rust-lang-ci/rust/actions/runs/8831529677/job/24251135366)
- [Failure](https://github.com/rust-lang-ci/rust/actions/runs/8833052319/job/24251628792)

r? `@ghost`

.github/workflows/ci.yml

@@ -1,21 +1,14 @@
-#############################################################
+# This file defines our primary CI workflow that runs on pull requests
-#   WARNING: automatically generated file, DO NOT CHANGE!   #
+# and also on pushes to special branches (auto, try).
-#############################################################
+#
+# The actual definition of the executed jobs is calculated by a Python
+# script located at src/ci/github-actions/calculate-job-matrix.py, which
+# uses job definition data from src/ci/github-actions/jobs.yml.
+# You should primarily modify the `jobs.yml` file if you want to modify
+# what jobs are executed in CI.
 
-# This file was automatically generated by the expand-yaml-anchors tool. The
-# source file that generated this one is:
-#
-#   src/ci/github-actions/ci.yml
-#
-# Once you make changes to that file you need to run:
-#
-#   ./x.py run src/tools/expand-yaml-anchors/
-#
-# The CI build will fail if the tool is not run after changes to this file.
-
----
 name: CI
-"on":
+on:
   push:
     branches:
       - auto
@@ -25,23 +18,36 @@ name: CI
   pull_request:
     branches:
       - "**"
+
 permissions:
   contents: read
   packages: write
+
 defaults:
   run:
+    # On Linux, macOS, and Windows, use the system-provided bash as the default
+    # shell. (This should only make a difference on Windows, where the default
+    # shell is PowerShell.)
     shell: bash
+
 concurrency:
-  group: "${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}"
+  # For a given workflow, if we push to the same branch, cancel all previous builds on that branch.
+  # We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which
+  # are all triggered on the same branch, but which should be able to run concurrently.
+  group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}
   cancel-in-progress: true
 env:
   TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
 jobs:
+  # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml.
+  # It calculates which jobs should be executed, based on the data of the ${{ github }} context.
+  # If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml.
   calculate_matrix:
     name: Calculate job matrix
     runs-on: ubuntu-latest
     outputs:
-      jobs: "${{ steps.jobs.outputs.jobs }}"
+      jobs: ${{ steps.jobs.outputs.jobs }}
+      run_type: ${{ steps.jobs.outputs.run_type }}
     steps:
       - name: Checkout the source code
         uses: actions/checkout@v4
@@ -49,152 +55,195 @@ jobs:
         run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
         id: jobs
   job:
-    name: "${{ matrix.name }}"
+    name: ${{ matrix.name }}
-    needs:
+    needs: [ calculate_matrix ]
-      - calculate_matrix
     runs-on: "${{ matrix.os }}"
     defaults:
       run:
-        shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}"
+        shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}
     timeout-minutes: 600
     env:
-      CI_JOB_NAME: "${{ matrix.image }}"
+      CI_JOB_NAME: ${{ matrix.image }}
       CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
-      HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
+      # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
-      DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+      HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+      DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       SCCACHE_BUCKET: rust-lang-ci-sccache2
       CACHE_DOMAIN: ci-caches.rust-lang.org
-    continue-on-error: "${{ matrix.continue_on_error || false }}"
+    continue-on-error: ${{ matrix.continue_on_error || false }}
     strategy:
       matrix:
-        include: "${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}"
+        # Check the `calculate_matrix` job to see how is the matrix defined.
-    if: "fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null"
+        include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}
+    # GitHub Actions fails the workflow if an empty list of jobs is provided to
+    # the workflow, so we need to skip this job if nothing was produced by
+    # the Python script.
+    #
+    # Unfortunately checking whether a list is empty is not possible in a nice
+    # way due to GitHub Actions expressions limits.
+    # This hack is taken from https://github.com/ferrocene/ferrocene/blob/d43edc6b7697cf1719ec1c17c54904ab94825763/.github/workflows/release.yml#L75-L82
+    if: fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null
     steps:
-      - if: "contains(matrix.os, 'windows')"
+      - if: contains(matrix.os, 'windows')
         uses: msys2/setup-msys2@v2.22.0
         with:
-          msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}"
+          # i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64.
+          msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}
+          # don't try to download updates for already installed packages
           update: false
+          # don't try to use the msys that comes built-in to the github runner,
+          # so we can control what is installed (i.e. not python)
           release: true
+          # Inherit the full path from the Windows environment, with MSYS2's */bin/
+          # dirs placed in front. This lets us run Windows-native Python etc.
           path-type: inherit
-          install: "make dos2unix diffutils\n"
+          install: >
+            make
+            dos2unix
+            diffutils
+
       - name: disable git crlf conversion
         run: git config --global core.autocrlf false
+
       - name: checkout the source code
         uses: actions/checkout@v4
         with:
           fetch-depth: 2
+
+      # Rust Log Analyzer can't currently detect the PR number of a GitHub
+      # Actions build on its own, so a hint in the log message is needed to
+      # point it in the right direction.
       - name: configure the PR in which the error message will be posted
-        run: "echo \"[CI_PR_NUMBER=$num]\""
+        run: echo "[CI_PR_NUMBER=$num]"
         env:
-          num: "${{ github.event.number }}"
+          num: ${{ github.event.number }}
-        if: "success() && github.event_name == 'pull_request'"
+        if: needs.calculate_matrix.outputs.run_type == 'pr'
+
       - name: add extra environment variables
         run: src/ci/scripts/setup-environment.sh
         env:
-          EXTRA_VARIABLES: "${{ toJson(matrix.env) }}"
+          # Since it's not possible to merge `${{ matrix.env }}` with the other
+          # variables in `job.<name>.env`, the variables defined in the matrix
+          # are passed to the `setup-environment.sh` script encoded in JSON,
+          # which then uses log commands to actually set them.
+          EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
+
       - name: ensure the channel matches the target branch
         run: src/ci/scripts/verify-channel.sh
+
       - name: collect CPU statistics
         run: src/ci/scripts/collect-cpu-stats.sh
+
       - name: show the current environment
         run: src/ci/scripts/dump-environment.sh
+
       - name: install awscli
         run: src/ci/scripts/install-awscli.sh
+
       - name: install sccache
         run: src/ci/scripts/install-sccache.sh
+
       - name: select Xcode
         run: src/ci/scripts/select-xcode.sh
+
       - name: install clang
         run: src/ci/scripts/install-clang.sh
+
       - name: install tidy
         run: src/ci/scripts/install-tidy.sh
+
       - name: install WIX
         run: src/ci/scripts/install-wix.sh
+
       - name: disable git crlf conversion
         run: src/ci/scripts/disable-git-crlf-conversion.sh
+
       - name: checkout submodules
         run: src/ci/scripts/checkout-submodules.sh
+
       - name: install MSYS2
         run: src/ci/scripts/install-msys2.sh
+
       - name: install MinGW
         run: src/ci/scripts/install-mingw.sh
+
       - name: install ninja
         run: src/ci/scripts/install-ninja.sh
+
       - name: enable ipv6 on Docker
         run: src/ci/scripts/enable-docker-ipv6.sh
+
+      # Disable automatic line ending conversion (again). On Windows, when we're
+      # installing dependencies, something switches the git configuration directory or
+      # re-enables autocrlf. We've not tracked down the exact cause -- and there may
+      # be multiple -- but this should ensure submodules are checked out with the
+      # appropriate line endings.
       - name: disable git crlf conversion
         run: src/ci/scripts/disable-git-crlf-conversion.sh
+
       - name: ensure line endings are correct
         run: src/ci/scripts/verify-line-endings.sh
+
       - name: ensure backported commits are in upstream branches
         run: src/ci/scripts/verify-backported-commits.sh
+
       - name: ensure the stable version number is correct
         run: src/ci/scripts/verify-stable-version-number.sh
+
       - name: run the build
+        # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
         run: src/ci/scripts/run-build-from-ci.sh 2>&1
         env:
-          AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}"
+          AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
+          AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
-          TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
+          TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
+
       - name: create github artifacts
         run: src/ci/scripts/create-doc-artifacts.sh
+
       - name: upload artifacts to github
         uses: actions/upload-artifact@v4
         with:
-          name: "${{ env.DOC_ARTIFACT_NAME }}"
+          # name is set in previous step
+          name: ${{ env.DOC_ARTIFACT_NAME }}
           path: obj/artifacts/doc
           if-no-files-found: ignore
           retention-days: 5
+
       - name: upload artifacts to S3
         run: src/ci/scripts/upload-artifacts.sh
         env:
-          AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
+          AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
+          AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
-        if: "success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
+        # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
-  try-success:
+        # builders *should* have the AWS credentials available. Still, explicitly
-    needs:
+        # adding the condition is helpful as this way CI will not silently skip
-      - job
+        # deploying artifacts from a dist builder if the variables are misconfigured,
-    if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
+        # erroring about invalid credentials instead.
-    steps:
+        if: github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1'
-      - name: mark the job as a success
+
-        run: exit 0
+  # This job isused to tell bors the final status of the build, as there is no practical way to detect
-        shell: bash
+  # when a workflow is successful listening to webhooks only in our current bors implementation (homu).
+  outcome:
     name: bors build finished
     runs-on: ubuntu-latest
-  try-failure:
+    needs: [ calculate_matrix, job ]
-    needs:
+    # !cancelled() executes the job regardless of whether the previous jobs passed or failed
-      - job
+    if: ${{ !cancelled() && contains(fromJSON('["auto", "try"]'), needs.calculate_matrix.outputs.run_type) }}
-    if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
-    steps:
-      - name: mark the job as a failure
-        run: exit 1
-        shell: bash
-    name: bors build finished
-    runs-on: ubuntu-latest
-  auto-success:
-    needs:
-      - job
-    if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
     steps:
       - name: checkout the source code
         uses: actions/checkout@v4
         with:
           fetch-depth: 2
+      # Calculate the exit status of the whole CI workflow.
+      # If all dependent jobs were successful, this exits with 0 (and the outcome job continues successfully).
+      # If a some dependent job has failed, this exits with 1.
+      - name: calculate the correct exit status
+        run: jq --exit-status 'all(.result == "success" or .result == "skipped")' <<< '${{ toJson(needs) }}'
+      # Publish the toolstate if an auto build succeeds (just before push to master)
       - name: publish toolstate
         run: src/ci/publish_toolstate.sh
         shell: bash
+        if: needs.calculate_matrix.outputs.run_type == 'auto'
         env:
-          TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
+          TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
-    name: bors build finished
-    runs-on: ubuntu-latest
-  auto-failure:
-    needs:
-      - job
-    if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
-    steps:
-      - name: mark the job as a failure
-        run: exit 1
-        shell: bash
-    name: bors build finished
-    runs-on: ubuntu-latest

Cargo.lock

@@ -1254,14 +1254,6 @@ dependencies = [
  "mdbook",
 ]
 
-[[package]]
-name = "expand-yaml-anchors"
-version = "0.1.0"
-dependencies = [
- "yaml-merge-keys",
- "yaml-rust",
-]
-
 [[package]]
 name = "expect-test"
 version = "1.5.0"
@@ -2234,12 +2226,6 @@ dependencies = [
  "regex",
 ]
 
-[[package]]
-name = "linked-hash-map"
-version = "0.5.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
-
 [[package]]
 name = "lint-docs"
 version = "0.1.0"
@@ -6540,26 +6526,6 @@ dependencies = [
  "lzma-sys",
 ]
 
-[[package]]
-name = "yaml-merge-keys"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd236a7dc9bb598f349fe4a8754f49181fee50284daa15cd1ba652d722280004"
-dependencies = [
- "lazy_static",
- "thiserror",
- "yaml-rust",
-]
-
-[[package]]
-name = "yaml-rust"
-version = "0.4.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
-dependencies = [
- "linked-hash-map",
-]
-
 [[package]]
 name = "yansi-term"
 version = "0.1.2"

Cargo.toml

@@ -31,7 +31,6 @@ members = [
   "src/tools/miri/cargo-miri",
   "src/tools/rustdoc-themes",
   "src/tools/unicode-table-generator",
-  "src/tools/expand-yaml-anchors",
   "src/tools/jsondocck",
   "src/tools/jsondoclint",
   "src/tools/llvm-bitcode-linker",

src/bootstrap/src/core/build_steps/run.rs

@@ -15,32 +15,6 @@ use crate::core::config::TargetSelection;
 use crate::utils::helpers::output;
 use crate::Mode;
 
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct ExpandYamlAnchors;
-
-impl Step for ExpandYamlAnchors {
-    type Output = ();
-
-    /// Runs the `expand-yaml_anchors` tool.
-    ///
-    /// This tool in `src/tools` reads the CI configuration files written in YAML and expands the
-    /// anchors in them, since GitHub Actions doesn't support them.
-    fn run(self, builder: &Builder<'_>) {
-        builder.info("Expanding YAML anchors in the GitHub Actions configuration");
-        builder.run_delaying_failure(
-            builder.tool_cmd(Tool::ExpandYamlAnchors).arg("generate").arg(&builder.src),
-        );
-    }
-
-    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
-        run.path("src/tools/expand-yaml-anchors")
-    }
-
-    fn make_run(run: RunConfig<'_>) {
-        run.builder.ensure(ExpandYamlAnchors);
-    }
-}
-
 #[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)]
 pub struct BuildManifest;
 

src/bootstrap/src/core/build_steps/test.rs

@@ -1146,8 +1146,6 @@ HELP: to skip test's attempt to check tidiness, pass `--skip src/tools/tidy` to
         builder.info("tidy check");
         builder.run_delaying_failure(&mut cmd);
 
-        builder.ensure(ExpandYamlAnchors);
-
         builder.info("x.py completions check");
         let [bash, zsh, fish, powershell] = ["x.py.sh", "x.py.zsh", "x.py.fish", "x.py.ps1"]
             .map(|filename| builder.src.join("src/etc/completions").join(filename));
@@ -1175,39 +1173,6 @@ HELP: to skip test's attempt to check tidiness, pass `--skip src/tools/tidy` to
     }
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct ExpandYamlAnchors;
-
-impl Step for ExpandYamlAnchors {
-    type Output = ();
-    const ONLY_HOSTS: bool = true;
-
-    /// Ensure the `generate-ci-config` tool was run locally.
-    ///
-    /// The tool in `src/tools` reads the CI definition in `src/ci/builders.yml` and generates the
-    /// appropriate configuration for all our CI providers. This step ensures the tool was called
-    /// by the user before committing CI changes.
-    fn run(self, builder: &Builder<'_>) {
-        // NOTE: `.github/` is not included in dist-src tarballs
-        if !builder.src.join(".github/workflows/ci.yml").exists() {
-            builder.info("Skipping YAML anchors check: GitHub Actions config not found");
-            return;
-        }
-        builder.info("Ensuring the YAML anchors in the GitHub Actions config were expanded");
-        builder.run_delaying_failure(
-            builder.tool_cmd(Tool::ExpandYamlAnchors).arg("check").arg(&builder.src),
-        );
-    }
-
-    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
-        run.path("src/tools/expand-yaml-anchors")
-    }
-
-    fn make_run(run: RunConfig<'_>) {
-        run.builder.ensure(ExpandYamlAnchors);
-    }
-}
-
 fn testdir(builder: &Builder<'_>, host: TargetSelection) -> PathBuf {
     builder.out.join(host.triple).join("test")
 }

src/bootstrap/src/core/build_steps/tool.rs

@@ -302,7 +302,6 @@ bootstrap_tool!(
     RemoteTestClient, "src/tools/remote-test-client", "remote-test-client";
     RustInstaller, "src/tools/rust-installer", "rust-installer";
     RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes";
-    ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors";
     LintDocs, "src/tools/lint-docs", "lint-docs";
     JsonDocCk, "src/tools/jsondocck", "jsondocck";
     JsonDocLint, "src/tools/jsondoclint", "jsondoclint";

src/bootstrap/src/core/builder.rs

@@ -791,7 +791,6 @@ impl<'a> Builder<'a> {
             ),
             Kind::Test => describe!(
                 crate::core::build_steps::toolstate::ToolStateCheck,
-                test::ExpandYamlAnchors,
                 test::Tidy,
                 test::Ui,
                 test::Crashes,
@@ -933,7 +932,6 @@ impl<'a> Builder<'a> {
                 install::Src,
             ),
             Kind::Run => describe!(
-                run::ExpandYamlAnchors,
                 run::BuildManifest,
                 run::BumpStage0,
                 run::ReplaceVersionPlaceholder,

src/bootstrap/src/core/config/flags.rs

@@ -420,7 +420,7 @@ pub enum Subcommand {
     Arguments:
         This subcommand accepts a number of paths to tools to build and run. For
         example:
-            ./x.py run src/tools/expand-yaml-anchors
+            ./x.py run src/tools/bump-stage0
         At least a tool needs to be called.")]
     /// Run tools contained in this repository
     Run {

src/ci/docker/host-x86_64/mingw-check/Dockerfile

@@ -42,11 +42,10 @@ COPY host-x86_64/mingw-check/validate-error-codes.sh /scripts/
 
 ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1
 
-ENV SCRIPT python3 ../x.py --stage 2 test src/tools/expand-yaml-anchors && \
+# Check library crates on all tier 1 targets.
-           # Check library crates on all tier 1 targets.
+# We disable optimized compiler built-ins because that requires a C toolchain for the target.
-           # We disable optimized compiler built-ins because that requires a C toolchain for the target.
+# We also skip the x86_64-unknown-linux-gnu target as it is well-tested by other jobs.
-           # We also skip the x86_64-unknown-linux-gnu target as it is well-tested by other jobs.
+ENV SCRIPT python3 ../x.py check --stage 0 --set build.optimized-compiler-builtins=false core alloc std --target=aarch64-unknown-linux-gnu,i686-pc-windows-msvc,i686-unknown-linux-gnu,x86_64-apple-darwin,x86_64-pc-windows-gnu,x86_64-pc-windows-msvc && \
-           python3 ../x.py check --stage 0 --set build.optimized-compiler-builtins=false core alloc std --target=aarch64-unknown-linux-gnu,i686-pc-windows-msvc,i686-unknown-linux-gnu,x86_64-apple-darwin,x86_64-pc-windows-gnu,x86_64-pc-windows-msvc && \
            python3 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \
            python3 ../x.py clippy bootstrap -Dwarnings && \
            python3 ../x.py clippy compiler library -Aclippy::all -Dclippy::correctness && \

src/ci/github-actions/calculate-job-matrix.py

@@ -44,7 +44,7 @@ def add_base_env(jobs: List[Job], environment: Dict[str, str]) -> List[Job]:
     return jobs
 
 
-class JobType(enum.Enum):
+class WorkflowRunType(enum.Enum):
     PR = enum.auto()
     Try = enum.auto()
     Auto = enum.auto()
@@ -57,9 +57,9 @@ class GitHubCtx:
     repository: str
 
 
-def find_job_type(ctx: GitHubCtx) -> Optional[JobType]:
+def find_run_type(ctx: GitHubCtx) -> Optional[WorkflowRunType]:
     if ctx.event_name == "pull_request":
-        return JobType.PR
+        return WorkflowRunType.PR
     elif ctx.event_name == "push":
         old_bors_try_build = (
             ctx.ref in ("refs/heads/try", "refs/heads/try-perf") and
@@ -72,20 +72,20 @@ def find_job_type(ctx: GitHubCtx) -> Optional[JobType]:
         try_build = old_bors_try_build or new_bors_try_build
 
         if try_build:
-            return JobType.Try
+            return WorkflowRunType.Try
 
         if ctx.ref == "refs/heads/auto" and ctx.repository == "rust-lang-ci/rust":
-            return JobType.Auto
+            return WorkflowRunType.Auto
 
     return None
 
 
-def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Job]:
+def calculate_jobs(run_type: WorkflowRunType, job_data: Dict[str, Any]) -> List[Job]:
-    if job_type == JobType.PR:
+    if run_type == WorkflowRunType.PR:
         return add_base_env(name_jobs(job_data["pr"], "PR"), job_data["envs"]["pr"])
-    elif job_type == JobType.Try:
+    elif run_type == WorkflowRunType.Try:
         return add_base_env(name_jobs(job_data["try"], "try"), job_data["envs"]["try"])
-    elif job_type == JobType.Auto:
+    elif run_type == WorkflowRunType.Auto:
         return add_base_env(name_jobs(job_data["auto"], "auto"), job_data["envs"]["auto"])
 
     return []
@@ -106,6 +106,17 @@ def get_github_ctx() -> GitHubCtx:
     )
 
 
+def format_run_type(run_type: WorkflowRunType) -> str:
+    if run_type == WorkflowRunType.PR:
+        return "pr"
+    elif run_type == WorkflowRunType.Auto:
+        return "auto"
+    elif run_type == WorkflowRunType.Try:
+        return "try"
+    else:
+        raise AssertionError()
+
+
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
 
@@ -114,16 +125,18 @@ if __name__ == "__main__":
 
     github_ctx = get_github_ctx()
 
-    job_type = find_job_type(github_ctx)
+    run_type = find_run_type(github_ctx)
-    logging.info(f"Job type: {job_type}")
+    logging.info(f"Job type: {run_type}")
 
     with open(CI_DIR / "channel") as f:
         channel = f.read().strip()
 
     jobs = []
-    if job_type is not None:
+    if run_type is not None:
-        jobs = calculate_jobs(job_type, data)
+        jobs = calculate_jobs(run_type, data)
     jobs = skip_jobs(jobs, channel)
+    run_type = format_run_type(run_type)
 
-    logging.info(f"Output:\n{yaml.dump(jobs, indent=4)}")
+    logging.info(f"Output:\n{yaml.dump(dict(jobs=jobs, run_type=run_type), indent=4)}")
     print(f"jobs={json.dumps(jobs)}")
+    print(f"run_type={run_type}")

src/ci/github-actions/ci.yml

@@ -1,302 +0,0 @@
-######################################################
-#   WARNING! Action needed when changing this file   #
-######################################################
-
-# Due to GitHub Actions limitations, we can't use YAML Anchors directly in the
-# CI configuration stored on the repository. To work around that this file is
-# expanded by a tool in the repository, and the expansion is committed as well.
-#
-# After you make any change to the file you'll need to run this command:
-#
-#   ./x.py run src/tools/expand-yaml-anchors
-#
-# ...and commit the file it updated in addition to this one. If you forget this
-# step CI will fail.
-
----
-###############################
-#   YAML Anchors Definition   #
-###############################
-
-# This key contains most of the YAML anchors that will be used later in the
-# document. YAML anchors allows us to greatly reduce duplication inside the CI
-# configuration by reusing parts of the configuration.
-#
-# YAML anchors work by defining an anchor with `&anchor-name` and reusing its
-# content in another place with `*anchor-name`. The special `<<` map key merges
-# the content of the map with the content of the anchor (or list of anchors).
-#
-# The expand-yaml-anchors tool will automatically remove this block from the
-# output YAML file.
-x--expand-yaml-anchors--remove:
-  # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs.
-  # Check out their documentation for more information on why they're needed.
-
-  - &base-outcome-job
-    name: bors build finished
-    runs-on: ubuntu-latest
-
-  - &base-success-job
-    steps:
-      - name: mark the job as a success
-        run: exit 0
-        shell: bash
-    <<: *base-outcome-job
-
-  - &base-failure-job
-    steps:
-      - name: mark the job as a failure
-        run: exit 1
-        shell: bash
-    <<: *base-outcome-job
-
-###########################
-#   Builders definition   #
-###########################
-
-name: CI
-on:
-  push:
-    branches:
-      - auto
-      - try
-      - try-perf
-      - automation/bors/try
-  pull_request:
-    branches:
-      - "**"
-
-permissions:
-  contents: read
-  packages: write
-
-defaults:
-  run:
-    # On Linux, macOS, and Windows, use the system-provided bash as the default
-    # shell. (This should only make a difference on Windows, where the default
-    # shell is PowerShell.)
-    shell: bash
-
-concurrency:
-  # For a given workflow, if we push to the same branch, cancel all previous builds on that branch.
-  # We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which
-  # are all triggered on the same branch, but which should be able to run concurrently.
-  group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}
-  cancel-in-progress: true
-
-env:
-  TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
-
-jobs:
-  # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml.
-  # It calculates which jobs should be executed, based on the data of the ${{ github }} context.
-  # If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml.
-  calculate_matrix:
-    name: Calculate job matrix
-    runs-on: ubuntu-latest
-    outputs:
-      jobs: ${{ steps.jobs.outputs.jobs }}
-    steps:
-      - name: Checkout the source code
-        uses: actions/checkout@v4
-      - name: Calculate the CI job matrix
-        run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
-        id: jobs
-  job:
-    name: ${{ matrix.name }}
-    needs: [ calculate_matrix ]
-    runs-on: "${{ matrix.os }}"
-    defaults:
-      run:
-        shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}
-    timeout-minutes: 600
-    env:
-      CI_JOB_NAME: ${{ matrix.image }}
-      CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
-      # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
-      HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
-      DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      SCCACHE_BUCKET: rust-lang-ci-sccache2
-      CACHE_DOMAIN: ci-caches.rust-lang.org
-    continue-on-error: ${{ matrix.continue_on_error || false }}
-    strategy:
-      matrix:
-        # Check the `calculate_matrix` job to see how is the matrix defined.
-        include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}
-    # GitHub Actions fails the workflow if an empty list of jobs is provided to
-    # the workflow, so we need to skip this job if nothing was produced by
-    # the Python script.
-    #
-    # Unfortunately checking whether a list is empty is not possible in a nice
-    # way due to GitHub Actions expressions limits.
-    # This hack is taken from https://github.com/ferrocene/ferrocene/blob/d43edc6b7697cf1719ec1c17c54904ab94825763/.github/workflows/release.yml#L75-L82
-    if: fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null
-    steps:
-      - if: contains(matrix.os, 'windows')
-        uses: msys2/setup-msys2@v2.22.0
-        with:
-          # i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64.
-          msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}
-          # don't try to download updates for already installed packages
-          update: false
-          # don't try to use the msys that comes built-in to the github runner,
-          # so we can control what is installed (i.e. not python)
-          release: true
-          # Inherit the full path from the Windows environment, with MSYS2's */bin/
-          # dirs placed in front. This lets us run Windows-native Python etc.
-          path-type: inherit
-          install: >
-            make
-            dos2unix
-            diffutils
-
-      - name: disable git crlf conversion
-        run: git config --global core.autocrlf false
-
-      - name: checkout the source code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 2
-
-      # Rust Log Analyzer can't currently detect the PR number of a GitHub
-      # Actions build on its own, so a hint in the log message is needed to
-      # point it in the right direction.
-      - name: configure the PR in which the error message will be posted
-        run: echo "[CI_PR_NUMBER=$num]"
-        env:
-          num: ${{ github.event.number }}
-        if: success() && github.event_name == 'pull_request'
-
-      - name: add extra environment variables
-        run: src/ci/scripts/setup-environment.sh
-        env:
-          # Since it's not possible to merge `${{ matrix.env }}` with the other
-          # variables in `job.<name>.env`, the variables defined in the matrix
-          # are passed to the `setup-environment.sh` script encoded in JSON,
-          # which then uses log commands to actually set them.
-          EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
-
-      - name: ensure the channel matches the target branch
-        run: src/ci/scripts/verify-channel.sh
-
-      - name: collect CPU statistics
-        run: src/ci/scripts/collect-cpu-stats.sh
-
-      - name: show the current environment
-        run: src/ci/scripts/dump-environment.sh
-
-      - name: install awscli
-        run: src/ci/scripts/install-awscli.sh
-
-      - name: install sccache
-        run: src/ci/scripts/install-sccache.sh
-
-      - name: select Xcode
-        run: src/ci/scripts/select-xcode.sh
-
-      - name: install clang
-        run: src/ci/scripts/install-clang.sh
-
-      - name: install tidy
-        run: src/ci/scripts/install-tidy.sh
-
-      - name: install WIX
-        run: src/ci/scripts/install-wix.sh
-
-      - name: disable git crlf conversion
-        run: src/ci/scripts/disable-git-crlf-conversion.sh
-
-      - name: checkout submodules
-        run: src/ci/scripts/checkout-submodules.sh
-
-      - name: install MSYS2
-        run: src/ci/scripts/install-msys2.sh
-
-      - name: install MinGW
-        run: src/ci/scripts/install-mingw.sh
-
-      - name: install ninja
-        run: src/ci/scripts/install-ninja.sh
-
-      - name: enable ipv6 on Docker
-        run: src/ci/scripts/enable-docker-ipv6.sh
-
-      # Disable automatic line ending conversion (again). On Windows, when we're
-      # installing dependencies, something switches the git configuration directory or
-      # re-enables autocrlf. We've not tracked down the exact cause -- and there may
-      # be multiple -- but this should ensure submodules are checked out with the
-      # appropriate line endings.
-      - name: disable git crlf conversion
-        run: src/ci/scripts/disable-git-crlf-conversion.sh
-
-      - name: ensure line endings are correct
-        run: src/ci/scripts/verify-line-endings.sh
-
-      - name: ensure backported commits are in upstream branches
-        run: src/ci/scripts/verify-backported-commits.sh
-
-      - name: ensure the stable version number is correct
-        run: src/ci/scripts/verify-stable-version-number.sh
-
-      - name: run the build
-        # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
-        run: src/ci/scripts/run-build-from-ci.sh 2>&1
-        env:
-          AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
-          TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
-
-      - name: create github artifacts
-        run: src/ci/scripts/create-doc-artifacts.sh
-
-      - name: upload artifacts to github
-        uses: actions/upload-artifact@v4
-        with:
-          # name is set in previous step
-          name: ${{ env.DOC_ARTIFACT_NAME }}
-          path: obj/artifacts/doc
-          if-no-files-found: ignore
-          retention-days: 5
-
-      - name: upload artifacts to S3
-        run: src/ci/scripts/upload-artifacts.sh
-        env:
-          AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
-        # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
-        # builders *should* have the AWS credentials available. Still, explicitly
-        # adding the condition is helpful as this way CI will not silently skip
-        # deploying artifacts from a dist builder if the variables are misconfigured,
-        # erroring about invalid credentials instead.
-        if: success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
-
-  # These jobs don't actually test anything, but they're used to tell bors the
-  # build completed, as there is no practical way to detect when a workflow is
-  # successful listening to webhooks only.
-  try-success:
-    needs: [ job ]
-    if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
-    <<: *base-success-job
-  try-failure:
-    needs: [ job ]
-    if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
-    <<: *base-failure-job
-  auto-success:
-    needs: [ job ]
-    if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
-    <<: *base-outcome-job
-    steps:
-      - name: checkout the source code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 2
-      - name: publish toolstate
-        run: src/ci/publish_toolstate.sh
-        shell: bash
-        env:
-          TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
-
-  auto-failure:
-    needs: [ job ]
-    if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
-    <<: *base-failure-job

src/ci/github-actions/jobs.yml

@@ -1,7 +1,5 @@
 # This file contains definitions of CI job parameters that are loaded
 # dynamically in CI from ci.yml.
-# You *do not* need to re-run `src/tools/expand-yaml-anchors` when you
-# modify this file.
 runners:
   - &base-job
     env: { }

src/tools/expand-yaml-anchors/Cargo.toml

@@ -1,8 +0,0 @@
-[package]
-name = "expand-yaml-anchors"
-version = "0.1.0"
-edition = "2021"
-
-[dependencies]
-yaml-rust = "0.4.3"
-yaml-merge-keys = "0.4.0"

src/tools/expand-yaml-anchors/src/main.rs

@@ -1,198 +0,0 @@
-use std::error::Error;
-use std::path::{Path, PathBuf};
-use yaml_rust::{Yaml, YamlEmitter, YamlLoader};
-
-/// List of files to expand. The first tuple element is the source
-/// file, while the second tuple element is the destination file.
-#[rustfmt::skip]
-static TO_EXPAND: &[(&str, &str)] = &[
-    ("src/ci/github-actions/ci.yml", ".github/workflows/ci.yml"),
-];
-
-/// Name of a special key that will be removed from all the maps in expanded configuration files.
-/// This key can then be used to contain shared anchors.
-static REMOVE_MAP_KEY: &str = "x--expand-yaml-anchors--remove";
-
-/// Message that will be included at the top of all the expanded files. {source} will be replaced
-/// with the source filename relative to the base path.
-static HEADER_MESSAGE: &str = "\
-#############################################################
-#   WARNING: automatically generated file, DO NOT CHANGE!   #
-#############################################################
-
-# This file was automatically generated by the expand-yaml-anchors tool. The
-# source file that generated this one is:
-#
-#   {source}
-#
-# Once you make changes to that file you need to run:
-#
-#   ./x.py run src/tools/expand-yaml-anchors/
-#
-# The CI build will fail if the tool is not run after changes to this file.
-
-";
-
-enum Mode {
-    Check,
-    Generate,
-}
-
-struct App {
-    mode: Mode,
-    base: PathBuf,
-}
-
-impl App {
-    fn from_args() -> Result<Self, Box<dyn Error>> {
-        // Parse CLI arguments
-        let args = std::env::args().skip(1).collect::<Vec<_>>();
-        let (mode, base) = match args.iter().map(|s| s.as_str()).collect::<Vec<_>>().as_slice() {
-            ["generate", ref base] => (Mode::Generate, PathBuf::from(base)),
-            ["check", ref base] => (Mode::Check, PathBuf::from(base)),
-            _ => {
-                eprintln!("usage: expand-yaml-anchors <generate|check> <base-dir>");
-                std::process::exit(1);
-            }
-        };
-
-        Ok(App { mode, base })
-    }
-
-    fn run(&self) -> Result<(), Box<dyn Error>> {
-        for (source, dest) in TO_EXPAND {
-            let source = self.base.join(source);
-            let dest_path = self.base.join(dest);
-
-            self.expand(&source, &dest_path).with_context(|| match self.mode {
-                Mode::Generate => format!(
-                    "failed to expand {} into {}",
-                    self.path(&source),
-                    self.path(&dest_path)
-                ),
-                Mode::Check => format!(
-                    "{} is not up to date; please run \
-                        `x.py run src/tools/expand-yaml-anchors`.",
-                    self.path(&dest_path)
-                ),
-            })?;
-        }
-        Ok(())
-    }
-
-    fn expand(&self, source: &Path, dest: &Path) -> Result<(), Box<dyn Error>> {
-        let content = std::fs::read_to_string(source)
-            .with_context(|| format!("failed to read {}", self.path(source)))?;
-
-        let mut buf =
-            HEADER_MESSAGE.replace("{source}", &self.path(source).to_string().replace("\\", "/"));
-
-        let documents = YamlLoader::load_from_str(&content)
-            .with_context(|| format!("failed to parse {}", self.path(source)))?;
-        for mut document in documents.into_iter() {
-            document = yaml_merge_keys::merge_keys(document)
-                .with_context(|| format!("failed to expand {}", self.path(source)))?;
-            document = filter_document(document);
-
-            YamlEmitter::new(&mut buf).dump(&document).map_err(|err| WithContext {
-                context: "failed to serialize the expanded yaml".into(),
-                source: Box::new(err),
-            })?;
-            buf.push('\n');
-        }
-
-        match self.mode {
-            Mode::Check => {
-                let old = std::fs::read_to_string(dest)
-                    .with_context(|| format!("failed to read {}", self.path(dest)))?;
-                if old != buf {
-                    return Err(Box::new(StrError(format!(
-                        "{} and {} are different",
-                        self.path(source),
-                        self.path(dest),
-                    ))));
-                }
-            }
-            Mode::Generate => {
-                std::fs::write(dest, buf.as_bytes())
-                    .with_context(|| format!("failed to write to {}", self.path(dest)))?;
-            }
-        }
-        Ok(())
-    }
-
-    fn path<'a>(&self, path: &'a Path) -> impl std::fmt::Display + 'a {
-        path.strip_prefix(&self.base).unwrap_or(path).display()
-    }
-}
-
-fn filter_document(document: Yaml) -> Yaml {
-    match document {
-        Yaml::Hash(map) => Yaml::Hash(
-            map.into_iter()
-                .filter(|(key, _)| {
-                    if let Yaml::String(string) = &key { string != REMOVE_MAP_KEY } else { true }
-                })
-                .map(|(key, value)| (filter_document(key), filter_document(value)))
-                .collect(),
-        ),
-        Yaml::Array(vec) => Yaml::Array(vec.into_iter().map(filter_document).collect()),
-        other => other,
-    }
-}
-
-fn main() {
-    if let Err(err) = App::from_args().and_then(|app| app.run()) {
-        eprintln!("error: {}", err);
-
-        let mut source = err.as_ref() as &dyn Error;
-        while let Some(err) = source.source() {
-            eprintln!("caused by: {}", err);
-            source = err;
-        }
-
-        std::process::exit(1);
-    }
-}
-
-#[derive(Debug)]
-struct StrError(String);
-
-impl Error for StrError {}
-
-impl std::fmt::Display for StrError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        std::fmt::Display::fmt(&self.0, f)
-    }
-}
-
-#[derive(Debug)]
-struct WithContext {
-    context: String,
-    source: Box<dyn Error>,
-}
-
-impl std::fmt::Display for WithContext {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.context)
-    }
-}
-
-impl Error for WithContext {
-    fn source(&self) -> Option<&(dyn Error + 'static)> {
-        Some(self.source.as_ref())
-    }
-}
-
-pub(crate) trait ResultExt<T> {
-    fn with_context<F: FnOnce() -> String>(self, f: F) -> Result<T, Box<dyn Error>>;
-}
-
-impl<T, E: Into<Box<dyn Error>>> ResultExt<T> for Result<T, E> {
-    fn with_context<F: FnOnce() -> String>(self, f: F) -> Result<T, Box<dyn Error>> {
-        match self {
-            Ok(ok) => Ok(ok),
-            Err(err) => Err(WithContext { source: err.into(), context: f() }.into()),
-        }
-    }
-}