From 775c3c0493e9a383a7f1c521b06d36f2e3d0d886 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 31 Jul 2022 14:02:31 -0500 Subject: [PATCH] Add `x.sh` and `x.ps1` shell scripts This is a more ambitious version of https://github.com/rust-lang/rust/pull/98716. It still changes the shebang back to python3, for compatibility with non-Unix systems, but also adds alternative entrypoints for systems without `python3` installed. These scripts will be necessary for the rust entrypoint (#94829), so I see little downside in adding them early. --- src/bootstrap/mk/Makefile.in | 11 +++++-- .../host-x86_64/x86_64-gnu-llvm-12/Dockerfile | 18 ++++++++-- src/tools/tidy/src/bins.rs | 8 +++-- x.ps1 | 28 ++++++++++++++++ x.py | 30 +++-------------- x.sh | 33 +++++++++++++++++++ 6 files changed, 94 insertions(+), 34 deletions(-) create mode 100755 x.ps1 create mode 100755 x.sh diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index 5a1f2e70413..1e0f7e9acf4 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -66,16 +66,21 @@ TESTS_IN_2 := \ src/test/ui \ src/tools/linkchecker +## MSVC native builders + +# these intentionally don't use `$(BOOTSTRAP)` so we can test the shebang on Windows ci-subset-1: - $(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_2:%=--exclude %) + $(Q)$(CFG_SRC_DIR)/x.py test --stage 2 $(TESTS_IN_2:%=--exclude %) ci-subset-2: - $(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_2) + $(Q)$(CFG_SRC_DIR)/x.ps1 test --stage 2 $(TESTS_IN_2) + +## MingW native builders TESTS_IN_MINGW_2 := \ src/test/ui ci-mingw-subset-1: - $(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_MINGW_2:%=--exclude %) + $(Q)$(CFG_SRC_DIR)/x.sh test --stage 2 $(TESTS_IN_MINGW_2:%=--exclude %) ci-mingw-subset-2: $(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_MINGW_2) diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-12/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-12/Dockerfile index df1fbc29cf5..8de9045c3ba 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-12/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-12/Dockerfile @@ -1,6 +1,8 @@ FROM ubuntu:20.04 ARG DEBIAN_FRONTEND=noninteractive + +# NOTE: intentionally installs both python2 and python3 so we can test support for both. RUN apt-get update && apt-get install -y --no-install-recommends \ g++ \ gcc-multilib \ @@ -10,6 +12,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ ca-certificates \ python2.7 \ + python3.9 \ git \ cmake \ sudo \ @@ -23,6 +26,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ xz-utils \ nodejs +# Install powershell so we can test x.ps1 on Linux +RUN apt-get update && \ + apt-get install -y apt-transport-https software-properties-common && \ + curl -s "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb" > packages-microsoft-prod.deb && \ + dpkg -i packages-microsoft-prod.deb && \ + apt-get update && \ + apt-get install -y powershell + COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh @@ -33,21 +44,22 @@ ENV RUST_CONFIGURE_ARGS \ --enable-llvm-link-shared \ --set rust.thin-lto-import-instr-limit=10 -ENV SCRIPT python2.7 ../x.py --stage 2 test --exclude src/tools/tidy && \ +# NOTE: intentionally uses all of `x.py`, `x.sh`, and `x.ps1` to make sure they all work on Linux. +ENV SCRIPT ../x.py --stage 2 test --exclude src/tools/tidy && \ # Run the `mir-opt` tests again but this time for a 32-bit target. # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have # both 32-bit and 64-bit outputs updated by the PR author, before # the PR is approved and tested for merging. # It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`, # despite having different output on 32-bit vs 64-bit targets. - python2.7 ../x.py --stage 2 test src/test/mir-opt \ + ../x.sh --stage 2 test src/test/mir-opt \ --host='' --target=i686-unknown-linux-gnu && \ # Run the UI test suite again, but in `--pass=check` mode # # This is intended to make sure that both `--pass=check` continues to # work. # - python2.7 ../x.py --stage 2 test src/test/ui --pass=check \ + ../x.ps1 --stage 2 test src/test/ui --pass=check \ --host='' --target=i686-unknown-linux-gnu && \ # Run tidy at the very end, after all the other tests. python2.7 ../x.py --stage 2 test src/tools/tidy diff --git a/src/tools/tidy/src/bins.rs b/src/tools/tidy/src/bins.rs index 9615c4db6b4..025b8ab9f0a 100644 --- a/src/tools/tidy/src/bins.rs +++ b/src/tools/tidy/src/bins.rs @@ -96,6 +96,8 @@ mod os_impl { #[cfg(unix)] pub fn check(path: &Path, bad: &mut bool) { + use std::ffi::OsStr; + const ALLOWED: &[&str] = &["configure"]; crate::walk_no_read( @@ -117,9 +119,9 @@ mod os_impl { }, &mut |entry| { let file = entry.path(); - let filename = file.file_name().unwrap().to_string_lossy(); - let extensions = [".py", ".sh"]; - if extensions.iter().any(|e| filename.ends_with(e)) { + let extension = file.extension(); + let scripts = ["py", "sh", "ps1"]; + if scripts.into_iter().any(|e| extension == Some(OsStr::new(e))) { return; } diff --git a/x.ps1 b/x.ps1 new file mode 100755 index 00000000000..1225443735f --- /dev/null +++ b/x.ps1 @@ -0,0 +1,28 @@ +#!/usr/bin/env pwsh + +# See x.sh for why these scripts exist. + +$xpy = Join-Path $PSScriptRoot x.py +# Start-Process for some reason splits arguments on spaces. (Isn't powershell supposed to be simpler than bash?) +# Double-quote all the arguments so it doesn't do that. +$xpy_args = @("""$xpy""") +foreach ($arg in $args) { + $xpy_args += """$arg""" +} + +foreach ($python in "py", "python3", "python", "python2") { + # NOTE: this only tests that the command exists in PATH, not that it's actually + # executable. The latter is not possible in a portable way, see + # https://github.com/PowerShell/PowerShell/issues/12625. + if (Get-Command $python -ErrorAction SilentlyContinue) { + if ($python -eq "py") { + # Use python3, not python2 + $xpy_args = @("-3") + $xpy_args + } + $process = Start-Process -NoNewWindow -Wait -PassThru $python $xpy_args + Exit $process.ExitCode + } +} + +Write-Error "${PSCommandPath}: error: did not find python installed" +Exit 1 diff --git a/x.py b/x.py index 0289056fdcb..6c68907c581 100755 --- a/x.py +++ b/x.py @@ -1,36 +1,16 @@ -#!/usr/bin/env bash +#!/usr/bin/env python3 +# Some systems don't have `python3` in their PATH. This isn't supported by x.py directly; +# they should use `x.sh` or `x.ps1` instead. -# Modern Linux and macOS systems commonly only have a thing called `python3` and -# not `python`, while Windows commonly does not have `python3`, so we cannot -# directly use python in the shebang and have it consistently work. Instead we -# embed some bash to look for a python to run the rest of the script. -# -# On Windows, `py -3` sometimes works. We need to try it first because `python3` -# sometimes tries to launch the app store on Windows. -'''': -for PYTHON in "py -3" python3 python python2; do - if command -v $PYTHON >/dev/null; then - exec $PYTHON "$0" "$@" - break - fi -done -echo "$0: error: did not find python installed" >&2 -exit 1 -''' - -# The rest of this file is Python. -# # This file is only a "symlink" to bootstrap.py, all logic should go there. import os import sys # If this is python2, check if python3 is available and re-execute with that -# interpreter. +# interpreter. Only python3 allows downloading CI LLVM. # -# `./x.py` would not normally benefit from this because the bash above tries -# python3 before 2, but this matters if someone ran `python x.py` and their -# system's `python` is python2. +# This matters if someone's system `python` is python2. if sys.version_info.major < 3: try: os.execvp("py", ["py", "-3"] + sys.argv) diff --git a/x.sh b/x.sh new file mode 100755 index 00000000000..704d0f791f3 --- /dev/null +++ b/x.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +# Modern Linux and macOS systems commonly only have a thing called `python3` and +# not `python`, while Windows commonly does not have `python3`, so we cannot +# directly use python in the x.py shebang and have it consistently work. Instead we +# have a shell script to look for a python to run x.py. + +set -eu + +realpath() { + if [ -d "$1" ]; then + CDPATH='' command cd "$1" && pwd -P + else + echo "$(realpath "$(dirname "$1")")/$(basename "$1")" + fi +} + +xpy=$(dirname "$(realpath "$0")")/x.py + +# On Windows, `py -3` sometimes works. We need to try it first because `python3` +# sometimes tries to launch the app store on Windows. +for SEARCH_PYTHON in py python3 python python2; do + if python=$(command -v $SEARCH_PYTHON) && [ -x "$python" ]; then + if [ $SEARCH_PYTHON = py ]; then + extra_arg="-3" + else + extra_arg="" + fi + exec "$python" $extra_arg "$xpy" "$@" + fi +done +echo "$0: error: did not find python installed" >&2 +exit 1