nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-11-21 22:34:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Auto merge of #112482 - tgross35:ci-non-rust-linters, r=pietroalbini

Browse Source 
Add support for tidy linting via external tools for non-rust files

This change adds the flag `--check-extras` to `tidy`. It accepts a comma separated list of any of the options:

* py (test everything applicable for python files)
* py:lint (lint python files using `ruff`)
* py:fmt (check formatting for python files using `black`)
* shell or shell:lint (lint shell files using `shellcheck`)

Specific files to check can also be specified via positional args. Examples:

* `./x test tidy --check-extras=shell,py`
* `./x test tidy --check-extras=py:fmt -- src/bootstrap/bootstrap.py`
* `./x test tidy --check-extras=shell -- src/ci/*.sh`
* Python formatting can be applied with bless: `./x test tidy --ckeck-extras=py:fmt --bless`

`ruff` and `black` need to be installed via pip; this tool manages these within a virtual environment at `build/venv`. `shellcheck` needs to be installed on the system already.

---

This PR doesn't fix any of the errors that show up (I will likely go through those at some point) and it doesn't enforce anything new in CI. Relevant zulip discussion: https://rust-lang.zulipchat.com/#narrow/stream/242791-t-infra/topic/Other.20linters.20in.20CI
This commit is contained in:
bors 2023-08-10 13:07:18 +00:00
commit 9fa6bdd764
18 changed files with 682 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/bootstrap/bootstrap.py
View File

@ -472,7 +472,9 @@ class FakeArgs:
class RustBuild(object):
"""Provide all the methods required to build Rust"""
def __init__(self, config_toml="", args=FakeArgs()):
def __init__(self, config_toml="", args=None):
if args is None:
args = FakeArgs()
self.git_version = None
self.nix_deps_dir = None
self._should_fix_bins_and_dylibs = None

19
src/bootstrap/bootstrap_test.py
View File

@ -4,7 +4,6 @@ Run these with `x test bootstrap`, or `python -m unittest src/bootstrap/bootstra
from __future__ import absolute_import, division, print_function
import os
import doctest
import unittest
import tempfile
import hashlib
@ -16,12 +15,15 @@ from shutil import rmtree
bootstrap_dir = os.path.dirname(os.path.abspath(__file__))
# For the import below, have Python search in src/bootstrap first.
sys.path.insert(0, bootstrap_dir)
import bootstrap
import configure
import bootstrap # noqa: E402
import configure # noqa: E402
def serialize_and_parse(configure_args, bootstrap_args=bootstrap.FakeArgs()):
def serialize_and_parse(configure_args, bootstrap_args=None):
from io import StringIO
if bootstrap_args is None:
bootstrap_args = bootstrap.FakeArgs()
section_order, sections, targets = configure.parse_args(configure_args)
buffer = StringIO()
configure.write_config_toml(buffer, section_order, targets, sections)
@ -129,7 +131,14 @@ class GenerateAndParseConfig(unittest.TestCase):
class BuildBootstrap(unittest.TestCase):
"""Test that we generate the appropriate arguments when building bootstrap"""
def build_args(self, configure_args=[], args=[], env={}):
def build_args(self, configure_args=None, args=None, env=None):
if configure_args is None:
configure_args = []
if args is None:
args = []
if env is None:
env = {}
env = env.copy()
env["PATH"] = os.environ["PATH"]

2
src/bootstrap/builder/tests.rs
View File

@ -587,6 +587,7 @@ mod dist {
run: None,
only_modified: false,
skip: vec![],
extra_checks: None,
};
let build = Build::new(config);
@ -658,6 +659,7 @@ mod dist {
pass: None,
run: None,
only_modified: false,
extra_checks: None,
};
// Make sure rustfmt binary not being found isn't an error.
config.channel = "beta".to_string();

11
src/bootstrap/flags.rs
View File

@ -339,6 +339,10 @@ pub enum Subcommand {
/// whether to automatically update stderr/stdout files
bless: bool,
#[arg(long)]
/// comma-separated list of other files types to check (accepts py, py:lint,
/// py:fmt, shell)
extra_checks: Option<String>,
#[arg(long)]
/// rerun tests even if the inputs are unchanged
force_rerun: bool,
#[arg(long)]
@ -476,6 +480,13 @@ impl Subcommand {
}
}
pub fn extra_checks(&self) -> Option<&str> {
match *self {
Subcommand::Test { ref extra_checks, .. } => extra_checks.as_ref().map(String::as_str),
_ => None,
}
}
pub fn only_modified(&self) -> bool {
match *self {
Subcommand::Test { only_modified, .. } => only_modified,

9
src/bootstrap/test.rs
View File

@ -4,6 +4,7 @@
//! our CI.
use std::env;
use std::ffi::OsStr;
use std::ffi::OsString;
use std::fs;
use std::iter;
@ -1094,6 +1095,14 @@ impl Step for Tidy {
if builder.config.cmd.bless() {
cmd.arg("--bless");
}
if let Some(s) = builder.config.cmd.extra_checks() {
cmd.arg(format!("--extra-checks={s}"));
}
let mut args = std::env::args_os();
if let Some(_) = args.find(|arg| arg == OsStr::new("--")) {
cmd.arg("--");
cmd.args(args);
}
if builder.config.channel == "dev" || builder.config.channel == "nightly" {
builder.info("fmt check");

6
src/ci/docker/host-x86_64/mingw-check-tidy/Dockerfile
View File

@ -26,11 +26,13 @@ COPY scripts/sccache.sh /scripts/
RUN sh /scripts/sccache.sh
COPY host-x86_64/mingw-check/reuse-requirements.txt /tmp/
RUN pip3 install --no-deps --no-cache-dir --require-hashes -r /tmp/reuse-requirements.txt
RUN pip3 install --no-deps --no-cache-dir --require-hashes -r /tmp/reuse-requirements.txt \
&& pip3 install virtualenv
COPY host-x86_64/mingw-check/validate-toolstate.sh /scripts/
COPY host-x86_64/mingw-check/validate-error-codes.sh /scripts/
# NOTE: intentionally uses python2 for x.py so we can test it still works.
# validate-toolstate only runs in our CI, so it's ok for it to only support python3.
ENV SCRIPT python2.7 ../x.py test --stage 0 src/tools/tidy tidyselftest
ENV SCRIPT TIDY_PRINT_DIFF=1 python2.7 ../x.py test \
--stage 0 src/tools/tidy tidyselftest --extra-checks=py:lint

6
src/ci/docker/scripts/fuchsia-test-runner.py
View File

@ -15,12 +15,10 @@ import hashlib
import json
import os
import platform
import re
import shutil
import signal
import subprocess
import sys
from typing import ClassVar, List, Optional
from typing import ClassVar, List
@dataclass
@ -523,7 +521,7 @@ class TestEnvironment:
env_vars += '\n "RUST_BACKTRACE=0",'
# Use /tmp as the test temporary directory
env_vars += f'\n "RUST_TEST_TMPDIR=/tmp",'
env_vars += '\n "RUST_TEST_TMPDIR=/tmp",'
cml.write(
self.CML_TEMPLATE.format(env_vars=env_vars, exe_name=exe_name)

1
src/etc/completions/x.py.fish
View File

@ -234,6 +234,7 @@ complete -c x.py -n "__fish_seen_subcommand_from doc" -s h -l help -d 'Print hel
complete -c x.py -n "__fish_seen_subcommand_from test" -l skip -d 'skips tests matching SUBSTRING, if supported by test tool. May be passed multiple times' -r -F
complete -c x.py -n "__fish_seen_subcommand_from test" -l test-args -d 'extra arguments to be passed for the test tool being used (e.g. libtest, compiletest or rustdoc)' -r
complete -c x.py -n "__fish_seen_subcommand_from test" -l rustc-args -d 'extra options to pass the compiler when running tests' -r
complete -c x.py -n "__fish_seen_subcommand_from test" -l extra-checks -d 'comma-separated list of other files types to check (accepts py, py:lint, py:fmt, shell)' -r
complete -c x.py -n "__fish_seen_subcommand_from test" -l compare-mode -d 'mode describing what file the actual ui output will be compared to' -r
complete -c x.py -n "__fish_seen_subcommand_from test" -l pass -d 'force {check,build,run}-pass tests to this mode' -r
complete -c x.py -n "__fish_seen_subcommand_from test" -l run -d 'whether to execute run-* tests' -r

1
src/etc/completions/x.py.ps1
View File

@ -306,6 +306,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
[CompletionResult]::new('--skip', 'skip', [CompletionResultType]::ParameterName, 'skips tests matching SUBSTRING, if supported by test tool. May be passed multiple times')
[CompletionResult]::new('--test-args', 'test-args', [CompletionResultType]::ParameterName, 'extra arguments to be passed for the test tool being used (e.g. libtest, compiletest or rustdoc)')
[CompletionResult]::new('--rustc-args', 'rustc-args', [CompletionResultType]::ParameterName, 'extra options to pass the compiler when running tests')
[CompletionResult]::new('--extra-checks', 'extra-checks', [CompletionResultType]::ParameterName, 'comma-separated list of other files types to check (accepts py, py:lint, py:fmt, shell)')
[CompletionResult]::new('--compare-mode', 'compare-mode', [CompletionResultType]::ParameterName, 'mode describing what file the actual ui output will be compared to')
[CompletionResult]::new('--pass', 'pass', [CompletionResultType]::ParameterName, 'force {check,build,run}-pass tests to this mode')
[CompletionResult]::new('--run', 'run', [CompletionResultType]::ParameterName, 'whether to execute run-* tests')

6
src/etc/completions/x.py.sh
View File

@ -1625,7 +1625,7 @@ _x.py() {
return 0
;;
x.py__test)
opts="-v -i -j -h --no-fail-fast --skip --test-args --rustc-args --no-doc --doc --bless --force-rerun --only-modified --compare-mode --pass --run --rustfix-coverage --verbose --incremental --config --build-dir --build --host --target --exclude --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..."
opts="-v -i -j -h --no-fail-fast --skip --test-args --rustc-args --no-doc --doc --bless --extra-checks --force-rerun --only-modified --compare-mode --pass --run --rustfix-coverage --verbose --incremental --config --build-dir --build --host --target --exclude --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..."
if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
return 0
@ -1643,6 +1643,10 @@ _x.py() {
COMPREPLY=($(compgen -f "${cur}"))
return 0
;;
--extra-checks)
COMPREPLY=($(compgen -f "${cur}"))
return 0
;;
--compare-mode)
COMPREPLY=($(compgen -f "${cur}"))
return 0

15
src/tools/tidy/config/black.toml Normal file
View File

@ -0,0 +1,15 @@
[tool.black]
# Ignore all submodules
extend-exclude = """(\
src/doc/nomicon|\
src/tools/cargo/|\
src/doc/reference/|\
src/doc/book/|\
src/doc/rust-by-example/|\
library/stdarch/|\
src/doc/rustc-dev-guide/|\
src/doc/edition-guide/|\
src/llvm-project/|\
src/doc/embedded-book/|\
library/backtrace/
)"""

10
src/tools/tidy/config/requirements.in Normal file
View File

@ -0,0 +1,10 @@
# requirements.in This is the source file for our pinned version requirements
# file "requirements.txt" To regenerate that file, pip-tools is required
# (`python -m pip install pip-tools`). Once installed, run: `pip-compile
# --generate-hashes src/tools/tidy/config/requirements.in`
#
# Note: this generation step should be run with the oldest supported python
# version (currently 3.7) to ensure backward compatibility
black==23.3.0
ruff==0.0.272

117
src/tools/tidy/config/requirements.txt Normal file
View File

@ -0,0 +1,117 @@
#
# This file is autogenerated by pip-compile with Python 3.11
# by the following command:
#
# pip-compile --generate-hashes src/tools/tidy/config/requirements.in
#
black==23.3.0 \
--hash=sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5 \
--hash=sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915 \
--hash=sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326 \
--hash=sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940 \
--hash=sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b \
--hash=sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30 \
--hash=sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c \
--hash=sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c \
--hash=sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab \
--hash=sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27 \
--hash=sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2 \
--hash=sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961 \
--hash=sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9 \
--hash=sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb \
--hash=sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70 \
--hash=sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331 \
--hash=sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2 \
--hash=sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266 \
--hash=sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d \
--hash=sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6 \
--hash=sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b \
--hash=sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925 \
--hash=sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8 \
--hash=sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4 \
--hash=sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3
# via -r src/tools/tidy/config/requirements.in
click==8.1.3 \
--hash=sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e \
--hash=sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48
# via black
importlib-metadata==6.7.0 \
--hash=sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4 \
--hash=sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5
# via click
mypy-extensions==1.0.0 \
--hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \
--hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782
# via black
packaging==23.1 \
--hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \
--hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f
# via black
pathspec==0.11.1 \
--hash=sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687 \
--hash=sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293
# via black
platformdirs==3.6.0 \
--hash=sha256:57e28820ca8094678b807ff529196506d7a21e17156cb1cddb3e74cebce54640 \
--hash=sha256:ffa199e3fbab8365778c4a10e1fbf1b9cd50707de826eb304b50e57ec0cc8d38
# via black
ruff==0.0.272 \
--hash=sha256:06b8ee4eb8711ab119db51028dd9f5384b44728c23586424fd6e241a5b9c4a3b \
--hash=sha256:1609b864a8d7ee75a8c07578bdea0a7db75a144404e75ef3162e0042bfdc100d \
--hash=sha256:19643d448f76b1eb8a764719072e9c885968971bfba872e14e7257e08bc2f2b7 \
--hash=sha256:273a01dc8c3c4fd4c2af7ea7a67c8d39bb09bce466e640dd170034da75d14cab \
--hash=sha256:27b2ea68d2aa69fff1b20b67636b1e3e22a6a39e476c880da1282c3e4bf6ee5a \
--hash=sha256:48eccf225615e106341a641f826b15224b8a4240b84269ead62f0afd6d7e2d95 \
--hash=sha256:677284430ac539bb23421a2b431b4ebc588097ef3ef918d0e0a8d8ed31fea216 \
--hash=sha256:691d72a00a99707a4e0b2846690961157aef7b17b6b884f6b4420a9f25cd39b5 \
--hash=sha256:86bc788245361a8148ff98667da938a01e1606b28a45e50ac977b09d3ad2c538 \
--hash=sha256:905ff8f3d6206ad56fcd70674453527b9011c8b0dc73ead27618426feff6908e \
--hash=sha256:9c4bfb75456a8e1efe14c52fcefb89cfb8f2a0d31ed8d804b82c6cf2dc29c42c \
--hash=sha256:a37ec80e238ead2969b746d7d1b6b0d31aa799498e9ba4281ab505b93e1f4b28 \
--hash=sha256:ae9b57546e118660175d45d264b87e9b4c19405c75b587b6e4d21e6a17bf4fdf \
--hash=sha256:bd2bbe337a3f84958f796c77820d55ac2db1e6753f39d1d1baed44e07f13f96d \
--hash=sha256:d5a208f8ef0e51d4746930589f54f9f92f84bb69a7d15b1de34ce80a7681bc00 \
--hash=sha256:dc406e5d756d932da95f3af082814d2467943631a587339ee65e5a4f4fbe83eb \
--hash=sha256:ee76b4f05fcfff37bd6ac209d1370520d509ea70b5a637bdf0a04d0c99e13dff
# via -r src/tools/tidy/config/requirements.in
tomli==2.0.1 \
--hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \
--hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
# via black
typed-ast==1.5.4 \
--hash=sha256:0261195c2062caf107831e92a76764c81227dae162c4f75192c0d489faf751a2 \
--hash=sha256:0fdbcf2fef0ca421a3f5912555804296f0b0960f0418c440f5d6d3abb549f3e1 \
--hash=sha256:183afdf0ec5b1b211724dfef3d2cad2d767cbefac291f24d69b00546c1837fb6 \
--hash=sha256:211260621ab1cd7324e0798d6be953d00b74e0428382991adfddb352252f1d62 \
--hash=sha256:267e3f78697a6c00c689c03db4876dd1efdfea2f251a5ad6555e82a26847b4ac \
--hash=sha256:2efae9db7a8c05ad5547d522e7dbe62c83d838d3906a3716d1478b6c1d61388d \
--hash=sha256:370788a63915e82fd6f212865a596a0fefcbb7d408bbbb13dea723d971ed8bdc \
--hash=sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2 \
--hash=sha256:3e123d878ba170397916557d31c8f589951e353cc95fb7f24f6bb69adc1a8a97 \
--hash=sha256:4879da6c9b73443f97e731b617184a596ac1235fe91f98d279a7af36c796da35 \
--hash=sha256:4e964b4ff86550a7a7d56345c7864b18f403f5bd7380edf44a3c1fb4ee7ac6c6 \
--hash=sha256:639c5f0b21776605dd6c9dbe592d5228f021404dafd377e2b7ac046b0349b1a1 \
--hash=sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4 \
--hash=sha256:6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c \
--hash=sha256:683407d92dc953c8a7347119596f0b0e6c55eb98ebebd9b23437501b28dcbb8e \
--hash=sha256:79b1e0869db7c830ba6a981d58711c88b6677506e648496b1f64ac7d15633aec \
--hash=sha256:7d5d014b7daa8b0bf2eaef684295acae12b036d79f54178b92a2b6a56f92278f \
--hash=sha256:98f80dee3c03455e92796b58b98ff6ca0b2a6f652120c263efdba4d6c5e58f72 \
--hash=sha256:a94d55d142c9265f4ea46fab70977a1944ecae359ae867397757d836ea5a3f47 \
--hash=sha256:a9916d2bb8865f973824fb47436fa45e1ebf2efd920f2b9f99342cb7fab93f72 \
--hash=sha256:c542eeda69212fa10a7ada75e668876fdec5f856cd3d06829e6aa64ad17c8dfe \
--hash=sha256:cf4afcfac006ece570e32d6fa90ab74a17245b83dfd6655a6f68568098345ff6 \
--hash=sha256:ebd9d7f80ccf7a82ac5f88c521115cc55d84e35bf8b446fcd7836eb6b98929a3 \
--hash=sha256:ed855bbe3eb3715fca349c80174cfcfd699c2f9de574d40527b8429acae23a66
# via black
typing-extensions==4.6.3 \
--hash=sha256:88a4153d8505aabbb4e13aacb7c486c2b4a33ca3b3f807914a9b4c844c471c26 \
--hash=sha256:d91d5919357fe7f681a9f2b5b4cb2a5f1ef0a1e9f59c4d8ff0d3491e05c0ffd5
# via
# black
# importlib-metadata
# platformdirs
zipp==3.15.0 \
--hash=sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b \
--hash=sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556
# via importlib-metadata

41
src/tools/tidy/config/ruff.toml Normal file
View File

@ -0,0 +1,41 @@
# Configuration for ruff python linter, run as part of tidy external tools
# B (bugbear), E (pycodestyle, standard), EXE (executables) F (flakes, standard)
# ERM for error messages would be beneficial at some point
select = ["B", "E", "EXE", "F"]
ignore = [
"E501", # line-too-long
"F403", # undefined-local-with-import-star
"F405", # undefined-local-with-import-star-usage
]
# lowest possible for ruff
target-version = "py37"
# Ignore all submodules
extend-exclude = [
"src/doc/nomicon/",
"src/tools/cargo/",
"src/doc/reference/",
"src/doc/book/",
"src/doc/rust-by-example/",
"library/stdarch/",
"src/doc/rustc-dev-guide/",
"src/doc/edition-guide/",
"src/llvm-project/",
"src/doc/embedded-book/",
"library/backtrace/",
# Hack: CI runs from a subdirectory under the main checkout
"../src/doc/nomicon/",
"../src/tools/cargo/",
"../src/doc/reference/",
"../src/doc/book/",
"../src/doc/rust-by-example/",
"../library/stdarch/",
"../src/doc/rustc-dev-guide/",
"../src/doc/edition-guide/",
"../src/llvm-project/",
"../src/doc/embedded-book/",
"../library/backtrace/",
]

435
src/tools/tidy/src/ext_tool_checks.rs Normal file
View File

@ -0,0 +1,435 @@
//! Optional checks for file types other than Rust source
//!
//! Handles python tool version managment via a virtual environment in
//! `build/venv`.
//!
//! # Functional outline
//!
//! 1. Run tidy with an extra option: `--extra-checks=py,shell`,
//! `--extra-checks=py:lint`, or similar. Optionally provide specific
//! configuration after a double dash (`--extra-checks=py -- foo.py`)
//! 2. Build configuration based on args/environment:
//! - Formatters by default are in check only mode
//! - If in CI (TIDY_PRINT_DIFF=1 is set), check and print the diff
//! - If `--bless` is provided, formatters may run
//! - Pass any additional config after the `--`. If no files are specified,
//! use a default.
//! 3. Print the output of the given command. If it fails and `TIDY_PRINT_DIFF`
//! is set, rerun the tool to print a suggestion diff (for e.g. CI)
use std::ffi::OsStr;
use std::fmt;
use std::fs;
use std::io;
use std::path::{Path, PathBuf};
use std::process::Command;
/// Minimum python revision is 3.7 for ruff
const MIN_PY_REV: (u32, u32) = (3, 7);
const MIN_PY_REV_STR: &str = "≥3.7";
/// Path to find the python executable within a virtual environment
#[cfg(target_os = "windows")]
const REL_PY_PATH: &[&str] = &["Scripts", "python3.exe"];
#[cfg(not(target_os = "windows"))]
const REL_PY_PATH: &[&str] = &["bin", "python3"];
const RUFF_CONFIG_PATH: &[&str] = &["src", "tools", "tidy", "config", "ruff.toml"];
const BLACK_CONFIG_PATH: &[&str] = &["src", "tools", "tidy", "config", "black.toml"];
/// Location within build directory
const RUFF_CACH_PATH: &[&str] = &["cache", "ruff_cache"];
const PIP_REQ_PATH: &[&str] = &["src", "tools", "tidy", "config", "requirements.txt"];
pub fn check(
root_path: &Path,
outdir: &Path,
bless: bool,
extra_checks: Option<&str>,
pos_args: &[String],
bad: &mut bool,
) {
if let Err(e) = check_impl(root_path, outdir, bless, extra_checks, pos_args) {
tidy_error!(bad, "{e}");
}
}
fn check_impl(
root_path: &Path,
outdir: &Path,
bless: bool,
extra_checks: Option<&str>,
pos_args: &[String],
) -> Result<(), Error> {
let show_diff = std::env::var("TIDY_PRINT_DIFF")
.map_or(false, |v| v.eq_ignore_ascii_case("true") || v == "1");
// Split comma-separated args up
let lint_args = match extra_checks {
Some(s) => s.strip_prefix("--extra-checks=").unwrap().split(',').collect(),
None => vec![],
};
let python_all = lint_args.contains(&"py");
let python_lint = lint_args.contains(&"py:lint") || python_all;
let python_fmt = lint_args.contains(&"py:fmt") || python_all;
let shell_all = lint_args.contains(&"shell");
let shell_lint = lint_args.contains(&"shell:lint") || shell_all;
let mut py_path = None;
let (cfg_args, file_args): (Vec<_>, Vec<_>) = pos_args
.into_iter()
.map(OsStr::new)
.partition(|arg| arg.to_str().is_some_and(|s| s.starts_with("-")));
if python_lint || python_fmt {
let venv_path = outdir.join("venv");
let mut reqs_path = root_path.to_owned();
reqs_path.extend(PIP_REQ_PATH);
py_path = Some(get_or_create_venv(&venv_path, &reqs_path)?);
}
if python_lint {
eprintln!("linting python files");
let mut cfg_args_ruff = cfg_args.clone();
let mut file_args_ruff = file_args.clone();
let mut cfg_path = root_path.to_owned();
cfg_path.extend(RUFF_CONFIG_PATH);
let mut cache_dir = outdir.to_owned();
cache_dir.extend(RUFF_CACH_PATH);
cfg_args_ruff.extend([
"--config".as_ref(),
cfg_path.as_os_str(),
"--cache-dir".as_ref(),
cache_dir.as_os_str(),
]);
if file_args_ruff.is_empty() {
file_args_ruff.push(root_path.as_os_str());
}
let mut args = merge_args(&cfg_args_ruff, &file_args_ruff);
let res = py_runner(py_path.as_ref().unwrap(), "ruff", &args);
if res.is_err() && show_diff {
eprintln!("\npython linting failed! Printing diff suggestions:");
args.insert(0, "--diff".as_ref());
let _ = py_runner(py_path.as_ref().unwrap(), "ruff", &args);
}
// Rethrow error
let _ = res?;
}
if python_fmt {
let mut cfg_args_black = cfg_args.clone();
let mut file_args_black = file_args.clone();
if bless {
eprintln!("formatting python files");
} else {
eprintln!("checking python file formatting");
cfg_args_black.push("--check".as_ref());
}
let mut cfg_path = root_path.to_owned();
cfg_path.extend(BLACK_CONFIG_PATH);
cfg_args_black.extend(["--config".as_ref(), cfg_path.as_os_str()]);
if file_args_black.is_empty() {
file_args_black.push(root_path.as_os_str());
}
let mut args = merge_args(&cfg_args_black, &file_args_black);
let res = py_runner(py_path.as_ref().unwrap(), "black", &args);
if res.is_err() && show_diff {
eprintln!("\npython formatting does not match! Printing diff:");
args.insert(0, "--diff".as_ref());
let _ = py_runner(py_path.as_ref().unwrap(), "black", &args);
}
// Rethrow error
let _ = res?;
}
if shell_lint {
eprintln!("linting shell files");
let mut file_args_shc = file_args.clone();
let files;
if file_args_shc.is_empty() {
files = find_with_extension(root_path, "sh")?;
file_args_shc.extend(files.iter().map(|p| p.as_os_str()));
}
shellcheck_runner(&merge_args(&cfg_args, &file_args_shc))?;
}
Ok(())
}
/// Helper to create `cfg1 cfg2 -- file1 file2` output
fn merge_args<'a>(cfg_args: &[&'a OsStr], file_args: &[&'a OsStr]) -> Vec<&'a OsStr> {
let mut args = cfg_args.to_owned();
args.push("--".as_ref());
args.extend(file_args);
args
}
/// Run a python command with given arguments. `py_path` should be a virtualenv.
fn py_runner(py_path: &Path, bin: &'static str, args: &[&OsStr]) -> Result<(), Error> {
let status = Command::new(py_path).arg("-m").arg(bin).args(args).status()?;
if status.success() { Ok(()) } else { Err(Error::FailedCheck(bin)) }
}
/// Create a virtuaenv at a given path if it doesn't already exist, or validate
/// the install if it does. Returns the path to that venv's python executable.
fn get_or_create_venv(venv_path: &Path, src_reqs_path: &Path) -> Result<PathBuf, Error> {
let mut should_create = true;
let dst_reqs_path = venv_path.join("requirements.txt");
let mut py_path = venv_path.to_owned();
py_path.extend(REL_PY_PATH);
if let Ok(req) = fs::read_to_string(&dst_reqs_path) {
if req == fs::read_to_string(src_reqs_path)? {
// found existing environment
should_create = false;
} else {
eprintln!("requirements.txt file mismatch, recreating environment");
}
}
if should_create {
eprintln!("removing old virtual environment");
if venv_path.is_dir() {
fs::remove_dir_all(venv_path).unwrap_or_else(|_| {
panic!("failed to remove directory at {}", venv_path.display())
});
}
create_venv_at_path(venv_path)?;
install_requirements(&py_path, src_reqs_path, &dst_reqs_path)?;
}
verify_py_version(&py_path)?;
Ok(py_path)
}
/// Attempt to create a virtualenv at this path. Cycles through all expected
/// valid python versions to find one that is installed.
fn create_venv_at_path(path: &Path) -> Result<(), Error> {
/// Preferred python versions in order. Newest to oldest then current
/// development versions
const TRY_PY: &[&str] = &[
"python3.11",
"python3.10",
"python3.9",
"python3.8",
"python3.7",
"python3",
"python",
"python3.12",
"python3.13",
];
let mut sys_py = None;
let mut found = Vec::new();
for py in TRY_PY {
match verify_py_version(Path::new(py)) {
Ok(_) => {
sys_py = Some(*py);
break;
}
// Skip not found errors
Err(Error::Io(e)) if e.kind() == io::ErrorKind::NotFound => (),
// Skip insufficient version errors
Err(Error::Version { installed, .. }) => found.push(installed),
// just log and skip unrecognized errors
Err(e) => eprintln!("note: error running '{py}': {e}"),
}
}
let Some(sys_py) = sys_py else {
let ret = if found.is_empty() {
Error::MissingReq("python3", "python file checks", None)
} else {
found.sort();
found.dedup();
Error::Version {
program: "python3",
required: MIN_PY_REV_STR,
installed: found.join(", "),
}
};
return Err(ret);
};
eprintln!("creating virtual environment at '{}' using '{sys_py}'", path.display());
let out = Command::new(sys_py).args(["-m", "virtualenv"]).arg(path).output().unwrap();
if out.status.success() {
return Ok(());
}
let err = if String::from_utf8_lossy(&out.stderr).contains("No module named virtualenv") {
Error::Generic(format!(
"virtualenv not found: you may need to install it \
(`python3 -m pip install venv`)"
))
} else {
Error::Generic(format!("failed to create venv at '{}' using {sys_py}", path.display()))
};
Err(err)
}
/// Parse python's version output (`Python x.y.z`) and ensure we have a
/// suitable version.
fn verify_py_version(py_path: &Path) -> Result<(), Error> {
let out = Command::new(py_path).arg("--version").output()?;
let outstr = String::from_utf8_lossy(&out.stdout);
let vers = outstr.trim().split_ascii_whitespace().nth(1).unwrap().trim();
let mut vers_comps = vers.split('.');
let major: u32 = vers_comps.next().unwrap().parse().unwrap();
let minor: u32 = vers_comps.next().unwrap().parse().unwrap();
if (major, minor) < MIN_PY_REV {
Err(Error::Version {
program: "python",
required: MIN_PY_REV_STR,
installed: vers.to_owned(),
})
} else {
Ok(())
}
}
fn install_requirements(
py_path: &Path,
src_reqs_path: &Path,
dst_reqs_path: &Path,
) -> Result<(), Error> {
let stat = Command::new(py_path)
.args(["-m", "pip", "install", "--upgrade", "pip"])
.status()
.expect("failed to launch pip");
if !stat.success() {
return Err(Error::Generic(format!("pip install failed with status {stat}")));
}
let stat = Command::new(py_path)
.args(["-m", "pip", "install", "--require-hashes", "-r"])
.arg(src_reqs_path)
.status()?;
if !stat.success() {
return Err(Error::Generic(format!(
"failed to install requirements at {}",
src_reqs_path.display()
)));
}
fs::copy(src_reqs_path, dst_reqs_path)?;
assert_eq!(
fs::read_to_string(src_reqs_path).unwrap(),
fs::read_to_string(dst_reqs_path).unwrap()
);
Ok(())
}
/// Check that shellcheck is installed then run it at the given path
fn shellcheck_runner(args: &[&OsStr]) -> Result<(), Error> {
match Command::new("shellcheck").arg("--version").status() {
Ok(_) => (),
Err(e) if e.kind() == io::ErrorKind::NotFound => {
return Err(Error::MissingReq(
"shellcheck",
"shell file checks",
Some(
"see <https://github.com/koalaman/shellcheck#installing> \
for installation instructions"
.to_owned(),
),
));
}
Err(e) => return Err(e.into()),
}
let status = Command::new("shellcheck").args(args).status()?;
if status.success() { Ok(()) } else { Err(Error::FailedCheck("black")) }
}
/// Check git for tracked files matching an extension
fn find_with_extension(root_path: &Path, extension: &str) -> Result<Vec<PathBuf>, Error> {
// Untracked files show up for short status and are indicated with a leading `?`
// -C changes git to be as if run from that directory
let stat_output =
Command::new("git").arg("-C").arg(root_path).args(["status", "--short"]).output()?.stdout;
if String::from_utf8_lossy(&stat_output).lines().filter(|ln| ln.starts_with('?')).count() > 0 {
eprintln!("found untracked files, ignoring");
}
let mut output = Vec::new();
let binding = Command::new("git").arg("-C").arg(root_path).args(["ls-files"]).output()?;
let tracked = String::from_utf8_lossy(&binding.stdout);
for line in tracked.lines() {
let line = line.trim();
let path = Path::new(line);
if path.extension() == Some(OsStr::new(extension)) {
output.push(path.to_owned());
}
}
Ok(output)
}
#[derive(Debug)]
enum Error {
Io(io::Error),
/// a is required to run b. c is extra info
MissingReq(&'static str, &'static str, Option<String>),
/// Tool x failed the check
FailedCheck(&'static str),
/// Any message, just print it
Generic(String),
/// Installed but wrong version
Version {
program: &'static str,
required: &'static str,
installed: String,
},
}
impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::MissingReq(a, b, ex) => {
write!(
f,
"{a} is required to run {b} but it could not be located. Is it installed?"
)?;
if let Some(s) = ex {
write!(f, "\n{s}")?;
};
Ok(())
}
Self::Version { program, required, installed } => write!(
f,
"insufficient version of '{program}' to run external tools: \
{required} required but found {installed}",
),
Self::Generic(s) => f.write_str(s),
Self::Io(e) => write!(f, "IO error: {e}"),
Self::FailedCheck(s) => write!(f, "checks with external tool '{s}' failed"),
}
}
}
impl From<io::Error> for Error {
fn from(value: io::Error) -> Self {
Self::Io(value)
}
}

1
src/tools/tidy/src/lib.rs
View File

@ -57,6 +57,7 @@ pub mod debug_artifacts;
pub mod deps;
pub mod edition;
pub mod error_codes;
pub mod ext_tool_checks;
pub mod extdeps;
pub mod features;
pub mod fluent_alphabetical;

13
src/tools/tidy/src/main.rs
View File

@ -37,9 +37,14 @@ fn main() {
let librustdoc_path = src_path.join("librustdoc");
let args: Vec<String> = env::args().skip(1).collect();
let verbose = args.iter().any(|s| *s == "--verbose");
let bless = args.iter().any(|s| *s == "--bless");
let (cfg_args, pos_args) = match args.iter().position(|arg| arg == "--") {
Some(pos) => (&args[..pos], &args[pos + 1..]),
None => (&args[..], [].as_slice()),
};
let verbose = cfg_args.iter().any(|s| *s == "--verbose");
let bless = cfg_args.iter().any(|s| *s == "--bless");
let extra_checks =
cfg_args.iter().find(|s| s.starts_with("--extra-checks=")).map(String::as_str);
let bad = std::sync::Arc::new(AtomicBool::new(false));
@ -150,6 +155,8 @@ fn main() {
r
};
check!(unstable_book, &src_path, collected);
check!(ext_tool_checks, &root_path, &output_directory, bless, extra_checks, pos_args);
});
if bad.load(Ordering::Relaxed) {

2
x.py
View File

@ -40,7 +40,7 @@ if __name__ == '__main__':
This message can be suppressed by setting `RUST_IGNORE_OLD_PYTHON=1`
""".format(major, minor))
warnings.warn(msg)
warnings.warn(msg, stacklevel=1)
rust_dir = os.path.dirname(os.path.abspath(__file__))
# For the import below, have Python search in src/bootstrap first.