nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2025-05-14 02:49:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rollup merge of #109267 - jyn514:test-configure, r=Mark-Simulacrum

Browse Source 
Add tests for configure.py

I highly recommend reviewing this with whitespace disabled.

Notably, verifying that we generate valid toml relies on python 3.11 so
we can use `tomllib`, so this also switches`x86_64-gnu-llvm-14` (one of the PR builders) to use 3.11.

While fixing that, I noticed that we stopped testing python2.7 support on PR CI in https://github.com/rust-lang/rust/pull/106085. `@fee1-dead` `@pietroalbini` please be more careful in the future, there is no CI for CI itself that verifies we are testing everything we should be.

- Separate out functions so that each unit test doesn't create a file on disk
- Add a few unit tests
This commit is contained in:
Matthias Krüger 2023-03-20 09:46:52 +01:00 committed by GitHub
commit 023079fb86
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 249 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
src/bootstrap/bootstrap_test.py
View File

@ -11,6 +11,7 @@ import sys
from shutil import rmtree from shutil import rmtree
import bootstrap import bootstrap
import configure
class VerifyTestCase(unittest.TestCase): class VerifyTestCase(unittest.TestCase):
@ -74,12 +75,50 @@ class ProgramOutOfDate(unittest.TestCase):
self.assertFalse(self.build.program_out_of_date(self.rustc_stamp_path, self.key)) self.assertFalse(self.build.program_out_of_date(self.rustc_stamp_path, self.key))
class GenerateAndParseConfig(unittest.TestCase):
"""Test that we can serialize and deserialize a config.toml file"""
def serialize_and_parse(self, args):
from io import StringIO
section_order, sections, targets = configure.parse_args(args)
buffer = StringIO()
configure.write_config_toml(buffer, section_order, targets, sections)
build = bootstrap.RustBuild()
build.config_toml = buffer.getvalue()
try:
import tomllib
# Verify this is actually valid TOML.
tomllib.loads(build.config_toml)
except ImportError:
print("warning: skipping TOML validation, need at least python 3.11", file=sys.stderr)
return build
def test_no_args(self):
build = self.serialize_and_parse([])
self.assertEqual(build.get_toml("changelog-seen"), '2')
self.assertIsNone(build.get_toml("llvm.download-ci-llvm"))
def test_set_section(self):
build = self.serialize_and_parse(["--set", "llvm.download-ci-llvm"])
self.assertEqual(build.get_toml("download-ci-llvm", section="llvm"), 'true')
def test_set_target(self):
build = self.serialize_and_parse(["--set", "target.x86_64-unknown-linux-gnu.cc=gcc"])
self.assertEqual(build.get_toml("cc", section="target.x86_64-unknown-linux-gnu"), 'gcc')
# Uncomment when #108928 is fixed.
# def test_set_top_level(self):
# build = self.serialize_and_parse(["--set", "profile=compiler"])
# self.assertEqual(build.get_toml("profile"), 'compiler')
if __name__ == '__main__': if __name__ == '__main__':
SUITE = unittest.TestSuite() SUITE = unittest.TestSuite()
TEST_LOADER = unittest.TestLoader() TEST_LOADER = unittest.TestLoader()
SUITE.addTest(doctest.DocTestSuite(bootstrap)) SUITE.addTest(doctest.DocTestSuite(bootstrap))
SUITE.addTests([ SUITE.addTests([
TEST_LOADER.loadTestsFromTestCase(VerifyTestCase), TEST_LOADER.loadTestsFromTestCase(VerifyTestCase),
TEST_LOADER.loadTestsFromTestCase(GenerateAndParseConfig),
TEST_LOADER.loadTestsFromTestCase(ProgramOutOfDate)]) TEST_LOADER.loadTestsFromTestCase(ProgramOutOfDate)])
RUNNER = unittest.TextTestRunner(stream=sys.stdout, verbosity=2) RUNNER = unittest.TextTestRunner(stream=sys.stdout, verbosity=2)

389
src/bootstrap/configure.py
View File

@ -205,77 +205,78 @@ if '--help' in sys.argv or '-h' in sys.argv:
# Parse all command line arguments into one of these three lists, handling # Parse all command line arguments into one of these three lists, handling
# boolean and value-based options separately # boolean and value-based options separately
unknown_args = [] def parse_args(args):
need_value_args = [] unknown_args = []
known_args = {} need_value_args = []
known_args = {}
p("processing command line") i = 0
i = 1 while i < len(args):
while i < len(sys.argv): arg = args[i]
arg = sys.argv[i] i += 1
i += 1 if not arg.startswith('--'):
if not arg.startswith('--'): unknown_args.append(arg)
unknown_args.append(arg) continue
continue
found = False found = False
for option in options: for option in options:
value = None value = None
if option.value: if option.value:
keyval = arg[2:].split('=', 1) keyval = arg[2:].split('=', 1)
key = keyval[0] key = keyval[0]
if option.name != key: if option.name != key:
continue continue
if len(keyval) > 1: if len(keyval) > 1:
value = keyval[1] value = keyval[1]
elif i < len(sys.argv): elif i < len(args):
value = sys.argv[i] value = args[i]
i += 1 i += 1
else:
need_value_args.append(arg)
continue
else: else:
need_value_args.append(arg) if arg[2:] == 'enable-' + option.name:
continue value = True
else: elif arg[2:] == 'disable-' + option.name:
if arg[2:] == 'enable-' + option.name: value = False
value = True else:
elif arg[2:] == 'disable-' + option.name: continue
value = False
else:
continue
found = True found = True
if option.name not in known_args: if option.name not in known_args:
known_args[option.name] = [] known_args[option.name] = []
known_args[option.name].append((option, value)) known_args[option.name].append((option, value))
break break
if not found: if not found:
unknown_args.append(arg) unknown_args.append(arg)
p("")
# Note: here and a few other places, we use [-1] to apply the *last* value # Note: here and a few other places, we use [-1] to apply the *last* value
# passed. But if option-checking is enabled, then the known_args loop will # passed. But if option-checking is enabled, then the known_args loop will
# also assert that options are only passed once. # also assert that options are only passed once.
option_checking = ('option-checking' not in known_args option_checking = ('option-checking' not in known_args
or known_args['option-checking'][-1][1]) or known_args['option-checking'][-1][1])
if option_checking: if option_checking:
if len(unknown_args) > 0: if len(unknown_args) > 0:
err("Option '" + unknown_args[0] + "' is not recognized") err("Option '" + unknown_args[0] + "' is not recognized")
if len(need_value_args) > 0: if len(need_value_args) > 0:
err("Option '{0}' needs a value ({0}=val)".format(need_value_args[0])) err("Option '{0}' needs a value ({0}=val)".format(need_value_args[0]))
# Parse all known arguments into a configuration structure that reflects the config = {}
# TOML we're going to write out
config = {} set('build.configure-args', sys.argv[1:], config)
apply_args(known_args, option_checking, config)
return parse_example_config(known_args, config)
def build(): def build(known_args):
if 'build' in known_args: if 'build' in known_args:
return known_args['build'][-1][1] return known_args['build'][-1][1]
return bootstrap.default_build_triple(verbose=False) return bootstrap.default_build_triple(verbose=False)
def set(key, value): def set(key, value, config):
if isinstance(value, list): if isinstance(value, list):
# Remove empty values, which value.split(',') tends to generate. # Remove empty values, which value.split(',') tends to generate.
value = [v for v in value if v] value = [v for v in value if v]
@ -297,75 +298,76 @@ def set(key, value):
arr = arr[part] arr = arr[part]
for key in known_args: def apply_args(known_args, option_checking, config):
# The `set` option is special and can be passed a bunch of times for key in known_args:
if key == 'set': # The `set` option is special and can be passed a bunch of times
for option, value in known_args[key]: if key == 'set':
keyval = value.split('=', 1) for option, value in known_args[key]:
if len(keyval) == 1 or keyval[1] == "true": keyval = value.split('=', 1)
value = True if len(keyval) == 1 or keyval[1] == "true":
elif keyval[1] == "false": value = True
value = False elif keyval[1] == "false":
else: value = False
value = keyval[1] else:
set(keyval[0], value) value = keyval[1]
continue set(keyval[0], value, config)
continue
# Ensure each option is only passed once # Ensure each option is only passed once
arr = known_args[key] arr = known_args[key]
if option_checking and len(arr) > 1: if option_checking and len(arr) > 1:
err("Option '{}' provided more than once".format(key)) err("Option '{}' provided more than once".format(key))
option, value = arr[-1] option, value = arr[-1]
# If we have a clear avenue to set our value in rustbuild, do so # If we have a clear avenue to set our value in rustbuild, do so
if option.rustbuild is not None: if option.rustbuild is not None:
set(option.rustbuild, value) set(option.rustbuild, value, config)
continue continue
# Otherwise we're a "special" option and need some extra handling, so do # Otherwise we're a "special" option and need some extra handling, so do
# that here. # that here.
if option.name == 'sccache': build_triple = build(known_args)
set('llvm.ccache', 'sccache')
elif option.name == 'local-rust':
for path in os.environ['PATH'].split(os.pathsep):
if os.path.exists(path + '/rustc'):
set('build.rustc', path + '/rustc')
break
for path in os.environ['PATH'].split(os.pathsep):
if os.path.exists(path + '/cargo'):
set('build.cargo', path + '/cargo')
break
elif option.name == 'local-rust-root':
set('build.rustc', value + '/bin/rustc')
set('build.cargo', value + '/bin/cargo')
elif option.name == 'llvm-root':
set('target.{}.llvm-config'.format(build()), value + '/bin/llvm-config')
elif option.name == 'llvm-config':
set('target.{}.llvm-config'.format(build()), value)
elif option.name == 'llvm-filecheck':
set('target.{}.llvm-filecheck'.format(build()), value)
elif option.name == 'tools':
set('build.tools', value.split(','))
elif option.name == 'codegen-backends':
set('rust.codegen-backends', value.split(','))
elif option.name == 'host':
set('build.host', value.split(','))
elif option.name == 'target':
set('build.target', value.split(','))
elif option.name == 'full-tools':
set('rust.codegen-backends', ['llvm'])
set('rust.lld', True)
set('rust.llvm-tools', True)
set('build.extended', True)
elif option.name == 'option-checking':
# this was handled above
pass
elif option.name == 'dist-compression-formats':
set('dist.compression-formats', value.split(','))
else:
raise RuntimeError("unhandled option {}".format(option.name))
set('build.configure-args', sys.argv[1:]) if option.name == 'sccache':
set('llvm.ccache', 'sccache', config)
elif option.name == 'local-rust':
for path in os.environ['PATH'].split(os.pathsep):
if os.path.exists(path + '/rustc'):
set('build.rustc', path + '/rustc', config)
break
for path in os.environ['PATH'].split(os.pathsep):
if os.path.exists(path + '/cargo'):
set('build.cargo', path + '/cargo', config)
break
elif option.name == 'local-rust-root':
set('build.rustc', value + '/bin/rustc', config)
set('build.cargo', value + '/bin/cargo', config)
elif option.name == 'llvm-root':
set('target.{}.llvm-config'.format(build_triple), value + '/bin/llvm-config', config)
elif option.name == 'llvm-config':
set('target.{}.llvm-config'.format(build_triple), value, config)
elif option.name == 'llvm-filecheck':
set('target.{}.llvm-filecheck'.format(build_triple), value, config)
elif option.name == 'tools':
set('build.tools', value.split(','), config)
elif option.name == 'codegen-backends':
set('rust.codegen-backends', value.split(','), config)
elif option.name == 'host':
set('build.host', value.split(','), config)
elif option.name == 'target':
set('build.target', value.split(','), config)
elif option.name == 'full-tools':
set('rust.codegen-backends', ['llvm'], config)
set('rust.lld', True, config)
set('rust.llvm-tools', True, config)
set('build.extended', True, config)
elif option.name == 'option-checking':
# this was handled above
pass
elif option.name == 'dist-compression-formats':
set('dist.compression-formats', value.split(','), config)
else:
raise RuntimeError("unhandled option {}".format(option.name))
# "Parse" the `config.example.toml` file into the various sections, and we'll # "Parse" the `config.example.toml` file into the various sections, and we'll
# use this as a template of a `config.toml` to write out which preserves # use this as a template of a `config.toml` to write out which preserves
@ -373,46 +375,50 @@ set('build.configure-args', sys.argv[1:])
# #
# Note that the `target` section is handled separately as we'll duplicate it # Note that the `target` section is handled separately as we'll duplicate it
# per configured target, so there's a bit of special handling for that here. # per configured target, so there's a bit of special handling for that here.
sections = {} def parse_example_config(known_args, config):
cur_section = None sections = {}
sections[None] = [] cur_section = None
section_order = [None] sections[None] = []
targets = {} section_order = [None]
top_level_keys = [] targets = {}
top_level_keys = []
for line in open(rust_dir + '/config.example.toml').read().split("\n"): for line in open(rust_dir + '/config.example.toml').read().split("\n"):
if cur_section == None: if cur_section == None:
if line.count('=') == 1: if line.count('=') == 1:
top_level_key = line.split('=')[0] top_level_key = line.split('=')[0]
top_level_key = top_level_key.strip(' #') top_level_key = top_level_key.strip(' #')
top_level_keys.append(top_level_key) top_level_keys.append(top_level_key)
if line.startswith('['): if line.startswith('['):
cur_section = line[1:-1] cur_section = line[1:-1]
if cur_section.startswith('target'): if cur_section.startswith('target'):
cur_section = 'target' cur_section = 'target'
elif '.' in cur_section: elif '.' in cur_section:
raise RuntimeError("don't know how to deal with section: {}".format(cur_section)) raise RuntimeError("don't know how to deal with section: {}".format(cur_section))
sections[cur_section] = [line] sections[cur_section] = [line]
section_order.append(cur_section) section_order.append(cur_section)
else: else:
sections[cur_section].append(line) sections[cur_section].append(line)
# Fill out the `targets` array by giving all configured targets a copy of the # Fill out the `targets` array by giving all configured targets a copy of the
# `target` section we just loaded from the example config # `target` section we just loaded from the example config
configured_targets = [build()] configured_targets = [build(known_args)]
if 'build' in config: if 'build' in config:
if 'host' in config['build']: if 'host' in config['build']:
configured_targets += config['build']['host'] configured_targets += config['build']['host']
if 'target' in config['build']: if 'target' in config['build']:
configured_targets += config['build']['target'] configured_targets += config['build']['target']
if 'target' in config: if 'target' in config:
for target in config['target']: for target in config['target']:
configured_targets.append(target) configured_targets.append(target)
for target in configured_targets: for target in configured_targets:
targets[target] = sections['target'][:] targets[target] = sections['target'][:]
# For `.` to be valid TOML, it needs to be quoted. But `bootstrap.py` doesn't use a proper TOML parser and fails to parse the target. # For `.` to be valid TOML, it needs to be quoted. But `bootstrap.py` doesn't use a proper TOML parser and fails to parse the target.
# Avoid using quotes unless it's necessary. # Avoid using quotes unless it's necessary.
targets[target][0] = targets[target][0].replace("x86_64-unknown-linux-gnu", "'{}'".format(target) if "." in target else target) targets[target][0] = targets[target][0].replace("x86_64-unknown-linux-gnu", "'{}'".format(target) if "." in target else target)
configure_file(sections, top_level_keys, targets, config)
return section_order, sections, targets
def is_number(value): def is_number(value):
@ -475,17 +481,20 @@ def configure_top_level_key(lines, top_level_key, value):
raise RuntimeError("failed to find config line for {}".format(top_level_key)) raise RuntimeError("failed to find config line for {}".format(top_level_key))
for section_key, section_config in config.items(): # Modify `sections` to reflect the parsed arguments and example configs.
if section_key not in sections and section_key not in top_level_keys: def configure_file(sections, top_level_keys, targets, config):
raise RuntimeError("config key {} not in sections or top_level_keys".format(section_key)) for section_key, section_config in config.items():
if section_key in top_level_keys: if section_key not in sections and section_key not in top_level_keys:
configure_top_level_key(sections[None], section_key, section_config) raise RuntimeError("config key {} not in sections or top_level_keys".format(section_key))
if section_key in top_level_keys:
configure_top_level_key(sections[None], section_key, section_config)
elif section_key == 'target':
for target in section_config:
configure_section(targets[target], section_config[target])
else:
configure_section(sections[section_key], section_config)
elif section_key == 'target':
for target in section_config:
configure_section(targets[target], section_config[target])
else:
configure_section(sections[section_key], section_config)
def write_uncommented(target, f): def write_uncommented(target, f):
block = [] block = []
@ -503,24 +512,36 @@ def write_uncommented(target, f):
is_comment = is_comment and line.startswith('#') is_comment = is_comment and line.startswith('#')
return f return f
# Now that we've built up our `config.toml`, write it all out in the same
# order that we read it in. def write_config_toml(writer, section_order, targets, sections):
p("")
p("writing `config.toml` in current directory")
with bootstrap.output('config.toml') as f:
for section in section_order: for section in section_order:
if section == 'target': if section == 'target':
for target in targets: for target in targets:
f = write_uncommented(targets[target], f) writer = write_uncommented(targets[target], writer)
else: else:
f = write_uncommented(sections[section], f) writer = write_uncommented(sections[section], writer)
with bootstrap.output('Makefile') as f:
contents = os.path.join(rust_dir, 'src', 'bootstrap', 'mk', 'Makefile.in')
contents = open(contents).read()
contents = contents.replace("$(CFG_SRC_DIR)", rust_dir + '/')
contents = contents.replace("$(CFG_PYTHON)", sys.executable)
f.write(contents)
p("") if __name__ == "__main__":
p("run `python {}/x.py --help`".format(rust_dir)) p("processing command line")
# Parse all known arguments into a configuration structure that reflects the
# TOML we're going to write out
p("")
section_order, sections, targets = parse_args(sys.argv[1:])
# Now that we've built up our `config.toml`, write it all out in the same
# order that we read it in.
p("")
p("writing `config.toml` in current directory")
with bootstrap.output('config.toml') as f:
write_config_toml(f, section_order, targets, sections)
with bootstrap.output('Makefile') as f:
contents = os.path.join(rust_dir, 'src', 'bootstrap', 'mk', 'Makefile.in')
contents = open(contents).read()
contents = contents.replace("$(CFG_SRC_DIR)", rust_dir + '/')
contents = contents.replace("$(CFG_PYTHON)", sys.executable)
f.write(contents)
p("")
p("run `python {}/x.py --help`".format(rust_dir))

5
src/ci/docker/host-x86_64/mingw-check-tidy/Dockerfile
View File

@ -1,6 +1,8 @@
FROM ubuntu:22.04 FROM ubuntu:22.04
ARG DEBIAN_FRONTEND=noninteractive ARG DEBIAN_FRONTEND=noninteractive
# NOTE: intentionally uses python2 for x.py so we can test it still works.
# validate-toolstate only runs in our CI, so it's ok for it to only support python3.
RUN apt-get update && apt-get install -y --no-install-recommends \ RUN apt-get update && apt-get install -y --no-install-recommends \
g++ \ g++ \
make \ make \
@ -8,6 +10,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
file \ file \
curl \ curl \
ca-certificates \ ca-certificates \
python2.7 \
python3 \ python3 \
python3-pip \ python3-pip \
python3-pkg-resources \ python3-pkg-resources \
@ -30,4 +33,4 @@ RUN pip3 install --no-deps --no-cache-dir --require-hashes -r /tmp/reuse-require
COPY host-x86_64/mingw-check/validate-toolstate.sh /scripts/ COPY host-x86_64/mingw-check/validate-toolstate.sh /scripts/
COPY host-x86_64/mingw-check/validate-error-codes.sh /scripts/ COPY host-x86_64/mingw-check/validate-error-codes.sh /scripts/
ENV SCRIPT python3 ../x.py test --stage 0 src/tools/tidy tidyselftest ENV SCRIPT python2.7 ../x.py test --stage 0 src/tools/tidy tidyselftest

4
src/ci/docker/host-x86_64/x86_64-gnu-llvm-14/Dockerfile
View File

@ -2,7 +2,6 @@ FROM ubuntu:22.04
ARG DEBIAN_FRONTEND=noninteractive ARG DEBIAN_FRONTEND=noninteractive
# NOTE: intentionally installs both python2 and python3 so we can test support for both.
RUN apt-get update && apt-get install -y --no-install-recommends \ RUN apt-get update && apt-get install -y --no-install-recommends \
g++ \ g++ \
gcc-multilib \ gcc-multilib \
@ -11,8 +10,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
file \ file \
curl \ curl \
ca-certificates \ ca-certificates \
python2.7 \ python3.11 \
python3 \
git \ git \
cmake \ cmake \
sudo \ sudo \