Auto merge of #110427 - Nilstrieb:parallel-bootstrap-startup, r=albertlarsan68

Parallelize initial Rust download in bootstrap

Parallelize the initial download of Rust in `bootstrap.py`

`time ./x.py --help` after `rm -r build`
Before: 33s
After: 27s
This commit is contained in:
bors 2023-04-30 20:02:00 +00:00
commit 9ecda8de85

View File

@ -13,6 +13,7 @@ import tarfile
import tempfile
from time import time
from multiprocessing import Pool, cpu_count
try:
import lzma
@ -406,6 +407,48 @@ class Stage0Toolchain:
return self.version + "-" + self.date
class DownloadInfo:
"""A helper class that can be pickled into a parallel subprocess"""
def __init__(
self,
base_download_url,
download_path,
bin_root,
tarball_path,
tarball_suffix,
checksums_sha256,
pattern,
verbose,
):
self.base_download_url = base_download_url
self.download_path = download_path
self.bin_root = bin_root
self.tarball_path = tarball_path
self.tarball_suffix = tarball_suffix
self.checksums_sha256 = checksums_sha256
self.pattern = pattern
self.verbose = verbose
def download_component(download_info):
if not os.path.exists(download_info.tarball_path):
get(
download_info.base_download_url,
download_info.download_path,
download_info.tarball_path,
download_info.checksums_sha256,
verbose=download_info.verbose,
)
def unpack_component(download_info):
unpack(
download_info.tarball_path,
download_info.tarball_suffix,
download_info.bin_root,
match=download_info.pattern,
verbose=download_info.verbose,
)
class RustBuild(object):
"""Provide all the methods required to build Rust"""
def __init__(self):
@ -460,17 +503,49 @@ class RustBuild(object):
)
run_powershell([script])
shutil.rmtree(bin_root)
key = self.stage0_compiler.date
cache_dst = os.path.join(self.build_dir, "cache")
rustc_cache = os.path.join(cache_dst, key)
if not os.path.exists(rustc_cache):
os.makedirs(rustc_cache)
tarball_suffix = '.tar.gz' if lzma is None else '.tar.xz'
filename = "rust-std-{}-{}{}".format(
rustc_channel, self.build, tarball_suffix)
pattern = "rust-std-{}".format(self.build)
self._download_component_helper(filename, pattern, tarball_suffix)
filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
tarball_suffix)
self._download_component_helper(filename, "rustc", tarball_suffix)
filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
tarball_suffix)
self._download_component_helper(filename, "cargo", tarball_suffix)
toolchain_suffix = "{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
tarballs_to_download = [
("rust-std-{}".format(toolchain_suffix), "rust-std-{}".format(self.build)),
("rustc-{}".format(toolchain_suffix), "rustc"),
("cargo-{}".format(toolchain_suffix), "cargo"),
]
tarballs_download_info = [
DownloadInfo(
base_download_url=self.download_url,
download_path="dist/{}/{}".format(self.stage0_compiler.date, filename),
bin_root=self.bin_root(),
tarball_path=os.path.join(rustc_cache, filename),
tarball_suffix=tarball_suffix,
checksums_sha256=self.checksums_sha256,
pattern=pattern,
verbose=self.verbose,
)
for filename, pattern in tarballs_to_download
]
# Download the components serially to show the progress bars properly.
for download_info in tarballs_download_info:
download_component(download_info)
# Unpack the tarballs in parallle.
# In Python 2.7, Pool cannot be used as a context manager.
p = Pool(min(len(tarballs_download_info), cpu_count()))
try:
p.map(unpack_component, tarballs_download_info)
finally:
p.close()
if self.should_fix_bins_and_dylibs():
self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
@ -486,13 +561,9 @@ class RustBuild(object):
rust_stamp.write(key)
def _download_component_helper(
self, filename, pattern, tarball_suffix,
self, filename, pattern, tarball_suffix, rustc_cache,
):
key = self.stage0_compiler.date
cache_dst = os.path.join(self.build_dir, "cache")
rustc_cache = os.path.join(cache_dst, key)
if not os.path.exists(rustc_cache):
os.makedirs(rustc_cache)
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):