mirror of
https://github.com/rust-lang/rust.git
synced 2025-06-04 19:29:07 +00:00
Auto merge of #110427 - Nilstrieb:parallel-bootstrap-startup, r=albertlarsan68
Parallelize initial Rust download in bootstrap Parallelize the initial download of Rust in `bootstrap.py` `time ./x.py --help` after `rm -r build` Before: 33s After: 27s
This commit is contained in:
commit
9ecda8de85
@ -13,6 +13,7 @@ import tarfile
|
||||
import tempfile
|
||||
|
||||
from time import time
|
||||
from multiprocessing import Pool, cpu_count
|
||||
|
||||
try:
|
||||
import lzma
|
||||
@ -406,6 +407,48 @@ class Stage0Toolchain:
|
||||
return self.version + "-" + self.date
|
||||
|
||||
|
||||
class DownloadInfo:
|
||||
"""A helper class that can be pickled into a parallel subprocess"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_download_url,
|
||||
download_path,
|
||||
bin_root,
|
||||
tarball_path,
|
||||
tarball_suffix,
|
||||
checksums_sha256,
|
||||
pattern,
|
||||
verbose,
|
||||
):
|
||||
self.base_download_url = base_download_url
|
||||
self.download_path = download_path
|
||||
self.bin_root = bin_root
|
||||
self.tarball_path = tarball_path
|
||||
self.tarball_suffix = tarball_suffix
|
||||
self.checksums_sha256 = checksums_sha256
|
||||
self.pattern = pattern
|
||||
self.verbose = verbose
|
||||
|
||||
def download_component(download_info):
|
||||
if not os.path.exists(download_info.tarball_path):
|
||||
get(
|
||||
download_info.base_download_url,
|
||||
download_info.download_path,
|
||||
download_info.tarball_path,
|
||||
download_info.checksums_sha256,
|
||||
verbose=download_info.verbose,
|
||||
)
|
||||
|
||||
def unpack_component(download_info):
|
||||
unpack(
|
||||
download_info.tarball_path,
|
||||
download_info.tarball_suffix,
|
||||
download_info.bin_root,
|
||||
match=download_info.pattern,
|
||||
verbose=download_info.verbose,
|
||||
)
|
||||
|
||||
class RustBuild(object):
|
||||
"""Provide all the methods required to build Rust"""
|
||||
def __init__(self):
|
||||
@ -460,17 +503,49 @@ class RustBuild(object):
|
||||
)
|
||||
run_powershell([script])
|
||||
shutil.rmtree(bin_root)
|
||||
|
||||
key = self.stage0_compiler.date
|
||||
cache_dst = os.path.join(self.build_dir, "cache")
|
||||
rustc_cache = os.path.join(cache_dst, key)
|
||||
if not os.path.exists(rustc_cache):
|
||||
os.makedirs(rustc_cache)
|
||||
|
||||
tarball_suffix = '.tar.gz' if lzma is None else '.tar.xz'
|
||||
filename = "rust-std-{}-{}{}".format(
|
||||
rustc_channel, self.build, tarball_suffix)
|
||||
pattern = "rust-std-{}".format(self.build)
|
||||
self._download_component_helper(filename, pattern, tarball_suffix)
|
||||
filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
|
||||
tarball_suffix)
|
||||
self._download_component_helper(filename, "rustc", tarball_suffix)
|
||||
filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
|
||||
tarball_suffix)
|
||||
self._download_component_helper(filename, "cargo", tarball_suffix)
|
||||
|
||||
toolchain_suffix = "{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
|
||||
|
||||
tarballs_to_download = [
|
||||
("rust-std-{}".format(toolchain_suffix), "rust-std-{}".format(self.build)),
|
||||
("rustc-{}".format(toolchain_suffix), "rustc"),
|
||||
("cargo-{}".format(toolchain_suffix), "cargo"),
|
||||
]
|
||||
|
||||
tarballs_download_info = [
|
||||
DownloadInfo(
|
||||
base_download_url=self.download_url,
|
||||
download_path="dist/{}/{}".format(self.stage0_compiler.date, filename),
|
||||
bin_root=self.bin_root(),
|
||||
tarball_path=os.path.join(rustc_cache, filename),
|
||||
tarball_suffix=tarball_suffix,
|
||||
checksums_sha256=self.checksums_sha256,
|
||||
pattern=pattern,
|
||||
verbose=self.verbose,
|
||||
)
|
||||
for filename, pattern in tarballs_to_download
|
||||
]
|
||||
|
||||
# Download the components serially to show the progress bars properly.
|
||||
for download_info in tarballs_download_info:
|
||||
download_component(download_info)
|
||||
|
||||
# Unpack the tarballs in parallle.
|
||||
# In Python 2.7, Pool cannot be used as a context manager.
|
||||
p = Pool(min(len(tarballs_download_info), cpu_count()))
|
||||
try:
|
||||
p.map(unpack_component, tarballs_download_info)
|
||||
finally:
|
||||
p.close()
|
||||
|
||||
if self.should_fix_bins_and_dylibs():
|
||||
self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
|
||||
|
||||
@ -486,13 +561,9 @@ class RustBuild(object):
|
||||
rust_stamp.write(key)
|
||||
|
||||
def _download_component_helper(
|
||||
self, filename, pattern, tarball_suffix,
|
||||
self, filename, pattern, tarball_suffix, rustc_cache,
|
||||
):
|
||||
key = self.stage0_compiler.date
|
||||
cache_dst = os.path.join(self.build_dir, "cache")
|
||||
rustc_cache = os.path.join(cache_dst, key)
|
||||
if not os.path.exists(rustc_cache):
|
||||
os.makedirs(rustc_cache)
|
||||
|
||||
tarball = os.path.join(rustc_cache, filename)
|
||||
if not os.path.exists(tarball):
|
||||
|
Loading…
Reference in New Issue
Block a user