Make x.py work again in most (all?) cases

Wrap all of x.py in `if __name__ == '__main__':` to avoid problems with `multiprocessing`
Make the pool sizing better
This commit is contained in:
Albert Larsan 2023-05-01 13:46:31 +00:00
parent 4b87ed9d0f
commit 30119498be
No known key found for this signature in database
GPG Key ID: 92709B88BB8F13EA
2 changed files with 40 additions and 19 deletions

View File

@ -28,6 +28,20 @@ if platform_is_win32():
else:
EXE_SUFFIX = ""
def get_cpus():
if hasattr(os, "sched_getaffinity"):
return len(os.sched_getaffinity(0))
if hasattr(os, "cpu_count"):
cpus = os.cpu_count()
if cpus is not None:
return cpus
try:
return cpu_count()
except NotImplementedError:
return 1
def get(base, url, path, checksums, verbose=False):
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
temp_path = temp_file.name
@ -540,11 +554,15 @@ class RustBuild(object):
# Unpack the tarballs in parallle.
# In Python 2.7, Pool cannot be used as a context manager.
p = Pool(min(len(tarballs_download_info), cpu_count()))
pool_size = min(len(tarballs_download_info), get_cpus())
if self.verbose:
print('Choosing a pool size of', pool_size, 'for the unpacking of the tarballs')
p = Pool(pool_size)
try:
p.map(unpack_component, tarballs_download_info)
finally:
p.close()
p.join()
if self.should_fix_bins_and_dylibs():
self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))

3
x.py
View File

@ -4,6 +4,9 @@
# This file is only a "symlink" to bootstrap.py, all logic should go there.
# Parts of `bootstrap.py` use the `multiprocessing` module, so this entry point
# must use the normal `if __name__ == '__main__':` convention to avoid problems.
if __name__ == '__main__':
import os
import sys