mathieu/granite-rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Auto merge of #111067 - albertlarsan68:fix-multiprocessing-x-py, r=jyn514

Browse source 
Make x.py work again in most (all?) cases

Fixes #111046.

Wrap all of x.py in `if __name__ == '__main__':` to avoid problems with `multiprocessing`
Make the pool sizing better
This commit is contained in:
bors 2023-05-01 17:34:24 +00:00
commit dbba594575
2 changed files with 40 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
src/bootstrap/bootstrap.py
View file

@ -28,6 +28,20 @@ if platform_is_win32():
else: else:
EXE_SUFFIX = "" EXE_SUFFIX = ""
def get_cpus():
if hasattr(os, "sched_getaffinity"):
return len(os.sched_getaffinity(0))
if hasattr(os, "cpu_count"):
cpus = os.cpu_count()
if cpus is not None:
return cpus
try:
return cpu_count()
except NotImplementedError:
return 1
def get(base, url, path, checksums, verbose=False): def get(base, url, path, checksums, verbose=False):
with tempfile.NamedTemporaryFile(delete=False) as temp_file: with tempfile.NamedTemporaryFile(delete=False) as temp_file:
temp_path = temp_file.name temp_path = temp_file.name
@ -540,11 +554,15 @@ class RustBuild(object):
# Unpack the tarballs in parallle. # Unpack the tarballs in parallle.
# In Python 2.7, Pool cannot be used as a context manager. # In Python 2.7, Pool cannot be used as a context manager.
p = Pool(min(len(tarballs_download_info), cpu_count())) pool_size = min(len(tarballs_download_info), get_cpus())
if self.verbose:
print('Choosing a pool size of', pool_size, 'for the unpacking of the tarballs')
p = Pool(pool_size)
try: try:
p.map(unpack_component, tarballs_download_info) p.map(unpack_component, tarballs_download_info)
finally: finally:
p.close() p.close()
p.join()
if self.should_fix_bins_and_dylibs(): if self.should_fix_bins_and_dylibs():
self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root)) self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))

39
x.py
View file

@ -4,26 +4,29 @@
# This file is only a "symlink" to bootstrap.py, all logic should go there. # This file is only a "symlink" to bootstrap.py, all logic should go there.
import os # Parts of `bootstrap.py` use the `multiprocessing` module, so this entry point
import sys # must use the normal `if __name__ == '__main__':` convention to avoid problems.
if __name__ == '__main__':
import os
import sys
# If this is python2, check if python3 is available and re-execute with that # If this is python2, check if python3 is available and re-execute with that
# interpreter. Only python3 allows downloading CI LLVM. # interpreter. Only python3 allows downloading CI LLVM.
# #
# This matters if someone's system `python` is python2. # This matters if someone's system `python` is python2.
if sys.version_info.major < 3: if sys.version_info.major < 3:
try:
os.execvp("py", ["py", "-3"] + sys.argv)
except OSError:
try: try:
os.execvp("python3", ["python3"] + sys.argv) os.execvp("py", ["py", "-3"] + sys.argv)
except OSError: except OSError:
# Python 3 isn't available, fall back to python 2 try:
pass os.execvp("python3", ["python3"] + sys.argv)
except OSError:
# Python 3 isn't available, fall back to python 2
pass
rust_dir = os.path.dirname(os.path.abspath(__file__)) rust_dir = os.path.dirname(os.path.abspath(__file__))
# For the import below, have Python search in src/bootstrap first. # For the import below, have Python search in src/bootstrap first.
sys.path.insert(0, os.path.join(rust_dir, "src", "bootstrap")) sys.path.insert(0, os.path.join(rust_dir, "src", "bootstrap"))
import bootstrap import bootstrap
bootstrap.main() bootstrap.main()