diff --git a/.gitignore b/.gitignore index 6e4789a5a612..e715e6973bb4 100644 --- a/.gitignore +++ b/.gitignore @@ -90,6 +90,12 @@ parser/html/java/translator.jar # Local Gradle configuration properties. /local.properties +# Python virtualenv artifacts. +third_party/python/psutil/**/*.so +third_party/python/psutil/**/*.pyd +third_party/python/psutil/build/ +third_party/python/psutil/tmp/ + # Ignore chrome.manifest files from the devtools loader devtools/client/chrome.manifest devtools/shared/chrome.manifest diff --git a/.hgignore b/.hgignore index bd7f77ddf662..6a379bacdedd 100644 --- a/.hgignore +++ b/.hgignore @@ -87,6 +87,12 @@ _OPT\.OBJ/ # Local Gradle configuration properties. ^local.properties$ +# Python stuff installed at build time. +^third_party/python/psutil/.*\.so +^third_party/python/psutil/.*\.pyd +^third_party/python/psutil/build/ +^third_party/python/psutil/tmp/ + # Git repositories .git/ diff --git a/build/mach_bootstrap.py b/build/mach_bootstrap.py index 5fa3c51dd37f..5b94b039b6e4 100644 --- a/build/mach_bootstrap.py +++ b/build/mach_bootstrap.py @@ -9,7 +9,6 @@ import json import math import os import platform -import shutil import subprocess import sys import uuid @@ -198,13 +197,6 @@ def bootstrap(topsrcdir, mozilla_dir=None): print("You are running Python", platform.python_version()) sys.exit(1) - # This directory was deleted in bug 1666345, but there may be some ignored - # files here. We can safely just delete it for the user so they don't have - # to clean the repo themselves. - deleted_dir = os.path.join(topsrcdir, "third_party", "python", "psutil") - if os.path.exists(deleted_dir): - shutil.rmtree(deleted_dir) - # Global build system and mach state is stored in a central directory. By # default, this is ~/.mozbuild. However, it can be defined via an # environment variable. We detect first run (by lack of this directory @@ -511,8 +503,7 @@ def _finalize_telemetry_glean(telemetry, is_bootstrap, success): has_psutil, logical_cores, physical_cores, memory_total = get_psutil_stats() if has_psutil: - # psutil may not be available (we allow `mach create-mach-environment` - # to fail to install it). + # psutil may not be available if a successful build hasn't occurred yet. system_metrics.logical_cores.add(logical_cores) system_metrics.physical_cores.add(physical_cores) if memory_total is not None: diff --git a/python/mozbuild/mozbuild/vendor/mach_commands.py b/python/mozbuild/mozbuild/vendor/mach_commands.py index 0818ba5af40f..f3f544dcfdfe 100644 --- a/python/mozbuild/mozbuild/vendor/mach_commands.py +++ b/python/mozbuild/mozbuild/vendor/mach_commands.py @@ -152,6 +152,12 @@ Please commit or stash these changes before vendoring, or re-run with `--ignore- description="Vendor Python packages from pypi.org into third_party/python. " "Some extra files like docs and tests will automatically be excluded.", ) + @CommandArgument( + "--with-windows-wheel", + action="store_true", + help="Vendor a wheel for Windows along with the source package", + default=False, + ) @CommandArgument( "--keep-extra-files", action="store_true", diff --git a/python/mozbuild/mozbuild/vendor/vendor_python.py b/python/mozbuild/mozbuild/vendor/vendor_python.py index 5c6b78e46aad..1f78b89756ca 100644 --- a/python/mozbuild/mozbuild/vendor/vendor_python.py +++ b/python/mozbuild/mozbuild/vendor/vendor_python.py @@ -16,13 +16,17 @@ from mozpack.files import FileFinder class VendorPython(MozbuildObject): - def vendor(self, packages=None, keep_extra_files=False): + def vendor(self, packages=None, with_windows_wheel=False, keep_extra_files=False): self.populate_logger() self.log_manager.enable_unstructured() vendor_dir = mozpath.join(self.topsrcdir, os.path.join("third_party", "python")) packages = packages or [] + if with_windows_wheel and len(packages) != 1: + raise Exception( + "--with-windows-wheel is only supported for a single package!" + ) self.activate_virtualenv() pip_compile = os.path.join(self.virtualenv_manager.bin_path, "pip-compile") @@ -71,6 +75,30 @@ class VendorPython(MozbuildObject): "--disable-pip-version-check", ] ) + if with_windows_wheel: + # This is hardcoded to CPython 2.7 for win64, which is good + # enough for what we need currently. If we need psutil for Python 3 + # in the future that could be added here as well. + self.virtualenv_manager._run_pip( + [ + "download", + "--dest", + tmp, + "--no-deps", + "--only-binary", + ":all:", + "--platform", + "win_amd64", + "--implementation", + "cp", + "--python-version", + "27", + "--abi", + "none", + "--disable-pip-version-check", + packages[0], + ] + ) self._extract(tmp, vendor_dir, keep_extra_files) shutil.copyfile(tmpspec_absolute, spec) @@ -119,14 +147,27 @@ class VendorPython(MozbuildObject): finder = FileFinder(src) for path, _ in finder.find("*"): base, ext = os.path.splitext(path) - # packages extract into package-version directory name and we strip the version - tld = mozfile.extract(os.path.join(finder.base, path), dest, ignore=ignore)[ - 0 - ] - target = os.path.join(dest, tld.rpartition("-")[0]) - mozfile.remove(target) # remove existing version of vendored package - mozfile.move(tld, target) + if ext == ".whl": + # Wheels would extract into a directory with the name of the package, but + # we want the platform signifiers, minus the version number. + # Wheel filenames look like: + # {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag} + bits = base.split("-") + # Remove the version number. + bits.pop(1) + target = os.path.join(dest, "-".join(bits)) + mozfile.remove(target) # remove existing version of vendored package + os.mkdir(target) + mozfile.extract(os.path.join(finder.base, path), target, ignore=ignore) + else: + # packages extract into package-version directory name and we strip the version + tld = mozfile.extract( + os.path.join(finder.base, path), dest, ignore=ignore + )[0] + target = os.path.join(dest, tld.rpartition("-")[0]) + mozfile.remove(target) # remove existing version of vendored package + mozfile.move(tld, target) # If any files inside the vendored package were symlinks, turn them into normal files # because hg.mozilla.org forbids symlinks in the repository. link_finder = FileFinder(target)