From 6f42e00ea5f92391db727d74ca3625657b89dfad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Mon, 9 Dec 2024 04:40:42 +0000 Subject: [PATCH] Bug 1935621 - Fix virtual environment sysconfig path calculation r=firefox-build-system-reviewers,ahochheiden a=pascalc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns Signed-off-by: Filipe Laíns Signed-off-by: Filipe Laíns Differential Revision: https://phabricator.services.mozilla.com/D231480 --- python/mach/mach/site.py | 89 +++++++++++++++++++++++++++++----------- 1 file changed, 66 insertions(+), 23 deletions(-) diff --git a/python/mach/mach/site.py b/python/mach/mach/site.py index 7041437bf8da..c48d282b2fc2 100644 --- a/python/mach/mach/site.py +++ b/python/mach/mach/site.py @@ -17,6 +17,7 @@ import subprocess import sys import sysconfig import tempfile +import warnings from contextlib import contextmanager from pathlib import Path from typing import Callable, Optional @@ -817,33 +818,75 @@ class PythonVirtualenv: """Calculates paths of interest for general python virtual environments""" def __init__(self, prefix): - if _is_windows: - self.bin_path = os.path.join(prefix, "Scripts") - self.python_path = os.path.join(self.bin_path, "python.exe") - else: - self.bin_path = os.path.join(prefix, "bin") - self.python_path = os.path.join(self.bin_path, "python") self.prefix = os.path.realpath(prefix) + self.paths = self._get_sysconfig_paths(self.prefix) - @functools.lru_cache(maxsize=None) - def resolve_sysconfig_packages_path(self, sysconfig_path): - # macOS uses a different default sysconfig scheme based on whether it's using the - # system Python or running in a virtualenv. - # Manually define the scheme (following the implementation in - # "sysconfig._get_default_scheme()") so that we're always following the - # code path for a virtualenv directory structure. - if os.name == "posix": - scheme = "posix_prefix" + # Name of the Python executable to use in virtual environments. + # An executable with the same name as sys.executable might not exist in + # virtual environments. An executable with 'python' as the steam — + # without version numbers or ABI flags — will always be present in + # virtual environments, so we use that. + python_exe_name = "python" + sysconfig.get_config_var("EXE") + + self.bin_path = self.paths["scripts"] + self.python_path = os.path.join(self.bin_path, python_exe_name) + + @staticmethod + def _get_sysconfig_paths(prefix): + """Calculate the sysconfig paths of a virtual environment in the given prefix. + + The virtual environment MUST be using the same Python distribution as us. + """ + # Determine the sysconfig scheme used in virtual environments + if "venv" in sysconfig.get_scheme_names(): + # A 'venv' scheme was added in Python 3.11 to allow users to + # calculate the paths for a virtual environment, since the default + # scheme may not always be the same as used on virtual environments. + # Some common examples are the system Python distributed by macOS, + # Debian, and Fedora. + # For more information, see https://github.com/python/cpython/issues/89576 + venv_scheme = "venv" + elif os.name == "nt": + # We know that before the 'venv' scheme was added, on Windows, + # the 'nt' scheme was used in virtual environments. + venv_scheme = "nt" + elif os.name == "posix": + # We know that before the 'venv' scheme was added, on POSIX, + # the 'posix_prefix' scheme was used in virtual environments. + venv_scheme = "posix_prefix" else: - scheme = os.name + # This should never happen with upstream Python, as the 'venv' + # scheme should always be available on >=3.11, and no other + # platforms are supported by the upstream on older Python versions. + # + # Since the 'venv' scheme isn't available, and we have no knowledge + # of this platform/distribution, fallback to the default scheme. + # + # Hitting this will likely be the result of running a custom Python + # distribution targetting a platform that is not supported by the + # upstream. + # In this case, unless the Python vendor patched the Python + # distribution in such a way as the default scheme may not always be + # the same scheme, using the default scheme should be correct. + # If the vendor did patch Python as such, to work around this issue, + # I would recommend them to define a 'venv' scheme that matches + # the layout used on virtual environments in their Python distribution. + # (rec. signed Filipe Laíns — upstream sysconfig maintainer) + venv_scheme = sysconfig.get_default_scheme() + warnings.warn( + f"Unknown platform '{os.name}', using the default install scheme '{venv_scheme}'. " + "If this is incorrect, please ask your Python vendor to add a 'venv' sysconfig scheme " + "(see https://github.com/python/cpython/issues/89576, or check the code comment).", + stacklevel=2, + ) + # Build the sysconfig config_vars dictionary for the virtual environment. + venv_vars = sysconfig.get_config_vars().copy() + venv_vars["base"] = venv_vars["platbase"] = prefix + # Get sysconfig paths for the virtual environment. + return sysconfig.get_paths(venv_scheme, vars=venv_vars) - sysconfig_paths = sysconfig.get_paths(scheme) - data_path = Path(sysconfig_paths["data"]) - path = Path(sysconfig_paths[sysconfig_path]) - relative_path = path.relative_to(data_path) - - # Path to virtualenv's "site-packages" directory for provided sysconfig path - return os.path.normpath(os.path.normcase(Path(self.prefix) / relative_path)) + def resolve_sysconfig_packages_path(self, sysconfig_path): + return self.paths[sysconfig_path] def site_packages_dirs(self): dirs = []