Bug 1717051: Automatically create and activate Mach virtualenv r=ahal

Consolidate Mach virtualenv management to the front of the
Mach process. This obsoletes `./mach create-mach-environment`
and simplifies the `sh` portion of the top-level `./mach` script.

This helps ensure that the Mach virtualenv doesn't become
out-of-sync and simplifies the mental model of the Mach
virtualenv situation.

Differential Revision: https://phabricator.services.mozilla.com/D120401

build/mach_initialize.py

@@ -179,38 +179,14 @@ def _activate_python_environment(topsrcdir, state_dir):
         )
     ]
 
-    from mach.site import (
+    from mach.site import MachSiteManager
-        MachSiteManager,
-        VirtualenvOutOfDateException,
-        MozSiteMetadata,
-        MozSiteMetadataOutOfDateError,
-    )
 
-    try:
+    mach_environment = MachSiteManager.from_environment(
-        active_metadata = MozSiteMetadata.from_runtime()
+        topsrcdir,
-        if not active_metadata and os.path.basename(sys.prefix) == "mach":
+        # normpath state_dir to normalize msys-style slashes.
-            # Until the new "MozVirtualenvMetadata" code has been live for long enough,
+        os.path.normpath(state_dir),
-            # there will be Mach virtualenvs without associated "metadata" files.
+    )
-            # Since "active_metadata" will be "None" in these cases, let's try
+    mach_environment.activate()
-            # to identify them with a dumb, but mostly correct, "parent directory name"
-            # check.
-            # If the check matches, then the virtualenv should be re-generated so it
-            # gets its metadata file.
-            raise MozSiteMetadataOutOfDateError(
-                "Mach virtualenv is missing metadata file."
-            )
-        mach_environment = MachSiteManager.from_environment(
-            topsrcdir,
-            # normpath state_dir to normalize msys-style slashes.
-            os.path.normpath(state_dir),
-        )
-        mach_environment.activate()
-    except (VirtualenvOutOfDateException, MozSiteMetadataOutOfDateError):
-        print(
-            'The "mach" virtualenv is not up-to-date, please run '
-            '"./mach create-mach-environment"'
-        )
-        sys.exit(1)
 
 
 def initialize(topsrcdir):
@@ -430,8 +406,8 @@ def _finalize_telemetry_glean(telemetry, is_bootstrap, success):
 
     has_psutil, logical_cores, physical_cores, memory_total = get_psutil_stats()
     if has_psutil:
-        # psutil may not be available (we allow `mach create-mach-environment`
+        # psutil may not be available (we may not have been able to download
-        # to fail to install it).
+        # a wheel or build it from source).
         system_metrics.logical_cores.add(logical_cores)
         system_metrics.physical_cores.add(physical_cores)
         if memory_total is not None:

js/src/make-source-package.py

@@ -226,7 +226,6 @@ be run over the binaries before deploying them.
 
 Building with default options may be performed as follows:
 
-  ./mach create-mach-environment
   ./mach build
 
 This will produce a debug build (much more suitable for developing against the

mach

@@ -11,32 +11,6 @@
 # shell because `''':'`, `':'` and `:` are all equivalent, and `:` is a no-op.
 ''':'
 
-# Commands that are to be run with the system Python 3 instead of the
-# virtualenv.
-nativecmds="
-    bootstrap
-    create-mach-environment
-"
-
-run_py() {
-    # Try to run a specific Python interpreter.
-    py_executable="$1"
-    shift
-    if command -v "$py_executable" > /dev/null
-    then
-        exec "$py_executable" $py_profile_command_args "$0" "$@"
-    else
-        echo "This mach command requires $py_executable, which wasn't found on the system!"
-        case "$py_executable" in
-            python3) ;;
-            *)
-                echo "Consider running 'mach bootstrap' or 'mach create-mach-environment' to create the mach virtualenvs, or set MACH_USE_SYSTEM_PYTHON to use the system Python installation over a virtualenv."
-                ;;
-        esac
-        exit 1
-    fi
-}
-
 get_command() {
     # Parse the name of the mach command out of the arguments. This is necessary
     # in the presence of global mach arguments that come before the name of the
@@ -78,7 +52,6 @@ get_command() {
     return ${py_profile_command}
 }
 
-state_dir=${MOZBUILD_STATE_PATH:-~/.mozbuild}
 command=$(get_command "$@")
 py_profile_command=$?
 
@@ -91,32 +64,17 @@ else
     # a string instead.
     py_profile_command_args="-m cProfile -o mach_profile_${command}.cProfile"
     echo "Running with --profile-command.  To visualize, use snakeviz:"
-    echo "$HOME/.mozbuild/_virtualenvs/mach/bin/python -m pip install snakeviz"
+    echo "python3 -m pip install snakeviz"
-    echo "$HOME/.mozbuild/_virtualenvs/mach/bin/python -m snakeviz mach_profile_${command}.cProfile"
+    echo "python3 -m snakeviz mach_profile_${command}.cProfile"
 fi
 
-# If MACH_USE_SYSTEM_PYTHON or MOZ_AUTOMATION are set, always use the
+if command -v python3 > /dev/null
-# python 3 executables and not the virtualenv locations.
-if [ -z ${MACH_USE_SYSTEM_PYTHON} ] && [ -z ${MOZ_AUTOMATION} ]
 then
-    case "$OSTYPE" in
+    exec python3 $py_profile_command_args "$0" "$@"
-        cygwin|msys|win32) bin_path=Scripts;;
-        *) bin_path=bin;;
-    esac
-    py3executable=$state_dir/_virtualenvs/mach/$bin_path/python
 else
-    py3executable=python3
+    echo "This mach command requires 'python3', which wasn't found on the system!"
+    exit 1
 fi
-
-# Check whether we need to run with the native Python 3 interpreter.
-case " $(echo $nativecmds) " in
-    *\ $command\ *)
-        run_py python3 "$@"
-        ;;
-esac
-
-# # Use the mach virtualenv's Python 3 for the rest of the commands.
-run_py "$py3executable" "$@"
 '''
 
 from __future__ import absolute_import, print_function, unicode_literals

python/mach/mach/site.py

@@ -242,21 +242,7 @@ class MachSiteManager:
             external_python = ExternalPythonSite(sys.executable)
 
         if not _system_python_env_variable_present():
-            # We can detect that this is for initializing a native command by:
+            source = SitePackagesSource.VENV
-            # * Checking if we're not using the Mach virtualenv's python, yet the
-            #   "use system python" environment variable isn't set.
-            # * We're still in the "initialization" phase because there's no "current"
-            #   MozEnvironment active yet (so, when the `./mach create-mach-environment`
-            #   command's function runs, this check will be Falsy).
-            # If true, then don't use any "site-packages" sources.
-            is_for_initializing_native_cmd = (
-                not active_metadata or active_metadata.site_name != "mach"
-            ) and not MozSiteMetadata.current
-            source = (
-                SitePackagesSource.NONE
-                if is_for_initializing_native_cmd
-                else SitePackagesSource.VENV
-            )
         elif not external_python.has_pip():
             source = SitePackagesSource.NONE
         else:

python/mozboot/mozboot/base.py

@@ -329,13 +329,6 @@ class BaseBootstrapper(object):
             % __name__
         )
 
-    def ensure_mach_environment(self, checkout_root):
-        mach_binary = os.path.abspath(os.path.join(checkout_root, "mach"))
-        if not os.path.exists(mach_binary):
-            raise ValueError("mach not found at %s" % mach_binary)
-        cmd = [sys.executable, mach_binary, "create-mach-environment"]
-        subprocess.check_call(cmd, cwd=checkout_root)
-
     def ensure_clang_static_analysis_package(self, state_dir, checkout_root):
         """
         Install the clang static analysis package

python/mozboot/mozboot/bootstrap.py

@@ -337,7 +337,6 @@ class Bootstrapper(object):
         self._validate_python_environment()
 
         if self.instance.no_system_changes:
-            self.instance.ensure_mach_environment(checkout_root)
             self.maybe_install_private_packages_or_exit(
                 state_dir, checkout_root, application
             )
@@ -345,10 +344,6 @@ class Bootstrapper(object):
             sys.exit(0)
 
         self.instance.install_system_packages()
-        # Install mach environment python packages after system packages.
-        # Some mach packages require building native modules, which require
-        # tools which are installed to the system.
-        self.instance.ensure_mach_environment(checkout_root)
 
         # Like 'install_browser_packages' or 'install_mobile_android_packages'.
         getattr(self.instance, "install_%s_packages" % application)(mozconfig_builder)

python/mozbuild/mozbuild/mach_commands.py

@@ -2444,33 +2444,6 @@ def package_l10n(command_context, verbose=False, locales=[]):
     return 0
 
 
-@Command(
-    "create-mach-environment",
-    category="devenv",
-    description="Create the `mach` virtualenv.",
-)
-@CommandArgument(
-    "-f",
-    "--force",
-    action="store_true",
-    help=("Force re-creating the virtualenv even if it is already up-to-date."),
-)
-def create_mach_environment(command_context, force=False):
-    """Create the mach virtualenv."""
-    from mozboot.util import get_state_dir
-    from mach.site import MachSiteManager
-
-    manager = MachSiteManager.from_environment(
-        command_context.topsrcdir,
-        get_state_dir(),
-    )
-
-    if manager.ensure(force=force):
-        print("Mach environment is already up to date.")
-    else:
-        print("Mach environment created.")
-
-
 def _prepend_debugger_args(args, debugger, debugger_args):
     """
     Given an array with program arguments, prepend arguments to run it under a

taskcluster/scripts/builder/build-sm-package.sh

@@ -26,8 +26,6 @@ status=0
 (
     # Build the freshly extracted, packaged SpiderMonkey.
     cd ./mozjs-*
-
-    ./mach create-mach-environment
     AUTOMATION=1 $PYTHON3 js/src/devtools/automation/autospider.py --skip-tests=checks $SPIDERMONKEY_VARIANT
 ) || status=$?
 

tools/update-verify/README.md

@@ -114,6 +114,5 @@ docker run \
 
 `total-chunks` and `this-chunk` refer to the number of lines in `update-verify.cfg`
 ```
-./mach create-mach-environment
 ./tools/update-verify/scripts/chunked-verify.sh --total-chunks=228 --this-chunk=4
 ```