Bug 1738848: Handle virtualenv case where metadata file is missing r=ahal

At Mach initialization time, it needs to know if it's not running from
the Mach virtualenv so it can decide to insulate itself from the
external sys.path.

However, the current mechanism of detecting the Mach virtualenv
struggles if the venv is old, and therefore missing its metadata file.
In such a case, it's recognized as "not the Mach venv" rather than "an
out-of-date Mach venv".

As part of this, I'm realizing that simply using the metadata file as a
"virtualenv has finished building" marker is insufficient, because it
will cause similar confusion here. This is solved with a "finalization"
key in in the metadata.

Differential Revision: https://phabricator.services.mozilla.com/D130794

build/mach_initialize.py

@@ -182,10 +182,23 @@ def _activate_python_environment(topsrcdir, state_dir):
     from mach.site import (
         MachSiteManager,
         VirtualenvOutOfDateException,
+        MozSiteMetadata,
         MozSiteMetadataOutOfDateError,
     )
 
     try:
+        active_metadata = MozSiteMetadata.from_runtime()
+        if not active_metadata and os.path.basename(sys.prefix) == "mach":
+            # Until the new "MozVirtualenvMetadata" code has been live for long enough,
+            # there will be Mach virtualenvs without associated "metadata" files.
+            # Since "active_metadata" will be "None" in these cases, let's try
+            # to identify them with a dumb, but mostly correct, "parent directory name"
+            # check.
+            # If the check matches, then the virtualenv should be re-generated so it
+            # gets its metadata file.
+            raise MozSiteMetadataOutOfDateError(
+                "Mach virtualenv is missing metadata file."
+            )
         mach_environment = MachSiteManager.from_environment(
             topsrcdir,
             # normpath state_dir to normalize msys-style slashes.

python/mach/mach/site.py

@@ -41,8 +41,12 @@ class MozSiteMetadata:
         self.site_name = site_name
         self.prefix = prefix
 
-    def write(self):
-        raw = {"hex_version": self.hex_version, "virtualenv_name": self.site_name}
+    def write(self, is_finalized):
+        raw = {
+            "hex_version": self.hex_version,
+            "virtualenv_name": self.site_name,
+            "is_finalized": is_finalized,
+        }
         with open(os.path.join(self.prefix, METADATA_FILENAME), "w") as file:
             json.dump(raw, file)
 
@@ -60,9 +64,16 @@ class MozSiteMetadata:
     @classmethod
     def from_path(cls, prefix):
         metadata_path = os.path.join(prefix, METADATA_FILENAME)
+        out_of_date_exception = MozSiteMetadataOutOfDateError(
+            f'The virtualenv at "{prefix}" is out-of-date.'
+        )
         try:
             with open(metadata_path, "r") as file:
                 raw = json.load(file)
+
+            if not raw.get("is_finalized", False):
+                raise out_of_date_exception
+
             return cls(
                 raw["hex_version"],
                 raw["virtualenv_name"],
@@ -71,9 +82,7 @@ class MozSiteMetadata:
         except FileNotFoundError:
             return None
         except KeyError:
-            raise MozSiteMetadataOutOfDateError(
-                f'The moz site metadata at "{metadata_path}" is out-of-date.'
-            )
+            raise out_of_date_exception
 
 
 class SitePackagesSource(enum.Enum):
@@ -557,6 +566,9 @@ def _create_venv_with_pthfile(
     if os.path.exists(virtualenv_root):
         shutil.rmtree(virtualenv_root)
 
+    os.makedirs(virtualenv_root)
+    metadata.write(is_finalized=False)
+
     subprocess.check_call(
         [
             sys.executable,
@@ -595,7 +607,7 @@ def _create_venv_with_pthfile(
                 )
 
     os.utime(target_venv.activate_path, None)
-    metadata.write()
+    metadata.write(is_finalized=True)
 
 
 def _is_venv_up_to_date(