Bug 1935333 - Avoid trying to install dist toolchains when bootstrapping from a spidermonkey standalone tree. a=pascalc

The auto-bootstrap code gracefully handles the lack of information as to
what toolchains are available in spidermonkey standalone trees, but the
fallback code path for when toolchains are not to be unpacked doesn't
handle the situation.

By making the toolchains themselves define whether they are to be
unpacked, we can remove the fallback code path and use auto-bootstrap in
all cases.

This makes `mach artifact toolchain --from-build` stop requiring a
manual `--no-unpack`, but `--from-task` still needs it because it
can't infer that information, as it doesn't have the task graph.

Original Revision: https://phabricator.services.mozilla.com/D231342

Differential Revision: https://phabricator.services.mozilla.com/D231846

build/moz.configure/bootstrap.configure

@@ -92,6 +92,7 @@ def bootstrap_toolchain_tasks(host):
         result = {
             "index": t.optimization["index-search"],
             "artifact": t.attributes["toolchain-artifact"],
+            "extract": t.attributes.get("toolchain-extract", True),
         }
         command = t.attributes.get("toolchain-command")
         if command:
@@ -219,6 +220,9 @@ def bootstrap_path(path, **kwargs):
                     "--from-task",
                     f"{task_id}:{artifact}",
                 ]
+                if not task["extract"]:
+                    command.append("--no-unpack")
+
             elif command:
                 # For private local toolchains, run the associated command.
                 command = (

python/mozboot/mozboot/base.py

@@ -16,7 +16,6 @@ from packaging.version import Version
 from mozboot import rust
 from mozboot.util import (
     MINIMUM_RUST_VERSION,
-    get_mach_virtualenv_binary,
     http_download_and_save,
 )
 
@@ -332,46 +331,9 @@ class BaseBootstrapper(object):
         """
         pass
 
-    def install_toolchain_artifact(self, toolchain_job, no_unpack=False):
+    def install_toolchain_artifact(self, toolchain_job):
-        if no_unpack:
-            return self.install_toolchain_artifact_impl(
-                self.state_dir, toolchain_job, no_unpack
-            )
         bootstrap_toolchain(toolchain_job)
 
-    def install_toolchain_artifact_impl(
-        self, install_dir: Path, toolchain_job, no_unpack=False
-    ):
-        if type(self.srcdir) is str:
-            mach_binary = Path(self.srcdir) / "mach"
-        else:
-            mach_binary = (self.srcdir / "mach").resolve()
-        if not mach_binary.exists():
-            raise ValueError(f"mach not found at {mach_binary}")
-
-        if not self.state_dir:
-            raise ValueError(
-                "Need a state directory (e.g. ~/.mozbuild) to download " "artifacts"
-            )
-        python_location = get_mach_virtualenv_binary()
-        if not python_location.exists():
-            raise ValueError(f"python not found at {python_location}")
-
-        cmd = [
-            str(python_location),
-            str(mach_binary),
-            "artifact",
-            "toolchain",
-            "--bootstrap",
-            "--from-build",
-            toolchain_job,
-        ]
-
-        if no_unpack:
-            cmd += ["--no-unpack"]
-
-        subprocess.check_call(cmd, cwd=str(install_dir))
-
     def auto_bootstrap(self, application, exclude=[]):
         args = ["--with-ccache=sccache"]
         if application.endswith("_artifact_mode"):

python/mozboot/mozboot/mozillabuild.py

@@ -217,8 +217,8 @@ class MozillaBuildBootstrapper(BaseBootstrapper):
     def ensure_sccache_packages(self):
         from mozboot import sccache
 
-        self.install_toolchain_artifact(sccache.RUSTC_DIST_TOOLCHAIN, no_unpack=True)
+        self.install_toolchain_artifact(sccache.RUSTC_DIST_TOOLCHAIN)
-        self.install_toolchain_artifact(sccache.CLANG_DIST_TOOLCHAIN, no_unpack=True)
+        self.install_toolchain_artifact(sccache.CLANG_DIST_TOOLCHAIN)
 
     def _update_package_manager(self):
         pass

python/mozboot/mozboot/osx.py

@@ -269,8 +269,8 @@ class OSXBootstrapper(OSXAndroidBootstrapper, BaseBootstrapper):
     def ensure_sccache_packages(self):
         from mozboot import sccache
 
-        self.install_toolchain_artifact(sccache.RUSTC_DIST_TOOLCHAIN, no_unpack=True)
+        self.install_toolchain_artifact(sccache.RUSTC_DIST_TOOLCHAIN)
-        self.install_toolchain_artifact(sccache.CLANG_DIST_TOOLCHAIN, no_unpack=True)
+        self.install_toolchain_artifact(sccache.CLANG_DIST_TOOLCHAIN)
 
     def install_homebrew(self):
         print(BREW_INSTALL)

python/mozbuild/mozbuild/artifact_commands.py

@@ -480,6 +480,7 @@ def artifact_toolchain(
             )
 
             record = ArtifactRecord(task_id, artifact_name)
+            record.unpack = task.attributes.get("toolchain-extract", True)
             records[record.filename] = record
 
     # Handle the list of files of the form task_id:path from --from-task.

python/mozbuild/mozbuild/test/configure/test_bootstrap.py

@@ -79,14 +79,17 @@ class TestBootstrap(BaseConfigureTest):
                 "toolchain-foo": {
                     "index": ["fake.index.foo"],
                     "artifact": "public/foo.artifact",
+                    "extract": True,
                 },
                 "toolchain-linux64-bar": {
                     "index": ["fake.index.bar"],
                     "artifact": "public/bar.artifact",
+                    "extract": True,
                 },
                 "toolchain-linux64-qux": {
                     "index": ["fake.index.qux"],
                     "artifact": "public/qux.artifact",
+                    "extract": True,
                 },
             },
         )

taskcluster/docs/attributes.rst

@@ -243,6 +243,11 @@ toolchain-artifact
 ==================
 For toolchain jobs, this is the path to the artifact for that toolchain.
 
+toolchain-extract
+=================
+Control whether toolchain should be automatically extracted after download.
+Default is true.
+
 toolchain-alias
 ===============
 An alias that can be used instead of the real toolchain job name in fetch

taskcluster/gecko_taskgraph/transforms/job/__init__.py

@@ -239,7 +239,7 @@ def get_attribute(dict, key, attributes, attribute_name):
     """Get `attribute_name` from the given `attributes` dict, and if there
     is a corresponding value, set `key` in `dict` to that value."""
     value = attributes.get(attribute_name)
-    if value:
+    if value is not None:
         dict[key] = value
 
 
@@ -282,6 +282,7 @@ def use_system_python(config, jobs):
 def use_fetches(config, jobs):
     artifact_names = {}
     extra_env = {}
+    should_extract = {}
     aliases = {}
     tasks = []
 
@@ -299,6 +300,9 @@ def use_fetches(config, jobs):
             artifact_names, task["label"], task["attributes"], f"{kind}-artifact"
         )
         get_attribute(extra_env, task["label"], task["attributes"], f"{kind}-env")
+        get_attribute(
+            should_extract, task["label"], task["attributes"], f"{kind}-extract"
+        )
         value = task["attributes"].get(f"{kind}-alias")
         if not value:
             value = []
@@ -348,7 +352,7 @@ def use_fetches(config, jobs):
                         {
                             "artifact": path,
                             "task": f"<{label}>",
-                            "extract": True,
+                            "extract": should_extract.get(label, True),
                         }
                     )
 

taskcluster/gecko_taskgraph/transforms/job/toolchain.py

@@ -5,7 +5,6 @@
 Support for running toolchain-building jobs via dedicated scripts
 """
 
-
 import os
 
 import taskgraph
@@ -63,6 +62,11 @@ toolchain_run_schema = Schema(
             "toolchain-env",
             description="Additional env variables to add to the worker when using this toolchain",
         ): {str: object},
+        Optional(
+            "toolchain-extract",
+            description="Whether the toolchain should be extracted after it is fetched "
+            + "(default: True)",
+        ): bool,
         # Base work directory used to set up the task.
         Optional("workdir"): str,
     }
@@ -157,6 +161,8 @@ def common_toolchain(config, job, taskdesc, is_docker):
         attributes["toolchain-alias"] = alias
     if "toolchain-env" in run:
         attributes["toolchain-env"] = run.pop("toolchain-env")
+    if "toolchain-extract" in run:
+        attributes["toolchain-extract"] = run.pop("toolchain-extract")
 
     # Allow the job to specify where artifacts come from, but add
     # public/build if it's not there already.

taskcluster/kinds/toolchain/dist-toolchains.yml

@@ -19,6 +19,7 @@ clang-dist-toolchain:
     run:
         arguments: ['clang']
         toolchain-artifact: public/build/clang-dist-toolchain.tar.xz
+        toolchain-extract: false
     use-sccache: true
     fetches:
         toolchain:
@@ -33,6 +34,7 @@ rustc-dist-toolchain:
     run:
         arguments: ['rustc']
         toolchain-artifact: public/build/rustc-dist-toolchain.tar.xz
+        toolchain-extract: false
     use-sccache: true
     fetches:
         toolchain: