diff --git a/build/moz.configure/bootstrap.configure b/build/moz.configure/bootstrap.configure index 6e8e59812ff6..fb1275dad03b 100644 --- a/build/moz.configure/bootstrap.configure +++ b/build/moz.configure/bootstrap.configure @@ -92,6 +92,7 @@ def bootstrap_toolchain_tasks(host): result = { "index": t.optimization["index-search"], "artifact": t.attributes["toolchain-artifact"], + "extract": t.attributes.get("toolchain-extract", True), } command = t.attributes.get("toolchain-command") if command: @@ -219,6 +220,9 @@ def bootstrap_path(path, **kwargs): "--from-task", f"{task_id}:{artifact}", ] + if not task["extract"]: + command.append("--no-unpack") + elif command: # For private local toolchains, run the associated command. command = ( diff --git a/python/mozboot/mozboot/base.py b/python/mozboot/mozboot/base.py index 23f9f7cf67ca..81f9b52c6014 100644 --- a/python/mozboot/mozboot/base.py +++ b/python/mozboot/mozboot/base.py @@ -16,7 +16,6 @@ from packaging.version import Version from mozboot import rust from mozboot.util import ( MINIMUM_RUST_VERSION, - get_mach_virtualenv_binary, http_download_and_save, ) @@ -332,46 +331,9 @@ class BaseBootstrapper(object): """ pass - def install_toolchain_artifact(self, toolchain_job, no_unpack=False): - if no_unpack: - return self.install_toolchain_artifact_impl( - self.state_dir, toolchain_job, no_unpack - ) + def install_toolchain_artifact(self, toolchain_job): bootstrap_toolchain(toolchain_job) - def install_toolchain_artifact_impl( - self, install_dir: Path, toolchain_job, no_unpack=False - ): - if type(self.srcdir) is str: - mach_binary = Path(self.srcdir) / "mach" - else: - mach_binary = (self.srcdir / "mach").resolve() - if not mach_binary.exists(): - raise ValueError(f"mach not found at {mach_binary}") - - if not self.state_dir: - raise ValueError( - "Need a state directory (e.g. ~/.mozbuild) to download " "artifacts" - ) - python_location = get_mach_virtualenv_binary() - if not python_location.exists(): - raise ValueError(f"python not found at {python_location}") - - cmd = [ - str(python_location), - str(mach_binary), - "artifact", - "toolchain", - "--bootstrap", - "--from-build", - toolchain_job, - ] - - if no_unpack: - cmd += ["--no-unpack"] - - subprocess.check_call(cmd, cwd=str(install_dir)) - def auto_bootstrap(self, application, exclude=[]): args = ["--with-ccache=sccache"] if application.endswith("_artifact_mode"): diff --git a/python/mozboot/mozboot/mozillabuild.py b/python/mozboot/mozboot/mozillabuild.py index 55a3cca1f53c..a1ecec92e20b 100644 --- a/python/mozboot/mozboot/mozillabuild.py +++ b/python/mozboot/mozboot/mozillabuild.py @@ -217,8 +217,8 @@ class MozillaBuildBootstrapper(BaseBootstrapper): def ensure_sccache_packages(self): from mozboot import sccache - self.install_toolchain_artifact(sccache.RUSTC_DIST_TOOLCHAIN, no_unpack=True) - self.install_toolchain_artifact(sccache.CLANG_DIST_TOOLCHAIN, no_unpack=True) + self.install_toolchain_artifact(sccache.RUSTC_DIST_TOOLCHAIN) + self.install_toolchain_artifact(sccache.CLANG_DIST_TOOLCHAIN) def _update_package_manager(self): pass diff --git a/python/mozboot/mozboot/osx.py b/python/mozboot/mozboot/osx.py index 8cd180f4ab08..078d2abb06cc 100644 --- a/python/mozboot/mozboot/osx.py +++ b/python/mozboot/mozboot/osx.py @@ -269,8 +269,8 @@ class OSXBootstrapper(OSXAndroidBootstrapper, BaseBootstrapper): def ensure_sccache_packages(self): from mozboot import sccache - self.install_toolchain_artifact(sccache.RUSTC_DIST_TOOLCHAIN, no_unpack=True) - self.install_toolchain_artifact(sccache.CLANG_DIST_TOOLCHAIN, no_unpack=True) + self.install_toolchain_artifact(sccache.RUSTC_DIST_TOOLCHAIN) + self.install_toolchain_artifact(sccache.CLANG_DIST_TOOLCHAIN) def install_homebrew(self): print(BREW_INSTALL) diff --git a/python/mozbuild/mozbuild/artifact_commands.py b/python/mozbuild/mozbuild/artifact_commands.py index e42b5a38d97e..5fe8a472d5a6 100644 --- a/python/mozbuild/mozbuild/artifact_commands.py +++ b/python/mozbuild/mozbuild/artifact_commands.py @@ -480,6 +480,7 @@ def artifact_toolchain( ) record = ArtifactRecord(task_id, artifact_name) + record.unpack = task.attributes.get("toolchain-extract", True) records[record.filename] = record # Handle the list of files of the form task_id:path from --from-task. diff --git a/python/mozbuild/mozbuild/test/configure/test_bootstrap.py b/python/mozbuild/mozbuild/test/configure/test_bootstrap.py index e3e3d3c744dd..a14007010283 100644 --- a/python/mozbuild/mozbuild/test/configure/test_bootstrap.py +++ b/python/mozbuild/mozbuild/test/configure/test_bootstrap.py @@ -79,14 +79,17 @@ class TestBootstrap(BaseConfigureTest): "toolchain-foo": { "index": ["fake.index.foo"], "artifact": "public/foo.artifact", + "extract": True, }, "toolchain-linux64-bar": { "index": ["fake.index.bar"], "artifact": "public/bar.artifact", + "extract": True, }, "toolchain-linux64-qux": { "index": ["fake.index.qux"], "artifact": "public/qux.artifact", + "extract": True, }, }, ) diff --git a/taskcluster/docs/attributes.rst b/taskcluster/docs/attributes.rst index 8b82fa9986ed..0163bfe73c59 100644 --- a/taskcluster/docs/attributes.rst +++ b/taskcluster/docs/attributes.rst @@ -243,6 +243,11 @@ toolchain-artifact ================== For toolchain jobs, this is the path to the artifact for that toolchain. +toolchain-extract +================= +Control whether toolchain should be automatically extracted after download. +Default is true. + toolchain-alias =============== An alias that can be used instead of the real toolchain job name in fetch diff --git a/taskcluster/gecko_taskgraph/transforms/job/__init__.py b/taskcluster/gecko_taskgraph/transforms/job/__init__.py index 421dbbff9fb8..ef41112e6474 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/__init__.py +++ b/taskcluster/gecko_taskgraph/transforms/job/__init__.py @@ -239,7 +239,7 @@ def get_attribute(dict, key, attributes, attribute_name): """Get `attribute_name` from the given `attributes` dict, and if there is a corresponding value, set `key` in `dict` to that value.""" value = attributes.get(attribute_name) - if value: + if value is not None: dict[key] = value @@ -282,6 +282,7 @@ def use_system_python(config, jobs): def use_fetches(config, jobs): artifact_names = {} extra_env = {} + should_extract = {} aliases = {} tasks = [] @@ -299,6 +300,9 @@ def use_fetches(config, jobs): artifact_names, task["label"], task["attributes"], f"{kind}-artifact" ) get_attribute(extra_env, task["label"], task["attributes"], f"{kind}-env") + get_attribute( + should_extract, task["label"], task["attributes"], f"{kind}-extract" + ) value = task["attributes"].get(f"{kind}-alias") if not value: value = [] @@ -348,7 +352,7 @@ def use_fetches(config, jobs): { "artifact": path, "task": f"<{label}>", - "extract": True, + "extract": should_extract.get(label, True), } ) diff --git a/taskcluster/gecko_taskgraph/transforms/job/toolchain.py b/taskcluster/gecko_taskgraph/transforms/job/toolchain.py index 715c1577c50d..c4d96269faa5 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/toolchain.py +++ b/taskcluster/gecko_taskgraph/transforms/job/toolchain.py @@ -5,7 +5,6 @@ Support for running toolchain-building jobs via dedicated scripts """ - import os import taskgraph @@ -63,6 +62,11 @@ toolchain_run_schema = Schema( "toolchain-env", description="Additional env variables to add to the worker when using this toolchain", ): {str: object}, + Optional( + "toolchain-extract", + description="Whether the toolchain should be extracted after it is fetched " + + "(default: True)", + ): bool, # Base work directory used to set up the task. Optional("workdir"): str, } @@ -157,6 +161,8 @@ def common_toolchain(config, job, taskdesc, is_docker): attributes["toolchain-alias"] = alias if "toolchain-env" in run: attributes["toolchain-env"] = run.pop("toolchain-env") + if "toolchain-extract" in run: + attributes["toolchain-extract"] = run.pop("toolchain-extract") # Allow the job to specify where artifacts come from, but add # public/build if it's not there already. diff --git a/taskcluster/kinds/toolchain/dist-toolchains.yml b/taskcluster/kinds/toolchain/dist-toolchains.yml index b3b001f5e024..c5688efa10ac 100644 --- a/taskcluster/kinds/toolchain/dist-toolchains.yml +++ b/taskcluster/kinds/toolchain/dist-toolchains.yml @@ -19,6 +19,7 @@ clang-dist-toolchain: run: arguments: ['clang'] toolchain-artifact: public/build/clang-dist-toolchain.tar.xz + toolchain-extract: false use-sccache: true fetches: toolchain: @@ -33,6 +34,7 @@ rustc-dist-toolchain: run: arguments: ['rustc'] toolchain-artifact: public/build/rustc-dist-toolchain.tar.xz + toolchain-extract: false use-sccache: true fetches: toolchain: