nagai/fune
3
0
Fork 0
forked from mirrors/gecko-dev
Code Pull requests Activity

Bug 1935333 - Avoid trying to install dist toolchains when bootstrapping from a spidermonkey standalone tree. a=pascalc

Browse source 
The auto-bootstrap code gracefully handles the lack of information as to
what toolchains are available in spidermonkey standalone trees, but the
fallback code path for when toolchains are not to be unpacked doesn't
handle the situation.

By making the toolchains themselves define whether they are to be
unpacked, we can remove the fallback code path and use auto-bootstrap in
all cases.

This makes `mach artifact toolchain --from-build` stop requiring a
manual `--no-unpack`, but `--from-task` still needs it because it
can't infer that information, as it doesn't have the task graph.

Original Revision: https://phabricator.services.mozilla.com/D231342

Differential Revision: https://phabricator.services.mozilla.com/D231846
This commit is contained in:
Mike Hommey 2024-12-12 10:07:16 +00:00
parent 2ac03ecb16
commit 6707a5c1ab
10 changed files with 33 additions and 46 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

4
build/moz.configure/bootstrap.configure
View file

@ -92,6 +92,7 @@ def bootstrap_toolchain_tasks(host):
result = {
"index": t.optimization["index-search"],
"artifact": t.attributes["toolchain-artifact"],
"extract": t.attributes.get("toolchain-extract", True),
}
command = t.attributes.get("toolchain-command")
if command:
@ -219,6 +220,9 @@ def bootstrap_path(path, **kwargs):
"--from-task",
f"{task_id}:{artifact}",
]
if not task["extract"]:
command.append("--no-unpack")
elif command:
# For private local toolchains, run the associated command.
command = (

40
python/mozboot/mozboot/base.py
View file

@ -16,7 +16,6 @@ from packaging.version import Version
from mozboot import rust
from mozboot.util import (
MINIMUM_RUST_VERSION,
get_mach_virtualenv_binary,
http_download_and_save,
)
@ -332,46 +331,9 @@ class BaseBootstrapper(object):
"""
pass
def install_toolchain_artifact(self, toolchain_job, no_unpack=False):
if no_unpack:
return self.install_toolchain_artifact_impl(
self.state_dir, toolchain_job, no_unpack
)
def install_toolchain_artifact(self, toolchain_job):
bootstrap_toolchain(toolchain_job)
def install_toolchain_artifact_impl(
self, install_dir: Path, toolchain_job, no_unpack=False
):
if type(self.srcdir) is str:
mach_binary = Path(self.srcdir) / "mach"
else:
mach_binary = (self.srcdir / "mach").resolve()
if not mach_binary.exists():
raise ValueError(f"mach not found at {mach_binary}")
if not self.state_dir:
raise ValueError(
"Need a state directory (e.g. ~/.mozbuild) to download " "artifacts"
)
python_location = get_mach_virtualenv_binary()
if not python_location.exists():
raise ValueError(f"python not found at {python_location}")
cmd = [
str(python_location),
str(mach_binary),
"artifact",
"toolchain",
"--bootstrap",
"--from-build",
toolchain_job,
]
if no_unpack:
cmd += ["--no-unpack"]
subprocess.check_call(cmd, cwd=str(install_dir))
def auto_bootstrap(self, application, exclude=[]):
args = ["--with-ccache=sccache"]
if application.endswith("_artifact_mode"):

4
python/mozboot/mozboot/mozillabuild.py
View file

@ -217,8 +217,8 @@ class MozillaBuildBootstrapper(BaseBootstrapper):
def ensure_sccache_packages(self):
from mozboot import sccache
self.install_toolchain_artifact(sccache.RUSTC_DIST_TOOLCHAIN, no_unpack=True)
self.install_toolchain_artifact(sccache.CLANG_DIST_TOOLCHAIN, no_unpack=True)
self.install_toolchain_artifact(sccache.RUSTC_DIST_TOOLCHAIN)
self.install_toolchain_artifact(sccache.CLANG_DIST_TOOLCHAIN)
def _update_package_manager(self):
pass

4
python/mozboot/mozboot/osx.py
View file

@ -269,8 +269,8 @@ class OSXBootstrapper(OSXAndroidBootstrapper, BaseBootstrapper):
def ensure_sccache_packages(self):
from mozboot import sccache
self.install_toolchain_artifact(sccache.RUSTC_DIST_TOOLCHAIN, no_unpack=True)
self.install_toolchain_artifact(sccache.CLANG_DIST_TOOLCHAIN, no_unpack=True)
self.install_toolchain_artifact(sccache.RUSTC_DIST_TOOLCHAIN)
self.install_toolchain_artifact(sccache.CLANG_DIST_TOOLCHAIN)
def install_homebrew(self):
print(BREW_INSTALL)

1
python/mozbuild/mozbuild/artifact_commands.py
View file

@ -480,6 +480,7 @@ def artifact_toolchain(
)
record = ArtifactRecord(task_id, artifact_name)
record.unpack = task.attributes.get("toolchain-extract", True)
records[record.filename] = record
# Handle the list of files of the form task_id:path from --from-task.

3
python/mozbuild/mozbuild/test/configure/test_bootstrap.py
View file

@ -79,14 +79,17 @@ class TestBootstrap(BaseConfigureTest):
"toolchain-foo": {
"index": ["fake.index.foo"],
"artifact": "public/foo.artifact",
"extract": True,
},
"toolchain-linux64-bar": {
"index": ["fake.index.bar"],
"artifact": "public/bar.artifact",
"extract": True,
},
"toolchain-linux64-qux": {
"index": ["fake.index.qux"],
"artifact": "public/qux.artifact",
"extract": True,
},
},
)

5
taskcluster/docs/attributes.rst
View file

@ -243,6 +243,11 @@ toolchain-artifact
==================
For toolchain jobs, this is the path to the artifact for that toolchain.
toolchain-extract
=================
Control whether toolchain should be automatically extracted after download.
Default is true.
toolchain-alias
===============
An alias that can be used instead of the real toolchain job name in fetch

8
taskcluster/gecko_taskgraph/transforms/job/__init__.py
View file

@ -239,7 +239,7 @@ def get_attribute(dict, key, attributes, attribute_name):
"""Get `attribute_name` from the given `attributes` dict, and if there
is a corresponding value, set `key` in `dict` to that value."""
value = attributes.get(attribute_name)
if value:
if value is not None:
dict[key] = value
@ -282,6 +282,7 @@ def use_system_python(config, jobs):
def use_fetches(config, jobs):
artifact_names = {}
extra_env = {}
should_extract = {}
aliases = {}
tasks = []
@ -299,6 +300,9 @@ def use_fetches(config, jobs):
artifact_names, task["label"], task["attributes"], f"{kind}-artifact"
)
get_attribute(extra_env, task["label"], task["attributes"], f"{kind}-env")
get_attribute(
should_extract, task["label"], task["attributes"], f"{kind}-extract"
)
value = task["attributes"].get(f"{kind}-alias")
if not value:
value = []
@ -348,7 +352,7 @@ def use_fetches(config, jobs):
{
"artifact": path,
"task": f"<{label}>",
"extract": True,
"extract": should_extract.get(label, True),
}
)

8
taskcluster/gecko_taskgraph/transforms/job/toolchain.py
View file

@ -5,7 +5,6 @@
Support for running toolchain-building jobs via dedicated scripts
"""
import os
import taskgraph
@ -63,6 +62,11 @@ toolchain_run_schema = Schema(
"toolchain-env",
description="Additional env variables to add to the worker when using this toolchain",
): {str: object},
Optional(
"toolchain-extract",
description="Whether the toolchain should be extracted after it is fetched "
+ "(default: True)",
): bool,
# Base work directory used to set up the task.
Optional("workdir"): str,
}
@ -157,6 +161,8 @@ def common_toolchain(config, job, taskdesc, is_docker):
attributes["toolchain-alias"] = alias
if "toolchain-env" in run:
attributes["toolchain-env"] = run.pop("toolchain-env")
if "toolchain-extract" in run:
attributes["toolchain-extract"] = run.pop("toolchain-extract")
# Allow the job to specify where artifacts come from, but add
# public/build if it's not there already.

2
taskcluster/kinds/toolchain/dist-toolchains.yml
View file

@ -19,6 +19,7 @@ clang-dist-toolchain:
run:
arguments: ['clang']
toolchain-artifact: public/build/clang-dist-toolchain.tar.xz
toolchain-extract: false
use-sccache: true
fetches:
toolchain:
@ -33,6 +34,7 @@ rustc-dist-toolchain:
run:
arguments: ['rustc']
toolchain-artifact: public/build/rustc-dist-toolchain.tar.xz
toolchain-extract: false
use-sccache: true
fetches:
toolchain: