Backed out changeset a9befcc1f72b (bug 1752111) for causint l10n bumper bustages on autoland.

This commit is contained in:
Sandor Molnar 2022-08-25 01:40:25 +03:00
parent 6916e07b1a
commit 413fd984e6
14 changed files with 65 additions and 98 deletions

View file

@ -29,6 +29,8 @@ jobs:
- mozilla-central - mozilla-central
when: when:
by-project: by-project:
# `l10n-bumper` job should have enough time to finish before this
# job runs
mozilla-central: [{hour: 10, minute: 0}, {hour: 22, minute: 0}] mozilla-central: [{hour: 10, minute: 0}, {hour: 22, minute: 0}]
# No default # No default
@ -275,14 +277,12 @@ jobs:
treeherder-symbol: l10n-bump treeherder-symbol: l10n-bump
target-tasks-method: l10n_bump target-tasks-method: l10n_bump
run-on-projects: run-on-projects:
- autoland
- mozilla-central - mozilla-central
- mozilla-beta - mozilla-beta
when: when:
by-project: by-project:
# 3h15m before launch of `nightly-desktop` # 3h15m before launch of `nightly-desktop`
autoland: [{hour: 6, minute: 45}, {hour: 18, minute: 45}] mozilla-central: [{hour: 6, minute: 45}, {hour: 18, minute: 45}]
mozilla-central: []
# 3h15m before launch of `daily-releases` # 3h15m before launch of `daily-releases`
mozilla-beta: [{hour: 18, minute: 45}] mozilla-beta: [{hour: 18, minute: 45}]
# No default # No default

View file

@ -555,10 +555,7 @@ workers:
worker-type: worker-type:
by-release-level: by-release-level:
production: '{trust-domain}-3-tree' production: '{trust-domain}-3-tree'
staging: staging: '{trust-domain}-1-tree'
by-project:
autoland: '{trust-domain}-3-tree'
default: '{trust-domain}-1-tree'
tree-dev: tree-dev:
provisioner: scriptworker-k8s provisioner: scriptworker-k8s
implementation: treescript implementation: treescript

View file

@ -8,6 +8,7 @@ transforms:
- gecko_taskgraph.transforms.release_version_bump:transforms - gecko_taskgraph.transforms.release_version_bump:transforms
- gecko_taskgraph.transforms.task:transforms - gecko_taskgraph.transforms.task:transforms
job-defaults: job-defaults:
worker-type: tree worker-type: tree
worker: worker:
@ -18,20 +19,15 @@ job-defaults:
by-project: by-project:
mozilla-beta: false mozilla-beta: false
default: true default: true
ignore-closed-tree: ignore-closed-tree: false
by-project:
autoland: true
default: false
push: push:
by-project: by-project:
mozilla-(central|beta): true mozilla-(central|beta): true
autoland: true
default: false default: false
source-repo: source-repo:
by-release-type: by-release-type:
beta: https://hg.mozilla.org/releases/mozilla-beta beta: https://hg.mozilla.org/releases/mozilla-beta
nightly: https://hg.mozilla.org/mozilla-central default: https://hg.mozilla.org/mozilla-central
default: https://hg.mozilla.org/integration/autoland
l10n-bump-info: l10n-bump-info:
by-release-type: by-release-type:
# XXX whenever the `beta` config changes, make sure to make # XXX whenever the `beta` config changes, make sure to make
@ -56,11 +52,8 @@ job-defaults:
- win64-devedition - win64-devedition
- win64-aarch64 - win64-aarch64
- win64-aarch64-devedition - win64-aarch64-devedition
platform-configs: platform-configs: [{
[ "platforms": [
{
"platforms":
[
"linux", "linux",
"linux-devedition", "linux-devedition",
"linux64", "linux64",
@ -75,8 +68,7 @@ job-defaults:
"win64-aarch64-devedition", "win64-aarch64-devedition",
], ],
"path": "browser/locales/shipped-locales", "path": "browser/locales/shipped-locales",
}, }]
]
default: default:
name: Firefox l10n changesets name: Firefox l10n changesets
path: browser/locales/l10n-changesets.json path: browser/locales/l10n-changesets.json
@ -97,11 +89,8 @@ job-defaults:
- win64-devedition - win64-devedition
- win64-aarch64 - win64-aarch64
- win64-aarch64-devedition - win64-aarch64-devedition
platform-configs: platform-configs: [{
[ "platforms": [
{
"platforms":
[
"linux", "linux",
"linux-devedition", "linux-devedition",
"linux64", "linux64",
@ -116,8 +105,7 @@ job-defaults:
"win64-aarch64-devedition", "win64-aarch64-devedition",
], ],
"path": "browser/locales/all-locales", "path": "browser/locales/all-locales",
}, }]
]
jobs: jobs:
l10n-bumper: l10n-bumper:

View file

@ -107,7 +107,7 @@ graph_config_schema = Schema(
Required("implementation"): str, Required("implementation"): str,
Required("os"): str, Required("os"): str,
Required("worker-type"): optionally_keyed_by( Required("worker-type"): optionally_keyed_by(
"level", "release-level", "project", str "level", "release-level", str
), ),
} }
}, },

View file

@ -29,6 +29,7 @@ from taskgraph.graph import Graph
from taskgraph.task import Task from taskgraph.task import Task
from taskgraph.taskgraph import TaskGraph from taskgraph.taskgraph import TaskGraph
from .util.attributes import release_level
from .util.workertypes import get_worker_type from .util.workertypes import get_worker_type
here = os.path.abspath(os.path.dirname(__file__)) here = os.path.abspath(os.path.dirname(__file__))
@ -70,9 +71,7 @@ def derive_misc_task(
image_taskid = label_to_taskid["docker-image-" + image] image_taskid = label_to_taskid["docker-image-" + image]
provisioner_id, worker_type = get_worker_type( provisioner_id, worker_type = get_worker_type(
graph_config, graph_config, "misc", parameters["level"], release_level(parameters["project"])
parameters,
"misc",
) )
deps = copy.copy(dependencies) deps = copy.copy(dependencies)

View file

@ -45,7 +45,6 @@ def config():
"head_repository": "http://hg.example.com", "head_repository": "http://hg.example.com",
"head_rev": "abcdef", "head_rev": "abcdef",
"level": 1, "level": 1,
"project": "example",
} }
) )
return TransformConfig( return TransformConfig(

View file

@ -27,7 +27,7 @@ def set_defaults(config, jobs):
job["treeherder"].setdefault("kind", "build") job["treeherder"].setdefault("kind", "build")
job["treeherder"].setdefault("tier", 1) job["treeherder"].setdefault("tier", 1)
_, worker_os = worker_type_implementation( _, worker_os = worker_type_implementation(
config.graph_config, config.params, job["worker-type"] config.graph_config, job["worker-type"]
) )
worker = job.setdefault("worker", {}) worker = job.setdefault("worker", {})
worker.setdefault("env", {}) worker.setdefault("env", {})
@ -172,7 +172,7 @@ def use_profile_data(config, jobs):
job["worker"]["env"].update({"TASKCLUSTER_PGO_PROFILE_USE": "1"}) job["worker"]["env"].update({"TASKCLUSTER_PGO_PROFILE_USE": "1"})
_, worker_os = worker_type_implementation( _, worker_os = worker_type_implementation(
config.graph_config, config.params, job["worker-type"] config.graph_config, job["worker-type"]
) )
if worker_os == "linux": if worker_os == "linux":
# LTO linkage needs more open files than the default from run-task. # LTO linkage needs more open files than the default from run-task.

View file

@ -134,9 +134,7 @@ def rewrite_when_to_optimization(config, jobs):
@transforms.add @transforms.add
def set_implementation(config, jobs): def set_implementation(config, jobs):
for job in jobs: for job in jobs:
impl, os = worker_type_implementation( impl, os = worker_type_implementation(config.graph_config, job["worker-type"])
config.graph_config, config.params, job["worker-type"]
)
if os: if os:
job.setdefault("tags", {})["os"] = os job.setdefault("tags", {})["os"] = os
if impl: if impl:
@ -166,7 +164,7 @@ def add_resource_monitor(config, jobs):
for job in jobs: for job in jobs:
if job.get("attributes", {}).get("resource-monitor"): if job.get("attributes", {}).get("resource-monitor"):
worker_implementation, worker_os = worker_type_implementation( worker_implementation, worker_os = worker_type_implementation(
config.graph_config, config.params, job["worker-type"] config.graph_config, job["worker-type"]
) )
# Normalise worker os so that linux-bitbar and similar use linux tools. # Normalise worker os so that linux-bitbar and similar use linux tools.
worker_os = worker_os.split("-")[0] worker_os = worker_os.split("-")[0]

View file

@ -21,12 +21,7 @@ def handle_keyed_by(config, tasks):
] ]
for task in tasks: for task in tasks:
fields = default_fields[:] fields = default_fields[:]
for additional_field in ( for additional_field in ("l10n-bump-info", "source-repo", "dontbuild"):
"l10n-bump-info",
"source-repo",
"dontbuild",
"ignore-closed-tree",
):
if additional_field in task["worker"]: if additional_field in task["worker"]:
fields.append(f"worker.{additional_field}") fields.append(f"worker.{additional_field}")
for field in fields: for field in fields:

View file

@ -475,7 +475,7 @@ def make_job_description(config, jobs):
worker["artifacts"] = _generate_task_output_files( worker["artifacts"] = _generate_task_output_files(
dep_job, dep_job,
worker_type_implementation(config.graph_config, config.params, worker_type), worker_type_implementation(config.graph_config, worker_type),
repackage_config=repackage_config, repackage_config=repackage_config,
locale=locale, locale=locale,
) )

View file

@ -177,7 +177,7 @@ def make_job_description(config, jobs):
worker["artifacts"] = _generate_task_output_files( worker["artifacts"] = _generate_task_output_files(
dep_job, dep_job,
worker_type_implementation(config.graph_config, config.params, worker_type), worker_type_implementation(config.graph_config, worker_type),
repackage_config, repackage_config,
partner=repack_id, partner=repack_id,
) )

View file

@ -1446,9 +1446,7 @@ def set_implementation(config, tasks):
yield task yield task
continue continue
impl, os = worker_type_implementation( impl, os = worker_type_implementation(config.graph_config, task["worker-type"])
config.graph_config, config.params, task["worker-type"]
)
tags = task.setdefault("tags", {}) tags = task.setdefault("tags", {})
tags["worker-implementation"] = impl tags["worker-implementation"] = impl
@ -1859,8 +1857,9 @@ def build_task(config, tasks):
else: else:
provisioner_id, worker_type = get_worker_type( provisioner_id, worker_type = get_worker_type(
config.graph_config, config.graph_config,
config.params,
task["worker-type"], task["worker-type"],
level=level,
release_level=release_level(config.params["project"]),
) )
task["worker-type"] = "/".join([provisioner_id, worker_type]) task["worker-type"] = "/".join([provisioner_id, worker_type])
project = config.params["project"] project = config.params["project"]

View file

@ -2,7 +2,7 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this # License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/. # file, You can obtain one at http://mozilla.org/MPL/2.0/.
from gecko_taskgraph.util.attributes import release_level as _release_level
from mozbuild.util import memoize from mozbuild.util import memoize
from taskgraph.util.attributes import keymatch from taskgraph.util.attributes import keymatch
from taskgraph.util.keyed_by import evaluate_keyed_by from taskgraph.util.keyed_by import evaluate_keyed_by
@ -20,7 +20,7 @@ WORKER_TYPES = {
@memoize @memoize
def _get(graph_config, alias, level, release_level, project): def _get(graph_config, alias, level, release_level):
"""Get the configuration for this worker_type alias: {provisioner, """Get the configuration for this worker_type alias: {provisioner,
worker-type, implementation, os}""" worker-type, implementation, os}"""
level = str(level) level = str(level)
@ -61,13 +61,10 @@ def _get(graph_config, alias, level, release_level, project):
"alias": alias, "alias": alias,
} }
) )
attrs = {"level": level, "release-level": release_level}
if project:
attrs["project"] = project
worker_config["worker-type"] = evaluate_keyed_by( worker_config["worker-type"] = evaluate_keyed_by(
worker_config["worker-type"], worker_config["worker-type"],
f"worker-type alias {alias} field worker-type", f"worker-type alias {alias} field worker-type",
attrs, {"level": level, "release-level": release_level},
).format( ).format(
**{ **{
"trust-domain": graph_config["trust-domain"], "trust-domain": graph_config["trust-domain"],
@ -79,26 +76,18 @@ def _get(graph_config, alias, level, release_level, project):
return worker_config return worker_config
def worker_type_implementation(graph_config, parameters, worker_type): def worker_type_implementation(graph_config, worker_type):
"""Get the worker implementation and OS for the given workerType, where the """Get the worker implementation and OS for the given workerType, where the
OS represents the host system, not the target OS, in the case of OS represents the host system, not the target OS, in the case of
cross-compiles.""" cross-compiles."""
worker_config = _get( worker_config = _get(graph_config, worker_type, "1", "staging")
graph_config, worker_type, "1", "staging", parameters["project"]
)
return worker_config["implementation"], worker_config.get("os") return worker_config["implementation"], worker_config.get("os")
def get_worker_type(graph_config, parameters, worker_type): def get_worker_type(graph_config, worker_type, level, release_level):
""" """
Get the worker type provisioner and worker-type, optionally evaluating Get the worker type provisioner and worker-type, optionally evaluating
aliases from the graph config. aliases from the graph config.
""" """
worker_config = _get( worker_config = _get(graph_config, worker_type, level, release_level)
graph_config,
worker_type,
parameters["level"],
_release_level(parameters.get("project")),
parameters.get("project"),
)
return worker_config["provisioner"], worker_config["worker-type"] return worker_config["provisioner"], worker_config["worker-type"]

View file

@ -505,7 +505,10 @@ class WorkerOverrides(TryConfig):
) )
sys.exit(1) sys.exit(1)
provisioner, worker_type = get_worker_type( provisioner, worker_type = get_worker_type(
graph_config, worker_type=alias, parameters={"level": "1"} graph_config,
alias,
level="1",
release_level="staging",
) )
overrides[alias] = "{provisioner}/{worker_type}{suffix}".format( overrides[alias] = "{provisioner}/{worker_type}{suffix}".format(
provisioner=provisioner, worker_type=worker_type, suffix=suffix provisioner=provisioner, worker_type=worker_type, suffix=suffix