diff --git a/taskcluster/ci/attribution-l10n/kind.yml b/taskcluster/ci/attribution-l10n/kind.yml index 34cddec74d62..b6962a598674 100644 --- a/taskcluster/ci/attribution-l10n/kind.yml +++ b/taskcluster/ci/attribution-l10n/kind.yml @@ -7,6 +7,7 @@ loader: gecko_taskgraph.loader.transform:loader transforms: - gecko_taskgraph.transforms.split_by_locale:transforms - gecko_taskgraph.transforms.attribution:transforms + - taskgraph.transforms.task_context - gecko_taskgraph.transforms.job:transforms - gecko_taskgraph.transforms.task:transforms @@ -37,6 +38,10 @@ job-defaults: fetches: repackage-signing-l10n: - "{locale}/target.installer.exe" + task-context: + from-file: browser/installer/attribution.yml + substitution-fields: + - run.command worker: artifacts: - name: public/build @@ -62,8 +67,6 @@ job-defaults: # attribution of other files, which they can append themselves. - --input - /builds/worker/fetches/target.installer.exe - command-context: - from-file: browser/installer/attribution.yml jobs: win32-devedition/opt: diff --git a/taskcluster/ci/attribution/kind.yml b/taskcluster/ci/attribution/kind.yml index 3d9c903910ff..cb097af47396 100644 --- a/taskcluster/ci/attribution/kind.yml +++ b/taskcluster/ci/attribution/kind.yml @@ -6,6 +6,7 @@ loader: gecko_taskgraph.loader.transform:loader transforms: - gecko_taskgraph.transforms.attribution:transforms + - taskgraph.transforms.task_context - gecko_taskgraph.transforms.job:transforms - gecko_taskgraph.transforms.task:transforms @@ -29,6 +30,10 @@ job-defaults: symbol: Attr kind: other tier: 1 + task-context: + from-file: browser/installer/attribution.yml + substitution-fields: + - run.command worker: artifacts: - name: public/build @@ -54,8 +59,6 @@ job-defaults: # attribution of other files, which they can append themselves. - --input - /builds/worker/fetches/target.installer.exe - command-context: - from-file: browser/installer/attribution.yml jobs: win32-devedition/opt: diff --git a/taskcluster/ci/fuzzing/kind.yml b/taskcluster/ci/fuzzing/kind.yml index 1a0e715fd9c3..68e212bb9c86 100644 --- a/taskcluster/ci/fuzzing/kind.yml +++ b/taskcluster/ci/fuzzing/kind.yml @@ -6,6 +6,7 @@ loader: gecko_taskgraph.loader.transform:loader transforms: - gecko_taskgraph.transforms.source_test:transforms + - taskgraph.transforms.task_context - gecko_taskgraph.transforms.job:transforms - gecko_taskgraph.transforms.task:transforms @@ -23,6 +24,9 @@ job-defaults: treeherder: kind: test tier: 3 + task-context: + from-object: {} + substitution-fields: [] if-dependencies: [build] jobs: diff --git a/taskcluster/ci/perftest/kind.yml b/taskcluster/ci/perftest/kind.yml index e93c7ada3949..5d8f253653c9 100644 --- a/taskcluster/ci/perftest/kind.yml +++ b/taskcluster/ci/perftest/kind.yml @@ -11,6 +11,7 @@ kind-dependencies: transforms: - gecko_taskgraph.transforms.perftest:transforms - gecko_taskgraph.transforms.source_test:transforms + - taskgraph.transforms.task_context - gecko_taskgraph.transforms.job:transforms - gecko_taskgraph.transforms.task:transforms @@ -28,6 +29,9 @@ job-defaults: treeherder: kind: other tier: 3 + task-context: + from-object: {} + substitution-fields: [] worker: taskcluster-proxy: true max-run-time: 10800 diff --git a/taskcluster/ci/source-test/kind.yml b/taskcluster/ci/source-test/kind.yml index 984f0648d336..f49b511ded89 100644 --- a/taskcluster/ci/source-test/kind.yml +++ b/taskcluster/ci/source-test/kind.yml @@ -7,6 +7,7 @@ loader: gecko_taskgraph.loader.transform:loader transforms: - gecko_taskgraph.transforms.try_job:transforms - gecko_taskgraph.transforms.source_test:transforms + - taskgraph.transforms.task_context - gecko_taskgraph.transforms.release_notifications:transforms - gecko_taskgraph.transforms.job:transforms - gecko_taskgraph.transforms.task:transforms @@ -38,3 +39,6 @@ jobs-from: job-defaults: attributes: retrigger: true + task-context: + from-object: {} + substitution-fields: [] diff --git a/taskcluster/gecko_taskgraph/test/test_transforms_job.py b/taskcluster/gecko_taskgraph/test/test_transforms_job.py index 8bf85dcf62c1..b032307ea6b6 100644 --- a/taskcluster/gecko_taskgraph/test/test_transforms_job.py +++ b/taskcluster/gecko_taskgraph/test/test_transforms_job.py @@ -107,44 +107,5 @@ def test_worker_caches(task, transform): validate_schema(partial_schema, taskdesc["worker"][key], "validation error") -@pytest.mark.parametrize( - "workerfn", [fn for fn, *_ in job.registry["run-task"].values()] -) -@pytest.mark.parametrize( - "task", - ( - { - "worker-type": "b-linux", - "run": { - "checkout": True, - "comm-checkout": False, - "command": "echo '{output}'", - "command-context": {"output": "hello", "extra": None}, - "run-as-root": False, - "sparse-profile": False, - "tooltool-downloads": False, - }, - }, - ), -) -def test_run_task_command_context(task, transform, workerfn): - config, job_, taskdesc, _ = transform(task) - job_ = deepcopy(job_) - - def assert_cmd(expected): - cmd = taskdesc["worker"]["command"] - while isinstance(cmd, list): - cmd = cmd[-1] - assert cmd == expected - - workerfn(config, job_, taskdesc) - assert_cmd("echo 'hello'") - - job_copy = job_.copy() - del job_copy["run"]["command-context"] - workerfn(config, job_copy, taskdesc) - assert_cmd("echo '{output}'") - - if __name__ == "__main__": main() diff --git a/taskcluster/gecko_taskgraph/transforms/job/mach.py b/taskcluster/gecko_taskgraph/transforms/job/mach.py index a418b44794c7..afd8acdad1f7 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/mach.py +++ b/taskcluster/gecko_taskgraph/transforms/job/mach.py @@ -27,10 +27,6 @@ mach_schema = Schema( Required("comm-checkout"): bool, # Base work directory used to set up the task. Optional("workdir"): str, - # Context to substitute into the command using format string - # substitution (e.g {value}). This is useful if certain aspects of the - # command need to be generated in transforms. - Optional("command-context"): dict, } ) diff --git a/taskcluster/gecko_taskgraph/transforms/job/run_task.py b/taskcluster/gecko_taskgraph/transforms/job/run_task.py index 201c0b825a91..ce411693d335 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/run_task.py +++ b/taskcluster/gecko_taskgraph/transforms/job/run_task.py @@ -12,9 +12,8 @@ from mozbuild.util import memoize from mozpack import path from taskgraph.util.schema import Schema from taskgraph.util.yaml import load_yaml -from voluptuous import Any, Extra, Optional, Required +from voluptuous import Any, Optional, Required -from gecko_taskgraph import GECKO from gecko_taskgraph.transforms.job import run_job_using from gecko_taskgraph.transforms.job.common import add_tooltool, support_vcs_checkout from gecko_taskgraph.transforms.task import taskref_or_string @@ -47,16 +46,6 @@ run_task_schema = Schema( # checkout arguments. If a list, it will be passed directly; otherwise # it will be included in a single argument to `bash -cx`. Required("command"): Any([taskref_or_string], taskref_or_string), - # Context to substitute into the command using format string - # substitution (e.g {value}). This is useful if certain aspects of the - # command need to be generated in transforms. - Optional("command-context"): { - # If present, loads a set of context variables from an unnested yaml - # file. If a value is present in both the provided file and directly - # in command-context, the latter will take priority. - Optional("from-file"): str, - Extra: object, - }, # Base work directory used to set up the task. Optional("workdir"): str, # If not false, tooltool downloads will be enabled via relengAPIProxy @@ -112,25 +101,6 @@ def script_url(config, script): return f"{tc_url}/api/queue/v1/task/{task_id}/artifacts/public/{script}" -def substitute_command_context(command_context, command): - from_file = command_context.pop("from-file", None) - full_context = {} - if from_file: - full_context = load_yaml(os.path.join(GECKO, from_file)) - else: - full_context = {} - - full_context.update(command_context) - - if isinstance(command, list): - for i in range(len(command)): - command[i] = command[i].format(**full_context) - else: - command = command.format(**full_context) - - return command - - @run_job_using( "docker-worker", "run-task", schema=run_task_schema, defaults=worker_defaults ) @@ -154,12 +124,7 @@ def docker_worker_run_task(config, job, taskdesc): } ) - if run.get("command-context"): - run_command = substitute_command_context( - run.get("command-context"), run["command"] - ) - else: - run_command = run["command"] + run_command = run["command"] run_cwd = run.get("cwd") if run_cwd and run["checkout"]: @@ -263,11 +228,6 @@ def generic_worker_run_task(config, job, taskdesc): run_command = f'"{run_command}"' run_command = ["bash", "-cx", run_command] - if run.get("command-context"): - run_command = substitute_command_context( - run.get("command-context"), run_command - ) - if run["comm-checkout"]: command.append( "--comm-checkout={}/comm".format(taskdesc["worker"]["env"]["GECKO_PATH"]) diff --git a/taskcluster/gecko_taskgraph/transforms/source_test.py b/taskcluster/gecko_taskgraph/transforms/source_test.py index e5359ebd8bee..5c561e8114ff 100644 --- a/taskcluster/gecko_taskgraph/transforms/source_test.py +++ b/taskcluster/gecko_taskgraph/transforms/source_test.py @@ -254,8 +254,13 @@ def set_base_revision_in_tgdiff(config, jobs): yield job continue - job["run"]["command-context"] = { - "base_rev": data["changesets"][0]["parents"][0] + job["task-context"] = { + "from-object": { + "base_rev": data["changesets"][0]["parents"][0], + }, + "substitution-fields": [ + "run.command", + ], } yield job