Bug 1850045: replace command-context with task-context r=taskgraph-reviewers,releng-reviewers,gabriel

Most of this patch is ripping `command-context` out from Gecko. The other parts are the fairly straightforward conversions from `command-context` to `task-context`.

Differential Revision: https://phabricator.services.mozilla.com/D186822

taskcluster/ci/attribution-l10n/kind.yml

@@ -7,6 +7,7 @@ loader: gecko_taskgraph.loader.transform:loader
 transforms:
     - gecko_taskgraph.transforms.split_by_locale:transforms
     - gecko_taskgraph.transforms.attribution:transforms
+    - taskgraph.transforms.task_context
     - gecko_taskgraph.transforms.job:transforms
     - gecko_taskgraph.transforms.task:transforms
 
@@ -37,6 +38,10 @@ job-defaults:
     fetches:
         repackage-signing-l10n:
             - "{locale}/target.installer.exe"
+    task-context:
+        from-file: browser/installer/attribution.yml
+        substitution-fields:
+            - run.command
     worker:
         artifacts:
             - name: public/build
@@ -62,8 +67,6 @@ job-defaults:
             # attribution of other files, which they can append themselves.
             - --input
             - /builds/worker/fetches/target.installer.exe
-        command-context:
-            from-file: browser/installer/attribution.yml
 
 jobs:
     win32-devedition/opt:

taskcluster/ci/attribution/kind.yml

@@ -6,6 +6,7 @@ loader: gecko_taskgraph.loader.transform:loader
 
 transforms:
     - gecko_taskgraph.transforms.attribution:transforms
+    - taskgraph.transforms.task_context
     - gecko_taskgraph.transforms.job:transforms
     - gecko_taskgraph.transforms.task:transforms
 
@@ -29,6 +30,10 @@ job-defaults:
         symbol: Attr
         kind: other
         tier: 1
+    task-context:
+        from-file: browser/installer/attribution.yml
+        substitution-fields:
+            - run.command
     worker:
         artifacts:
             - name: public/build
@@ -54,8 +59,6 @@ job-defaults:
             # attribution of other files, which they can append themselves.
             - --input
             - /builds/worker/fetches/target.installer.exe
-        command-context:
-            from-file: browser/installer/attribution.yml
 
 jobs:
     win32-devedition/opt:

taskcluster/ci/fuzzing/kind.yml

@@ -6,6 +6,7 @@ loader: gecko_taskgraph.loader.transform:loader
 
 transforms:
     - gecko_taskgraph.transforms.source_test:transforms
+    - taskgraph.transforms.task_context
     - gecko_taskgraph.transforms.job:transforms
     - gecko_taskgraph.transforms.task:transforms
 
@@ -23,6 +24,9 @@ job-defaults:
     treeherder:
         kind: test
         tier: 3
+    task-context:
+        from-object: {}
+        substitution-fields: []
     if-dependencies: [build]
 
 jobs:

taskcluster/ci/perftest/kind.yml

@@ -11,6 +11,7 @@ kind-dependencies:
 transforms:
     - gecko_taskgraph.transforms.perftest:transforms
     - gecko_taskgraph.transforms.source_test:transforms
+    - taskgraph.transforms.task_context
     - gecko_taskgraph.transforms.job:transforms
     - gecko_taskgraph.transforms.task:transforms
 
@@ -28,6 +29,9 @@ job-defaults:
     treeherder:
         kind: other
         tier: 3
+    task-context:
+        from-object: {}
+        substitution-fields: []
     worker:
         taskcluster-proxy: true
         max-run-time: 10800

taskcluster/ci/source-test/kind.yml

@@ -7,6 +7,7 @@ loader: gecko_taskgraph.loader.transform:loader
 transforms:
     - gecko_taskgraph.transforms.try_job:transforms
     - gecko_taskgraph.transforms.source_test:transforms
+    - taskgraph.transforms.task_context
     - gecko_taskgraph.transforms.release_notifications:transforms
     - gecko_taskgraph.transforms.job:transforms
     - gecko_taskgraph.transforms.task:transforms
@@ -38,3 +39,6 @@ jobs-from:
 job-defaults:
     attributes:
         retrigger: true
+    task-context:
+        from-object: {}
+        substitution-fields: []

taskcluster/gecko_taskgraph/test/test_transforms_job.py

@@ -107,44 +107,5 @@ def test_worker_caches(task, transform):
     validate_schema(partial_schema, taskdesc["worker"][key], "validation error")
 
 
-@pytest.mark.parametrize(
-    "workerfn", [fn for fn, *_ in job.registry["run-task"].values()]
-)
-@pytest.mark.parametrize(
-    "task",
-    (
-        {
-            "worker-type": "b-linux",
-            "run": {
-                "checkout": True,
-                "comm-checkout": False,
-                "command": "echo '{output}'",
-                "command-context": {"output": "hello", "extra": None},
-                "run-as-root": False,
-                "sparse-profile": False,
-                "tooltool-downloads": False,
-            },
-        },
-    ),
-)
-def test_run_task_command_context(task, transform, workerfn):
-    config, job_, taskdesc, _ = transform(task)
-    job_ = deepcopy(job_)
-
-    def assert_cmd(expected):
-        cmd = taskdesc["worker"]["command"]
-        while isinstance(cmd, list):
-            cmd = cmd[-1]
-        assert cmd == expected
-
-    workerfn(config, job_, taskdesc)
-    assert_cmd("echo 'hello'")
-
-    job_copy = job_.copy()
-    del job_copy["run"]["command-context"]
-    workerfn(config, job_copy, taskdesc)
-    assert_cmd("echo '{output}'")
-
-
 if __name__ == "__main__":
     main()

taskcluster/gecko_taskgraph/transforms/job/mach.py

@@ -27,10 +27,6 @@ mach_schema = Schema(
         Required("comm-checkout"): bool,
         # Base work directory used to set up the task.
         Optional("workdir"): str,
-        # Context to substitute into the command using format string
-        # substitution (e.g {value}). This is useful if certain aspects of the
-        # command need to be generated in transforms.
-        Optional("command-context"): dict,
     }
 )
 

taskcluster/gecko_taskgraph/transforms/job/run_task.py

@@ -12,9 +12,8 @@ from mozbuild.util import memoize
 from mozpack import path
 from taskgraph.util.schema import Schema
 from taskgraph.util.yaml import load_yaml
-from voluptuous import Any, Extra, Optional, Required
+from voluptuous import Any, Optional, Required
 
-from gecko_taskgraph import GECKO
 from gecko_taskgraph.transforms.job import run_job_using
 from gecko_taskgraph.transforms.job.common import add_tooltool, support_vcs_checkout
 from gecko_taskgraph.transforms.task import taskref_or_string
@@ -47,16 +46,6 @@ run_task_schema = Schema(
         # checkout arguments.  If a list, it will be passed directly; otherwise
         # it will be included in a single argument to `bash -cx`.
         Required("command"): Any([taskref_or_string], taskref_or_string),
-        # Context to substitute into the command using format string
-        # substitution (e.g {value}). This is useful if certain aspects of the
-        # command need to be generated in transforms.
-        Optional("command-context"): {
-            # If present, loads a set of context variables from an unnested yaml
-            # file. If a value is present in both the provided file and directly
-            # in command-context, the latter will take priority.
-            Optional("from-file"): str,
-            Extra: object,
-        },
         # Base work directory used to set up the task.
         Optional("workdir"): str,
         # If not false, tooltool downloads will be enabled via relengAPIProxy
@@ -112,25 +101,6 @@ def script_url(config, script):
     return f"{tc_url}/api/queue/v1/task/{task_id}/artifacts/public/{script}"
 
 
-def substitute_command_context(command_context, command):
-    from_file = command_context.pop("from-file", None)
-    full_context = {}
-    if from_file:
-        full_context = load_yaml(os.path.join(GECKO, from_file))
-    else:
-        full_context = {}
-
-    full_context.update(command_context)
-
-    if isinstance(command, list):
-        for i in range(len(command)):
-            command[i] = command[i].format(**full_context)
-    else:
-        command = command.format(**full_context)
-
-    return command
-
-
 @run_job_using(
     "docker-worker", "run-task", schema=run_task_schema, defaults=worker_defaults
 )
@@ -154,12 +124,7 @@ def docker_worker_run_task(config, job, taskdesc):
             }
         )
 
-    if run.get("command-context"):
+    run_command = run["command"]
-        run_command = substitute_command_context(
-            run.get("command-context"), run["command"]
-        )
-    else:
-        run_command = run["command"]
 
     run_cwd = run.get("cwd")
     if run_cwd and run["checkout"]:
@@ -263,11 +228,6 @@ def generic_worker_run_task(config, job, taskdesc):
                 run_command = f'"{run_command}"'
         run_command = ["bash", "-cx", run_command]
 
-    if run.get("command-context"):
-        run_command = substitute_command_context(
-            run.get("command-context"), run_command
-        )
-
     if run["comm-checkout"]:
         command.append(
             "--comm-checkout={}/comm".format(taskdesc["worker"]["env"]["GECKO_PATH"])

taskcluster/gecko_taskgraph/transforms/source_test.py

@@ -254,8 +254,13 @@ def set_base_revision_in_tgdiff(config, jobs):
             yield job
             continue
 
-        job["run"]["command-context"] = {
+        job["task-context"] = {
-            "base_rev": data["changesets"][0]["parents"][0]
+            "from-object": {
+                "base_rev": data["changesets"][0]["parents"][0],
+            },
+            "substitution-fields": [
+                "run.command",
+            ],
         }
         yield job