Bug 1751050 - Add a test to ensure params under 'taskcluster/test/params' aren't broken, r=releng-reviewers,aki

This will ensure we don't accidentally cause bustage to graph generation with
any of the parameters checked into `taskcluster/test/params`.

This was previously being (sort of) tested by the `tgdiff` task. Now that this
test exists, we no longer need to rely on it.

I removed 'always-target' from the task since this now takes ~25 min to run and
which is the new bottleneck for reviewbot turn around times.

Differential Revision: https://phabricator.services.mozilla.com/D136514

taskcluster/ci/source-test/python.yml

@@ -36,6 +36,7 @@ job-defaults:
 
 firefox-ci:
     description: taskcluster/gecko_taskgraph unit tests
+    always-target: false
     python-version: [3]
     treeherder:
         symbol: ci

taskcluster/test/python.ini

@@ -4,5 +4,6 @@ skip-if = python == 2
 
 [test_autoland.py]
 [test_autoland_backstop.py]
+[test_generate_params.py]
 [test_mach_try_auto.py]
 [test_mozilla_central.py]

taskcluster/test/test_generate_params.py

@@ -0,0 +1,58 @@
+import json
+import os
+import subprocess
+
+import pytest
+from mozunit import main
+
+from gecko_taskgraph import GECKO
+from gecko_taskgraph.taskgraph import TaskGraph
+
+pytestmark = pytest.mark.slow
+PARAMS_DIR = os.path.join(GECKO, "taskcluster", "test", "params")
+
+
+@pytest.fixture(scope="module")
+def get_graph_from_spec(tmpdir_factory):
+    outdir = tmpdir_factory.mktemp("graphs")
+
+    # Use a mach subprocess to leverage the auto parallelization of
+    # parameters when specifying a directory.
+    cmd = [
+        "./mach",
+        "taskgraph",
+        "morphed",
+        "--json",
+        f"--parameters={PARAMS_DIR}",
+        f"--output-file={outdir}/graph.json",
+    ]
+    subprocess.run(cmd, cwd=GECKO)
+    assert len(outdir.listdir()) > 0
+
+    def inner(param_spec):
+        outfile = f"{outdir}/graph_{param_spec}.json"
+        with open(outfile) as fh:
+            output = fh.read()
+            try:
+                return TaskGraph.from_json(json.loads(output))[1]
+            except ValueError:
+                return output
+
+    return inner
+
+
+@pytest.mark.parametrize(
+    "param_spec", [os.path.splitext(p)[0] for p in os.listdir(PARAMS_DIR)]
+)
+def test_generate_graphs(get_graph_from_spec, param_spec):
+    ret = get_graph_from_spec(param_spec)
+    if isinstance(ret, str):
+        print(ret)
+        pytest.fail("An exception was raised during graph generation!")
+
+    assert isinstance(ret, TaskGraph)
+    assert len(ret.tasks) > 0
+
+
+if __name__ == "__main__":
+    main()