forked from mirrors/gecko-dev
765 lines
24 KiB
Python
765 lines
24 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
import argparse
|
|
import atexit
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import traceback
|
|
from collections import namedtuple
|
|
from concurrent.futures import ProcessPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
from typing import Any, List
|
|
|
|
import appdirs
|
|
import yaml
|
|
|
|
Command = namedtuple("Command", ["func", "args", "kwargs", "defaults"])
|
|
commands = {}
|
|
|
|
|
|
def command(*args, **kwargs):
|
|
defaults = kwargs.pop("defaults", {})
|
|
|
|
def decorator(func):
|
|
commands[args[0]] = Command(func, args, kwargs, defaults)
|
|
return func
|
|
|
|
return decorator
|
|
|
|
|
|
def argument(*args, **kwargs):
|
|
def decorator(func):
|
|
if not hasattr(func, "args"):
|
|
func.args = []
|
|
func.args.append((args, kwargs))
|
|
return func
|
|
|
|
return decorator
|
|
|
|
|
|
def format_taskgraph_labels(taskgraph):
|
|
return "\n".join(
|
|
sorted(
|
|
taskgraph.tasks[index].label for index in taskgraph.graph.visit_postorder()
|
|
)
|
|
)
|
|
|
|
|
|
def format_taskgraph_json(taskgraph):
|
|
return json.dumps(
|
|
taskgraph.to_json(), sort_keys=True, indent=2, separators=(",", ": ")
|
|
)
|
|
|
|
|
|
def format_taskgraph_yaml(taskgraph):
|
|
return yaml.safe_dump(taskgraph.to_json(), default_flow_style=False)
|
|
|
|
|
|
def get_filtered_taskgraph(taskgraph, tasksregex):
|
|
"""
|
|
Filter all the tasks on basis of a regular expression
|
|
and returns a new TaskGraph object
|
|
"""
|
|
from taskgraph.graph import Graph
|
|
from taskgraph.taskgraph import TaskGraph
|
|
|
|
# return original taskgraph if no regular expression is passed
|
|
if not tasksregex:
|
|
return taskgraph
|
|
named_links_dict = taskgraph.graph.named_links_dict()
|
|
filteredtasks = {}
|
|
filterededges = set()
|
|
regexprogram = re.compile(tasksregex)
|
|
|
|
for key in taskgraph.graph.visit_postorder():
|
|
task = taskgraph.tasks[key]
|
|
if regexprogram.match(task.label):
|
|
filteredtasks[key] = task
|
|
for depname, dep in named_links_dict[key].items():
|
|
if regexprogram.match(dep):
|
|
filterededges.add((key, dep, depname))
|
|
filtered_taskgraph = TaskGraph(
|
|
filteredtasks, Graph(set(filteredtasks), filterededges)
|
|
)
|
|
return filtered_taskgraph
|
|
|
|
|
|
FORMAT_METHODS = {
|
|
"labels": format_taskgraph_labels,
|
|
"json": format_taskgraph_json,
|
|
"yaml": format_taskgraph_yaml,
|
|
}
|
|
|
|
|
|
def get_taskgraph_generator(root, parameters):
|
|
"""Helper function to make testing a little easier."""
|
|
from taskgraph.generator import TaskGraphGenerator
|
|
|
|
return TaskGraphGenerator(root_dir=root, parameters=parameters)
|
|
|
|
|
|
def format_taskgraph(options, parameters, logfile=None):
|
|
import taskgraph
|
|
from taskgraph.parameters import parameters_loader
|
|
|
|
if logfile:
|
|
handler = logging.FileHandler(logfile, mode="w")
|
|
if logging.root.handlers:
|
|
oldhandler = logging.root.handlers[-1]
|
|
logging.root.removeHandler(oldhandler)
|
|
handler.setFormatter(oldhandler.formatter)
|
|
logging.root.addHandler(handler)
|
|
|
|
if options["fast"]:
|
|
taskgraph.fast = True
|
|
|
|
if isinstance(parameters, str):
|
|
parameters = parameters_loader(
|
|
parameters,
|
|
overrides={"target-kind": options.get("target_kind")},
|
|
strict=False,
|
|
)
|
|
|
|
tgg = get_taskgraph_generator(options.get("root"), parameters)
|
|
|
|
tg = getattr(tgg, options["graph_attr"])
|
|
tg = get_filtered_taskgraph(tg, options["tasks_regex"])
|
|
format_method = FORMAT_METHODS[options["format"] or "labels"]
|
|
return format_method(tg)
|
|
|
|
|
|
def dump_output(out, path=None, params_spec=None):
|
|
from taskgraph.parameters import Parameters
|
|
|
|
params_name = Parameters.format_spec(params_spec)
|
|
fh = None
|
|
if path:
|
|
# Substitute params name into file path if necessary
|
|
if params_spec and "{params}" not in path:
|
|
name, ext = os.path.splitext(path)
|
|
name += "_{params}"
|
|
path = name + ext
|
|
|
|
path = path.format(params=params_name)
|
|
fh = open(path, "w")
|
|
else:
|
|
print(
|
|
"Dumping result with parameters from {}:".format(params_name),
|
|
file=sys.stderr,
|
|
)
|
|
print(out + "\n", file=fh)
|
|
|
|
|
|
def generate_taskgraph(options, parameters, logdir):
|
|
from taskgraph.parameters import Parameters
|
|
|
|
def logfile(spec):
|
|
"""Determine logfile given a parameters specification."""
|
|
if logdir is None:
|
|
return None
|
|
return os.path.join(
|
|
logdir,
|
|
"{}_{}.log".format(options["graph_attr"], Parameters.format_spec(spec)),
|
|
)
|
|
|
|
# Don't bother using futures if there's only one parameter. This can make
|
|
# tracebacks a little more readable and avoids additional process overhead.
|
|
if len(parameters) == 1:
|
|
spec = parameters[0]
|
|
out = format_taskgraph(options, spec, logfile(spec))
|
|
dump_output(out, options["output_file"])
|
|
return
|
|
|
|
futures = {}
|
|
with ProcessPoolExecutor() as executor:
|
|
for spec in parameters:
|
|
f = executor.submit(format_taskgraph, options, spec, logfile(spec))
|
|
futures[f] = spec
|
|
|
|
for future in as_completed(futures):
|
|
output_file = options["output_file"]
|
|
spec = futures[future]
|
|
e = future.exception()
|
|
if e:
|
|
out = "".join(traceback.format_exception(type(e), e, e.__traceback__))
|
|
if options["diff"]:
|
|
# Dump to console so we don't accidentally diff the tracebacks.
|
|
output_file = None
|
|
else:
|
|
out = future.result()
|
|
|
|
dump_output(
|
|
out,
|
|
path=output_file,
|
|
params_spec=spec if len(parameters) > 1 else None,
|
|
)
|
|
|
|
|
|
@command(
|
|
"tasks",
|
|
help="Show all tasks in the taskgraph.",
|
|
defaults={"graph_attr": "full_task_set"},
|
|
)
|
|
@command(
|
|
"full", help="Show the full taskgraph.", defaults={"graph_attr": "full_task_graph"}
|
|
)
|
|
@command(
|
|
"target",
|
|
help="Show the set of target tasks.",
|
|
defaults={"graph_attr": "target_task_set"},
|
|
)
|
|
@command(
|
|
"target-graph",
|
|
help="Show the target graph.",
|
|
defaults={"graph_attr": "target_task_graph"},
|
|
)
|
|
@command(
|
|
"optimized",
|
|
help="Show the optimized graph.",
|
|
defaults={"graph_attr": "optimized_task_graph"},
|
|
)
|
|
@command(
|
|
"morphed",
|
|
help="Show the morphed graph.",
|
|
defaults={"graph_attr": "morphed_task_graph"},
|
|
)
|
|
@argument("--root", "-r", help="root of the taskgraph definition relative to topsrcdir")
|
|
@argument("--quiet", "-q", action="store_true", help="suppress all logging output")
|
|
@argument(
|
|
"--verbose", "-v", action="store_true", help="include debug-level logging output"
|
|
)
|
|
@argument(
|
|
"--json",
|
|
"-J",
|
|
action="store_const",
|
|
dest="format",
|
|
const="json",
|
|
help="Output task graph as a JSON object",
|
|
)
|
|
@argument(
|
|
"--yaml",
|
|
"-Y",
|
|
action="store_const",
|
|
dest="format",
|
|
const="yaml",
|
|
help="Output task graph as a YAML object",
|
|
)
|
|
@argument(
|
|
"--labels",
|
|
"-L",
|
|
action="store_const",
|
|
dest="format",
|
|
const="labels",
|
|
help="Output the label for each task in the task graph (default)",
|
|
)
|
|
@argument(
|
|
"--parameters",
|
|
"-p",
|
|
default=None,
|
|
action="append",
|
|
help="Parameters to use for the generation. Can be a path to file (.yml or "
|
|
".json; see `taskcluster/docs/parameters.rst`), a directory (containing "
|
|
"parameters files), a url, of the form `project=mozilla-central` to download "
|
|
"latest parameters file for the specified project from CI, or of the form "
|
|
"`task-id=<decision task id>` to download parameters from the specified "
|
|
"decision task. Can be specified multiple times, in which case multiple "
|
|
"generations will happen from the same invocation (one per parameters "
|
|
"specified).",
|
|
)
|
|
@argument(
|
|
"--no-optimize",
|
|
dest="optimize",
|
|
action="store_false",
|
|
default="true",
|
|
help="do not remove tasks from the graph that are found in the "
|
|
"index (a.k.a. optimize the graph)",
|
|
)
|
|
@argument(
|
|
"-o",
|
|
"--output-file",
|
|
default=None,
|
|
help="file path to store generated output.",
|
|
)
|
|
@argument(
|
|
"--tasks-regex",
|
|
"--tasks",
|
|
default=None,
|
|
help="only return tasks with labels matching this regular " "expression.",
|
|
)
|
|
@argument(
|
|
"--target-kind",
|
|
default=None,
|
|
help="only return tasks that are of the given kind, or their dependencies.",
|
|
)
|
|
@argument(
|
|
"-F",
|
|
"--fast",
|
|
default=False,
|
|
action="store_true",
|
|
help="enable fast task generation for local debugging.",
|
|
)
|
|
@argument(
|
|
"--diff",
|
|
const="default",
|
|
nargs="?",
|
|
default=None,
|
|
help="Generate and diff the current taskgraph against another revision. "
|
|
"Without args the base revision will be used. A revision specifier such as "
|
|
"the hash or `.~1` (hg) or `HEAD~1` (git) can be used as well.",
|
|
)
|
|
def show_taskgraph(options):
|
|
from mozversioncontrol import get_repository_object as get_repository
|
|
from taskgraph.parameters import Parameters, parameters_loader
|
|
|
|
if options.pop("verbose", False):
|
|
logging.root.setLevel(logging.DEBUG)
|
|
|
|
repo = None
|
|
cur_ref = None
|
|
diffdir = None
|
|
output_file = options["output_file"]
|
|
|
|
if options["diff"]:
|
|
repo = get_repository(os.getcwd())
|
|
|
|
if not repo.working_directory_clean():
|
|
print(
|
|
"abort: can't diff taskgraph with dirty working directory",
|
|
file=sys.stderr,
|
|
)
|
|
return 1
|
|
|
|
# We want to return the working directory to the current state
|
|
# as best we can after we're done. In all known cases, using
|
|
# branch or bookmark (which are both available on the VCS object)
|
|
# as `branch` is preferable to a specific revision.
|
|
cur_ref = repo.branch or repo.head_ref[:12]
|
|
|
|
diffdir = tempfile.mkdtemp()
|
|
atexit.register(
|
|
shutil.rmtree, diffdir
|
|
) # make sure the directory gets cleaned up
|
|
options["output_file"] = os.path.join(
|
|
diffdir, f"{options['graph_attr']}_{cur_ref}"
|
|
)
|
|
print(f"Generating {options['graph_attr']} @ {cur_ref}", file=sys.stderr)
|
|
|
|
parameters: List[Any[str, Parameters]] = options.pop("parameters")
|
|
if not parameters:
|
|
overrides = {
|
|
"target-kind": options.get("target_kind"),
|
|
}
|
|
parameters = [
|
|
parameters_loader(None, strict=False, overrides=overrides)
|
|
] # will use default values
|
|
|
|
for param in parameters[:]:
|
|
if isinstance(param, str) and os.path.isdir(param):
|
|
parameters.remove(param)
|
|
parameters.extend(
|
|
[
|
|
p.as_posix()
|
|
for p in Path(param).iterdir()
|
|
if p.suffix in (".yml", ".json")
|
|
]
|
|
)
|
|
|
|
logdir = None
|
|
if len(parameters) > 1:
|
|
# Log to separate files for each process instead of stderr to
|
|
# avoid interleaving.
|
|
basename = os.path.basename(os.getcwd())
|
|
logdir = os.path.join(appdirs.user_log_dir("taskgraph"), basename)
|
|
if not os.path.isdir(logdir):
|
|
os.makedirs(logdir)
|
|
else:
|
|
# Only setup logging if we have a single parameter spec. Otherwise
|
|
# logging will go to files. This is also used as a hook for Gecko
|
|
# to setup its `mach` based logging.
|
|
setup_logging()
|
|
|
|
generate_taskgraph(options, parameters, logdir)
|
|
|
|
if options["diff"]:
|
|
assert diffdir is not None
|
|
assert repo is not None
|
|
|
|
# Reload taskgraph modules to pick up changes and clear global state.
|
|
for mod in sys.modules.copy():
|
|
if mod != __name__ and mod.split(".", 1)[0].endswith(
|
|
("taskgraph", "mozbuild")
|
|
):
|
|
del sys.modules[mod]
|
|
|
|
# Ensure gecko_taskgraph is ahead of taskcluster_taskgraph in sys.path.
|
|
# Without this, we may end up validating some things against the wrong
|
|
# schema.
|
|
import gecko_taskgraph # noqa
|
|
|
|
if options["diff"] == "default":
|
|
base_ref = repo.base_ref
|
|
else:
|
|
base_ref = options["diff"]
|
|
|
|
try:
|
|
repo.update(base_ref)
|
|
base_ref = repo.head_ref[:12]
|
|
options["output_file"] = os.path.join(
|
|
diffdir, f"{options['graph_attr']}_{base_ref}"
|
|
)
|
|
print(f"Generating {options['graph_attr']} @ {base_ref}", file=sys.stderr)
|
|
generate_taskgraph(options, parameters, logdir)
|
|
finally:
|
|
repo.update(cur_ref)
|
|
|
|
# Generate diff(s)
|
|
diffcmd = [
|
|
"diff",
|
|
"-U20",
|
|
"--report-identical-files",
|
|
f"--label={options['graph_attr']}@{base_ref}",
|
|
f"--label={options['graph_attr']}@{cur_ref}",
|
|
]
|
|
|
|
non_fatal_failures = []
|
|
for spec in parameters:
|
|
base_path = os.path.join(diffdir, f"{options['graph_attr']}_{base_ref}")
|
|
cur_path = os.path.join(diffdir, f"{options['graph_attr']}_{cur_ref}")
|
|
|
|
params_name = None
|
|
if len(parameters) > 1:
|
|
params_name = Parameters.format_spec(spec)
|
|
base_path += f"_{params_name}"
|
|
cur_path += f"_{params_name}"
|
|
|
|
# If the base or cur files are missing it means that generation
|
|
# failed. If one of them failed but not the other, the failure is
|
|
# likely due to the patch making changes to taskgraph in modules
|
|
# that don't get reloaded (safe to ignore). If both generations
|
|
# failed, there's likely a real issue.
|
|
base_missing = not os.path.isfile(base_path)
|
|
cur_missing = not os.path.isfile(cur_path)
|
|
if base_missing != cur_missing: # != is equivalent to XOR for booleans
|
|
non_fatal_failures.append(os.path.basename(base_path))
|
|
continue
|
|
|
|
try:
|
|
# If the output file(s) are missing, this command will raise
|
|
# CalledProcessError with a returncode > 1.
|
|
proc = subprocess.run(
|
|
diffcmd + [base_path, cur_path],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
universal_newlines=True,
|
|
check=True,
|
|
)
|
|
diff_output = proc.stdout
|
|
returncode = 0
|
|
except subprocess.CalledProcessError as e:
|
|
# returncode 1 simply means diffs were found
|
|
if e.returncode != 1:
|
|
print(e.stderr, file=sys.stderr)
|
|
raise
|
|
diff_output = e.output
|
|
returncode = e.returncode
|
|
|
|
dump_output(
|
|
diff_output,
|
|
# Don't bother saving file if no diffs were found. Log to
|
|
# console in this case instead.
|
|
path=None if returncode == 0 else output_file,
|
|
params_spec=spec if len(parameters) > 1 else None,
|
|
)
|
|
|
|
if non_fatal_failures:
|
|
failstr = "\n ".join(sorted(non_fatal_failures))
|
|
print(
|
|
"WARNING: Diff skipped for the following generation{s} "
|
|
"due to failures:\n {failstr}".format(
|
|
s="s" if len(non_fatal_failures) > 1 else "", failstr=failstr
|
|
),
|
|
file=sys.stderr,
|
|
)
|
|
|
|
if options["format"] != "json":
|
|
print(
|
|
"If you were expecting differences in task bodies "
|
|
'you should pass "-J"\n',
|
|
file=sys.stderr,
|
|
)
|
|
|
|
if len(parameters) > 1:
|
|
print("See '{}' for logs".format(logdir), file=sys.stderr)
|
|
|
|
|
|
@command("build-image", help="Build a Docker image")
|
|
@argument("image_name", help="Name of the image to build")
|
|
@argument(
|
|
"-t", "--tag", help="tag that the image should be built as.", metavar="name:tag"
|
|
)
|
|
@argument(
|
|
"--context-only",
|
|
help="File name the context tarball should be written to."
|
|
"with this option it will only build the context.tar.",
|
|
metavar="context.tar",
|
|
)
|
|
def build_image(args):
|
|
from gecko_taskgraph.docker import build_context, build_image
|
|
|
|
if args["context_only"] is None:
|
|
build_image(args["image_name"], args["tag"], os.environ)
|
|
else:
|
|
build_context(args["image_name"], args["context_only"], os.environ)
|
|
|
|
|
|
@command(
|
|
"load-image",
|
|
help="Load a pre-built Docker image. Note that you need to "
|
|
"have docker installed and running for this to work.",
|
|
)
|
|
@argument(
|
|
"--task-id",
|
|
help="Load the image at public/image.tar.zst in this task, "
|
|
"rather than searching the index",
|
|
)
|
|
@argument(
|
|
"-t",
|
|
"--tag",
|
|
help="tag that the image should be loaded as. If not "
|
|
"image will be loaded with tag from the tarball",
|
|
metavar="name:tag",
|
|
)
|
|
@argument(
|
|
"image_name",
|
|
nargs="?",
|
|
help="Load the image of this name based on the current "
|
|
"contents of the tree (as built for mozilla-central "
|
|
"or mozilla-inbound)",
|
|
)
|
|
def load_image(args):
|
|
from gecko_taskgraph.docker import load_image_by_name, load_image_by_task_id
|
|
|
|
if not args.get("image_name") and not args.get("task_id"):
|
|
print("Specify either IMAGE-NAME or TASK-ID")
|
|
sys.exit(1)
|
|
try:
|
|
if args["task_id"]:
|
|
ok = load_image_by_task_id(args["task_id"], args.get("tag"))
|
|
else:
|
|
ok = load_image_by_name(args["image_name"], args.get("tag"))
|
|
if not ok:
|
|
sys.exit(1)
|
|
except Exception:
|
|
traceback.print_exc()
|
|
sys.exit(1)
|
|
|
|
|
|
@command("image-digest", help="Print the digest of a docker image.")
|
|
@argument(
|
|
"image_name",
|
|
help="Print the digest of the image of this name based on the current "
|
|
"contents of the tree.",
|
|
)
|
|
def image_digest(args):
|
|
from gecko_taskgraph.docker import get_image_digest
|
|
|
|
try:
|
|
digest = get_image_digest(args["image_name"])
|
|
print(digest)
|
|
except Exception:
|
|
traceback.print_exc()
|
|
sys.exit(1)
|
|
|
|
|
|
@command("decision", help="Run the decision task")
|
|
@argument("--root", "-r", help="root of the taskgraph definition relative to topsrcdir")
|
|
@argument(
|
|
"--message",
|
|
required=False,
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
@argument(
|
|
"--project",
|
|
required=True,
|
|
help="Project to use for creating task graph. Example: --project=try",
|
|
)
|
|
@argument("--pushlog-id", dest="pushlog_id", required=True, default="0")
|
|
@argument("--pushdate", dest="pushdate", required=True, type=int, default=0)
|
|
@argument("--owner", required=True, help="email address of who owns this graph")
|
|
@argument("--level", required=True, help="SCM level of this repository")
|
|
@argument(
|
|
"--target-tasks-method", help="method for selecting the target tasks to generate"
|
|
)
|
|
@argument(
|
|
"--repository-type",
|
|
required=True,
|
|
help='Type of repository, either "hg" or "git"',
|
|
)
|
|
@argument("--base-repository", required=True, help='URL for "base" repository to clone')
|
|
@argument(
|
|
"--base-ref", default="", help='Reference of the revision in the "base" repository'
|
|
)
|
|
@argument(
|
|
"--base-rev",
|
|
default="",
|
|
help="Taskgraph decides what to do based on the revision range between "
|
|
"`--base-rev` and `--head-rev`. Value is determined automatically if not provided",
|
|
)
|
|
@argument(
|
|
"--head-repository",
|
|
required=True,
|
|
help='URL for "head" repository to fetch revision from',
|
|
)
|
|
@argument(
|
|
"--head-ref", required=True, help="Reference (this is same as rev usually for hg)"
|
|
)
|
|
@argument(
|
|
"--head-rev", required=True, help="Commit revision to use from head repository"
|
|
)
|
|
@argument("--head-tag", help="Tag attached to the revision", default="")
|
|
@argument(
|
|
"--tasks-for", required=True, help="the tasks_for value used to generate this task"
|
|
)
|
|
@argument("--try-task-config-file", help="path to try task configuration file")
|
|
def decision(options):
|
|
from gecko_taskgraph.decision import taskgraph_decision
|
|
|
|
taskgraph_decision(options)
|
|
|
|
|
|
@command("action-callback", description="Run action callback used by action tasks")
|
|
@argument(
|
|
"--root",
|
|
"-r",
|
|
default="taskcluster/ci",
|
|
help="root of the taskgraph definition relative to topsrcdir",
|
|
)
|
|
def action_callback(options):
|
|
from gecko_taskgraph.actions import trigger_action_callback
|
|
from gecko_taskgraph.actions.util import get_parameters
|
|
|
|
try:
|
|
# the target task for this action (or null if it's a group action)
|
|
task_id = json.loads(os.environ.get("ACTION_TASK_ID", "null"))
|
|
# the target task group for this action
|
|
task_group_id = os.environ.get("ACTION_TASK_GROUP_ID", None)
|
|
input = json.loads(os.environ.get("ACTION_INPUT", "null"))
|
|
callback = os.environ.get("ACTION_CALLBACK", None)
|
|
root = options["root"]
|
|
|
|
parameters = get_parameters(task_group_id)
|
|
|
|
return trigger_action_callback(
|
|
task_group_id=task_group_id,
|
|
task_id=task_id,
|
|
input=input,
|
|
callback=callback,
|
|
parameters=parameters,
|
|
root=root,
|
|
test=False,
|
|
)
|
|
except Exception:
|
|
traceback.print_exc()
|
|
sys.exit(1)
|
|
|
|
|
|
@command("test-action-callback", description="Run an action callback in a testing mode")
|
|
@argument(
|
|
"--root",
|
|
"-r",
|
|
default="taskcluster/ci",
|
|
help="root of the taskgraph definition relative to topsrcdir",
|
|
)
|
|
@argument(
|
|
"--parameters",
|
|
"-p",
|
|
default="",
|
|
help="parameters file (.yml or .json; see " "`taskcluster/docs/parameters.rst`)`",
|
|
)
|
|
@argument("--task-id", default=None, help="TaskId to which the action applies")
|
|
@argument(
|
|
"--task-group-id", default=None, help="TaskGroupId to which the action applies"
|
|
)
|
|
@argument("--input", default=None, help="Action input (.yml or .json)")
|
|
@argument("callback", default=None, help="Action callback name (Python function name)")
|
|
def test_action_callback(options):
|
|
import taskgraph.parameters
|
|
from taskgraph.config import load_graph_config
|
|
from taskgraph.util import yaml
|
|
|
|
import gecko_taskgraph.actions
|
|
|
|
def load_data(filename):
|
|
with open(filename) as f:
|
|
if filename.endswith(".yml"):
|
|
return yaml.load_stream(f)
|
|
if filename.endswith(".json"):
|
|
return json.load(f)
|
|
raise Exception(f"unknown filename {filename}")
|
|
|
|
try:
|
|
task_id = options["task_id"]
|
|
|
|
if options["input"]:
|
|
input = load_data(options["input"])
|
|
else:
|
|
input = None
|
|
|
|
root = options["root"]
|
|
graph_config = load_graph_config(root)
|
|
trust_domain = graph_config["trust-domain"]
|
|
graph_config.register()
|
|
|
|
parameters = taskgraph.parameters.load_parameters_file(
|
|
options["parameters"], strict=False, trust_domain=trust_domain
|
|
)
|
|
parameters.check()
|
|
|
|
return gecko_taskgraph.actions.trigger_action_callback(
|
|
task_group_id=options["task_group_id"],
|
|
task_id=task_id,
|
|
input=input,
|
|
callback=options["callback"],
|
|
parameters=parameters,
|
|
root=root,
|
|
test=True,
|
|
)
|
|
except Exception:
|
|
traceback.print_exc()
|
|
sys.exit(1)
|
|
|
|
|
|
def create_parser():
|
|
parser = argparse.ArgumentParser(description="Interact with taskgraph")
|
|
subparsers = parser.add_subparsers()
|
|
for _, (func, args, kwargs, defaults) in commands.items():
|
|
subparser = subparsers.add_parser(*args, **kwargs)
|
|
for arg in func.args:
|
|
subparser.add_argument(*arg[0], **arg[1])
|
|
subparser.set_defaults(command=func, **defaults)
|
|
return parser
|
|
|
|
|
|
def setup_logging():
|
|
logging.basicConfig(
|
|
format="%(asctime)s - %(levelname)s - %(message)s", level=logging.INFO
|
|
)
|
|
|
|
|
|
def main(args=sys.argv[1:]):
|
|
setup_logging()
|
|
parser = create_parser()
|
|
args = parser.parse_args(args)
|
|
try:
|
|
args.command(vars(args))
|
|
except Exception:
|
|
traceback.print_exc()
|
|
sys.exit(1)
|