From 91f4fe8c61a04d8e15cd9af11f3c7366323cfb5f Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Wed, 4 Jul 2018 02:46:59 +0000 Subject: [PATCH] Bug 1398277: special-case retriggering of tasks not in the taskgraph; r=bstack This will apply to cron tasks, action tasks, and decision tasks. It is a distinct retrigger implementation because (a) we do not want to follow dependencies, and (b) it takes a lot of scopes to create a decision task, so we need to limit access to this action. MozReview-Commit-ID: 21DVSiagcrO --HG-- extra : rebase_source : 6f027e349e245e4aa4dbed81145a0a5d75218cb1 extra : histedit_source : eff99aee5a0e7496b0734748b29739480eb0e3fb --- .taskcluster.yml | 8 +++- taskcluster/taskgraph/actions/retrigger.py | 38 +++++++++++++-- taskcluster/taskgraph/actions/util.py | 36 ++++++++++++++- taskcluster/taskgraph/taskgraph.py | 3 ++ taskcluster/taskgraph/test/python.ini | 2 + .../taskgraph/test/test_actions_util.py | 46 +++++++++++++++++++ taskcluster/taskgraph/test/test_taskgraph.py | 21 +++++++++ .../taskgraph/test/test_util_taskcluster.py | 24 ++++++++++ taskcluster/taskgraph/util/taskcluster.py | 11 +++-- 9 files changed, 181 insertions(+), 8 deletions(-) create mode 100644 taskcluster/taskgraph/test/test_actions_util.py create mode 100644 taskcluster/taskgraph/test/test_util_taskcluster.py diff --git a/.taskcluster.yml b/.taskcluster.yml index f8bbc9939f78..f5ce9752aba1 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -48,12 +48,18 @@ tasks: tags: $if: 'tasks_for == "hg-push"' - then: {createdForUser: "${ownerEmail}"} + then: + createdForUser: "${ownerEmail}" + kind: decision-task else: $if: 'tasks_for == "action"' then: createdForUser: '${ownerEmail}' kind: 'action-callback' + else: + $if: 'tasks_for == "cron"' + then: + kind: cron-task routes: $flatten: diff --git a/taskcluster/taskgraph/actions/retrigger.py b/taskcluster/taskgraph/actions/retrigger.py index 6dad3869e831..6885d6e11bed 100644 --- a/taskcluster/taskgraph/actions/retrigger.py +++ b/taskcluster/taskgraph/actions/retrigger.py @@ -8,13 +8,15 @@ from __future__ import absolute_import, print_function, unicode_literals import json import logging +import textwrap from slugid import nice as slugid from .util import ( combine_task_graph_files, create_tasks, + fetch_graph_and_labels, + relativize_datestamps, create_task_from_def, - fetch_graph_and_labels ) from ..util.parameterization import resolve_task_references from .registry import register_callback_action @@ -143,6 +145,35 @@ def mochitest_retrigger_action(parameters, graph_config, input, task_group_id, t create_task_from_def(new_task_id, new_task_definition, parameters['level']) +@register_callback_action( + title='Retrigger', + name='retrigger', + symbol='rt', + kind='hook', + cb_name='retrigger-decision', + description=textwrap.dedent('''\ + Create a clone of the task (retriggering decision, action, and cron tasks requires + special scopes).'''), + order=11, + context=[ + {'kind': 'decision-task'}, + {'kind': 'action-callback'}, + {'kind': 'cron-task'}, + ], +) +def retrigger_decision_action(parameters, graph_config, input, task_group_id, task_id, task): + decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( + parameters, graph_config) + """For a single task, we try to just run exactly the same task once more. + It's quite possible that we don't have the scopes to do so (especially for + an action), but this is best-effort.""" + + # make all of the timestamps relative; they will then be turned back into + # absolute timestamps relative to the current time. + task = relativize_datestamps(task) + create_task_from_def(slugid(), task, parameters['level']) + + @register_callback_action( title='Retrigger', name='retrigger', @@ -150,9 +181,9 @@ def mochitest_retrigger_action(parameters, graph_config, input, task_group_id, t kind='hook', generic=True, description=( - 'Create a clone of the task.\n\n' + 'Create a clone of the task.' ), - order=11, # must be greater than other orders in this file, as this is the fallback version + order=19, # must be greater than other orders in this file, as this is the fallback version context=[{}], schema={ 'type': 'object', @@ -181,6 +212,7 @@ def retrigger_action(parameters, graph_config, input, task_group_id, task_id, ta parameters, graph_config) label = task['metadata']['name'] + with_downstream = ' ' to_run = [label] diff --git a/taskcluster/taskgraph/actions/util.py b/taskcluster/taskgraph/actions/util.py index f6e2aba54e2d..8656fa1a7be4 100644 --- a/taskcluster/taskgraph/actions/util.py +++ b/taskcluster/taskgraph/actions/util.py @@ -10,6 +10,7 @@ import copy import logging import requests import os +import re from requests.exceptions import HTTPError @@ -17,7 +18,13 @@ from taskgraph import create from taskgraph.decision import read_artifact, write_artifact from taskgraph.taskgraph import TaskGraph from taskgraph.optimize import optimize_task_graph -from taskgraph.util.taskcluster import get_session, find_task_id, get_artifact, list_tasks +from taskgraph.util.taskcluster import ( + get_session, + find_task_id, + get_artifact, + list_tasks, + parse_time, +) logger = logging.getLogger(__name__) @@ -164,3 +171,30 @@ def combine_task_graph_files(suffixes): for suffix in suffixes: all.update(read_artifact('task-graph-{}.json'.format(suffix))) write_artifact('task-graph.json', all) + + +def relativize_datestamps(task_def): + """ + Given a task definition as received from the queue, convert all datestamps + to {relative_datestamp: ..} format, with the task creation time as "now". + The result is useful for handing to ``create_task``. + """ + base = parse_time(task_def['created']) + # borrowed from https://github.com/epoberezkin/ajv/blob/master/lib/compile/formats.js + ts_pattern = re.compile( + r'^\d\d\d\d-[0-1]\d-[0-3]\d[t\s]' + r'(?:[0-2]\d:[0-5]\d:[0-5]\d|23:59:60)(?:\.\d+)?' + r'(?:z|[+-]\d\d:\d\d)$', re.I) + + def recurse(value): + if isinstance(value, basestring): + if ts_pattern.match(value): + value = parse_time(value) + diff = value - base + return {'relative-datestamp': '{} seconds'.format(int(diff.total_seconds()))} + if isinstance(value, list): + return [recurse(e) for e in value] + if isinstance(value, dict): + return {k: recurse(v) for k, v in value.items()} + return value + return recurse(task_def) diff --git a/taskcluster/taskgraph/taskgraph.py b/taskcluster/taskgraph/taskgraph.py index b772dd0805b8..e0d98a2f4020 100644 --- a/taskcluster/taskgraph/taskgraph.py +++ b/taskcluster/taskgraph/taskgraph.py @@ -30,6 +30,9 @@ class TaskGraph(object): "Get a task by label" return self.tasks[label] + def __contains__(self, label): + return label in self.tasks + def __iter__(self): "Iterate over tasks in undefined order" return self.tasks.itervalues() diff --git a/taskcluster/taskgraph/test/python.ini b/taskcluster/taskgraph/test/python.ini index dba67c507817..4482f8846f42 100644 --- a/taskcluster/taskgraph/test/python.ini +++ b/taskcluster/taskgraph/test/python.ini @@ -2,6 +2,7 @@ subsuite = taskgraph skip-if = python == 3 +[test_actions_util.py] [test_create.py] [test_cron_util.py] [test_decision.py] @@ -21,6 +22,7 @@ skip-if = python == 3 [test_util_python_path.py] [test_util_runnable_jobs.py] [test_util_schema.py] +[test_util_taskcluster.py] [test_util_templates.py] [test_util_time.py] [test_util_treeherder.py] diff --git a/taskcluster/taskgraph/test/test_actions_util.py b/taskcluster/taskgraph/test/test_actions_util.py new file mode 100644 index 000000000000..0fbdda9fb7c6 --- /dev/null +++ b/taskcluster/taskgraph/test/test_actions_util.py @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest +from mozunit import main +from taskgraph.actions.util import ( + relativize_datestamps +) + +TASK_DEF = { + 'created': '2017-10-10T18:33:03.460Z', + # note that this is not an even number of seconds off! + 'deadline': '2017-10-11T18:33:03.461Z', + 'dependencies': [], + 'expires': '2018-10-10T18:33:04.461Z', + 'payload': { + 'artifacts': { + 'public': { + 'expires': '2018-10-10T18:33:03.463Z', + 'path': '/builds/worker/artifacts', + 'type': 'directory', + }, + }, + 'maxRunTime': 1800, + }, +} + + +class TestRelativize(unittest.TestCase): + + def test_relativize(self): + rel = relativize_datestamps(TASK_DEF) + import pprint + pprint.pprint(rel) + assert rel['created'] == {'relative-datestamp': '0 seconds'} + assert rel['deadline'] == {'relative-datestamp': '86400 seconds'} + assert rel['expires'] == {'relative-datestamp': '31536001 seconds'} + assert rel['payload']['artifacts']['public']['expires'] == \ + {'relative-datestamp': '31536000 seconds'} + + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_taskgraph.py b/taskcluster/taskgraph/test/test_taskgraph.py index 21ea24d77e3f..3f461ab34bd5 100644 --- a/taskcluster/taskgraph/test/test_taskgraph.py +++ b/taskcluster/taskgraph/test/test_taskgraph.py @@ -74,6 +74,27 @@ class TestTaskGraph(unittest.TestCase): tasks, new_graph = TaskGraph.from_json(graph.to_json()) self.assertEqual(graph, new_graph) + simple_graph = TaskGraph(tasks={ + 'a': Task( + kind='fancy', + label='a', + attributes={}, + dependencies={'prereq': 'b'}, # must match edges, below + optimization={'seta': None}, + task={'task': 'def'}), + 'b': Task( + kind='pre', + label='b', + attributes={}, + dependencies={}, + optimization={'seta': None}, + task={'task': 'def2'}), + }, graph=Graph(nodes={'a', 'b'}, edges={('a', 'b', 'prereq')})) + + def test_contains(self): + assert 'a' in self.simple_graph + assert 'c' not in self.simple_graph + if __name__ == '__main__': main() diff --git a/taskcluster/taskgraph/test/test_util_taskcluster.py b/taskcluster/taskgraph/test/test_util_taskcluster.py new file mode 100644 index 000000000000..0d355a5d8fdc --- /dev/null +++ b/taskcluster/taskgraph/test/test_util_taskcluster.py @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import unittest + +import mozunit +from taskgraph.util.taskcluster import ( + parse_time +) + + +class TestTCUtils(unittest.TestCase): + + def test_parse_time(self): + exp = datetime.datetime(2018, 10, 10, 18, 33, 3, 463000) + assert parse_time('2018-10-10T18:33:03.463Z') == exp + + +if __name__ == '__main__': + mozunit.main() diff --git a/taskcluster/taskgraph/util/taskcluster.py b/taskcluster/taskgraph/util/taskcluster.py index aeed2afc6c72..c4f2b28fee32 100644 --- a/taskcluster/taskgraph/util/taskcluster.py +++ b/taskcluster/taskgraph/util/taskcluster.py @@ -17,10 +17,10 @@ from requests.adapters import HTTPAdapter from taskgraph.task import Task _PUBLIC_TC_ARTIFACT_LOCATION = \ - 'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/{artifact_prefix}/{postfix}' + 'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/{artifact_prefix}/{postfix}' _PRIVATE_TC_ARTIFACT_LOCATION = \ - 'http://taskcluster/queue/v1/task/{task_id}/artifacts/{artifact_prefix}/{postfix}' + 'http://taskcluster/queue/v1/task/{task_id}/artifacts/{artifact_prefix}/{postfix}' logger = logging.getLogger(__name__) @@ -151,10 +151,15 @@ def list_tasks(index_path, use_proxy=False): # all of these tasks should be created with the same expires time so they end up in # order from earliest to latest action. If more correctness is needed, consider # fetching each task and sorting on the created date. - results.sort(key=lambda t: datetime.datetime.strptime(t['expires'], '%Y-%m-%dT%H:%M:%S.%fZ')) + results.sort(key=lambda t: parse_time(t['expires'])) return [t['taskId'] for t in results] +def parse_time(timestamp): + """Turn a "JSON timestamp" as used in TC APIs into a datetime""" + return datetime.datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ') + + def get_task_url(task_id, use_proxy=False): if use_proxy: TASK_URL = 'http://taskcluster/queue/v1/task/{}'