nagai/fune
3
0
Fork 0
forked from mirrors/gecko-dev
Code Pull requests Activity

No bug: [taskgraph] Make verify_test_packaging handle --target-kind gracefully; r=Callek

Browse source 
If we are generating only a part of the graph, to given kind, don't fail if a
build is packaging tests and there is no corresponding test task, as the tests
may not have been generated.

Differential Revision: https://phabricator.services.mozilla.com/D82097
This commit is contained in:
Tom Prince 2020-07-14 07:17:26 +00:00
parent 51f106a3a3
commit e92bbf5dd8
7 changed files with 57 additions and 30 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

7
taskcluster/docs/parameters.rst
View file

@ -235,3 +235,10 @@ Code Review
``phabricator_diff``
The code review process needs to know the Phabricator Differential diff that
started the analysis. This parameter must start with `PHID-DIFF-`
Local configuration
-------------------
``target-kind``
Generate only the given kind and its kind-dependencies. This is used for local inspection of the graph
and is not supported at run-time.

7
taskcluster/mach_commands.py
View file

@ -337,12 +337,15 @@ class MachCommands(MachCommandBase):
try:
self.setup_logging(quiet=options['quiet'], verbose=options['verbose'])
parameters = taskgraph.parameters.parameters_loader(options['parameters'])
parameters = taskgraph.parameters.parameters_loader(
options['parameters'],
overrides={'target-kind': options.get('target_kind')},
strict=False,
)
tgg = taskgraph.generator.TaskGraphGenerator(
root_dir=options.get('root'),
parameters=parameters,
target_kind=options.get('target_kind'),
)
tg = getattr(tgg, graph_attr)

14
taskcluster/taskgraph/generator.py
View file

@ -114,7 +114,6 @@ class TaskGraphGenerator(object):
parameters,
decision_task_id="DECISION-TASK",
write_artifacts=False,
target_kind=None,
):
"""
@param root_dir: root directory, with subdirectories for each kind
@ -126,7 +125,6 @@ class TaskGraphGenerator(object):
root_dir = 'taskcluster/ci'
self.root_dir = ensure_text(root_dir)
self._parameters = parameters
self._target_kind = target_kind
self._decision_task_id = decision_task_id
self._write_artifacts = write_artifacts
@ -262,11 +260,12 @@ class TaskGraphGenerator(object):
edges.add((kind.name, dep, 'kind-dependency'))
kind_graph = Graph(set(kinds), edges)
if self._target_kind:
if parameters.get('target-kind'):
target_kind = parameters['target-kind']
logger.info(
"Limiting kinds to {target_kind} and dependencies".format(
target_kind=self._target_kind))
kind_graph = kind_graph.transitive_closure({self._target_kind, 'docker-image'})
target_kind=target_kind))
kind_graph = kind_graph.transitive_closure({target_kind, 'docker-image'})
logger.info("Generating full task set")
all_tasks = {}
@ -419,7 +418,10 @@ def load_tasks_for_kind(parameters, kind, root_dir=None):
This function is designed to be called from outside of taskgraph.
"""
tgg = TaskGraphGenerator(root_dir=root_dir, parameters=parameters, target_kind=kind)
# make parameters read-write
parameters = dict(parameters)
parameters['target-kind'] = kind
tgg = TaskGraphGenerator(root_dir=root_dir, parameters=parameters)
return {
task.task['metadata']['name']: task
for task in tgg.full_task_set

2
taskcluster/taskgraph/parameters.py
View file

@ -96,6 +96,8 @@ base_schema = Schema({
Required('release_product'): Any(None, text_type),
Required('required_signoffs'): [text_type],
Required('signoff_urls'): dict,
# target-kind is not included, since it should never be
# used at run-time
Required('target_tasks_method'): text_type,
Required('tasks_for'): text_type,
Required('test_manifest_loader'): text_type,

49
taskcluster/taskgraph/util/verify.py
View file

@ -45,9 +45,18 @@ class VerificationSequence(object):
continue
scratch_pad = {}
graph.for_each_task(
verification.verify, scratch_pad=scratch_pad, graph_config=graph_config
verification.verify,
scratch_pad=scratch_pad,
graph_config=graph_config,
parameters=parameters,
)
verification.verify(
None,
graph,
scratch_pad=scratch_pad,
graph_config=graph_config,
parameters=parameters,
)
verification.verify(None, graph, scratch_pad=scratch_pad, graph_config=graph_config)
return graph_name, graph
def add(self, graph_name, run_on_projects={"all"}):
@ -128,7 +137,7 @@ def verify_docs(filename, identifiers, appearing_as):
@verifications.add('full_task_graph')
def verify_task_graph_symbol(task, taskgraph, scratch_pad, graph_config):
def verify_task_graph_symbol(task, taskgraph, scratch_pad, graph_config, parameters):
"""
This function verifies that tuple
(collection.keys(), machine.platform, groupSymbol, symbol) is unique
@ -169,7 +178,7 @@ def verify_task_graph_symbol(task, taskgraph, scratch_pad, graph_config):
@verifications.add('full_task_graph')
def verify_trust_domain_v2_routes(task, taskgraph, scratch_pad, graph_config):
def verify_trust_domain_v2_routes(task, taskgraph, scratch_pad, graph_config, parameters):
"""
This function ensures that any two tasks have distinct ``index.{trust-domain}.v2`` routes.
"""
@ -191,7 +200,7 @@ def verify_trust_domain_v2_routes(task, taskgraph, scratch_pad, graph_config):
@verifications.add('full_task_graph')
def verify_routes_notification_filters(task, taskgraph, scratch_pad, graph_config):
def verify_routes_notification_filters(task, taskgraph, scratch_pad, graph_config, parameters):
"""
This function ensures that only understood filters for notifications are
specified.
@ -217,7 +226,7 @@ def verify_routes_notification_filters(task, taskgraph, scratch_pad, graph_confi
@verifications.add('full_task_graph')
def verify_dependency_tiers(task, taskgraph, scratch_pad, graph_config):
def verify_dependency_tiers(task, taskgraph, scratch_pad, graph_config, parameters):
tiers = scratch_pad
if task is not None:
tiers[task.label] = task.task.get('extra', {}) \
@ -244,7 +253,7 @@ def verify_dependency_tiers(task, taskgraph, scratch_pad, graph_config):
@verifications.add('full_task_graph')
def verify_required_signoffs(task, taskgraph, scratch_pad, graph_config):
def verify_required_signoffs(task, taskgraph, scratch_pad, graph_config, parameters):
"""
Task with required signoffs can't be dependencies of tasks with less
required signoffs.
@ -271,7 +280,7 @@ def verify_required_signoffs(task, taskgraph, scratch_pad, graph_config):
@verifications.add('full_task_graph')
def verify_toolchain_alias(task, taskgraph, scratch_pad, graph_config):
def verify_toolchain_alias(task, taskgraph, scratch_pad, graph_config, parameters):
"""
This function verifies that toolchain aliases are not reused.
"""
@ -294,7 +303,7 @@ def verify_toolchain_alias(task, taskgraph, scratch_pad, graph_config):
@verifications.add('optimized_task_graph')
def verify_always_optimized(task, taskgraph, scratch_pad, graph_config):
def verify_always_optimized(task, taskgraph, scratch_pad, graph_config, parameters):
"""
This function ensures that always-optimized tasks have been optimized.
"""
@ -305,7 +314,7 @@ def verify_always_optimized(task, taskgraph, scratch_pad, graph_config):
@verifications.add('full_task_graph', run_on_projects=RELEASE_PROJECTS)
def verify_shippable_no_sccache(task, taskgraph, scratch_pad, graph_config):
def verify_shippable_no_sccache(task, taskgraph, scratch_pad, graph_config, parameters):
if task and task.attributes.get('shippable'):
if task.task.get('payload', {}).get('env', {}).get('USE_SCCACHE'):
raise Exception(
@ -313,8 +322,9 @@ def verify_shippable_no_sccache(task, taskgraph, scratch_pad, graph_config):
@verifications.add('full_task_graph')
def verify_test_packaging(task, taskgraph, scratch_pad, graph_config):
def verify_test_packaging(task, taskgraph, scratch_pad, graph_config, parameters):
if task is None:
has_target_kind = parameters.get('target-kind') is None
exceptions = []
for task in six.itervalues(taskgraph.tasks):
if task.kind == 'build' and not task.attributes.get('skip-verify-test-packaging'):
@ -342,11 +352,14 @@ def verify_test_packaging(task, taskgraph, scratch_pad, graph_config):
# With the caveat that we expect shippable jobs to always
# produce tests.
if not build_has_tests and not shippable:
exceptions.append(
'Build job {} has no tests, but specifies '
'MOZ_AUTOMATION_PACKAGE_TESTS={} in the environment. '
'Unset MOZ_AUTOMATION_PACKAGE_TESTS in the task definition '
'to fix.'.format(task.label, package_tests))
# If we have not generated all task kinds, we can't verify that
# there are no dependent tests.
if has_target_kind:
exceptions.append(
'Build job {} has no tests, but specifies '
'MOZ_AUTOMATION_PACKAGE_TESTS={} in the environment. '
'Unset MOZ_AUTOMATION_PACKAGE_TESTS in the task definition '
'to fix.'.format(task.label, package_tests))
if exceptions:
raise Exception("\n".join(exceptions))
return
@ -356,7 +369,7 @@ def verify_test_packaging(task, taskgraph, scratch_pad, graph_config):
@verifications.add('full_task_graph')
def verify_run_known_projects(task, taskgraph, scratch_pad, graph_config):
def verify_run_known_projects(task, taskgraph, scratch_pad, graph_config, parameters):
""" Validates the inputs in run-on-projects.
We should never let 'try' (or 'try-comm-central') be in run-on-projects even though it
@ -381,7 +394,7 @@ def verify_run_known_projects(task, taskgraph, scratch_pad, graph_config):
@verifications.add('full_task_graph')
def verify_local_toolchains(task, taskgraph, scratch_pad, graph_config):
def verify_local_toolchains(task, taskgraph, scratch_pad, graph_config, parameters):
"""
Toolchains that are used for local development need to be built on a
level-3 branch to installable via `mach bootstrap`. We ensure here that all

4
taskcluster/test/conftest.py
View file

@ -35,9 +35,9 @@ def create_tgg(responses, datadir):
lm = LoggingManager()
lm.add_terminal_logging()
def inner(parameters=None, overrides=None, target_kind=None):
def inner(parameters=None, overrides=None):
params = parameters_loader(parameters, strict=False, overrides=overrides)
tgg = TaskGraphGenerator(None, params, target_kind=target_kind)
tgg = TaskGraphGenerator(None, params)
# Mock out certain requests as they may depend on a revision that does
# not exist on hg.mozilla.org.

4
taskcluster/test/test_mach_try_auto.py
View file

@ -19,9 +19,9 @@ pytestmark = pytest.mark.slow
def tgg(create_tgg):
params = TRY_AUTO_PARAMETERS.copy()
params.update(
{"head_repository": "https://hg.mozilla.org/try", "project": "try"}
{"head_repository": "https://hg.mozilla.org/try", "project": "try", "target_kind": "test"}
)
tgg = create_tgg(overrides=params, target_kind="test")
tgg = create_tgg(overrides=params)
return tgg