forked from mirrors/gecko-dev
3c28444d2d
We want to be able to retroactively tell whether a push was a backstop or not. This patch stores whether or not a push was a "backstop" directly in the parameters. The optimization strategy now simply returns 'not params["backstop"]'. For simplicity, I'm not counting the 'optimized-backstop' as a backstop. It's unclear if we'll want to be able to detect these types of the pushes in the future or not, but we can cross that bridge when we get there. Differential Revision: https://phabricator.services.mozilla.com/D88151
322 lines
10 KiB
Python
322 lines
10 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
from __future__ import absolute_import, print_function, unicode_literals
|
|
|
|
import io
|
|
import logging
|
|
import os.path
|
|
import json
|
|
from datetime import datetime
|
|
|
|
from mozbuild.util import ReadOnlyDict, memoize
|
|
from mozversioncontrol import get_repository_object
|
|
from taskgraph.util.schema import validate_schema
|
|
from voluptuous import (
|
|
ALLOW_EXTRA,
|
|
Any,
|
|
Inclusive,
|
|
Required,
|
|
Schema,
|
|
)
|
|
|
|
import six
|
|
from six import text_type
|
|
|
|
from . import GECKO
|
|
from .util.attributes import release_level
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ParameterMismatch(Exception):
|
|
"""Raised when a parameters.yml has extra or missing parameters."""
|
|
|
|
|
|
@memoize
|
|
def get_head_ref():
|
|
return six.ensure_text(get_repository_object(GECKO).head_ref)
|
|
|
|
|
|
def get_contents(path):
|
|
with io.open(path, "r") as fh:
|
|
contents = fh.readline().rstrip()
|
|
return contents
|
|
|
|
|
|
def get_version(product_dir='browser'):
|
|
version_path = os.path.join(GECKO, product_dir, 'config',
|
|
'version_display.txt')
|
|
return get_contents(version_path)
|
|
|
|
|
|
def get_app_version(product_dir='browser'):
|
|
app_version_path = os.path.join(GECKO, product_dir, 'config',
|
|
'version.txt')
|
|
return get_contents(app_version_path)
|
|
|
|
|
|
base_schema = Schema({
|
|
Required('app_version'): text_type,
|
|
Required('backstop'): bool,
|
|
Required('base_repository'): text_type,
|
|
Required('build_date'): int,
|
|
Required('build_number'): int,
|
|
Inclusive('comm_base_repository', 'comm'): text_type,
|
|
Inclusive('comm_head_ref', 'comm'): text_type,
|
|
Inclusive('comm_head_repository', 'comm'): text_type,
|
|
Inclusive('comm_head_rev', 'comm'): text_type,
|
|
Required('do_not_optimize'): [text_type],
|
|
Required('existing_tasks'): {text_type: text_type},
|
|
Required('filters'): [text_type],
|
|
Required('head_ref'): text_type,
|
|
Required('head_repository'): text_type,
|
|
Required('head_rev'): text_type,
|
|
Required('hg_branch'): text_type,
|
|
Required('level'): text_type,
|
|
Required('message'): text_type,
|
|
Required('moz_build_date'): text_type,
|
|
Required('next_version'): Any(None, text_type),
|
|
Required('optimize_strategies'): Any(None, text_type),
|
|
Required('optimize_target_tasks'): bool,
|
|
Required('owner'): text_type,
|
|
Required('phabricator_diff'): Any(None, text_type),
|
|
Required('project'): text_type,
|
|
Required('pushdate'): int,
|
|
Required('pushlog_id'): text_type,
|
|
Required('release_enable_emefree'): bool,
|
|
Required('release_enable_partner_repack'): bool,
|
|
Required('release_enable_partner_attribution'): bool,
|
|
Required('release_eta'): Any(None, text_type),
|
|
Required('release_history'): {text_type: dict},
|
|
Required('release_partners'): Any(None, [text_type]),
|
|
Required('release_partner_config'): Any(None, dict),
|
|
Required('release_partner_build_number'): int,
|
|
Required('release_type'): text_type,
|
|
Required('release_product'): Any(None, text_type),
|
|
Required('required_signoffs'): [text_type],
|
|
Required('signoff_urls'): dict,
|
|
# target-kind is not included, since it should never be
|
|
# used at run-time
|
|
Required('target_tasks_method'): text_type,
|
|
Required('tasks_for'): text_type,
|
|
Required('test_manifest_loader'): text_type,
|
|
Required('try_mode'): Any(None, text_type),
|
|
Required('try_options'): Any(None, dict),
|
|
Required('try_task_config'): dict,
|
|
Required('version'): text_type,
|
|
})
|
|
|
|
|
|
COMM_PARAMETERS = [
|
|
'comm_base_repository',
|
|
'comm_head_ref',
|
|
'comm_head_repository',
|
|
'comm_head_rev',
|
|
]
|
|
|
|
|
|
def extend_parameters_schema(schema):
|
|
"""
|
|
Extend the schema for parameters to include per-project configuration.
|
|
|
|
This should be called by the `taskgraph.register` function in the
|
|
graph-configuration.
|
|
"""
|
|
global base_schema
|
|
base_schema = base_schema.extend(schema)
|
|
|
|
|
|
class Parameters(ReadOnlyDict):
|
|
"""An immutable dictionary with nicer KeyError messages on failure"""
|
|
|
|
def __init__(self, strict=True, **kwargs):
|
|
self.strict = strict
|
|
|
|
if not self.strict:
|
|
# apply defaults to missing parameters
|
|
kwargs = Parameters._fill_defaults(**kwargs)
|
|
|
|
ReadOnlyDict.__init__(self, **kwargs)
|
|
|
|
@staticmethod
|
|
def _fill_defaults(**kwargs):
|
|
now = datetime.utcnow()
|
|
epoch = datetime.utcfromtimestamp(0)
|
|
seconds_from_epoch = int((now - epoch).total_seconds())
|
|
|
|
defaults = {
|
|
'app_version': get_app_version(),
|
|
'backstop': False,
|
|
'base_repository': 'https://hg.mozilla.org/mozilla-unified',
|
|
'build_date': seconds_from_epoch,
|
|
'build_number': 1,
|
|
'do_not_optimize': [],
|
|
'existing_tasks': {},
|
|
'filters': ['target_tasks_method'],
|
|
'head_ref': get_head_ref(),
|
|
'head_repository': 'https://hg.mozilla.org/mozilla-central',
|
|
'head_rev': get_head_ref(),
|
|
'hg_branch': 'default',
|
|
'level': '3',
|
|
'message': '',
|
|
'moz_build_date': six.ensure_text(now.strftime("%Y%m%d%H%M%S")),
|
|
'next_version': None,
|
|
'optimize_strategies': None,
|
|
'optimize_target_tasks': True,
|
|
'owner': 'nobody@mozilla.com',
|
|
'phabricator_diff': None,
|
|
'project': 'mozilla-central',
|
|
'pushdate': seconds_from_epoch,
|
|
'pushlog_id': '0',
|
|
'release_enable_emefree': False,
|
|
'release_enable_partner_repack': False,
|
|
'release_enable_partner_attribution': False,
|
|
'release_eta': '',
|
|
'release_history': {},
|
|
'release_partners': [],
|
|
'release_partner_config': None,
|
|
'release_partner_build_number': 1,
|
|
'release_product': None,
|
|
'release_type': 'nightly',
|
|
'required_signoffs': [],
|
|
'signoff_urls': {},
|
|
'target_tasks_method': 'default',
|
|
'tasks_for': 'hg-push',
|
|
'test_manifest_loader': 'default',
|
|
'try_mode': None,
|
|
'try_options': None,
|
|
'try_task_config': {},
|
|
'version': get_version(),
|
|
}
|
|
|
|
if set(COMM_PARAMETERS) & set(kwargs):
|
|
defaults.update({
|
|
'comm_base_repository': 'https://hg.mozilla.org/comm-central',
|
|
'comm_head_repository': 'https://hg.mozilla.org/comm-central',
|
|
})
|
|
|
|
for name, default in defaults.items():
|
|
if name not in kwargs:
|
|
kwargs[name] = default
|
|
|
|
return kwargs
|
|
|
|
def check(self):
|
|
schema = base_schema if self.strict else base_schema.extend({}, extra=ALLOW_EXTRA)
|
|
validate_schema(schema, self.copy(), 'Invalid parameters:')
|
|
|
|
def __getitem__(self, k):
|
|
try:
|
|
return super(Parameters, self).__getitem__(k)
|
|
except KeyError:
|
|
raise KeyError("taskgraph parameter {!r} not found".format(k))
|
|
|
|
def is_try(self):
|
|
"""
|
|
Determine whether this graph is being built on a try project or for
|
|
`mach try fuzzy`.
|
|
"""
|
|
return 'try' in self['project'] or self['try_mode'] == 'try_select'
|
|
|
|
def file_url(self, path, pretty=False):
|
|
"""
|
|
Determine the VCS URL for viewing a file in the tree, suitable for
|
|
viewing by a human.
|
|
|
|
:param text_type path: The path, relative to the root of the repository.
|
|
:param bool pretty: Whether to return a link to a formatted version of the
|
|
file, or the raw file version.
|
|
:return text_type: The URL displaying the given path.
|
|
"""
|
|
if path.startswith('comm/'):
|
|
path = path[len('comm/'):]
|
|
repo = self['comm_head_repository']
|
|
rev = self['comm_head_rev']
|
|
else:
|
|
repo = self['head_repository']
|
|
rev = self['head_rev']
|
|
|
|
endpoint = 'file' if pretty else 'raw-file'
|
|
return '{}/{}/{}/{}'.format(repo, endpoint, rev, path)
|
|
|
|
def release_level(self):
|
|
"""
|
|
Whether this is a staging release or not.
|
|
|
|
:return six.text_type: One of "production" or "staging".
|
|
"""
|
|
return release_level(self['project'])
|
|
|
|
|
|
def load_parameters_file(filename, strict=True, overrides=None, trust_domain=None):
|
|
"""
|
|
Load parameters from a path, url, decision task-id or project.
|
|
|
|
Examples:
|
|
task-id=fdtgsD5DQUmAQZEaGMvQ4Q
|
|
project=mozilla-central
|
|
"""
|
|
import requests
|
|
from taskgraph.util.taskcluster import get_artifact_url, find_task_id
|
|
from taskgraph.util import yaml
|
|
|
|
if overrides is None:
|
|
overrides = {}
|
|
|
|
if not filename:
|
|
return Parameters(strict=strict, **overrides)
|
|
|
|
try:
|
|
# reading parameters from a local parameters.yml file
|
|
f = open(filename)
|
|
except IOError:
|
|
# fetching parameters.yml using task task-id, project or supplied url
|
|
task_id = None
|
|
if filename.startswith("task-id="):
|
|
task_id = filename.split("=")[1]
|
|
elif filename.startswith("project="):
|
|
if trust_domain is None:
|
|
raise ValueError(
|
|
"Can't specify parameters by project "
|
|
"if trust domain isn't supplied.",
|
|
)
|
|
index = "{trust_domain}.v2.{project}.latest.taskgraph.decision".format(
|
|
trust_domain=trust_domain,
|
|
project=filename.split("=")[1],
|
|
)
|
|
task_id = find_task_id(index)
|
|
|
|
if task_id:
|
|
filename = get_artifact_url(task_id, 'public/parameters.yml')
|
|
logger.info("Loading parameters from {}".format(filename))
|
|
resp = requests.get(filename, stream=True)
|
|
resp.raise_for_status()
|
|
f = resp.raw
|
|
|
|
if filename.endswith('.yml'):
|
|
kwargs = yaml.load_stream(f)
|
|
elif filename.endswith('.json'):
|
|
kwargs = json.load(f)
|
|
else:
|
|
raise TypeError("Parameters file `{}` is not JSON or YAML".format(filename))
|
|
|
|
kwargs.update(overrides)
|
|
|
|
return Parameters(strict=strict, **kwargs)
|
|
|
|
|
|
def parameters_loader(filename, strict=True, overrides=None):
|
|
def get_parameters(graph_config):
|
|
parameters = load_parameters_file(
|
|
filename,
|
|
strict=strict,
|
|
overrides=overrides,
|
|
trust_domain=graph_config["trust-domain"],
|
|
)
|
|
parameters.check()
|
|
return parameters
|
|
return get_parameters
|