forked from mirrors/gecko-dev
02a7b4ebdf
Allow-list all Python code in tree for use with the black linter, and re-format all code in-tree accordingly. To produce this patch I did all of the following: 1. Make changes to tools/lint/black.yml to remove include: stanza and update list of source extensions. 2. Run ./mach lint --linter black --fix 3. Make some ad-hoc manual updates to python/mozbuild/mozbuild/test/configure/test_configure.py -- it has some hard-coded line numbers that the reformat breaks. 4. Make some ad-hoc manual updates to `testing/marionette/client/setup.py`, `testing/marionette/harness/setup.py`, and `testing/firefox-ui/harness/setup.py`, which have hard-coded regexes that break after the reformat. 5. Add a set of exclusions to black.yml. These will be deleted in a follow-up bug (1672023). # ignore-this-changeset Differential Revision: https://phabricator.services.mozilla.com/D94045
327 lines
11 KiB
Python
327 lines
11 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
from __future__ import absolute_import, print_function, unicode_literals
|
|
|
|
import io
|
|
import logging
|
|
import os.path
|
|
import json
|
|
from datetime import datetime
|
|
|
|
from mozbuild.util import ReadOnlyDict, memoize
|
|
from mozversioncontrol import get_repository_object
|
|
from taskgraph.util.schema import validate_schema
|
|
from voluptuous import (
|
|
ALLOW_EXTRA,
|
|
Any,
|
|
Inclusive,
|
|
Required,
|
|
Schema,
|
|
)
|
|
|
|
import six
|
|
from six import text_type
|
|
|
|
from . import GECKO
|
|
from .util.attributes import release_level
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ParameterMismatch(Exception):
|
|
"""Raised when a parameters.yml has extra or missing parameters."""
|
|
|
|
|
|
@memoize
|
|
def get_head_ref():
|
|
return six.ensure_text(get_repository_object(GECKO).head_ref)
|
|
|
|
|
|
def get_contents(path):
|
|
with io.open(path, "r") as fh:
|
|
contents = fh.readline().rstrip()
|
|
return contents
|
|
|
|
|
|
def get_version(product_dir="browser"):
|
|
version_path = os.path.join(GECKO, product_dir, "config", "version_display.txt")
|
|
return get_contents(version_path)
|
|
|
|
|
|
def get_app_version(product_dir="browser"):
|
|
app_version_path = os.path.join(GECKO, product_dir, "config", "version.txt")
|
|
return get_contents(app_version_path)
|
|
|
|
|
|
base_schema = Schema(
|
|
{
|
|
Required("app_version"): text_type,
|
|
Required("backstop"): bool,
|
|
Required("base_repository"): text_type,
|
|
Required("build_date"): int,
|
|
Required("build_number"): int,
|
|
Inclusive("comm_base_repository", "comm"): text_type,
|
|
Inclusive("comm_head_ref", "comm"): text_type,
|
|
Inclusive("comm_head_repository", "comm"): text_type,
|
|
Inclusive("comm_head_rev", "comm"): text_type,
|
|
Required("do_not_optimize"): [text_type],
|
|
Required("existing_tasks"): {text_type: text_type},
|
|
Required("filters"): [text_type],
|
|
Required("head_ref"): text_type,
|
|
Required("head_repository"): text_type,
|
|
Required("head_rev"): text_type,
|
|
Required("hg_branch"): text_type,
|
|
Required("level"): text_type,
|
|
Required("message"): text_type,
|
|
Required("moz_build_date"): text_type,
|
|
Required("next_version"): Any(None, text_type),
|
|
Required("optimize_strategies"): Any(None, text_type),
|
|
Required("optimize_target_tasks"): bool,
|
|
Required("owner"): text_type,
|
|
Required("phabricator_diff"): Any(None, text_type),
|
|
Required("project"): text_type,
|
|
Required("pushdate"): int,
|
|
Required("pushlog_id"): text_type,
|
|
Required("release_enable_emefree"): bool,
|
|
Required("release_enable_partner_repack"): bool,
|
|
Required("release_enable_partner_attribution"): bool,
|
|
Required("release_eta"): Any(None, text_type),
|
|
Required("release_history"): {text_type: dict},
|
|
Required("release_partners"): Any(None, [text_type]),
|
|
Required("release_partner_config"): Any(None, dict),
|
|
Required("release_partner_build_number"): int,
|
|
Required("release_type"): text_type,
|
|
Required("release_product"): Any(None, text_type),
|
|
Required("required_signoffs"): [text_type],
|
|
Required("signoff_urls"): dict,
|
|
# target-kind is not included, since it should never be
|
|
# used at run-time
|
|
Required("target_tasks_method"): text_type,
|
|
Required("tasks_for"): text_type,
|
|
Required("test_manifest_loader"): text_type,
|
|
Required("try_mode"): Any(None, text_type),
|
|
Required("try_options"): Any(None, dict),
|
|
Required("try_task_config"): dict,
|
|
Required("version"): text_type,
|
|
}
|
|
)
|
|
|
|
|
|
COMM_PARAMETERS = [
|
|
"comm_base_repository",
|
|
"comm_head_ref",
|
|
"comm_head_repository",
|
|
"comm_head_rev",
|
|
]
|
|
|
|
|
|
def extend_parameters_schema(schema):
|
|
"""
|
|
Extend the schema for parameters to include per-project configuration.
|
|
|
|
This should be called by the `taskgraph.register` function in the
|
|
graph-configuration.
|
|
"""
|
|
global base_schema
|
|
base_schema = base_schema.extend(schema)
|
|
|
|
|
|
class Parameters(ReadOnlyDict):
|
|
"""An immutable dictionary with nicer KeyError messages on failure"""
|
|
|
|
def __init__(self, strict=True, **kwargs):
|
|
self.strict = strict
|
|
|
|
if not self.strict:
|
|
# apply defaults to missing parameters
|
|
kwargs = Parameters._fill_defaults(**kwargs)
|
|
|
|
ReadOnlyDict.__init__(self, **kwargs)
|
|
|
|
@staticmethod
|
|
def _fill_defaults(**kwargs):
|
|
now = datetime.utcnow()
|
|
epoch = datetime.utcfromtimestamp(0)
|
|
seconds_from_epoch = int((now - epoch).total_seconds())
|
|
|
|
defaults = {
|
|
"app_version": get_app_version(),
|
|
"backstop": False,
|
|
"base_repository": "https://hg.mozilla.org/mozilla-unified",
|
|
"build_date": seconds_from_epoch,
|
|
"build_number": 1,
|
|
"do_not_optimize": [],
|
|
"existing_tasks": {},
|
|
"filters": ["target_tasks_method"],
|
|
"head_ref": get_head_ref(),
|
|
"head_repository": "https://hg.mozilla.org/mozilla-central",
|
|
"head_rev": get_head_ref(),
|
|
"hg_branch": "default",
|
|
"level": "3",
|
|
"message": "",
|
|
"moz_build_date": six.ensure_text(now.strftime("%Y%m%d%H%M%S")),
|
|
"next_version": None,
|
|
"optimize_strategies": None,
|
|
"optimize_target_tasks": True,
|
|
"owner": "nobody@mozilla.com",
|
|
"phabricator_diff": None,
|
|
"project": "mozilla-central",
|
|
"pushdate": seconds_from_epoch,
|
|
"pushlog_id": "0",
|
|
"release_enable_emefree": False,
|
|
"release_enable_partner_repack": False,
|
|
"release_enable_partner_attribution": False,
|
|
"release_eta": "",
|
|
"release_history": {},
|
|
"release_partners": [],
|
|
"release_partner_config": None,
|
|
"release_partner_build_number": 1,
|
|
"release_product": None,
|
|
"release_type": "nightly",
|
|
"required_signoffs": [],
|
|
"signoff_urls": {},
|
|
"target_tasks_method": "default",
|
|
"tasks_for": "hg-push",
|
|
"test_manifest_loader": "default",
|
|
"try_mode": None,
|
|
"try_options": None,
|
|
"try_task_config": {},
|
|
"version": get_version(),
|
|
}
|
|
|
|
if set(COMM_PARAMETERS) & set(kwargs):
|
|
defaults.update(
|
|
{
|
|
"comm_base_repository": "https://hg.mozilla.org/comm-central",
|
|
"comm_head_repository": "https://hg.mozilla.org/comm-central",
|
|
}
|
|
)
|
|
|
|
for name, default in defaults.items():
|
|
if name not in kwargs:
|
|
kwargs[name] = default
|
|
|
|
return kwargs
|
|
|
|
def check(self):
|
|
schema = (
|
|
base_schema if self.strict else base_schema.extend({}, extra=ALLOW_EXTRA)
|
|
)
|
|
validate_schema(schema, self.copy(), "Invalid parameters:")
|
|
|
|
def __getitem__(self, k):
|
|
try:
|
|
return super(Parameters, self).__getitem__(k)
|
|
except KeyError:
|
|
raise KeyError("taskgraph parameter {!r} not found".format(k))
|
|
|
|
def is_try(self):
|
|
"""
|
|
Determine whether this graph is being built on a try project or for
|
|
`mach try fuzzy`.
|
|
"""
|
|
return "try" in self["project"] or self["try_mode"] == "try_select"
|
|
|
|
def file_url(self, path, pretty=False):
|
|
"""
|
|
Determine the VCS URL for viewing a file in the tree, suitable for
|
|
viewing by a human.
|
|
|
|
:param text_type path: The path, relative to the root of the repository.
|
|
:param bool pretty: Whether to return a link to a formatted version of the
|
|
file, or the raw file version.
|
|
:return text_type: The URL displaying the given path.
|
|
"""
|
|
if path.startswith("comm/"):
|
|
path = path[len("comm/") :]
|
|
repo = self["comm_head_repository"]
|
|
rev = self["comm_head_rev"]
|
|
else:
|
|
repo = self["head_repository"]
|
|
rev = self["head_rev"]
|
|
|
|
endpoint = "file" if pretty else "raw-file"
|
|
return "{}/{}/{}/{}".format(repo, endpoint, rev, path)
|
|
|
|
def release_level(self):
|
|
"""
|
|
Whether this is a staging release or not.
|
|
|
|
:return six.text_type: One of "production" or "staging".
|
|
"""
|
|
return release_level(self["project"])
|
|
|
|
|
|
def load_parameters_file(filename, strict=True, overrides=None, trust_domain=None):
|
|
"""
|
|
Load parameters from a path, url, decision task-id or project.
|
|
|
|
Examples:
|
|
task-id=fdtgsD5DQUmAQZEaGMvQ4Q
|
|
project=mozilla-central
|
|
"""
|
|
import requests
|
|
from taskgraph.util.taskcluster import get_artifact_url, find_task_id
|
|
from taskgraph.util import yaml
|
|
|
|
if overrides is None:
|
|
overrides = {}
|
|
|
|
if not filename:
|
|
return Parameters(strict=strict, **overrides)
|
|
|
|
try:
|
|
# reading parameters from a local parameters.yml file
|
|
f = open(filename)
|
|
except IOError:
|
|
# fetching parameters.yml using task task-id, project or supplied url
|
|
task_id = None
|
|
if filename.startswith("task-id="):
|
|
task_id = filename.split("=")[1]
|
|
elif filename.startswith("project="):
|
|
if trust_domain is None:
|
|
raise ValueError(
|
|
"Can't specify parameters by project "
|
|
"if trust domain isn't supplied.",
|
|
)
|
|
index = "{trust_domain}.v2.{project}.latest.taskgraph.decision".format(
|
|
trust_domain=trust_domain,
|
|
project=filename.split("=")[1],
|
|
)
|
|
task_id = find_task_id(index)
|
|
|
|
if task_id:
|
|
filename = get_artifact_url(task_id, "public/parameters.yml")
|
|
logger.info("Loading parameters from {}".format(filename))
|
|
resp = requests.get(filename, stream=True)
|
|
resp.raise_for_status()
|
|
f = resp.raw
|
|
|
|
if filename.endswith(".yml"):
|
|
kwargs = yaml.load_stream(f)
|
|
elif filename.endswith(".json"):
|
|
kwargs = json.load(f)
|
|
else:
|
|
raise TypeError("Parameters file `{}` is not JSON or YAML".format(filename))
|
|
|
|
kwargs.update(overrides)
|
|
|
|
return Parameters(strict=strict, **kwargs)
|
|
|
|
|
|
def parameters_loader(filename, strict=True, overrides=None):
|
|
def get_parameters(graph_config):
|
|
parameters = load_parameters_file(
|
|
filename,
|
|
strict=strict,
|
|
overrides=overrides,
|
|
trust_domain=graph_config["trust-domain"],
|
|
)
|
|
parameters.check()
|
|
return parameters
|
|
|
|
return get_parameters
|