forked from mirrors/gecko-dev
195 lines
7 KiB
Python
195 lines
7 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
"""
|
|
Apply some defaults and minor modifications to the jobs defined in the build
|
|
kind.
|
|
"""
|
|
|
|
from __future__ import absolute_import, print_function, unicode_literals
|
|
|
|
from taskgraph.transforms.base import TransformSequence
|
|
from taskgraph.util.attributes import RELEASE_PROJECTS
|
|
from taskgraph.util.schema import resolve_keyed_by
|
|
from taskgraph.util.treeherder import add_suffix
|
|
from taskgraph.util.workertypes import worker_type_implementation
|
|
|
|
from mozbuild.artifact_builds import JOB_CHOICES as ARTIFACT_JOBS
|
|
|
|
import logging
|
|
logger = logging.getLogger(__name__)
|
|
|
|
transforms = TransformSequence()
|
|
|
|
|
|
@transforms.add
|
|
def set_defaults(config, jobs):
|
|
"""Set defaults, including those that differ per worker implementation"""
|
|
for job in jobs:
|
|
job['treeherder'].setdefault('kind', 'build')
|
|
job['treeherder'].setdefault('tier', 1)
|
|
_, worker_os = worker_type_implementation(config.graph_config, job['worker-type'])
|
|
worker = job.setdefault('worker', {})
|
|
worker.setdefault('env', {})
|
|
if worker_os == "linux":
|
|
worker.setdefault('docker-image', {'in-tree': 'debian8-amd64-build'})
|
|
worker['chain-of-trust'] = True
|
|
elif worker_os == "windows":
|
|
worker['chain-of-trust'] = True
|
|
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def stub_installer(config, jobs):
|
|
for job in jobs:
|
|
resolve_keyed_by(
|
|
job, 'stub-installer', item_name=job['name'], project=config.params['project'],
|
|
**{
|
|
'release-type': config.params['release_type'],
|
|
}
|
|
)
|
|
job.setdefault('attributes', {})
|
|
if job.get('stub-installer'):
|
|
job['attributes']['stub-installer'] = job['stub-installer']
|
|
job['worker']['env'].update({"USE_STUB_INSTALLER": "1"})
|
|
if 'stub-installer' in job:
|
|
del job['stub-installer']
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def resolve_shipping_product(config, jobs):
|
|
for job in jobs:
|
|
resolve_keyed_by(
|
|
job, 'shipping-product', item_name=job['name'],
|
|
**{
|
|
'release-type': config.params['release_type'],
|
|
}
|
|
)
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def update_channel(config, jobs):
|
|
keys = [
|
|
'run.update-channel',
|
|
'run.mar-channel-id',
|
|
'run.accepted-mar-channel-ids',
|
|
]
|
|
for job in jobs:
|
|
job['worker'].setdefault('env', {})
|
|
for key in keys:
|
|
resolve_keyed_by(
|
|
job, key, item_name=job['name'],
|
|
**{
|
|
'project': config.params['project'],
|
|
'release-type': config.params['release_type'],
|
|
}
|
|
)
|
|
update_channel = job['run'].pop('update-channel', None)
|
|
if update_channel:
|
|
job['run'].setdefault('extra-config', {})['update_channel'] = update_channel
|
|
job['attributes']['update-channel'] = update_channel
|
|
mar_channel_id = job['run'].pop('mar-channel-id', None)
|
|
if mar_channel_id:
|
|
job['attributes']['mar-channel-id'] = mar_channel_id
|
|
job['worker']['env']['MAR_CHANNEL_ID'] = mar_channel_id
|
|
accepted_mar_channel_ids = job['run'].pop('accepted-mar-channel-ids', None)
|
|
if accepted_mar_channel_ids:
|
|
job['attributes']['accepted-mar-channel-ids'] = accepted_mar_channel_ids
|
|
job['worker']['env']['ACCEPTED_MAR_CHANNEL_IDS'] = accepted_mar_channel_ids
|
|
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def mozconfig(config, jobs):
|
|
for job in jobs:
|
|
resolve_keyed_by(
|
|
job, 'run.mozconfig-variant', item_name=job['name'],
|
|
**{
|
|
'release-type': config.params['release_type'],
|
|
}
|
|
)
|
|
mozconfig_variant = job['run'].pop('mozconfig-variant', None)
|
|
if mozconfig_variant:
|
|
job['run'].setdefault('extra-config', {})['mozconfig_variant'] = mozconfig_variant
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def use_artifact(config, jobs):
|
|
if config.params.is_try():
|
|
use_artifact = config.params['try_task_config'].get('use-artifact-builds', False)
|
|
else:
|
|
use_artifact = False
|
|
for job in jobs:
|
|
if (
|
|
config.kind == "build"
|
|
and use_artifact
|
|
and job.get("index", {}).get("job-name") in ARTIFACT_JOBS
|
|
# If tests aren't packaged, then we are not able to rebuild all the packages
|
|
and job['worker']['env'].get('MOZ_AUTOMATION_PACKAGE_TESTS') == '1'
|
|
):
|
|
job['treeherder']['symbol'] = add_suffix(job['treeherder']['symbol'], 'a')
|
|
job['worker']['env']['USE_ARTIFACT'] = '1'
|
|
job['attributes']['artifact-build'] = True
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def use_profile_data(config, jobs):
|
|
for job in jobs:
|
|
use_pgo = job.pop('use-pgo', False)
|
|
disable_pgo = config.params['try_task_config'].get('disable-pgo', False)
|
|
artifact_build = job['attributes'].get('artifact-build')
|
|
if not use_pgo or disable_pgo or artifact_build:
|
|
yield job
|
|
continue
|
|
|
|
# If use_pgo is True, the task uses the generate-profile task of the
|
|
# same name. Otherwise a task can specify a specific generate-profile
|
|
# task to use in the use_pgo field.
|
|
if use_pgo is True:
|
|
name = job['name']
|
|
else:
|
|
name = use_pgo
|
|
dependencies = 'generate-profile-{}'.format(name)
|
|
job.setdefault('dependencies', {})['generate-profile'] = dependencies
|
|
job.setdefault('fetches', {})['generate-profile'] = ['profdata.tar.xz']
|
|
job['worker']['env'].update({"TASKCLUSTER_PGO_PROFILE_USE": "1"})
|
|
|
|
_, worker_os = worker_type_implementation(config.graph_config, job['worker-type'])
|
|
if worker_os == "linux":
|
|
# LTO linkage needs more open files than the default from run-task.
|
|
job['worker']['env'].update({"MOZ_LIMIT_NOFILE": "8192"})
|
|
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def resolve_keys(config, jobs):
|
|
for job in jobs:
|
|
resolve_keyed_by(
|
|
job, 'use-sccache', item_name=job['name'],
|
|
**{'release-level': config.params.release_level()}
|
|
)
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def enable_full_crashsymbols(config, jobs):
|
|
"""Enable full crashsymbols on jobs with
|
|
'enable-full-crashsymbols' set to True and on release branches, or
|
|
on try"""
|
|
branches = RELEASE_PROJECTS | {'try', }
|
|
for job in jobs:
|
|
enable_full_crashsymbols = job['attributes'].get('enable-full-crashsymbols')
|
|
if enable_full_crashsymbols and config.params['project'] in branches:
|
|
logger.debug("Enabling full symbol generation for %s", job['name'])
|
|
job['worker']['env']['MOZ_ENABLE_FULL_SYMBOLS'] = '1'
|
|
else:
|
|
logger.debug("Disabling full symbol generation for %s", job['name'])
|
|
job['attributes'].pop('enable-full-crashsymbols', None)
|
|
yield job
|