mirror of
https://github.com/mozilla/gecko-dev.git
synced 2025-11-11 21:58:41 +02:00
260 lines
9.7 KiB
Python
260 lines
9.7 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
"""
|
|
Transform the beetmover task into an actual task description.
|
|
"""
|
|
|
|
from __future__ import absolute_import, print_function, unicode_literals
|
|
|
|
from taskgraph.loader.single_dep import schema
|
|
from taskgraph.transforms.base import TransformSequence
|
|
from taskgraph.transforms.beetmover import craft_release_properties
|
|
from taskgraph.util.attributes import copy_attributes_from_dependent_job
|
|
from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by
|
|
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
|
|
get_beetmover_action_scope,
|
|
generate_beetmover_upstream_artifacts,
|
|
generate_beetmover_artifact_map,
|
|
should_use_artifact_map)
|
|
from taskgraph.transforms.task import task_description_schema
|
|
from taskgraph.transforms.release_sign_and_push_langpacks import get_upstream_task_ref
|
|
from voluptuous import Required, Optional
|
|
|
|
import logging
|
|
import copy
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
transforms = TransformSequence()
|
|
|
|
|
|
beetmover_description_schema = schema.extend({
|
|
# depname is used in taskref's to identify the taskID of the unsigned things
|
|
Required('depname', default='build'): basestring,
|
|
|
|
# attributes is used for enabling artifact-map by declarative artifacts
|
|
Required('attributes'): {basestring: object},
|
|
|
|
# unique label to describe this beetmover task, defaults to {dep.label}-beetmover
|
|
Optional('label'): basestring,
|
|
|
|
# treeherder is allowed here to override any defaults we use for beetmover. See
|
|
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
|
|
# below transforms for defaults of various values.
|
|
Optional('treeherder'): task_description_schema['treeherder'],
|
|
|
|
Required('description'): basestring,
|
|
Required('worker-type'): optionally_keyed_by('release-level', basestring),
|
|
Required('run-on-projects'): [],
|
|
|
|
# locale is passed only for l10n beetmoving
|
|
Optional('locale'): basestring,
|
|
Optional('shipping-phase'): task_description_schema['shipping-phase'],
|
|
Optional('shipping-product'): task_description_schema['shipping-product'],
|
|
})
|
|
|
|
|
|
@transforms.add
|
|
def set_label(config, jobs):
|
|
for job in jobs:
|
|
job['label'] = job['primary-dependency'].label.replace(
|
|
'sign-and-push-langpacks', 'beetmover-signed-langpacks'
|
|
)
|
|
|
|
yield job
|
|
|
|
|
|
transforms.add_validate(beetmover_description_schema)
|
|
|
|
|
|
@transforms.add
|
|
def resolve_keys(config, jobs):
|
|
for job in jobs:
|
|
for field in ('worker-type', 'attributes.artifact_map'):
|
|
resolve_keyed_by(
|
|
job, field, item_name=job['label'],
|
|
**{
|
|
'release-level': config.params.release_level(),
|
|
'project': config.params['project']
|
|
}
|
|
)
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def make_task_description(config, jobs):
|
|
for job in jobs:
|
|
dep_job = job['primary-dependency']
|
|
attributes = dep_job.attributes
|
|
|
|
treeherder = job.get('treeherder', {})
|
|
treeherder.setdefault('symbol', 'langpack(BM{})'.format(attributes.get('l10n_chunk', '')))
|
|
dep_th_platform = dep_job.task.get('extra', {}).get(
|
|
'treeherder', {}).get('machine', {}).get('platform', '')
|
|
treeherder.setdefault('platform',
|
|
"{}/opt".format(dep_th_platform))
|
|
treeherder.setdefault('tier', 1)
|
|
treeherder.setdefault('kind', 'build')
|
|
|
|
job['attributes'].update(copy_attributes_from_dependent_job(dep_job))
|
|
job['attributes']['chunk_locales'] = dep_job.attributes['chunk_locales']
|
|
|
|
job['description'] = job['description'].format(
|
|
locales='/'.join(job['attributes']['chunk_locales']),
|
|
platform=job['attributes']['build_platform']
|
|
)
|
|
|
|
job['scopes'] = [
|
|
get_beetmover_bucket_scope(config),
|
|
get_beetmover_action_scope(config),
|
|
]
|
|
|
|
job['dependencies'] = {dep_job.kind: dep_job.label}
|
|
|
|
job['run-on-projects'] = dep_job.attributes['run_on_projects']
|
|
job['treeherder'] = treeherder
|
|
job['shipping-phase'] = dep_job.attributes['shipping_phase']
|
|
job['shipping-product'] = dep_job.attributes['shipping_product']
|
|
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def make_task_worker(config, jobs):
|
|
for job in jobs:
|
|
signing_task_ref = get_upstream_task_ref(
|
|
job, expected_kinds=('release-sign-and-push-langpacks',)
|
|
)
|
|
|
|
platform = job["attributes"]["build_platform"]
|
|
locale = job["attributes"]["chunk_locales"]
|
|
if should_use_artifact_map(platform, config.params['project']):
|
|
upstream_artifacts = generate_beetmover_upstream_artifacts(
|
|
config, job, platform, locale,
|
|
)
|
|
else:
|
|
upstream_artifacts = generate_upstream_artifacts(
|
|
signing_task_ref, job['attributes']['chunk_locales']
|
|
)
|
|
job['worker'] = {
|
|
'implementation': 'beetmover',
|
|
'release-properties': craft_release_properties(config, job),
|
|
'upstream-artifacts': upstream_artifacts,
|
|
}
|
|
|
|
if should_use_artifact_map(platform, config.params['project']):
|
|
job['worker']['artifact-map'] = generate_beetmover_artifact_map(
|
|
config, job, platform=platform, locale=locale)
|
|
|
|
yield job
|
|
|
|
|
|
def generate_upstream_artifacts(upstream_task_ref, locales):
|
|
return [{
|
|
'taskId': {'task-reference': upstream_task_ref},
|
|
'taskType': 'scriptworker',
|
|
'locale': locale,
|
|
'paths': [
|
|
# addonscript uploads en-US XPI in the en-US folder
|
|
'public/build/{}/target.langpack.xpi'.format(locale)
|
|
],
|
|
} for locale in locales]
|
|
|
|
|
|
@transforms.add
|
|
def strip_unused_data(config, jobs):
|
|
for job in jobs:
|
|
del job['primary-dependency']
|
|
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def yield_all_platform_jobs(config, jobs):
|
|
# Even though langpacks are now platform independent, we keep beetmoving them at old
|
|
# platform-specific locations. That's why this transform exist
|
|
for job in jobs:
|
|
if 'ja-JP-mac' in job['label']:
|
|
# This locale must not be copied on any other platform than macos
|
|
yield job
|
|
else:
|
|
platforms = ('linux', 'linux64', 'macosx64', 'win32', 'win64')
|
|
if 'devedition' in job['attributes']['build_platform']:
|
|
platforms = ('{}-devedition'.format(plat) for plat in platforms)
|
|
for platform in platforms:
|
|
platform_job = copy.deepcopy(job)
|
|
if 'ja' in platform_job['attributes']['chunk_locales'] and \
|
|
platform in ('macosx64', 'macosx64-devedition'):
|
|
platform_job = _strip_ja_data_from_linux_job(platform_job)
|
|
|
|
platform_job = _change_platform_data(config, platform_job, platform)
|
|
|
|
yield platform_job
|
|
|
|
|
|
def _strip_ja_data_from_linux_job(platform_job):
|
|
# Let's take "ja" out the description. This locale is in a substring like "aa/bb/cc/dd", where
|
|
# "ja" could be any of "aa", "bb", "cc", "dd"
|
|
platform_job['description'] = platform_job['description'].replace('ja/', '')
|
|
platform_job['description'] = platform_job['description'].replace('/ja', '')
|
|
|
|
platform_job['worker']['upstream-artifacts'] = [
|
|
artifact
|
|
for artifact in platform_job['worker']['upstream-artifacts']
|
|
if artifact['locale'] != 'ja'
|
|
]
|
|
|
|
return platform_job
|
|
|
|
|
|
def _change_platform_in_artifact_map_paths(paths, orig_platform, new_platform):
|
|
amended_paths = {}
|
|
for artifact, artifact_info in paths.iteritems():
|
|
amended_artifact_info = {
|
|
'checksums_path': artifact_info['checksums_path'].replace(orig_platform, new_platform),
|
|
'destinations': [
|
|
d.replace(orig_platform, new_platform) for d in artifact_info['destinations']
|
|
]
|
|
}
|
|
amended_paths[artifact] = amended_artifact_info
|
|
|
|
return amended_paths
|
|
|
|
|
|
def _change_platform_data(config, platform_job, platform):
|
|
orig_platform = 'linux64'
|
|
if 'devedition' in platform:
|
|
orig_platform = 'linux64-devedition'
|
|
backup_platform = platform_job['attributes']['build_platform']
|
|
platform_job['attributes']['build_platform'] = platform
|
|
platform_job['label'] = platform_job['label'].replace(orig_platform, platform)
|
|
platform_job['description'] = platform_job['description'].replace(orig_platform, platform)
|
|
platform_job['treeherder']['platform'] = platform_job['treeherder']['platform'].replace(
|
|
orig_platform, platform
|
|
)
|
|
platform_job['worker']['release-properties']['platform'] = platform
|
|
|
|
# amend artifactMap entries as well
|
|
if should_use_artifact_map(backup_platform, config.params['project']):
|
|
platform_mapping = {
|
|
'linux64': 'linux-x86_64',
|
|
'linux': 'linux-i686',
|
|
'macosx64': 'mac',
|
|
'win32': 'win32',
|
|
'win64': 'win64',
|
|
}
|
|
orig_platform = platform_mapping.get(orig_platform, orig_platform)
|
|
platform = platform_mapping.get(platform, platform)
|
|
platform_job['worker']['artifact-map'] = [
|
|
{
|
|
'locale': entry['locale'],
|
|
'taskId': entry['taskId'],
|
|
'paths': _change_platform_in_artifact_map_paths(entry['paths'],
|
|
orig_platform,
|
|
platform)
|
|
} for entry in platform_job['worker']['artifact-map']
|
|
]
|
|
|
|
return platform_job
|