mirror of
https://github.com/mozilla/gecko-dev.git
synced 2025-11-09 21:00:42 +02:00
43a2e97c21
We starting doing that because snapshot.debian.org would ban some AWS IP ranges, and we would get random failures, but that's not the case anymore. OTOH, when more "normal" errors happen, like when you change a Dockerfile to add a package, and that package actually doesn't exist, the image build is tried 5 times, with no chance it will succeed, and treeherder doesn't link to the log because it's purple, so you need to manually go to taskcluster. Removing the autoretry will make things smoother. Differential Revision: https://phabricator.services.mozilla.com/D73392
258 lines
9.5 KiB
Python
258 lines
9.5 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
from __future__ import absolute_import, print_function, unicode_literals
|
|
|
|
import os
|
|
import re
|
|
|
|
from collections import deque
|
|
from six import text_type
|
|
import taskgraph
|
|
from taskgraph.transforms.base import TransformSequence
|
|
from taskgraph.transforms.task import _run_task_suffix
|
|
from .. import GECKO
|
|
from taskgraph.util.docker import (
|
|
generate_context_hash,
|
|
)
|
|
from taskgraph.util.taskcluster import get_root_url
|
|
from taskgraph.util.schema import (
|
|
Schema,
|
|
)
|
|
from voluptuous import (
|
|
Optional,
|
|
Required,
|
|
)
|
|
from .task import task_description_schema
|
|
|
|
DIGEST_RE = re.compile('^[0-9a-f]{64}$')
|
|
|
|
transforms = TransformSequence()
|
|
|
|
docker_image_schema = Schema({
|
|
# Name of the docker image.
|
|
Required('name'): text_type,
|
|
|
|
# Name of the parent docker image.
|
|
Optional('parent'): text_type,
|
|
|
|
# Treeherder symbol.
|
|
Required('symbol'): text_type,
|
|
|
|
# relative path (from config.path) to the file the docker image was defined
|
|
# in.
|
|
Optional('job-from'): text_type,
|
|
|
|
# Arguments to use for the Dockerfile.
|
|
Optional('args'): {text_type: text_type},
|
|
|
|
# Name of the docker image definition under taskcluster/docker, when
|
|
# different from the docker image name.
|
|
Optional('definition'): text_type,
|
|
|
|
# List of package tasks this docker image depends on.
|
|
Optional('packages'): [text_type],
|
|
|
|
Optional(
|
|
"index",
|
|
description="information for indexing this build so its artifacts can be discovered",
|
|
): task_description_schema['index'],
|
|
|
|
Optional(
|
|
"cache",
|
|
description="Whether this image should be cached based on inputs.",
|
|
): bool,
|
|
})
|
|
|
|
|
|
transforms.add_validate(docker_image_schema)
|
|
|
|
|
|
def order_image_tasks(config, tasks):
|
|
"""Iterate image tasks in an order where parent images come first."""
|
|
pending = deque(tasks)
|
|
task_names = {task['name'] for task in pending}
|
|
emitted = set()
|
|
while True:
|
|
try:
|
|
task = pending.popleft()
|
|
except IndexError:
|
|
break
|
|
parent = task.get('parent')
|
|
if parent and parent not in emitted:
|
|
if parent not in task_names:
|
|
raise Exception('Missing parent image for {}-{}: {}'.format(
|
|
config.kind, task['name'], parent))
|
|
pending.append(task)
|
|
continue
|
|
emitted.add(task['name'])
|
|
yield task
|
|
|
|
|
|
@transforms.add
|
|
def fill_template(config, tasks):
|
|
available_packages = set()
|
|
for task in config.kind_dependencies_tasks:
|
|
if task.kind != 'packages':
|
|
continue
|
|
name = task.label.replace('packages-', '')
|
|
available_packages.add(name)
|
|
|
|
context_hashes = {}
|
|
|
|
for task in order_image_tasks(config, tasks):
|
|
image_name = task.pop('name')
|
|
job_symbol = task.pop('symbol')
|
|
args = task.pop('args', {})
|
|
definition = task.pop('definition', image_name)
|
|
packages = task.pop('packages', [])
|
|
parent = task.pop('parent', None)
|
|
|
|
for p in packages:
|
|
if p not in available_packages:
|
|
raise Exception('Missing package job for {}-{}: {}'.format(
|
|
config.kind, image_name, p))
|
|
|
|
# Generating the context hash relies on arguments being set, so we
|
|
# set this now, although it's not the final value (it's a
|
|
# task-reference value, see further below). We add the package routes
|
|
# containing a hash to get the overall docker image hash, so changes
|
|
# to packages will be reflected in the docker image hash.
|
|
args['DOCKER_IMAGE_PACKAGES'] = ' '.join('<{}>'.format(p)
|
|
for p in packages)
|
|
if parent:
|
|
args['DOCKER_IMAGE_PARENT'] = '{}:{}'.format(parent, context_hashes[parent])
|
|
|
|
args['TASKCLUSTER_ROOT_URL'] = get_root_url(False)
|
|
|
|
if not taskgraph.fast:
|
|
context_path = os.path.join('taskcluster', 'docker', definition)
|
|
context_hash = generate_context_hash(
|
|
GECKO, context_path, image_name, args)
|
|
else:
|
|
context_hash = '0'*40
|
|
digest_data = [context_hash]
|
|
context_hashes[image_name] = context_hash
|
|
|
|
description = 'Build the docker image {} for use by dependent tasks'.format(
|
|
image_name)
|
|
|
|
# Adjust the zstandard compression level based on the execution level.
|
|
# We use faster compression for level 1 because we care more about
|
|
# end-to-end times. We use slower/better compression for other levels
|
|
# because images are read more often and it is worth the trade-off to
|
|
# burn more CPU once to reduce image size.
|
|
zstd_level = '3' if int(config.params['level']) == 1 else '10'
|
|
|
|
# include some information that is useful in reconstructing this task
|
|
# from JSON
|
|
taskdesc = {
|
|
'label': 'build-docker-image-' + image_name,
|
|
'description': description,
|
|
'attributes': {'image_name': image_name},
|
|
'expires-after': '28 days' if config.params.is_try() else '1 year',
|
|
'scopes': [
|
|
'secrets:get:project/taskcluster/gecko/hgfingerprint',
|
|
'secrets:get:project/taskcluster/gecko/hgmointernal',
|
|
],
|
|
'treeherder': {
|
|
'symbol': job_symbol,
|
|
'platform': 'taskcluster-images/opt',
|
|
'kind': 'other',
|
|
'tier': 1,
|
|
},
|
|
'run-on-projects': [],
|
|
'worker-type': 'images',
|
|
'worker': {
|
|
'implementation': 'docker-worker',
|
|
'os': 'linux',
|
|
'artifacts': [{
|
|
'type': 'file',
|
|
'path': '/builds/worker/workspace/artifacts/image.tar.zst',
|
|
'name': 'public/image.tar.zst',
|
|
}],
|
|
'env': {
|
|
'HG_STORE_PATH': '/builds/worker/checkouts/hg-store',
|
|
'HASH': context_hash,
|
|
'PROJECT': config.params['project'],
|
|
'IMAGE_NAME': image_name,
|
|
'DOCKER_IMAGE_ZSTD_LEVEL': zstd_level,
|
|
'GECKO_BASE_REPOSITORY': config.params['base_repository'],
|
|
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
|
|
'GECKO_HEAD_REV': config.params['head_rev'],
|
|
},
|
|
'chain-of-trust': True,
|
|
'docker-in-docker': True,
|
|
'taskcluster-proxy': True,
|
|
'max-run-time': 7200,
|
|
},
|
|
}
|
|
# Retry for 'funsize-update-generator' if exit status code is -1
|
|
if image_name in ['funsize-update-generator']:
|
|
taskdesc['worker']['retry-exit-status'] = [-1]
|
|
|
|
worker = taskdesc['worker']
|
|
|
|
# We use the in-tree image_builder image to build docker images, but
|
|
# that can't be used to build the image_builder image itself,
|
|
# obviously. So we fall back to an image on docker hub, identified
|
|
# by hash. After the image-builder image is updated, it's best to push
|
|
# and update this hash as well, to keep image-builder builds up to date.
|
|
if image_name == 'image_builder':
|
|
hash = 'sha256:c6622fd3e5794842ad83d129850330b26e6ba671e39c58ee288a616a3a1c4c73'
|
|
worker['docker-image'] = 'taskcluster/image_builder@' + hash
|
|
# Keep in sync with the Dockerfile used to generate the
|
|
# docker image whose digest is referenced above.
|
|
worker['volumes'] = [
|
|
'/builds/worker/checkouts',
|
|
'/builds/worker/workspace',
|
|
]
|
|
cache_name = 'imagebuilder-v1'
|
|
else:
|
|
worker['docker-image'] = {'in-tree': 'image_builder'}
|
|
cache_name = 'imagebuilder-sparse-{}'.format(_run_task_suffix())
|
|
# Force images built against the in-tree image builder to
|
|
# have a different digest by adding a fixed string to the
|
|
# hashed data.
|
|
# Append to this data whenever the image builder's output behavior
|
|
# is changed, in order to force all downstream images to be rebuilt and
|
|
# cached distinctly.
|
|
digest_data.append('image_builder')
|
|
# Updated for squashing images in Bug 1527394
|
|
digest_data.append('squashing layers')
|
|
|
|
worker['caches'] = [{
|
|
'type': 'persistent',
|
|
'name': cache_name,
|
|
'mount-point': '/builds/worker/checkouts',
|
|
}]
|
|
|
|
for k, v in args.items():
|
|
if k == 'DOCKER_IMAGE_PACKAGES':
|
|
worker['env'][k] = {'task-reference': v}
|
|
else:
|
|
worker['env'][k] = v
|
|
|
|
if packages:
|
|
deps = taskdesc.setdefault('dependencies', {})
|
|
for p in sorted(packages):
|
|
deps[p] = 'packages-{}'.format(p)
|
|
|
|
if parent:
|
|
deps = taskdesc.setdefault('dependencies', {})
|
|
deps[parent] = 'build-docker-image-{}'.format(parent)
|
|
worker['env']['DOCKER_IMAGE_PARENT_TASK'] = {
|
|
'task-reference': '<{}>'.format(parent),
|
|
}
|
|
if 'index' in task:
|
|
taskdesc['index'] = task['index']
|
|
|
|
if task.get('cache', True) and not taskgraph.fast:
|
|
taskdesc['cache'] = {
|
|
'type': 'docker-images.v2',
|
|
'name': image_name,
|
|
'digest-data': digest_data,
|
|
}
|
|
|
|
yield taskdesc
|