nagai/fune
3
0
Fork 0
forked from mirrors/gecko-dev
Code Pull requests Activity

Bug 1484012: [mozharness] Use fetch-content to download inputs in repackage tasks; r=Callek

Browse source 
Differential Revision: https://phabricator.services.mozilla.com/D3559

--HG--
extra : rebase_source : c910810629d16dbf507241fc609f3955afcf50e8
This commit is contained in:
Tom Prince 2018-08-15 15:12:33 -06:00
parent c60045308b
commit 5b57b42215
11 changed files with 73 additions and 125 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

68
taskcluster/taskgraph/transforms/repackage.py
View file

@ -17,7 +17,7 @@ from taskgraph.util.schema import (
resolve_keyed_by,
Schema,
)
from taskgraph.util.taskcluster import get_taskcluster_artifact_prefix, get_artifact_prefix
from taskgraph.util.taskcluster import get_artifact_prefix
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
@ -165,8 +165,6 @@ def make_job_description(config, jobs):
dependencies['build'] = "build-{}/opt".format(
dependencies[build_task][13:dependencies[build_task].rfind('-')])
build_task = 'build'
signing_task_ref = "<{}>".format(signing_task)
build_task_ref = "<{}>".format(build_task)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage'
@ -184,14 +182,11 @@ def make_job_description(config, jobs):
'using': 'mozharness',
'script': 'mozharness/scripts/repackage.py',
'job-script': 'taskcluster/scripts/builder/repackage.sh',
'actions': ['download_input', 'setup', 'repackage'],
'actions': ['setup', 'repackage'],
'extra-workspace-cache-key': 'repackage',
})
worker = {
'env': _generate_task_env(dep_job, build_platform, build_task_ref,
signing_task_ref, locale=locale,
project=config.params["project"]),
'artifacts': _generate_task_output_files(dep_job, build_platform,
locale=locale,
project=config.params["project"]),
@ -203,7 +198,11 @@ def make_job_description(config, jobs):
if locale:
# Make sure we specify the locale-specific upload dir
worker['env'].update(LOCALE=locale)
worker.setdefault('env', {}).update(LOCALE=locale)
use_stub = attributes.get('stub-installer')
if not use_stub:
worker.setdefault('env', {})['NO_STUB_INSTALLER'] = '1'
if build_platform.startswith('win'):
worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level
@ -240,6 +239,9 @@ def make_job_description(config, jobs):
'extra': job.get('extra', {}),
'worker': worker,
'run': run,
'fetches': _generate_download_config(dep_job, build_platform, build_task,
signing_task, locale=locale,
project=config.params["project"]),
}
if build_platform.startswith('macosx'):
@ -251,38 +253,42 @@ def make_job_description(config, jobs):
yield task
def _generate_task_env(task, build_platform, build_task_ref, signing_task_ref, locale=None,
project=None):
mar_prefix = get_taskcluster_artifact_prefix(
task, build_task_ref, postfix='host/bin/', locale=None
)
signed_prefix = get_taskcluster_artifact_prefix(task, signing_task_ref, locale=locale)
def _generate_download_config(task, build_platform, build_task, signing_task, locale=None,
project=None):
locale_path = '{}/'.format(locale) if locale else ''
if build_platform.startswith('linux') or build_platform.startswith('macosx'):
tarball_extension = 'bz2' if build_platform.startswith('linux') else 'gz'
return {
'SIGNED_INPUT': {'task-reference': '{}target.tar.{}'.format(
signed_prefix, tarball_extension
)},
'UNSIGNED_MAR': {'task-reference': '{}mar'.format(mar_prefix)},
signing_task: [
{
'artifact': '{}target.tar.{}'.format(locale_path, tarball_extension),
'extract': False,
},
],
build_task: [
'host/bin/mar',
],
}
elif build_platform.startswith('win'):
task_env = {
'SIGNED_ZIP': {'task-reference': '{}target.zip'.format(signed_prefix)},
'SIGNED_SETUP': {'task-reference': '{}setup.exe'.format(signed_prefix)},
'UNSIGNED_MAR': {'task-reference': '{}mar.exe'.format(mar_prefix)},
fetch_config = {
signing_task: [
{
'artifact': '{}target.zip'.format(locale_path),
'extract': False,
},
'{}setup.exe'.format(locale_path),
],
build_task: [
'host/bin/mar.exe',
],
}
use_stub = task.attributes.get('stub-installer')
if use_stub:
task_env['SIGNED_SETUP_STUB'] = {
'task-reference': '{}setup-stub.exe'.format(signed_prefix),
}
elif '32' in build_platform:
# Stub installer is only attempted on win32
task_env['NO_STUB_INSTALLER'] = '1'
fetch_config[signing_task].append('{}setup-stub.exe'.format(locale_path))
return task_env
return fetch_config
raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))
@ -322,8 +328,8 @@ def _generate_task_output_files(task, build_platform, locale=None, project=None)
if use_stub:
output_files.append({
'type': 'file',
'path': '{}/{}target.stub-installer.exe'.format(
artifact_prefix, locale_output_path
'path': 'build/outputs/{}target.stub-installer.exe'.format(
locale_output_path
),
'name': '{}/{}target.stub-installer.exe'.format(
artifact_prefix, locale_output_path

41
taskcluster/taskgraph/transforms/repackage_partner.py
View file

@ -17,7 +17,7 @@ from taskgraph.util.schema import (
resolve_keyed_by,
Schema,
)
from taskgraph.util.taskcluster import get_taskcluster_artifact_prefix, get_artifact_prefix
from taskgraph.util.taskcluster import get_artifact_prefix
from taskgraph.util.partners import check_if_partners_enabled
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
@ -142,7 +142,6 @@ def make_job_description(config, jobs):
signing_task = dependency
elif build_platform.startswith('win') and dependency.endswith('repack'):
signing_task = dependency
signing_task_ref = "<{}>".format(signing_task)
attributes['repackage_type'] = 'repackage'
@ -159,12 +158,11 @@ def make_job_description(config, jobs):
})
worker = {
'env': _generate_task_env(build_platform, signing_task, signing_task_ref,
partner=repack_id),
'artifacts': _generate_task_output_files(dep_job, build_platform, partner=repack_id),
'chain-of-trust': True,
'max-run-time': 7200 if build_platform.startswith('win') else 3600,
'taskcluster-proxy': True if get_artifact_prefix(dep_job) else False,
'env': {},
# Don't add generic artifact directory.
'skip-artifacts': True,
}
@ -206,6 +204,9 @@ def make_job_description(config, jobs):
'extra': job.get('extra', {}),
'worker': worker,
'run': run,
'fetches': _generate_download_config(dep_job, build_platform,
signing_task, partner=repack_id,
project=config.params["project"]),
}
if build_platform.startswith('macosx'):
@ -217,28 +218,30 @@ def make_job_description(config, jobs):
yield task
def _generate_task_env(build_platform, signing_task, signing_task_ref, partner):
# Force private artifacts here, until we can populate our dependency map
# with actual task definitions rather than labels.
# (get_taskcluster_artifact_prefix requires the task definition to find
# the artifact_prefix attribute).
signed_prefix = get_taskcluster_artifact_prefix(
signing_task, signing_task_ref, locale=partner, force_private=True
)
signed_prefix = signed_prefix.replace('public/build', 'releng/partner')
def _generate_download_config(task, build_platform, signing_task, partner=None,
project=None):
locale_path = '{}/'.format(partner) if partner else ''
if build_platform.startswith('macosx'):
return {
'SIGNED_INPUT': {'task-reference': '{}target.tar.gz'.format(signed_prefix)},
signing_task: [
{
'artifact': '{}target.tar.gz'.format(locale_path),
'extract': False,
},
],
}
elif build_platform.startswith('win'):
task_env = {
'SIGNED_ZIP': {'task-reference': '{}target.zip'.format(signed_prefix)},
'SIGNED_SETUP': {'task-reference': '{}setup.exe'.format(signed_prefix)},
return {
signing_task: [
{
'artifact': '{}target.zip'.format(locale_path),
'extract': False,
},
'{}setup.exe'.format(locale_path),
],
}
return task_env
raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))

7
testing/mozharness/configs/repackage/linux32_signed.py
View file

@ -5,14 +5,9 @@ platform = "linux32"
config = {
"locale": os.environ.get("LOCALE"),
"download_config": {
"target.tar.gz": os.environ.get("SIGNED_INPUT"),
"mar": os.environ.get("UNSIGNED_MAR"),
},
"repackage_config": [[
"mar",
"-i", "{abs_input_dir}/target.tar.gz",
"-i", "{abs_input_dir}/target.tar.bz2",
"--mar", "{abs_input_dir}/mar",
"-o", "{abs_output_dir}/target.complete.mar"
]],

7
testing/mozharness/configs/repackage/linux64_signed.py
View file

@ -5,14 +5,9 @@ platform = "linux64"
config = {
"locale": os.environ.get("LOCALE"),
"download_config": {
"target.tar.gz": os.environ.get("SIGNED_INPUT"),
"mar": os.environ.get("UNSIGNED_MAR"),
},
"repackage_config": [[
"mar",
"-i", "{abs_input_dir}/target.tar.gz",
"-i", "{abs_input_dir}/target.tar.bz2",
"--mar", "{abs_input_dir}/mar",
"-o", "{abs_output_dir}/target.complete.mar"
]],

4
testing/mozharness/configs/repackage/osx_partner.py
View file

@ -5,10 +5,6 @@ config = {
"repack_id": os.environ.get("REPACK_ID"),
"download_config": {
"target.tar.gz": os.environ.get("SIGNED_INPUT"),
},
"repackage_config": [[
"dmg",
"-i", "{abs_input_dir}/target.tar.gz",

5
testing/mozharness/configs/repackage/osx_signed.py
View file

@ -5,11 +5,6 @@ config = {
"locale": os.environ.get("LOCALE"),
"download_config": {
"target.tar.gz": os.environ.get("SIGNED_INPUT"),
"mar": os.environ.get("UNSIGNED_MAR"),
},
"repackage_config": [[
"dmg",
"-i", "{abs_input_dir}/target.tar.gz",

7
testing/mozharness/configs/repackage/win32_partner.py
View file

@ -2,11 +2,6 @@ import os
platform = "win32"
download_config = {
"target.zip": os.environ.get("SIGNED_ZIP"),
"setup.exe": os.environ.get("SIGNED_SETUP"),
}
repackage_config = [[
"installer",
"--package-name", "firefox",
@ -20,8 +15,6 @@ repackage_config = [[
config = {
"repack_id": os.environ.get("REPACK_ID"),
"download_config": download_config,
"repackage_config": repackage_config,
# ToolTool

15
testing/mozharness/configs/repackage/win32_signed.py
View file

@ -2,19 +2,6 @@ import os
platform = "win32"
download_config = {
"target.zip": os.environ.get("SIGNED_ZIP"),
"setup.exe": os.environ.get("SIGNED_SETUP"),
"mar.exe": os.environ.get("UNSIGNED_MAR"),
}
if not os.environ.get("NO_STUB_INSTALLER"):
# Some channels, like esr don't build a stub installer
download_config.update({
# stub installer is only built on win32
"setup-stub.exe": os.environ.get("SIGNED_SETUP_STUB"),
})
repackage_config = [[
"installer",
"--package-name", "firefox",
@ -43,8 +30,6 @@ if not os.environ.get("NO_STUB_INSTALLER"):
config = {
"locale": os.environ.get("LOCALE"),
"download_config": download_config,
"repackage_config": repackage_config,
# ToolTool

7
testing/mozharness/configs/repackage/win64_partner.py
View file

@ -2,11 +2,6 @@ import os
platform = "win64"
download_config = {
"target.zip": os.environ.get("SIGNED_ZIP"),
"setup.exe": os.environ.get("SIGNED_SETUP"),
}
repackage_config = [[
"installer",
"--package-name", "firefox",
@ -20,8 +15,6 @@ repackage_config = [[
config = {
"repack_id": os.environ.get("REPACK_ID"),
"download_config": download_config,
"repackage_config": repackage_config,
# ToolTool

8
testing/mozharness/configs/repackage/win64_signed.py
View file

@ -2,12 +2,6 @@ import os
platform = "win64"
download_config = {
"target.zip": os.environ.get("SIGNED_ZIP"),
"setup.exe": os.environ.get("SIGNED_SETUP"),
"mar.exe": os.environ.get("UNSIGNED_MAR"),
}
repackage_config = [[
"installer",
"--package-name", "firefox",
@ -26,8 +20,6 @@ repackage_config = [[
config = {
"locale": os.environ.get("LOCALE"),
"download_config": download_config,
"repackage_config": repackage_config,
# ToolTool

29
testing/mozharness/scripts/repackage.py
View file

@ -23,25 +23,17 @@ class Repackage(BaseScript):
**script_kwargs
)
def download_input(self):
config = self.config
def setup(self):
dirs = self.query_abs_dirs()
input_dir = dirs['abs_input_dir']
for path, url in config["download_config"].items():
status = self.download_file(url=url,
file_name=path,
parent_dir=input_dir)
if not status:
self.fatal("Unable to fetch signed input from %s" % url)
if 'mar' in path:
# Ensure mar is executable
self.chmod(os.path.join(input_dir, path), 0755)
def setup(self):
self._run_tooltool()
mar_path = os.path.join(dirs['abs_input_dir'], 'mar')
if self._is_windows():
mar_path += '.exe'
if mar_path:
self.chmod(mar_path, 0755)
if self.config.get("run_configure", True):
self._get_mozconfig()
self._run_configure()
@ -55,7 +47,10 @@ class Repackage(BaseScript):
dirs = {}
dirs['abs_tools_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tools')
dirs['abs_mozilla_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'src')
dirs['abs_input_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'inputs')
dirs['abs_input_dir'] = os.path.join(
abs_dirs['base_work_dir'],
os.environ.get('MOZ_FETCHES_DIR', 'fetches'),
)
output_dir_suffix = []
if config.get('locale'):
output_dir_suffix.append(config['locale'])