diff --git a/taskcluster/taskgraph/transforms/repackage.py b/taskcluster/taskgraph/transforms/repackage.py index c8b673a7bf23..cb4fbca6fdac 100644 --- a/taskcluster/taskgraph/transforms/repackage.py +++ b/taskcluster/taskgraph/transforms/repackage.py @@ -17,7 +17,7 @@ from taskgraph.util.schema import ( resolve_keyed_by, Schema, ) -from taskgraph.util.taskcluster import get_taskcluster_artifact_prefix, get_artifact_prefix +from taskgraph.util.taskcluster import get_artifact_prefix from taskgraph.transforms.task import task_description_schema from voluptuous import Any, Required, Optional @@ -165,8 +165,6 @@ def make_job_description(config, jobs): dependencies['build'] = "build-{}/opt".format( dependencies[build_task][13:dependencies[build_task].rfind('-')]) build_task = 'build' - signing_task_ref = "<{}>".format(signing_task) - build_task_ref = "<{}>".format(build_task) attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage' @@ -184,14 +182,11 @@ def make_job_description(config, jobs): 'using': 'mozharness', 'script': 'mozharness/scripts/repackage.py', 'job-script': 'taskcluster/scripts/builder/repackage.sh', - 'actions': ['download_input', 'setup', 'repackage'], + 'actions': ['setup', 'repackage'], 'extra-workspace-cache-key': 'repackage', }) worker = { - 'env': _generate_task_env(dep_job, build_platform, build_task_ref, - signing_task_ref, locale=locale, - project=config.params["project"]), 'artifacts': _generate_task_output_files(dep_job, build_platform, locale=locale, project=config.params["project"]), @@ -203,7 +198,11 @@ def make_job_description(config, jobs): if locale: # Make sure we specify the locale-specific upload dir - worker['env'].update(LOCALE=locale) + worker.setdefault('env', {}).update(LOCALE=locale) + + use_stub = attributes.get('stub-installer') + if not use_stub: + worker.setdefault('env', {})['NO_STUB_INSTALLER'] = '1' if build_platform.startswith('win'): worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level @@ -240,6 +239,9 @@ def make_job_description(config, jobs): 'extra': job.get('extra', {}), 'worker': worker, 'run': run, + 'fetches': _generate_download_config(dep_job, build_platform, build_task, + signing_task, locale=locale, + project=config.params["project"]), } if build_platform.startswith('macosx'): @@ -251,38 +253,42 @@ def make_job_description(config, jobs): yield task -def _generate_task_env(task, build_platform, build_task_ref, signing_task_ref, locale=None, - project=None): - mar_prefix = get_taskcluster_artifact_prefix( - task, build_task_ref, postfix='host/bin/', locale=None - ) - signed_prefix = get_taskcluster_artifact_prefix(task, signing_task_ref, locale=locale) +def _generate_download_config(task, build_platform, build_task, signing_task, locale=None, + project=None): + locale_path = '{}/'.format(locale) if locale else '' if build_platform.startswith('linux') or build_platform.startswith('macosx'): tarball_extension = 'bz2' if build_platform.startswith('linux') else 'gz' return { - 'SIGNED_INPUT': {'task-reference': '{}target.tar.{}'.format( - signed_prefix, tarball_extension - )}, - 'UNSIGNED_MAR': {'task-reference': '{}mar'.format(mar_prefix)}, + signing_task: [ + { + 'artifact': '{}target.tar.{}'.format(locale_path, tarball_extension), + 'extract': False, + }, + ], + build_task: [ + 'host/bin/mar', + ], } elif build_platform.startswith('win'): - task_env = { - 'SIGNED_ZIP': {'task-reference': '{}target.zip'.format(signed_prefix)}, - 'SIGNED_SETUP': {'task-reference': '{}setup.exe'.format(signed_prefix)}, - 'UNSIGNED_MAR': {'task-reference': '{}mar.exe'.format(mar_prefix)}, + fetch_config = { + signing_task: [ + { + 'artifact': '{}target.zip'.format(locale_path), + 'extract': False, + }, + '{}setup.exe'.format(locale_path), + ], + build_task: [ + 'host/bin/mar.exe', + ], } use_stub = task.attributes.get('stub-installer') if use_stub: - task_env['SIGNED_SETUP_STUB'] = { - 'task-reference': '{}setup-stub.exe'.format(signed_prefix), - } - elif '32' in build_platform: - # Stub installer is only attempted on win32 - task_env['NO_STUB_INSTALLER'] = '1' + fetch_config[signing_task].append('{}setup-stub.exe'.format(locale_path)) - return task_env + return fetch_config raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform)) @@ -322,8 +328,8 @@ def _generate_task_output_files(task, build_platform, locale=None, project=None) if use_stub: output_files.append({ 'type': 'file', - 'path': '{}/{}target.stub-installer.exe'.format( - artifact_prefix, locale_output_path + 'path': 'build/outputs/{}target.stub-installer.exe'.format( + locale_output_path ), 'name': '{}/{}target.stub-installer.exe'.format( artifact_prefix, locale_output_path diff --git a/taskcluster/taskgraph/transforms/repackage_partner.py b/taskcluster/taskgraph/transforms/repackage_partner.py index 0c33027a578f..f3a52fa11f4d 100644 --- a/taskcluster/taskgraph/transforms/repackage_partner.py +++ b/taskcluster/taskgraph/transforms/repackage_partner.py @@ -17,7 +17,7 @@ from taskgraph.util.schema import ( resolve_keyed_by, Schema, ) -from taskgraph.util.taskcluster import get_taskcluster_artifact_prefix, get_artifact_prefix +from taskgraph.util.taskcluster import get_artifact_prefix from taskgraph.util.partners import check_if_partners_enabled from taskgraph.transforms.task import task_description_schema from voluptuous import Any, Required, Optional @@ -142,7 +142,6 @@ def make_job_description(config, jobs): signing_task = dependency elif build_platform.startswith('win') and dependency.endswith('repack'): signing_task = dependency - signing_task_ref = "<{}>".format(signing_task) attributes['repackage_type'] = 'repackage' @@ -159,12 +158,11 @@ def make_job_description(config, jobs): }) worker = { - 'env': _generate_task_env(build_platform, signing_task, signing_task_ref, - partner=repack_id), 'artifacts': _generate_task_output_files(dep_job, build_platform, partner=repack_id), 'chain-of-trust': True, 'max-run-time': 7200 if build_platform.startswith('win') else 3600, 'taskcluster-proxy': True if get_artifact_prefix(dep_job) else False, + 'env': {}, # Don't add generic artifact directory. 'skip-artifacts': True, } @@ -206,6 +204,9 @@ def make_job_description(config, jobs): 'extra': job.get('extra', {}), 'worker': worker, 'run': run, + 'fetches': _generate_download_config(dep_job, build_platform, + signing_task, partner=repack_id, + project=config.params["project"]), } if build_platform.startswith('macosx'): @@ -217,28 +218,30 @@ def make_job_description(config, jobs): yield task -def _generate_task_env(build_platform, signing_task, signing_task_ref, partner): - # Force private artifacts here, until we can populate our dependency map - # with actual task definitions rather than labels. - # (get_taskcluster_artifact_prefix requires the task definition to find - # the artifact_prefix attribute). - signed_prefix = get_taskcluster_artifact_prefix( - signing_task, signing_task_ref, locale=partner, force_private=True - ) - signed_prefix = signed_prefix.replace('public/build', 'releng/partner') +def _generate_download_config(task, build_platform, signing_task, partner=None, + project=None): + locale_path = '{}/'.format(partner) if partner else '' if build_platform.startswith('macosx'): return { - 'SIGNED_INPUT': {'task-reference': '{}target.tar.gz'.format(signed_prefix)}, + signing_task: [ + { + 'artifact': '{}target.tar.gz'.format(locale_path), + 'extract': False, + }, + ], } elif build_platform.startswith('win'): - task_env = { - 'SIGNED_ZIP': {'task-reference': '{}target.zip'.format(signed_prefix)}, - 'SIGNED_SETUP': {'task-reference': '{}setup.exe'.format(signed_prefix)}, + return { + signing_task: [ + { + 'artifact': '{}target.zip'.format(locale_path), + 'extract': False, + }, + '{}setup.exe'.format(locale_path), + ], } - return task_env - raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform)) diff --git a/testing/mozharness/configs/repackage/linux32_signed.py b/testing/mozharness/configs/repackage/linux32_signed.py index 8e96d2f0b000..6e4671be65c7 100644 --- a/testing/mozharness/configs/repackage/linux32_signed.py +++ b/testing/mozharness/configs/repackage/linux32_signed.py @@ -5,14 +5,9 @@ platform = "linux32" config = { "locale": os.environ.get("LOCALE"), - "download_config": { - "target.tar.gz": os.environ.get("SIGNED_INPUT"), - "mar": os.environ.get("UNSIGNED_MAR"), - }, - "repackage_config": [[ "mar", - "-i", "{abs_input_dir}/target.tar.gz", + "-i", "{abs_input_dir}/target.tar.bz2", "--mar", "{abs_input_dir}/mar", "-o", "{abs_output_dir}/target.complete.mar" ]], diff --git a/testing/mozharness/configs/repackage/linux64_signed.py b/testing/mozharness/configs/repackage/linux64_signed.py index 0d96ca45d13b..4ea1808e275f 100644 --- a/testing/mozharness/configs/repackage/linux64_signed.py +++ b/testing/mozharness/configs/repackage/linux64_signed.py @@ -5,14 +5,9 @@ platform = "linux64" config = { "locale": os.environ.get("LOCALE"), - "download_config": { - "target.tar.gz": os.environ.get("SIGNED_INPUT"), - "mar": os.environ.get("UNSIGNED_MAR"), - }, - "repackage_config": [[ "mar", - "-i", "{abs_input_dir}/target.tar.gz", + "-i", "{abs_input_dir}/target.tar.bz2", "--mar", "{abs_input_dir}/mar", "-o", "{abs_output_dir}/target.complete.mar" ]], diff --git a/testing/mozharness/configs/repackage/osx_partner.py b/testing/mozharness/configs/repackage/osx_partner.py index d063781f4596..a4c8db3b9251 100644 --- a/testing/mozharness/configs/repackage/osx_partner.py +++ b/testing/mozharness/configs/repackage/osx_partner.py @@ -5,10 +5,6 @@ config = { "repack_id": os.environ.get("REPACK_ID"), - "download_config": { - "target.tar.gz": os.environ.get("SIGNED_INPUT"), - }, - "repackage_config": [[ "dmg", "-i", "{abs_input_dir}/target.tar.gz", diff --git a/testing/mozharness/configs/repackage/osx_signed.py b/testing/mozharness/configs/repackage/osx_signed.py index 2661d57958a0..1b2a708c70cb 100644 --- a/testing/mozharness/configs/repackage/osx_signed.py +++ b/testing/mozharness/configs/repackage/osx_signed.py @@ -5,11 +5,6 @@ config = { "locale": os.environ.get("LOCALE"), - "download_config": { - "target.tar.gz": os.environ.get("SIGNED_INPUT"), - "mar": os.environ.get("UNSIGNED_MAR"), - }, - "repackage_config": [[ "dmg", "-i", "{abs_input_dir}/target.tar.gz", diff --git a/testing/mozharness/configs/repackage/win32_partner.py b/testing/mozharness/configs/repackage/win32_partner.py index 7768e349ef71..30055f80ea12 100644 --- a/testing/mozharness/configs/repackage/win32_partner.py +++ b/testing/mozharness/configs/repackage/win32_partner.py @@ -2,11 +2,6 @@ import os platform = "win32" -download_config = { - "target.zip": os.environ.get("SIGNED_ZIP"), - "setup.exe": os.environ.get("SIGNED_SETUP"), - } - repackage_config = [[ "installer", "--package-name", "firefox", @@ -20,8 +15,6 @@ repackage_config = [[ config = { "repack_id": os.environ.get("REPACK_ID"), - "download_config": download_config, - "repackage_config": repackage_config, # ToolTool diff --git a/testing/mozharness/configs/repackage/win32_signed.py b/testing/mozharness/configs/repackage/win32_signed.py index 8bc332eff5ca..c2a0956673ed 100644 --- a/testing/mozharness/configs/repackage/win32_signed.py +++ b/testing/mozharness/configs/repackage/win32_signed.py @@ -2,19 +2,6 @@ import os platform = "win32" -download_config = { - "target.zip": os.environ.get("SIGNED_ZIP"), - "setup.exe": os.environ.get("SIGNED_SETUP"), - "mar.exe": os.environ.get("UNSIGNED_MAR"), -} - -if not os.environ.get("NO_STUB_INSTALLER"): - # Some channels, like esr don't build a stub installer - download_config.update({ - # stub installer is only built on win32 - "setup-stub.exe": os.environ.get("SIGNED_SETUP_STUB"), - }) - repackage_config = [[ "installer", "--package-name", "firefox", @@ -43,8 +30,6 @@ if not os.environ.get("NO_STUB_INSTALLER"): config = { "locale": os.environ.get("LOCALE"), - "download_config": download_config, - "repackage_config": repackage_config, # ToolTool diff --git a/testing/mozharness/configs/repackage/win64_partner.py b/testing/mozharness/configs/repackage/win64_partner.py index 23a516bdd36b..22933a3f697c 100644 --- a/testing/mozharness/configs/repackage/win64_partner.py +++ b/testing/mozharness/configs/repackage/win64_partner.py @@ -2,11 +2,6 @@ import os platform = "win64" -download_config = { - "target.zip": os.environ.get("SIGNED_ZIP"), - "setup.exe": os.environ.get("SIGNED_SETUP"), - } - repackage_config = [[ "installer", "--package-name", "firefox", @@ -20,8 +15,6 @@ repackage_config = [[ config = { "repack_id": os.environ.get("REPACK_ID"), - "download_config": download_config, - "repackage_config": repackage_config, # ToolTool diff --git a/testing/mozharness/configs/repackage/win64_signed.py b/testing/mozharness/configs/repackage/win64_signed.py index 4ba61d870c82..dcad7d90b5ff 100644 --- a/testing/mozharness/configs/repackage/win64_signed.py +++ b/testing/mozharness/configs/repackage/win64_signed.py @@ -2,12 +2,6 @@ import os platform = "win64" -download_config = { - "target.zip": os.environ.get("SIGNED_ZIP"), - "setup.exe": os.environ.get("SIGNED_SETUP"), - "mar.exe": os.environ.get("UNSIGNED_MAR"), - } - repackage_config = [[ "installer", "--package-name", "firefox", @@ -26,8 +20,6 @@ repackage_config = [[ config = { "locale": os.environ.get("LOCALE"), - "download_config": download_config, - "repackage_config": repackage_config, # ToolTool diff --git a/testing/mozharness/scripts/repackage.py b/testing/mozharness/scripts/repackage.py index 7141d132ecd3..cb2435566847 100644 --- a/testing/mozharness/scripts/repackage.py +++ b/testing/mozharness/scripts/repackage.py @@ -23,25 +23,17 @@ class Repackage(BaseScript): **script_kwargs ) - def download_input(self): - config = self.config + def setup(self): dirs = self.query_abs_dirs() - input_dir = dirs['abs_input_dir'] - - for path, url in config["download_config"].items(): - status = self.download_file(url=url, - file_name=path, - parent_dir=input_dir) - if not status: - self.fatal("Unable to fetch signed input from %s" % url) - - if 'mar' in path: - # Ensure mar is executable - self.chmod(os.path.join(input_dir, path), 0755) - - def setup(self): self._run_tooltool() + + mar_path = os.path.join(dirs['abs_input_dir'], 'mar') + if self._is_windows(): + mar_path += '.exe' + if mar_path: + self.chmod(mar_path, 0755) + if self.config.get("run_configure", True): self._get_mozconfig() self._run_configure() @@ -55,7 +47,10 @@ class Repackage(BaseScript): dirs = {} dirs['abs_tools_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tools') dirs['abs_mozilla_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'src') - dirs['abs_input_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'inputs') + dirs['abs_input_dir'] = os.path.join( + abs_dirs['base_work_dir'], + os.environ.get('MOZ_FETCHES_DIR', 'fetches'), + ) output_dir_suffix = [] if config.get('locale'): output_dir_suffix.append(config['locale'])