nagai/fune
3
0
Fork 0
forked from mirrors/gecko-dev
Code Pull requests Activity

Bug 1527313: Move objdir out of source directory for all mozharness builds; r=glandium,Callek

Browse source 
This moves the object directory and source directories around in all mozharness
jobs, to allow enabling caching on windows builders.

This makes a number of changes that all need to land at once:
- Move the source checkout for the workspace cache mount, to the checkouts
  cache mount.
- Makes the object directoy from underneath the source directory, to directly
  in the work directory (which is still under `workspace`).
- Sets the object directory to `obj-build` instead of `obj-firefox`.
- Stops caching the workspace directory (it is still a volume in docker workers,
  so writes perform well; a followup revision add some checks around this).
- Removes one level of directory in the mozharness workdir (things were under
  `workspace/build`, but are now just under `workspace/`.
- Adjust paths in environment variables and artifact specifications to match
  the above changes.

Differential Revision: https://phabricator.services.mozilla.com/D62482

--HG--
extra : moz-landing-system : lando
This commit is contained in:
Tom Prince 2020-03-06 08:51:21 +00:00
parent 80b16bae55
commit fa69ebf563
24 changed files with 44 additions and 57 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

4
taskcluster/ci/build-fat-aar/kind.yml
View file

@ -27,12 +27,12 @@ job-defaults:
max-run-time: 7200
env:
# Online in order to download the per-architecture AARs.
GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-online"
GRADLE_USER_HOME: "/builds/worker/checkouts/build/gecko/mobile/android/gradle/dotgradle-online"
TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest"
MOZ_ANDROID_FAT_AAR_ARCHITECTURES: "armeabi-v7a,arm64-v8a,x86,x86_64"
artifacts:
- name: public/build/maven
path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/maven/
path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview/maven/
type: directory
- name: public/build
path: /builds/worker/artifacts/

4
taskcluster/ci/build/android-stuff.yml
View file

@ -23,12 +23,12 @@ android-geckoview-docs/opt:
worker:
docker-image: {in-tree: android-build}
env:
GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-offline"
GRADLE_USER_HOME: "/builds/worker/checkouts/gecko/mobile/android/gradle/dotgradle-offline"
PERFHERDER_EXTRA_OPTIONS: android-geckoview-docs
GECKOVIEW_DOCS_UPLOAD_SECRET: "project/releng/gecko/build/level-{level}/geckoview-docs-upload"
artifacts:
- name: public/android/geckoview-docs/geckoview-javadoc.jar
path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/libs/geckoview-javadoc.jar
path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview/libs/geckoview-javadoc.jar
type: file
max-run-time: 7200
run:

8
taskcluster/ci/build/android.yml
View file

@ -11,16 +11,16 @@ job-defaults:
docker-image: {in-tree: android-build}
max-run-time: 7200
env:
GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-offline"
GRADLE_USER_HOME: "/builds/worker/checkouts/gecko/mobile/android/gradle/dotgradle-offline"
artifacts:
- name: public/build/maven
path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/maven/
path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview/maven/
type: directory
- name: public/build/geckoview-androidTest.apk
path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk
path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk
type: file
- name: public/build/geckoview_example.apk
path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview_example/outputs/apk/withGeckoBinaries/debug/geckoview_example-withGeckoBinaries-debug.apk
path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview_example/outputs/apk/withGeckoBinaries/debug/geckoview_example-withGeckoBinaries-debug.apk
type: file
- name: public/build
path: /builds/worker/artifacts/

2
taskcluster/ci/build/linux.yml
View file

@ -1251,7 +1251,7 @@ linux64-ccov/opt:
worker:
artifacts:
- name: public/code-coverage-grcov.zip
path: /builds/worker/workspace/build/src/obj-firefox/code-coverage-grcov.zip
path: /builds/worker/workspace/obj-build/code-coverage-grcov.zip
type: file
max-run-time: 7200
env:

2
taskcluster/ci/build/windows.yml
View file

@ -737,7 +737,7 @@ win64-ccov/opt:
worker:
artifacts:
- name: public/code-coverage-grcov.zip
path: build\src\obj-firefox\code-coverage-grcov.zip
path: workspace\obj-build\code-coverage-grcov.zip
type: file
max-run-time: 7200
env:

3
taskcluster/ci/generate-profile/kind.yml
View file

@ -96,9 +96,8 @@ jobs:
max-run-time: 5400
docker-image: {in-tree: ubuntu1804-test}
env:
WORKSPACE: "/builds/worker/workspace"
WORKING_DIR: "/builds/worker"
MOZHARNESS_PATH: "/builds/worker/workspace/build/src/testing/mozharness"
MOZHARNESS_PATH: "/builds/worker/checkouts/gecko/testing/mozharness"
artifacts:
- type: directory
name: public/build

6
taskcluster/ci/instrumented-build/kind.yml
View file

@ -135,17 +135,17 @@ jobs:
worker:
docker-image: {in-tree: android-build}
env:
GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-offline"
GRADLE_USER_HOME: "/builds/worker/checkouts/gecko/mobile/android/gradle/dotgradle-offline"
TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest"
artifacts:
- name: public/build
path: /builds/worker/artifacts/
type: directory
- name: public/build/geckoview-androidTest.apk
path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk
path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk
type: file
- name: public/build/geckoview_example.apk
path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview_example/outputs/apk/withGeckoBinaries/debug/geckoview_example-withGeckoBinaries-debug.apk
path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview_example/outputs/apk/withGeckoBinaries/debug/geckoview_example-withGeckoBinaries-debug.apk
type: file
run:
config:

20
taskcluster/ci/openh264-plugin/kind.yml
View file

@ -36,7 +36,7 @@ jobs:
max-run-time: 1800
artifacts:
- name: private/openh264
path: /builds/worker/workspace/build/openh264/artifacts
path: /builds/worker/workspace/openh264/artifacts
type: directory
run:
using: mozharness
@ -60,7 +60,7 @@ jobs:
max-run-time: 1800
artifacts:
- name: private/openh264
path: /builds/worker/workspace/build/openh264/artifacts
path: /builds/worker/workspace/openh264/artifacts
type: directory
run:
using: mozharness
@ -84,7 +84,7 @@ jobs:
max-run-time: 1800
artifacts:
- name: private/openh264
path: /builds/worker/workspace/build/openh264/artifacts
path: /builds/worker/workspace/openh264/artifacts
type: directory
run:
using: mozharness
@ -107,7 +107,7 @@ jobs:
max-run-time: 1800
artifacts:
- name: private/openh264
path: build/openh264/artifacts
path: workspace/openh264/artifacts
type: directory
run:
using: mozharness
@ -128,7 +128,7 @@ jobs:
max-run-time: 1800
artifacts:
- name: private/openh264
path: build/openh264/artifacts
path: workspace/openh264/artifacts
type: directory
run:
using: mozharness
@ -149,7 +149,7 @@ jobs:
max-run-time: 1800
artifacts:
- name: private/openh264
path: build/openh264/artifacts
path: workspace/openh264/artifacts
type: directory
run:
using: mozharness
@ -170,7 +170,7 @@ jobs:
max-run-time: 1800
artifacts:
- name: private/openh264
path: /builds/worker/workspace/build/openh264/artifacts
path: /builds/worker/workspace/openh264/artifacts
type: directory
# This rev includes Android Makefile fixes from after
# the 1.8.1 release
@ -199,7 +199,7 @@ jobs:
max-run-time: 1800
artifacts:
- name: private/openh264
path: /builds/worker/workspace/build/openh264/artifacts
path: /builds/worker/workspace/openh264/artifacts
type: directory
# This rev includes Android Makefile fixes from after
# the 1.8.1 release
@ -228,7 +228,7 @@ jobs:
max-run-time: 1800
artifacts:
- name: private/openh264
path: /builds/worker/workspace/build/openh264/artifacts
path: /builds/worker/workspace/openh264/artifacts
type: directory
# This rev includes Android Makefile fixes from after
# the 1.8.1 release
@ -257,7 +257,7 @@ jobs:
max-run-time: 1800
artifacts:
- name: private/openh264
path: /builds/worker/workspace/build/openh264/artifacts
path: /builds/worker/workspace/openh264/artifacts
type: directory
# This rev includes Android Makefile fixes from after
# the 1.8.1 release

2
taskcluster/ci/searchfox/kind.yml
View file

@ -147,7 +147,7 @@ jobs:
PERFHERDER_EXTRA_OPTIONS: searchfox
RUSTC_BOOTSTRAP: "1"
MOZSEARCH_PLATFORM: "android-armv7"
GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-offline"
GRADLE_USER_HOME: "/builds/worker/checkouts/gecko/mobile/android/gradle/dotgradle-offline"
TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest"
run:
using: mozharness

2
taskcluster/ci/static-analysis-autotest/kind.yml
View file

@ -46,7 +46,7 @@ jobs:
env:
# clang-tidy needs a recent libstdc++, which can be found in the clang
# toolchain.
LD_LIBRARY_PATH: /builds/worker/workspace/build/src/clang/lib
LD_LIBRARY_PATH: /builds/worker/checkouts/gecko/clang/lib
PERFHERDER_EXTRA_OPTIONS: static-analysis-autotest
run:
config:

5
taskcluster/scripts/builder/build-l10n.sh
View file

@ -38,11 +38,10 @@ fail() {
}
export MOZ_CRASHREPORTER_NO_REPORT=1
export MOZ_OBJDIR=obj-firefox
export TINDERBOX_OUTPUT=1
# Ensure that in tree libraries can be found
export LIBRARY_PATH=$LIBRARY_PATH:$WORKSPACE/src/obj-firefox:$WORKSPACE/src/gcc/lib64
export LIBRARY_PATH=$LIBRARY_PATH:$WORKSPACE/obj-build:$WORKSPACE/src/gcc/lib64
# test required parameters are supplied
if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi
@ -103,4 +102,4 @@ $GECKO_PATH/mach python $GECKO_PATH/testing/${MOZHARNESS_SCRIPT} \
$actions \
$options \
--log-level=debug \
--work-dir=$WORKSPACE/build \
--work-dir=$WORKSPACE \

5
taskcluster/scripts/builder/build-linux.sh
View file

@ -37,7 +37,6 @@ fail() {
}
export MOZ_CRASHREPORTER_NO_REPORT=1
export MOZ_OBJDIR=obj-firefox
export TINDERBOX_OUTPUT=1
# use "simple" package names so that they can be hard-coded in the task's
@ -45,7 +44,7 @@ export TINDERBOX_OUTPUT=1
export MOZ_SIMPLE_PACKAGE_NAME=target
# Ensure that in tree libraries can be found
export LIBRARY_PATH=$LIBRARY_PATH:$WORKSPACE/src/obj-firefox:$WORKSPACE/src/gcc/lib64
export LIBRARY_PATH=$LIBRARY_PATH:$WORKSPACE/obj-build:$WORKSPACE/src/gcc/lib64
# test required parameters are supplied
if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi
@ -120,6 +119,6 @@ $GECKO_PATH/mach python $GECKO_PATH/testing/${MOZHARNESS_SCRIPT} \
$actions \
$options \
--log-level=debug \
--work-dir=$WORKSPACE/build \
--work-dir=$WORKSPACE \
--branch=${MH_BRANCH} \
--build-pool=${MH_BUILD_POOL}

3
taskcluster/scripts/builder/repackage.sh
View file

@ -33,7 +33,6 @@ fail() {
}
export MOZ_CRASHREPORTER_NO_REPORT=1
export MOZ_OBJDIR=obj-firefox
export TINDERBOX_OUTPUT=1
# use "simple" package names so that they can be hard-coded in the task's
@ -99,4 +98,4 @@ $GECKO_PATH/mach python $GECKO_PATH/testing/${MOZHARNESS_SCRIPT} \
$actions \
$options \
--log-level=debug \
--work-dir=$WORKSPACE/build \
--work-dir=$WORKSPACE \

21
taskcluster/taskgraph/transforms/job/mozharness.py
View file

@ -26,7 +26,6 @@ from taskgraph.transforms.job import (
run_job_using,
)
from taskgraph.transforms.job.common import (
docker_worker_add_workspace_cache,
setup_secrets,
docker_worker_add_artifacts,
generic_worker_add_artifacts,
@ -164,13 +163,10 @@ def mozharness_on_docker_worker_setup(config, job, taskdesc):
})
worker['taskcluster-proxy'] = run.pop('taskcluster-proxy', None)
docker_worker_add_artifacts(config, job, taskdesc)
docker_worker_add_workspace_cache(config, job, taskdesc,
extra=run.pop('extra-workspace-cache-key', None))
env = worker.setdefault('env', {})
env.update({
'WORKSPACE': '{workdir}/workspace'.format(**run),
'GECKO_PATH': '{workdir}/workspace/build/src'.format(**run),
'MOZHARNESS_CONFIG': ' '.join(run.pop('config')),
'MOZHARNESS_SCRIPT': run.pop('script'),
'MH_BRANCH': config.params['project'],
@ -194,9 +190,10 @@ def mozharness_on_docker_worker_setup(config, job, taskdesc):
if 'custom-build-variant-cfg' in run:
env['MH_CUSTOM_BUILD_VARIANT_CFG'] = run.pop('custom-build-variant-cfg')
if 'extra-config' in run:
env['EXTRA_MOZHARNESS_CONFIG'] = six.ensure_text(
json.dumps(run.pop('extra-config')))
extra_config = run.pop('extra-config', {})
extra_config['objdir'] = 'obj-build'
env['EXTRA_MOZHARNESS_CONFIG'] = six.ensure_text(
json.dumps(extra_config))
if 'job-script' in run:
env['JOB_SCRIPT'] = run['job-script']
@ -230,6 +227,7 @@ def mozharness_on_docker_worker_setup(config, job, taskdesc):
)
run.pop('secrets')
run.pop('requires-signed-builds')
run.pop('extra-workspace-cache-key', None)
configure_taskdesc_for_run(config, job, taskdesc, worker['implementation'])
@ -279,9 +277,10 @@ def mozharness_on_generic_worker(config, job, taskdesc):
if run.pop('use-simple-package'):
env.update({'MOZ_SIMPLE_PACKAGE_NAME': 'target'})
if 'extra-config' in run:
env['EXTRA_MOZHARNESS_CONFIG'] = six.ensure_text(
json.dumps(run.pop('extra-config')))
extra_config = run.pop('extra-config', {})
extra_config['objdir'] = 'obj-build'
env['EXTRA_MOZHARNESS_CONFIG'] = six.ensure_text(
json.dumps(extra_config))
# The windows generic worker uses batch files to pass environment variables
# to commands. Setting a variable to empty in a batch file unsets, so if
@ -307,7 +306,7 @@ def mozharness_on_generic_worker(config, job, taskdesc):
mh_command.append('--config ' + cfg)
if run.pop('use-magic-mh-args'):
mh_command.append('--branch ' + config.params['project'])
mh_command.append(r'--work-dir %cd:Z:=z:%\build')
mh_command.append(r'--work-dir %cd:Z:=z:%\workspace')
for action in run.pop('actions', []):
mh_command.append('--' + action)

4
taskcluster/taskgraph/transforms/repackage.py
View file

@ -382,7 +382,7 @@ def _generate_task_output_files(task, worker_implementation, repackage_config, l
if worker_implementation == ('docker-worker', 'linux'):
local_prefix = '/builds/worker/workspace/'
elif worker_implementation == ('generic-worker', 'windows'):
local_prefix = ''
local_prefix = 'workspace/'
else:
raise NotImplementedError(
'Unsupported worker implementation: "{}"'.format(worker_implementation))
@ -391,7 +391,7 @@ def _generate_task_output_files(task, worker_implementation, repackage_config, l
for config in repackage_config:
output_files.append({
'type': 'file',
'path': '{}build/outputs/{}{}'
'path': '{}outputs/{}{}'
.format(local_prefix, locale_output_path, config['output']),
'name': '{}/{}{}'.format(artifact_prefix, locale_output_path, config['output']),
})

1
testing/mozharness/configs/builds/releng_base_android_64_builds.py
View file

@ -22,7 +22,6 @@ config = {
'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
],
'vcs_share_base': '/builds/hg-shared',
'objdir': 'obj-firefox',
'multi_locale': True,
#########################################################################

1
testing/mozharness/configs/builds/releng_base_firefox.py
View file

@ -1,4 +1,3 @@
config = {
'objdir': 'obj-firefox',
'app_name': 'browser',
}

1
testing/mozharness/configs/builds/releng_sub_linux_configs/64_source.py
View file

@ -1,6 +1,5 @@
config = {
'default_actions': ['package-source'],
'objdir': 'obj-firefox',
'stage_platform': 'source', # Not used, but required by the script
'env': {
'HG_SHARE_BASE_DIR': '/builds/hg-shared',

1
testing/mozharness/configs/multi_locale/android-mozharness-build.json
View file

@ -1,6 +1,5 @@
{
"log_name": "multilocale",
"objdir": "obj-firefox",
"locales_dir": "mobile/android/locales",
"ignore_locales": ["en-US", "multi"],
"vcs_share_base": "/builds/hg-shared",

1
testing/mozharness/configs/single_locale/devedition.py
View file

@ -1,6 +1,5 @@
config = {
"app_name": "browser",
"mozconfig_variant": "l10n-mozconfig-devedition",
"objdir": "obj-firefox",
"locales_dir": "browser/locales",
}

1
testing/mozharness/configs/single_locale/firefox.py
View file

@ -1,6 +1,5 @@
config = {
"app_name": "browser",
"mozconfig_variant": "l10n-mozconfig",
"objdir": "obj-firefox",
"locales_dir": "browser/locales",
}

1
testing/mozharness/configs/single_locale/tc_android-api-16.py
View file

@ -2,7 +2,6 @@ config = {
"locales_dir": "mobile/android/locales",
"ignore_locales": ["en-US"],
"objdir": "obj-firefox",
"repack_env": {
"MOZ_OBJDIR": "%(abs_obj_dir)s",
},

2
testing/mozharness/mozharness/mozilla/l10n/locales.py
View file

@ -137,7 +137,7 @@ class LocalesMixin(object):
c['locales_dir'],
)
dirs['abs_obj_dir'] = os.path.join(dirs['abs_src_dir'],
dirs['abs_obj_dir'] = os.path.join(dirs['abs_work_dir'],
c['objdir'])
dirs['abs_locales_dir'] = os.path.join(dirs['abs_obj_dir'],
c['locales_dir'])

1
testing/mozharness/scripts/fx_desktop_build.py
View file

@ -81,7 +81,6 @@ class FxDesktopBuild(BuildScript, TryToolsMixin, object):
# mozharness_repo}/build/src/'
'abs_src_dir': os.environ['GECKO_PATH'],
'abs_obj_dir': os.path.join(abs_dirs['abs_work_dir'],
'src',
self._query_objdir()),
'upload_path': self.config["upload_env"]["UPLOAD_PATH"],
}