nagai/fune
3
0
Fork 0
forked from mirrors/gecko-dev
Code Pull requests Activity

Bug 1509962: [update-verify] Use in-tree update-verify code; r=mtabara

Browse source 
Differential Revision: https://phabricator.services.mozilla.com/D12977

--HG--
extra : moz-landing-system : lando
This commit is contained in:
Tom Prince 2018-12-03 06:44:28 +00:00
parent 5edec00d54
commit 6ffdb8908e
11 changed files with 46 additions and 102 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

4
build/sparse-profiles/update-verify Normal file
View file

@ -0,0 +1,4 @@
%include build/sparse-profiles/mach
[include]
path:tools/update-verify/

6
taskcluster/ci/release-final-verify/kind.yml
View file

@ -11,6 +11,7 @@ kind-dependencies:
transforms:
- taskgraph.transforms.release_deps:transforms
- taskgraph.transforms.final_verify:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
job-defaults:
@ -18,14 +19,11 @@ job-defaults:
run-on-projects: [] # to make sure this never runs as part of CI
worker-type: aws-provisioner-v1/gecko-{level}-b-linux
worker:
implementation: docker-worker
os: linux
docker-image:
in-tree: "update-verify"
max-run-time: 7200
retry-exit-status: [1]
env:
BUILD_TOOLS_REPO: https://hg.mozilla.org/build/tools
jobs:
firefox:

5
taskcluster/ci/release-secondary-final-verify/kind.yml
View file

@ -13,6 +13,7 @@ kind-dependencies:
transforms:
- taskgraph.transforms.release_deps:transforms
- taskgraph.transforms.final_verify:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
job-defaults:
@ -20,14 +21,10 @@ job-defaults:
run-on-projects: [] # to make sure this never runs as part of CI
worker-type: aws-provisioner-v1/gecko-{level}-b-linux
worker:
implementation: docker-worker
os: linux
docker-image:
in-tree: "update-verify"
max-run-time: 7200
retry-exit-status: [1]
env:
BUILD_TOOLS_REPO: https://hg.mozilla.org/build/tools
jobs:
firefox:

5
taskcluster/ci/release-secondary-update-verify/kind.yml
View file

@ -13,6 +13,7 @@ kind-dependencies:
transforms:
- taskgraph.transforms.release_deps:transforms
- taskgraph.transforms.update_verify:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
job-defaults:
@ -25,16 +26,12 @@ job-defaults:
- name: 'public/build/diff-summary.log'
path: '/builds/worker/tools/release/updates/diff-summary.log'
type: file
implementation: docker-worker
os: linux
docker-image:
in-tree: "update-verify"
max-run-time: 7200
retry-exit-status:
- 255
env:
NO_BBCONFIG: "1"
BUILD_TOOLS_REPO: https://hg.mozilla.org/build/tools
CHANNEL: "beta-localtest"
extra:
chunks: 12

6
taskcluster/ci/release-update-verify/kind.yml
View file

@ -13,6 +13,7 @@ kind-dependencies:
transforms:
- taskgraph.transforms.release_deps:transforms
- taskgraph.transforms.update_verify:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
job-defaults:
@ -25,16 +26,11 @@ job-defaults:
- name: 'public/build/diff-summary.log'
path: '/builds/worker/tools/release/updates/diff-summary.log'
type: file
implementation: docker-worker
os: linux
docker-image:
in-tree: "update-verify"
max-run-time: 5400
retry-exit-status:
- 255
env:
NO_BBCONFIG: "1"
BUILD_TOOLS_REPO: https://hg.mozilla.org/build/tools
extra:
chunks: 12

18
taskcluster/taskgraph/transforms/final_verify.py
View file

@ -28,15 +28,15 @@ def add_command(config, tasks):
get_taskcluster_artifact_prefix(task, "<{}>".format(upstream))
)
)
task["worker"]["command"] = [
"/bin/bash",
"-c",
{
"task-reference": "hg clone $BUILD_TOOLS_REPO tools && cd tools/release && " +
"./final-verification.sh " +
" ".join(final_verify_configs)
}
]
task['run'] = {
'using': 'run-task',
'command': {
'task-reference': 'cd /builds/worker/checkouts/gecko && '
'tools/update-verify/release/final-verification.sh '
+ ' '.join(final_verify_configs),
},
'sparse-profile': 'update-verify',
}
for thing in ("BUILD_TOOLS_REPO",):
thing = "worker.env.{}".format(thing)
resolve_keyed_by(task, thing, thing, **config.params)

6
taskcluster/taskgraph/transforms/job/run_task.py
View file

@ -7,6 +7,7 @@ Support for running jobs that are invoked via the `run-task` script.
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.task import taskref_or_string
from taskgraph.transforms.job import run_job_using
from taskgraph.util.schema import Schema
from taskgraph.transforms.job.common import support_vcs_checkout
@ -33,7 +34,7 @@ run_task_schema = Schema({
# The command arguments to pass to the `run-task` script, after the
# checkout arguments. If a list, it will be passed directly; otherwise
# it will be included in a single argument to `bash -cx`.
Required('command'): Any([basestring], basestring),
Required('command'): Any([taskref_or_string], taskref_or_string),
# Base work directory used to set up the task.
Required('workdir'): basestring,
@ -83,7 +84,8 @@ def docker_worker_run_task(config, job, taskdesc):
})
run_command = run['command']
if isinstance(run_command, basestring):
# dict is for the case of `{'task-reference': basestring}`.
if isinstance(run_command, (basestring, dict)):
run_command = ['bash', '-cx', run_command]
if run['comm-checkout']:
command.append('--comm-checkout={workdir}/checkouts/gecko/comm'.format(**run))

20
taskcluster/taskgraph/transforms/update_verify.py
View file

@ -31,16 +31,16 @@ def add_command(config, tasks):
)
if not chunked["worker"].get("env"):
chunked["worker"]["env"] = {}
chunked["worker"]["command"] = [
"/bin/bash",
"-c",
"hg clone $BUILD_TOOLS_REPO tools && " +
"tools/scripts/release/updates/chunked-verify.sh " +
"UNUSED UNUSED {} {}".format(
total_chunks,
this_chunk,
)
]
chunked["run"] = {
'using': 'run-task',
'command': 'cd /builds/worker/checkouts/gecko && '
'tools/update-verify/scripts/chunked-verify.sh '
'{} {}'.format(
total_chunks,
this_chunk,
),
'sparse-profile': 'update-verify',
}
for thing in ("CHANNEL", "VERIFY_CONFIG", "BUILD_TOOLS_REPO"):
thing = "worker.env.{}".format(thing)
resolve_keyed_by(

2
tools/update-verify/release/common/download_builds.sh
View file

@ -1,7 +1,7 @@
pushd `dirname $0` &>/dev/null
MY_DIR=$(pwd)
popd &>/dev/null
retry="$MY_DIR/../../buildfarm/utils/retry.py -s 1 -r 3"
retry="$MY_DIR/../../../../mach python -m redo.cmd -s 1 -a 3"
download_builds() {
# cleanup

2
tools/update-verify/release/updates/verify.sh
View file

@ -27,7 +27,7 @@ touch ${diff_summary_log}
pushd `dirname $0` &>/dev/null
MY_DIR=$(pwd)
popd &>/dev/null
retry="$MY_DIR/../../buildfarm/utils/retry.py -s 1 -r 3"
retry="$MY_DIR/../../../../mach python -m redo.cmd -s 1 -a 3"
cert_replacer="$MY_DIR/../replace-updater-certs.py"
dep_overrides="nightly_aurora_level3_primary.der dep1.der nightly_aurora_level3_secondary.der dep2.der release_primary.der dep1.der release_secondary.der dep2.der"

74
tools/update-verify/scripts/chunked-verify.sh
View file

@ -7,78 +7,28 @@ pushd `dirname $0` &>/dev/null
MY_DIR=$(pwd)
popd &>/dev/null
SCRIPTS_DIR="$MY_DIR/.."
PYTHON="/tools/python/bin/python"
if [ ! -x $PYTHON ]; then
PYTHON=python
fi
JSONTOOL="$PYTHON $SCRIPTS_DIR/buildfarm/utils/jsontool.py"
workdir=`pwd`
PYTHON='./mach python'
platform=$1
configDict=$2
chunks=$3
thisChunk=$4
channel=$5
if [ -n "$PROPERTIES_FILE" -a -f "$PROPERTIES_FILE" ]; then
# Buildbot only
if $JSONTOOL -k properties.NO_BBCONFIG $PROPERTIES_FILE; then
NO_BBCONFIG=$($JSONTOOL -k properties.NO_BBCONFIG $PROPERTIES_FILE);
fi
if $JSONTOOL -k properties.CHANNEL $PROPERTIES_FILE; then
CHANNEL=$($JSONTOOL -k properties.CHANNEL $PROPERTIES_FILE);
fi
if $JSONTOOL -k properties.VERIFY_CONFIG $PROPERTIES_FILE; then
VERIFY_CONFIG=$($JSONTOOL -k properties.VERIFY_CONFIG $PROPERTIES_FILE);
fi
if $JSONTOOL -k properties.TOTAL_CHUNKS $PROPERTIES_FILE; then
chunks=$($JSONTOOL -k properties.TOTAL_CHUNKS $PROPERTIES_FILE);
fi
if $JSONTOOL -k properties.THIS_CHUNK $PROPERTIES_FILE; then
thisChunk=$($JSONTOOL -k properties.THIS_CHUNK $PROPERTIES_FILE);
fi
if [ -z "$NO_BBCONFIG" -a -z "$BUILDBOT_CONFIGS" ]; then
export BUILDBOT_CONFIGS="https://hg.mozilla.org/build/buildbot-configs"
fi
# Get the assumed slavebuilddir, and read in from buildbot if this is not
# Release promotion
SLAVEBUILDDIR=$(basename $(cd "$SCRIPTS_DIR/.."; pwd))
if [ -z "$NO_BBCONFIG" ]; then
RELEASE_CONFIG=$($JSONTOOL -k properties.release_config $PROPERTIES_FILE)
TAG=$($JSONTOOL -k properties.release_tag $PROPERTIES_FILE)
SLAVEBUILDDIR=$($JSONTOOL -k properties.slavebuilddir $PROPERTIES_FILE)
fi
$PYTHON -u $SCRIPTS_DIR/buildfarm/maintenance/purge_builds.py \
-s 16 -n info -n 'rel-*' -n 'tb-rel-*' -n $SLAVEBUILDDIR
fi
chunks=$1
thisChunk=$2
if [ -n "$TASKCLUSTER_VERIFY_CONFIG" ]; then
wget -O "$SCRIPTS_DIR/release/updates/update-verify.cfg" "$TASKCLUSTER_VERIFY_CONFIG"
VERIFY_CONFIG="update-verify.cfg"
fi
if [ -z "$VERIFY_CONFIG" -a -n "$NO_BBCONFIG" ]; then
echo "Unable to run without VERIFY_CONFIG specified when using NO_BBCONFIG"
else
echo "Unable to run without TASKCLUSTER_VERIFY_CONFIG."
exit 1
fi
if [ -z "$NO_BBCONFIG" ]; then
$PYTHON $MY_DIR/chunked-verify.py -t $TAG -r $RELEASE_CONFIG \
-b $BUILDBOT_CONFIGS -p $platform --chunks $chunks --this-chunk $thisChunk \
--config-dict $configDict --release-channel $channel \
2>&1 | tee $SCRIPTS_DIR/../verify_log.txt
# release promotion
if [ -n "$CHANNEL" ]; then
EXTRA_PARAMS="--verify-channel $CHANNEL"
else
# release promotion
if [ -n "$CHANNEL" ]; then
EXTRA_PARAMS="--verify-channel $CHANNEL"
else
EXTRA_PARAMS=""
fi
$PYTHON $MY_DIR/chunked-verify.py --chunks $chunks --this-chunk $thisChunk \
--verify-config $VERIFY_CONFIG $EXTRA_PARAMS \
2>&1 | tee $SCRIPTS_DIR/../verify_log.txt
EXTRA_PARAMS=""
fi
$PYTHON $MY_DIR/chunked-verify.py --chunks $chunks --this-chunk $thisChunk \
--verify-config $VERIFY_CONFIG $EXTRA_PARAMS \
2>&1 | tee $SCRIPTS_DIR/../verify_log.txt
print_failed_msg()
{