Bug 1509962: [update-verify] Use in-tree update-verify code; r=mtabara

Differential Revision: https://phabricator.services.mozilla.com/D12977

--HG--
extra : moz-landing-system : lando

build/sparse-profiles/update-verify

@@ -0,0 +1,4 @@
+%include build/sparse-profiles/mach
+
+[include]
+path:tools/update-verify/

taskcluster/ci/release-final-verify/kind.yml

@@ -11,6 +11,7 @@ kind-dependencies:
 transforms:
    - taskgraph.transforms.release_deps:transforms
    - taskgraph.transforms.final_verify:transforms
+   - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 job-defaults:
@@ -18,14 +19,11 @@ job-defaults:
    run-on-projects: []  # to make sure this never runs as part of CI
    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
    worker:
-      implementation: docker-worker
-      os: linux
       docker-image:
          in-tree: "update-verify"
       max-run-time: 7200
       retry-exit-status: [1]
-      env:
+
-         BUILD_TOOLS_REPO: https://hg.mozilla.org/build/tools
 
 jobs:
    firefox:

taskcluster/ci/release-secondary-final-verify/kind.yml

@@ -13,6 +13,7 @@ kind-dependencies:
 transforms:
    - taskgraph.transforms.release_deps:transforms
    - taskgraph.transforms.final_verify:transforms
+   - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 job-defaults:
@@ -20,14 +21,10 @@ job-defaults:
    run-on-projects: []  # to make sure this never runs as part of CI
    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
    worker:
-      implementation: docker-worker
-      os: linux
       docker-image:
          in-tree: "update-verify"
       max-run-time: 7200
       retry-exit-status: [1]
-      env:
-         BUILD_TOOLS_REPO: https://hg.mozilla.org/build/tools
 
 jobs:
    firefox:

taskcluster/ci/release-secondary-update-verify/kind.yml

@@ -13,6 +13,7 @@ kind-dependencies:
 transforms:
    - taskgraph.transforms.release_deps:transforms
    - taskgraph.transforms.update_verify:transforms
+   - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 job-defaults:
@@ -25,16 +26,12 @@ job-defaults:
          - name: 'public/build/diff-summary.log'
            path: '/builds/worker/tools/release/updates/diff-summary.log'
            type: file
-      implementation: docker-worker
-      os: linux
       docker-image:
          in-tree: "update-verify"
       max-run-time: 7200
       retry-exit-status:
          - 255
       env:
-         NO_BBCONFIG: "1"
-         BUILD_TOOLS_REPO: https://hg.mozilla.org/build/tools
          CHANNEL: "beta-localtest"
    extra:
       chunks: 12

taskcluster/ci/release-update-verify/kind.yml

@@ -13,6 +13,7 @@ kind-dependencies:
 transforms:
    - taskgraph.transforms.release_deps:transforms
    - taskgraph.transforms.update_verify:transforms
+   - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 job-defaults:
@@ -25,16 +26,11 @@ job-defaults:
          - name: 'public/build/diff-summary.log'
            path: '/builds/worker/tools/release/updates/diff-summary.log'
            type: file
-      implementation: docker-worker
-      os: linux
       docker-image:
          in-tree: "update-verify"
       max-run-time: 5400
       retry-exit-status:
          - 255
-      env:
-         NO_BBCONFIG: "1"
-         BUILD_TOOLS_REPO: https://hg.mozilla.org/build/tools
    extra:
       chunks: 12
 

taskcluster/taskgraph/transforms/final_verify.py

@@ -28,15 +28,15 @@ def add_command(config, tasks):
                         get_taskcluster_artifact_prefix(task, "<{}>".format(upstream))
                     )
                 )
-        task["worker"]["command"] = [
+        task['run'] = {
-            "/bin/bash",
+            'using': 'run-task',
-            "-c",
+            'command': {
-            {
+                'task-reference': 'cd /builds/worker/checkouts/gecko && '
-                "task-reference": "hg clone $BUILD_TOOLS_REPO tools && cd tools/release && " +
+                                  'tools/update-verify/release/final-verification.sh '
-                                  "./final-verification.sh " +
+                                  + ' '.join(final_verify_configs),
-                                  " ".join(final_verify_configs)
+            },
-            }
+            'sparse-profile': 'update-verify',
-        ]
+        }
         for thing in ("BUILD_TOOLS_REPO",):
             thing = "worker.env.{}".format(thing)
             resolve_keyed_by(task, thing, thing, **config.params)

taskcluster/taskgraph/transforms/job/run_task.py

@@ -7,6 +7,7 @@ Support for running jobs that are invoked via the `run-task` script.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+from taskgraph.transforms.task import taskref_or_string
 from taskgraph.transforms.job import run_job_using
 from taskgraph.util.schema import Schema
 from taskgraph.transforms.job.common import support_vcs_checkout
@@ -33,7 +34,7 @@ run_task_schema = Schema({
     # The command arguments to pass to the `run-task` script, after the
     # checkout arguments.  If a list, it will be passed directly; otherwise
     # it will be included in a single argument to `bash -cx`.
-    Required('command'): Any([basestring], basestring),
+    Required('command'): Any([taskref_or_string], taskref_or_string),
 
     # Base work directory used to set up the task.
     Required('workdir'): basestring,
@@ -83,7 +84,8 @@ def docker_worker_run_task(config, job, taskdesc):
         })
 
     run_command = run['command']
-    if isinstance(run_command, basestring):
+    # dict is for the case of `{'task-reference': basestring}`.
+    if isinstance(run_command, (basestring, dict)):
         run_command = ['bash', '-cx', run_command]
     if run['comm-checkout']:
         command.append('--comm-checkout={workdir}/checkouts/gecko/comm'.format(**run))

taskcluster/taskgraph/transforms/update_verify.py

@@ -31,16 +31,16 @@ def add_command(config, tasks):
             )
             if not chunked["worker"].get("env"):
                 chunked["worker"]["env"] = {}
-            chunked["worker"]["command"] = [
+            chunked["run"] = {
-                "/bin/bash",
+                'using': 'run-task',
-                "-c",
+                'command': 'cd /builds/worker/checkouts/gecko && '
-                "hg clone $BUILD_TOOLS_REPO tools && " +
+                           'tools/update-verify/scripts/chunked-verify.sh '
-                "tools/scripts/release/updates/chunked-verify.sh " +
+                           '{} {}'.format(
-                "UNUSED UNUSED {} {}".format(
+                               total_chunks,
-                    total_chunks,
+                               this_chunk,
-                    this_chunk,
+                           ),
-                )
+                'sparse-profile': 'update-verify',
-            ]
+            }
             for thing in ("CHANNEL", "VERIFY_CONFIG", "BUILD_TOOLS_REPO"):
                 thing = "worker.env.{}".format(thing)
                 resolve_keyed_by(

tools/update-verify/release/common/download_builds.sh

@@ -1,7 +1,7 @@
 pushd `dirname $0` &>/dev/null
 MY_DIR=$(pwd)
 popd &>/dev/null
-retry="$MY_DIR/../../buildfarm/utils/retry.py -s 1 -r 3"
+retry="$MY_DIR/../../../../mach python -m redo.cmd -s 1 -a 3"
 
 download_builds() {
   # cleanup

tools/update-verify/release/updates/verify.sh

@@ -27,7 +27,7 @@ touch ${diff_summary_log}
 pushd `dirname $0` &>/dev/null
 MY_DIR=$(pwd)
 popd &>/dev/null
-retry="$MY_DIR/../../buildfarm/utils/retry.py -s 1 -r 3"
+retry="$MY_DIR/../../../../mach python -m redo.cmd -s 1 -a 3"
 cert_replacer="$MY_DIR/../replace-updater-certs.py"
 
 dep_overrides="nightly_aurora_level3_primary.der dep1.der nightly_aurora_level3_secondary.der dep2.der release_primary.der dep1.der release_secondary.der dep2.der"

tools/update-verify/scripts/chunked-verify.sh

@@ -7,78 +7,28 @@ pushd `dirname $0` &>/dev/null
 MY_DIR=$(pwd)
 popd &>/dev/null
 SCRIPTS_DIR="$MY_DIR/.."
-PYTHON="/tools/python/bin/python"
+PYTHON='./mach python'
-if [ ! -x $PYTHON ]; then
-    PYTHON=python
-fi
-JSONTOOL="$PYTHON $SCRIPTS_DIR/buildfarm/utils/jsontool.py"
-workdir=`pwd`
 
-platform=$1
+chunks=$1
-configDict=$2
+thisChunk=$2
-chunks=$3
-thisChunk=$4
-channel=$5
-
-if [ -n "$PROPERTIES_FILE" -a -f "$PROPERTIES_FILE" ]; then
-    # Buildbot only
-    if $JSONTOOL -k properties.NO_BBCONFIG $PROPERTIES_FILE; then
-       NO_BBCONFIG=$($JSONTOOL -k properties.NO_BBCONFIG $PROPERTIES_FILE);
-    fi
-    if $JSONTOOL -k properties.CHANNEL $PROPERTIES_FILE; then
-       CHANNEL=$($JSONTOOL -k properties.CHANNEL $PROPERTIES_FILE);
-    fi
-    if $JSONTOOL -k properties.VERIFY_CONFIG $PROPERTIES_FILE; then
-       VERIFY_CONFIG=$($JSONTOOL -k properties.VERIFY_CONFIG $PROPERTIES_FILE);
-    fi
-    if $JSONTOOL -k properties.TOTAL_CHUNKS $PROPERTIES_FILE; then
-       chunks=$($JSONTOOL -k properties.TOTAL_CHUNKS $PROPERTIES_FILE);
-    fi
-    if $JSONTOOL -k properties.THIS_CHUNK $PROPERTIES_FILE; then
-       thisChunk=$($JSONTOOL -k properties.THIS_CHUNK $PROPERTIES_FILE);
-    fi
-    if [ -z "$NO_BBCONFIG" -a -z "$BUILDBOT_CONFIGS" ]; then
-        export BUILDBOT_CONFIGS="https://hg.mozilla.org/build/buildbot-configs"
-    fi
-    # Get the assumed slavebuilddir, and read in from buildbot if this is not
-    # Release promotion
-    SLAVEBUILDDIR=$(basename $(cd "$SCRIPTS_DIR/.."; pwd))
-    if [ -z "$NO_BBCONFIG" ]; then
-        RELEASE_CONFIG=$($JSONTOOL -k properties.release_config $PROPERTIES_FILE)
-        TAG=$($JSONTOOL -k properties.release_tag $PROPERTIES_FILE)
-        SLAVEBUILDDIR=$($JSONTOOL -k properties.slavebuilddir $PROPERTIES_FILE)
-    fi
-
-    $PYTHON -u $SCRIPTS_DIR/buildfarm/maintenance/purge_builds.py \
-        -s 16 -n info -n 'rel-*' -n 'tb-rel-*' -n $SLAVEBUILDDIR
-fi
 
 if [ -n "$TASKCLUSTER_VERIFY_CONFIG" ]; then
     wget -O "$SCRIPTS_DIR/release/updates/update-verify.cfg" "$TASKCLUSTER_VERIFY_CONFIG"
     VERIFY_CONFIG="update-verify.cfg"
-fi
+else
-
+    echo "Unable to run without TASKCLUSTER_VERIFY_CONFIG."
-if [ -z "$VERIFY_CONFIG" -a -n "$NO_BBCONFIG" ]; then
-    echo "Unable to run without VERIFY_CONFIG specified when using NO_BBCONFIG"
     exit 1
 fi
 
-if [ -z "$NO_BBCONFIG" ]; then
+# release promotion
-  $PYTHON $MY_DIR/chunked-verify.py -t $TAG -r $RELEASE_CONFIG \
+if [ -n "$CHANNEL" ]; then
-  -b $BUILDBOT_CONFIGS -p $platform --chunks $chunks --this-chunk $thisChunk \
+  EXTRA_PARAMS="--verify-channel $CHANNEL"
-  --config-dict $configDict --release-channel $channel \
-    2>&1 | tee $SCRIPTS_DIR/../verify_log.txt
 else
-  # release promotion
+  EXTRA_PARAMS=""
-  if [ -n "$CHANNEL" ]; then
-    EXTRA_PARAMS="--verify-channel $CHANNEL"
-  else
-    EXTRA_PARAMS=""
-  fi
-  $PYTHON $MY_DIR/chunked-verify.py --chunks $chunks --this-chunk $thisChunk \
-  --verify-config $VERIFY_CONFIG $EXTRA_PARAMS \
-  2>&1 | tee $SCRIPTS_DIR/../verify_log.txt
 fi
+$PYTHON $MY_DIR/chunked-verify.py --chunks $chunks --this-chunk $thisChunk \
+--verify-config $VERIFY_CONFIG $EXTRA_PARAMS \
+2>&1 | tee $SCRIPTS_DIR/../verify_log.txt
 
 print_failed_msg()
 {