Bug 1900408 - Use CfT chromedriver for CaR tests. r=perftest-reviewers,taskgraph-reviewers,sparky,bhearsum

This patch replaces the chromedrivers retrieved from the chromium
snapshots with the Chrome for Testing (CfT) chromedrivers (through the
"Canary" channel) for our Chromium-as-Release tests.

There is an occasional intermittent issue that occurs where the
chromedriver from the snapshots don't always work. Instead we can use
the `last-known-good-versions-with-downloads.json` provided by CfT to
get guaranteed stable chromedrivers.

Differential Revision: https://phabricator.services.mozilla.com/D212707

taskcluster/docker/fetch/Dockerfile

@@ -34,4 +34,7 @@ ADD topsrcdir/taskcluster/scripts/misc/fetch-content /builds/worker/bin/fetch-co
 # %include taskcluster/scripts/misc/fetch-chromium.py
 ADD topsrcdir/taskcluster/scripts/misc/fetch-chromium.py /builds/worker/bin/fetch-chromium.py
 
+# %include taskcluster/scripts/misc/fetch-cft-chromedriver.py
+ADD topsrcdir/taskcluster/scripts/misc/fetch-cft-chromedriver.py /builds/worker/bin/fetch-cft-chromedriver.py
+
 RUN pip3 install redo==2.0.4 --break-system-packages

taskcluster/gecko_taskgraph/transforms/fetch.py

@@ -386,3 +386,38 @@ def create_chromium_fetch_task(config, name, fetch):
             f"artifact_name={artifact_name}",
         ],
     }
+
+
+@fetch_builder(
+    "cft-chromedriver-fetch",
+    schema={
+        Required("script"): str,
+        # Platform type for chromium build
+        Required("platform"): str,
+        # The name to give to the generated artifact.
+        Required("artifact-name"): str,
+    },
+)
+def create_cft_canary_fetch_task(config, name, fetch):
+    artifact_name = fetch.get("artifact-name")
+
+    workdir = "/builds/worker"
+
+    platform = fetch.get("platform")
+
+    args = "--platform " + shell_quote(platform)
+
+    cmd = [
+        "bash",
+        "-c",
+        "cd {} && " "/usr/bin/python3 {} {}".format(workdir, fetch["script"], args),
+    ]
+
+    return {
+        "command": cmd,
+        "artifact_name": artifact_name,
+        "digest_data": [
+            f"platform={platform}",
+            f"artifact_name={artifact_name}",
+        ],
+    }

taskcluster/gecko_taskgraph/transforms/test/other.py

@@ -422,13 +422,11 @@ def setup_browsertime(config, tasks):
         }
 
         chromium_fetches = {
-            "linux.*": ["linux64-chromiumdriver"],
+            "linux.*": ["linux64-cft-chromedriver"],
-            "macosx1015.*": ["mac-chromiumdriver"],
+            "macosx1015.*": ["mac-cft-chromedriver"],
-            "macosx1400.*": ["mac-chromiumdriver-arm"],
+            "macosx1400.*": ["mac-cft-chromedriver-arm"],
-            "windows.*aarch64.*": ["win32-chromiumdriver"],
+            "windows.*-64.*": ["win64-cft-chromedriver"],
-            "windows.*-32.*": ["win32-chromiumdriver"],
+            "android.*": ["linux64-cft-chromedriver"],
-            "windows.*-64.*": ["win64-chromiumdriver"],
-            "android.*": ["linux64-chromiumdriver"],
         }
 
         cd_extracted_name = {
@@ -445,13 +443,13 @@ def setup_browsertime(config, tasks):
             for platform in chromium_fetches:
                 fs["by-test-platform"][platform].extend(chromium_fetches[platform])
 
-            # The chromedrivers for chromium are repackaged into the archives
+            # The Chrome-for-Testing chromedrivers are repackaged into the following
-            # that we get the chromium binary from so we always have a compatible
+            # platform specific archives. The versions will always be compatible as
-            # version.
+            # these are fetched from the `Canary` channel.
             cd_extracted_name = {
-                "windows": "chrome-win/chromedriver.exe",
+                "windows": "cft-chromedriver-win64/chromedriver.exe",
-                "mac": "chrome-mac/chromedriver",
+                "mac": "cft-chromedriver-mac/chromedriver",
-                "default": "chrome-linux/chromedriver",
+                "default": "cft-chromedriver-linux/chromedriver",
             }
 
         # Disable the Raptor install step

taskcluster/kinds/fetch/cft-chromedriver-fetch.yml

@@ -0,0 +1,40 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+job-defaults:
+    fetch:
+        type: cft-chromedriver-fetch
+        script: /builds/worker/bin/fetch-cft-chromedriver.py
+
+linux64-cft-chromedriver:
+    description: 'Linux64 Chrome-for-Testing Chromedriver Fetch'
+    attributes:
+        cached_task: false
+    fetch:
+        platform: linux
+        artifact-name: cft-cd-linux.tar.bz2
+
+win64-cft-chromedriver:
+    description: 'Windows64 Chrome-for-Testing Chromedriver Fetch'
+    attributes:
+        cached_task: false
+    fetch:
+        platform: win64
+        artifact-name: cft-cd-win64.tar.bz2
+
+mac-cft-chromedriver:
+    description: 'MacOSX Chrome-for-Testing Chromedriver Fetch'
+    attributes:
+        cached_task: false
+    fetch:
+        platform: mac
+        artifact-name: cft-cd-mac.tar.bz2
+
+mac-cft-chromedriver-arm:
+    description: 'MacOS (arm) Chrome-for-Testing Chromedriver fetch'
+    attributes:
+        cached_task: false
+    fetch:
+        platform: mac-arm
+        artifact-name: cft-cd-mac-arm.tar.bz2

taskcluster/kinds/fetch/kind.yml

@@ -14,6 +14,7 @@ jobs-from:
     - benchmarks.yml
     - browsertime.yml
     - chromium-fetch.yml
+    - cft-chromedriver-fetch.yml
     - resource-monitor.yml
     - toolchain-clang-tidy-external.yml
     - toolchains.yml

taskcluster/scripts/misc/fetch-cft-chromedriver.py

@@ -0,0 +1,199 @@
+#!/usr/bin/python3 -u
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+This script downloads chromedriver for a given platform and then
+packages the driver along with the revision and uploads the archive.
+This is currently accomplished by using "last known good version" of
+the chromedrivers associated with Chrome for Testing. The `Canary`
+channel is specified as it is required for the Chromium-as-Release
+performance tests.
+"""
+
+
+import argparse
+import errno
+import os
+import shutil
+import subprocess
+import tempfile
+
+import requests
+from redo import retriable
+
+CHROME_FOR_TESTING_INFO = {
+    "linux": {
+        "platform": "linux64",
+        "dir": "cft-chromedriver-linux",
+        "result": "cft-cd-linux.tar.bz2",
+        "chromedriver": "chromedriver_linux64.zip",
+    },
+    "win64": {
+        "platform": "win64",
+        "dir": "cft-chromedriver-win64",
+        "result": "cft-cd-win64.tar.bz2",
+        "chromedriver": "chromedriver_win32.zip",
+    },
+    "mac": {
+        "platform": "mac-x64",
+        "dir": "cft-chromedriver-mac",
+        "result": "cft-cd-mac.tar.bz2",
+        "chromedriver": "chromedriver_mac64.zip",
+    },
+    "mac-arm": {
+        "platform": "mac-arm64",
+        "dir": "cft-chromedriver-mac",
+        "result": "cft-cd-mac-arm.tar.bz2",
+        "chromedriver": "chromedriver_mac64.zip",
+    },
+}
+
+# Bug 1869592
+# Potentially add another JSON endpoint to grab more than 1 chromedriver
+LAST_GOOD_CFT_JSON = (
+    "https://googlechromelabs.github.io/chrome-for-testing/"
+    "last-known-good-versions-with-downloads.json"
+)
+
+
+def log(msg):
+    print("build-cft-chromedriver: %s" % msg)
+
+
+@retriable(attempts=7, sleeptime=5, sleepscale=2)
+def fetch_file(url, filepath):
+    """Download a file from the given url to a given file."""
+    size = 4096
+    r = requests.get(url, stream=True)
+    r.raise_for_status()
+
+    with open(filepath, "wb") as fd:
+        for chunk in r.iter_content(size):
+            fd.write(chunk)
+
+
+def unzip(zippath, target):
+    """Unzips an archive to the target location."""
+    log("Unpacking archive at: %s to: %s" % (zippath, target))
+    unzip_command = ["unzip", "-q", "-o", zippath, "-d", target]
+    subprocess.check_call(unzip_command)
+
+
+def get_cft_metadata():
+    """Send a request to the Chrome for Testing's last
+    good json URL (default) and get the json payload which will have
+    the download URLs that we need.
+    """
+    res = requests.get(LAST_GOOD_CFT_JSON)
+    data = res.json()
+
+    return data
+
+
+def get_cd_url(data, cft_platform):
+    """Given the json data, get the download URL's for
+    the correct platform
+    """
+    for p in data["channels"]["Canary"]["downloads"]["chromedriver"]:
+        if p["platform"] == cft_platform:
+            return p["url"]
+    raise Exception("Platform not found")
+
+
+def get_chromedriver_revision(data):
+    """Grab revision metadata from payload"""
+    return data["channels"]["Canary"]["revision"]
+
+
+def fetch_chromedriver(download_url, cft_dir):
+    """Get the chromedriver for the given cft url repackage it."""
+
+    tmpzip = os.path.join(tempfile.mkdtemp(), "cd-tmp.zip")
+    log("Downloading chromedriver from %s" % download_url)
+    fetch_file(download_url, tmpzip)
+
+    tmppath = tempfile.mkdtemp()
+    unzip(tmpzip, tmppath)
+
+    # Find the chromedriver then copy it to the chromium directory
+    cd_path = None
+    for dirpath, _, filenames in os.walk(tmppath):
+        for filename in filenames:
+            if filename == "chromedriver" or filename == "chromedriver.exe":
+                cd_path = os.path.join(dirpath, filename)
+                break
+        if cd_path is not None:
+            break
+    if cd_path is None:
+        raise Exception("Could not find chromedriver binary in %s" % tmppath)
+    log("Copying chromedriver from: %s to: %s" % (cd_path, cft_dir))
+    shutil.copy(cd_path, cft_dir)
+
+
+def build_cft_archive(platform):
+    """Download and store a chromedriver for a given platform."""
+    upload_dir = os.environ.get("UPLOAD_DIR")
+    if upload_dir:
+        # Create the upload directory if it doesn't exist.
+        try:
+            log("Creating upload directory in %s..." % os.path.abspath(upload_dir))
+            os.makedirs(upload_dir)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+
+    cft_platform = CHROME_FOR_TESTING_INFO[platform]["platform"]
+
+    data = get_cft_metadata()
+    cft_chromedriver_url = get_cd_url(data, cft_platform)
+    revision = get_chromedriver_revision(data)
+    # Make a temporary location for the file
+    tmppath = tempfile.mkdtemp()
+
+    # Create the directory format expected for browsertime setup in taskgraph transform
+    artifact_dir = CHROME_FOR_TESTING_INFO[platform]["dir"]
+    cft_dir = os.path.join(tmppath, artifact_dir)
+    os.mkdir(cft_dir)
+
+    # Store the revision number and chromedriver
+    fetch_chromedriver(cft_chromedriver_url, cft_dir)
+    revision_file = os.path.join(cft_dir, ".REVISION")
+    with open(revision_file, "w+") as f:
+        f.write(str(revision))
+
+    tar_file = CHROME_FOR_TESTING_INFO[platform]["result"]
+    tar_command = ["tar", "cjf", tar_file, "-C", tmppath, artifact_dir]
+    log("Revision is %s" % revision)
+    log("Added revision to %s file." % revision_file)
+
+    log("Tarring with the command: %s" % str(tar_command))
+    subprocess.check_call(tar_command)
+
+    upload_dir = os.environ.get("UPLOAD_DIR")
+    if upload_dir:
+        # Move the tarball to the output directory for upload.
+        log("Moving %s to the upload directory..." % tar_file)
+        shutil.copy(tar_file, os.path.join(upload_dir, tar_file))
+
+    shutil.rmtree(tmppath)
+
+
+def parse_args():
+    """Read command line arguments and return options."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--platform",
+        help="Corresponding platform of CfT chromedriver to fetch.",
+        required=True,
+    )
+    # Bug 1869592 - Add optional flag to provide CfT channel e.g. Canary, Stable, etc.
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = vars(parse_args())
+    build_cft_archive(**args)