forked from mirrors/gecko-dev
b457d6c1b8
Bump the number of retries from 5 (5+10+20+40=75 seconds worst case) to 7 (5+10+20+40+80+160=315 seconds worst case). Also log the url being used for the first fetch, in case that helps diagnose future failures. Differential Revision: https://phabricator.services.mozilla.com/D167704
235 lines
7.2 KiB
Python
235 lines
7.2 KiB
Python
#!/usr/bin/python3 -u
|
|
# -*- coding: utf-8 -*-
|
|
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
"""
|
|
This script downloads the latest chromium build (or a manually
|
|
defined version) for a given platform. It then uploads the build,
|
|
with the revision of the build stored in a REVISION file.
|
|
"""
|
|
|
|
import argparse
|
|
import errno
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
|
|
import requests
|
|
from redo import retriable
|
|
|
|
LAST_CHANGE_URL = (
|
|
# formatted with platform
|
|
"https://www.googleapis.com/download/storage/v1/b/"
|
|
"chromium-browser-snapshots/o/{}%2FLAST_CHANGE?alt=media"
|
|
)
|
|
|
|
CHROMIUM_BASE_URL = (
|
|
# formatted with (platform/revision/archive)
|
|
"https://www.googleapis.com/download/storage/v1/b/"
|
|
"chromium-browser-snapshots/o/{}%2F{}%2F{}?alt=media"
|
|
)
|
|
|
|
|
|
CHROMIUM_INFO = {
|
|
"linux": {
|
|
"platform": "Linux_x64",
|
|
"chromium": "chrome-linux.zip",
|
|
"result": "chromium-linux.tar.bz2",
|
|
"chromedriver": "chromedriver_linux64.zip",
|
|
},
|
|
"win32": {
|
|
"platform": "Win",
|
|
"chromium": "chrome-win.zip",
|
|
"result": "chromium-win32.tar.bz2",
|
|
"chromedriver": "chromedriver_win32.zip",
|
|
},
|
|
"win64": {
|
|
"platform": "Win",
|
|
"chromium": "chrome-win.zip",
|
|
"result": "chromium-win64.tar.bz2",
|
|
"chromedriver": "chromedriver_win32.zip",
|
|
},
|
|
"mac": {
|
|
"platform": "Mac",
|
|
"chromium": "chrome-mac.zip",
|
|
"result": "chromium-mac.tar.bz2",
|
|
"chromedriver": "chromedriver_mac64.zip",
|
|
},
|
|
}
|
|
|
|
|
|
def log(msg):
|
|
print("build-chromium: %s" % msg)
|
|
|
|
|
|
@retriable(attempts=7, sleeptime=5, sleepscale=2)
|
|
def fetch_file(url, filepath):
|
|
"""Download a file from the given url to a given file."""
|
|
size = 4096
|
|
r = requests.get(url, stream=True)
|
|
r.raise_for_status()
|
|
|
|
with open(filepath, "wb") as fd:
|
|
for chunk in r.iter_content(size):
|
|
fd.write(chunk)
|
|
|
|
|
|
def unzip(zippath, target):
|
|
"""Unzips an archive to the target location."""
|
|
log("Unpacking archive at: %s to: %s" % (zippath, target))
|
|
unzip_command = ["unzip", "-q", "-o", zippath, "-d", target]
|
|
subprocess.check_call(unzip_command)
|
|
|
|
|
|
@retriable(attempts=7, sleeptime=5, sleepscale=2)
|
|
def fetch_chromium_revision(platform):
|
|
"""Get the revision of the latest chromium build."""
|
|
chromium_platform = CHROMIUM_INFO[platform]["platform"]
|
|
revision_url = LAST_CHANGE_URL.format(chromium_platform)
|
|
|
|
log("Getting revision number for latest %s chromium build..." % chromium_platform)
|
|
|
|
# Expecting a file with a single number indicating the latest
|
|
# chromium build with a chromedriver that we can download
|
|
r = requests.get(revision_url, timeout=30)
|
|
r.raise_for_status()
|
|
|
|
chromium_revision = r.content.decode("utf-8")
|
|
return chromium_revision.strip()
|
|
|
|
|
|
def fetch_chromium_build(platform, revision, zippath):
|
|
"""Download a chromium build for a given revision, or the latest."""
|
|
if not revision:
|
|
revision = fetch_chromium_revision(platform)
|
|
|
|
download_platform = CHROMIUM_INFO[platform]["platform"]
|
|
download_url = CHROMIUM_BASE_URL.format(
|
|
download_platform, revision, CHROMIUM_INFO[platform]["chromium"]
|
|
)
|
|
|
|
log("Downloading %s chromium build revision %s..." % (download_platform, revision))
|
|
log(download_url)
|
|
fetch_file(download_url, zippath)
|
|
return revision
|
|
|
|
|
|
def fetch_chromedriver(platform, revision, chromium_dir):
|
|
"""Get the chromedriver for the given revision and repackage it."""
|
|
download_url = CHROMIUM_BASE_URL.format(
|
|
CHROMIUM_INFO[platform]["platform"],
|
|
revision,
|
|
CHROMIUM_INFO[platform]["chromedriver"],
|
|
)
|
|
|
|
tmpzip = os.path.join(tempfile.mkdtemp(), "cd-tmp.zip")
|
|
log("Downloading chromedriver from %s" % download_url)
|
|
fetch_file(download_url, tmpzip)
|
|
|
|
tmppath = tempfile.mkdtemp()
|
|
unzip(tmpzip, tmppath)
|
|
|
|
# Find the chromedriver then copy it to the chromium directory
|
|
cd_path = None
|
|
for dirpath, _, filenames in os.walk(tmppath):
|
|
for filename in filenames:
|
|
if filename == "chromedriver" or filename == "chromedriver.exe":
|
|
cd_path = os.path.join(dirpath, filename)
|
|
break
|
|
if cd_path is not None:
|
|
break
|
|
if cd_path is None:
|
|
raise Exception("Could not find chromedriver binary in %s" % tmppath)
|
|
log("Copying chromedriver from: %s to: %s" % (cd_path, chromium_dir))
|
|
shutil.copy(cd_path, chromium_dir)
|
|
|
|
|
|
def build_chromium_archive(platform, revision=None):
|
|
"""
|
|
Download and store a chromium build for a given platform.
|
|
|
|
Retrieves either the latest version, or uses a pre-defined version if
|
|
the `--revision` option is given a revision.
|
|
"""
|
|
upload_dir = os.environ.get("UPLOAD_DIR")
|
|
if upload_dir:
|
|
# Create the upload directory if it doesn't exist.
|
|
try:
|
|
log("Creating upload directory in %s..." % os.path.abspath(upload_dir))
|
|
os.makedirs(upload_dir)
|
|
except OSError as e:
|
|
if e.errno != errno.EEXIST:
|
|
raise
|
|
|
|
# Make a temporary location for the file
|
|
tmppath = tempfile.mkdtemp()
|
|
tmpzip = os.path.join(tmppath, "tmp-chromium.zip")
|
|
|
|
revision = fetch_chromium_build(platform, revision, tmpzip)
|
|
|
|
# Unpack archive in `tmpzip` to store the revision number and
|
|
# the chromedriver
|
|
unzip(tmpzip, tmppath)
|
|
|
|
dirs = [
|
|
d
|
|
for d in os.listdir(tmppath)
|
|
if os.path.isdir(os.path.join(tmppath, d)) and d.startswith("chrome-")
|
|
]
|
|
|
|
if len(dirs) > 1:
|
|
raise Exception(
|
|
"Too many directories starting with `chrome-` after extracting."
|
|
)
|
|
elif len(dirs) == 0:
|
|
raise Exception(
|
|
"Could not find any directories after extraction of chromium zip."
|
|
)
|
|
|
|
chromium_dir = os.path.join(tmppath, dirs[0])
|
|
revision_file = os.path.join(chromium_dir, ".REVISION")
|
|
with open(revision_file, "w+") as f:
|
|
f.write(str(revision))
|
|
|
|
# Get and store the chromedriver
|
|
fetch_chromedriver(platform, revision, chromium_dir)
|
|
|
|
tar_file = CHROMIUM_INFO[platform]["result"]
|
|
tar_command = ["tar", "cjf", tar_file, "-C", tmppath, dirs[0]]
|
|
log("Added revision to %s file." % revision_file)
|
|
|
|
log("Tarring with the command: %s" % str(tar_command))
|
|
subprocess.check_call(tar_command)
|
|
|
|
upload_dir = os.environ.get("UPLOAD_DIR")
|
|
if upload_dir:
|
|
# Move the tarball to the output directory for upload.
|
|
log("Moving %s to the upload directory..." % tar_file)
|
|
shutil.copy(tar_file, os.path.join(upload_dir, tar_file))
|
|
|
|
shutil.rmtree(tmppath)
|
|
|
|
|
|
def parse_args():
|
|
"""Read command line arguments and return options."""
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
"--platform", help="Platform version of chromium to build.", required=True
|
|
)
|
|
parser.add_argument(
|
|
"--revision",
|
|
help="Revision of chromium to build to get. "
|
|
"(Defaults to the newest chromium build).",
|
|
default=None,
|
|
)
|
|
|
|
return parser.parse_args()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
args = vars(parse_args())
|
|
build_chromium_archive(**args)
|