gecko-dev/taskcluster/docker/updatebot/run.py

#!/usr/bin/env python3
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, # You can obtain one at http://mozilla.org/MPL/2.0/.

import sys

sys.path.append("/builds/worker/checkouts/gecko/third_party/python")
sys.path.append(".")

import base64
import os
import platform
import signal
import stat
import subprocess

import requests

import taskcluster

# Bump this number when you need to cause a commit for the job to re-run: 21

if len(sys.argv) < 3:
    print("Usage:", sys.argv[0], "gecko-dev-path updatebot-path [moz-fetches-dir]")
    sys.exit(1)

GECKO_DEV_PATH = sys.argv[1].replace("/", os.path.sep)
UPDATEBOT_PATH = sys.argv[2].replace("/", os.path.sep)

# Only needed on Windows
if len(sys.argv) > 3:
    FETCHES_PATH = sys.argv[3].replace("/", os.path.sep)
else:
    FETCHES_PATH = None

HOME_PATH = os.path.expanduser("~")

OPERATING_MODE = (
    "prod"
    if os.environ.get("GECKO_HEAD_REPOSITORY", "")
    == "https://hg.mozilla.org/mozilla-central"
    else "dev"
)

DEV_PHAB_URL = "https://phabricator-dev.allizom.org/"
PROD_PHAB_URL = "https://phabricator.services.mozilla.com/"

phabricator_url = DEV_PHAB_URL if OPERATING_MODE == "dev" else PROD_PHAB_URL


def log(*args):
    print(*args)


def get_secret(name):
    secret = None
    if "TASK_ID" in os.environ:
        secrets_url = (
            "http://taskcluster/secrets/v1/secret/project/updatebot/"
            + ("3" if OPERATING_MODE == "prod" else "2")
            + "/"
            + name
        )
        res = requests.get(secrets_url)
        res.raise_for_status()
        secret = res.json()
    else:
        secrets = taskcluster.Secrets(taskcluster.optionsFromEnvironment())
        secret = secrets.get("project/updatebot/" + OPERATING_MODE + "/" + name)
    secret = secret["secret"] if "secret" in secret else None
    secret = secret["value"] if "value" in secret else None
    return secret


# Get TC Secrets =======================================
log("Operating mode is ", OPERATING_MODE)
log("Getting secrets...")
bugzilla_api_key = get_secret("bugzilla-api-key")
phabricator_token = get_secret("phabricator-token")
try_sshkey = get_secret("try-sshkey")
database_config = get_secret("database-password")
sentry_url = get_secret("sentry-url")
sql_proxy_config = get_secret("sql-proxy-config")

# Update Updatebot =======================================
if OPERATING_MODE == "dev":
    """
    If we are in development mode, we will update from github.
    (This command will probably only work if we checkout a branch FWIW.)

    This allows us to iterate faster by committing to github and
    re-running the cron job on Taskcluster, without rebuilding the
    Docker image.

    However, this mechanism is bypassing the security feature we
    have in-tree, where upstream out-of-tree code is fixed at a known
    revision and cannot be changed without a commit to m-c.

    Therefore, we only do this in dev mode when running on try.
    """

    os.chdir(UPDATEBOT_PATH)
    log("Performing git repo update...")
    command = ["git", "symbolic-ref", "-q", "HEAD"]

    r = subprocess.run(command)
    if r.returncode == 0:
        # This indicates we are on a branch, and not a specific revision
        subprocess.check_call(["git", "pull", "origin"])

# Set Up SSH & Phabricator ==============================
os.chdir(HOME_PATH)
log("Setting up ssh and phab keys...")
with open("id_rsa", "w") as sshkey:
    sshkey.write(try_sshkey)
os.chmod("id_rsa", stat.S_IRUSR | stat.S_IWUSR)

arc_filename = ".arcrc"
if platform.system() == "Windows":
    arc_path = os.path.join(FETCHES_PATH, "..", "AppData", "Roaming")
    os.makedirs(arc_path, exist_ok=True)
    os.chdir(arc_path)
    log("Writing %s to %s" % (arc_filename, arc_path))
else:
    os.chdir(HOME_PATH)

arcrc = open(arc_filename, "w")
towrite = """
{
  "hosts": {
    "PHAB_URL_HERE": {
      "token": "TOKENHERE"
    }
  }
}
""".replace(
    "TOKENHERE", phabricator_token
).replace(
    "PHAB_URL_HERE", phabricator_url + "api/"
)
arcrc.write(towrite)
arcrc.close()
os.chmod(arc_filename, stat.S_IRUSR | stat.S_IWUSR)

# Set up the Cloud SQL Proxy =============================
os.chdir(HOME_PATH)
log("Setting up cloud_sql_proxy...")
with open("sql-proxy-key", "w") as proxy_key_file:
    proxy_key_file.write(
        base64.b64decode(sql_proxy_config["key-value"]).decode("utf-8")
    )

instance_name = sql_proxy_config["instance-name"]
if platform.system() == "Linux":
    sql_proxy_command = "cloud_sql_proxy"
else:
    sql_proxy_command = os.path.join(UPDATEBOT_PATH, "..", "cloud_sql_proxy.exe")

sql_proxy_command += (
    " -instances=" + instance_name + "=tcp:3306 -credential_file=sql-proxy-key"
)
sql_proxy_args = {
    "stdout": subprocess.PIPE,
    "stderr": subprocess.PIPE,
    "shell": True,
    "start_new_session": True,
}

if platform.system() == "Windows":
    si = subprocess.STARTUPINFO()
    si.dwFlags = subprocess.CREATE_NEW_PROCESS_GROUP

    sql_proxy_args["startupinfo"] = si

sql_proxy = subprocess.Popen((sql_proxy_command), **sql_proxy_args)

try:
    (stdout, stderr) = sql_proxy.communicate(input=None, timeout=2)
    log("sql proxy stdout:", stdout.decode("utf-8"))
    log("sql proxy stderr:", stderr.decode("utf-8"))
except subprocess.TimeoutExpired:
    log("no sqlproxy output in 2 seconds, this means it probably didn't error.")
    log("sqlproxy pid:", sql_proxy.pid)

database_config["host"] = "127.0.0.1"

# Vendor =================================================
log("Getting Updatebot ready...")
os.chdir(UPDATEBOT_PATH)
localconfig = {
    "General": {
        "env": OPERATING_MODE,
        "gecko-path": GECKO_DEV_PATH,
        "soft_timeout": 3600,
    },
    "Logging": {
        "local": True,
        "sentry": True,
        "sentry_config": {"url": sentry_url, "debug": False},
    },
    "Database": database_config,
    "Bugzilla": {
        "apikey": bugzilla_api_key,
    },
    "Taskcluster": {
        "url_treeherder": "https://treeherder.mozilla.org/",
        "url_taskcluster": "http://taskcluster/",
    },
}

log("Writing local config file")
config = open("localconfig.py", "w")
config.write("localconfig = " + str(localconfig))
config.close()

log("Running updatebot")
# On Windows, Updatebot is run by windows-setup.sh
if platform.system() == "Linux":
    subprocess.check_call(["python3", "-m", "poetry", "run", "./automation.py"])

    # Clean up ===============================================
    log("Killing cloud_sql_proxy")
    os.kill(sql_proxy.pid, signal.SIGTERM)