forked from mirrors/gecko-dev
69dbf231e6
This patch uses mozscreenshot to take a screenshot of the test right up to the moment of failure. This should be helpful in diagnosing failures in CI like benchmark tests (which don't output videos like pageload tests) as well as pageload tests if they fail before visual metrics can capture a video/screenshot to output. Differential Revision: https://phabricator.services.mozilla.com/D202634
460 lines
16 KiB
Python
460 lines
16 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
# Originally taken from /talos/mach_commands.py
|
|
|
|
# Integrates raptor mozharness with mach
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
|
|
from mach.decorators import Command
|
|
from mach.util import get_state_dir
|
|
from mozbuild.base import BinaryNotFoundException, MozbuildObject
|
|
from mozbuild.base import MachCommandConditions as Conditions
|
|
|
|
HERE = os.path.dirname(os.path.realpath(__file__))
|
|
|
|
ANDROID_BROWSERS = ["geckoview", "refbrow", "fenix", "chrome-m", "cstm-car-m"]
|
|
|
|
|
|
class RaptorRunner(MozbuildObject):
|
|
def run_test(self, raptor_args, kwargs):
|
|
"""Setup and run mozharness.
|
|
|
|
We want to do a few things before running Raptor:
|
|
|
|
1. Clone mozharness
|
|
2. Make the config for Raptor mozharness
|
|
3. Run mozharness
|
|
"""
|
|
# Validate that the user is using a supported python version before doing anything else
|
|
max_py_major, max_py_minor = 3, 11
|
|
sys_maj, sys_min = sys.version_info.major, sys.version_info.minor
|
|
if sys_min > max_py_minor:
|
|
raise PythonVersionException(
|
|
print(
|
|
f"\tPlease downgrade your Python version as Raptor does not yet support Python "
|
|
f"versions greater than {max_py_major}.{max_py_minor}."
|
|
f"\n\tYou seem to currently be using Python {sys_maj}.{sys_min}."
|
|
f"\n\tSee here for a possible solution in debugging your python environment: "
|
|
f"https://firefox-source-docs.mozilla.org/testing/perfdocs/"
|
|
f"debugging.html#debugging-local-python-environment"
|
|
)
|
|
)
|
|
self.init_variables(raptor_args, kwargs)
|
|
self.make_config()
|
|
self.write_config()
|
|
self.make_args()
|
|
return self.run_mozharness()
|
|
|
|
def init_variables(self, raptor_args, kwargs):
|
|
self.raptor_args = raptor_args
|
|
|
|
if kwargs.get("host") == "HOST_IP":
|
|
kwargs["host"] = os.environ["HOST_IP"]
|
|
self.host = kwargs["host"]
|
|
self.is_release_build = kwargs["is_release_build"]
|
|
self.live_sites = kwargs["live_sites"]
|
|
self.disable_perf_tuning = kwargs["disable_perf_tuning"]
|
|
self.conditioned_profile = kwargs["conditioned_profile"]
|
|
self.device_name = kwargs["device_name"]
|
|
self.enable_marionette_trace = kwargs["enable_marionette_trace"]
|
|
self.browsertime_visualmetrics = kwargs["browsertime_visualmetrics"]
|
|
self.browsertime_node = kwargs["browsertime_node"]
|
|
self.clean = kwargs["clean"]
|
|
self.screenshot_on_failure = kwargs["screenshot_on_failure"]
|
|
|
|
if Conditions.is_android(self) or kwargs["app"] in ANDROID_BROWSERS:
|
|
self.binary_path = None
|
|
else:
|
|
self.binary_path = kwargs.get("binary") or self.get_binary_path()
|
|
|
|
self.python = sys.executable
|
|
|
|
self.raptor_dir = os.path.join(self.topsrcdir, "testing", "raptor")
|
|
self.mozharness_dir = os.path.join(self.topsrcdir, "testing", "mozharness")
|
|
self.config_file_path = os.path.join(
|
|
self._topobjdir, "testing", "raptor-in_tree_conf.json"
|
|
)
|
|
|
|
self.virtualenv_script = os.path.join(
|
|
self.topsrcdir, "third_party", "python", "virtualenv", "virtualenv.py"
|
|
)
|
|
self.virtualenv_path = os.path.join(self._topobjdir, "testing", "raptor-venv")
|
|
|
|
def make_config(self):
|
|
default_actions = [
|
|
"populate-webroot",
|
|
"create-virtualenv",
|
|
"install-chromium-distribution",
|
|
"run-tests",
|
|
]
|
|
self.config = {
|
|
"run_local": True,
|
|
"binary_path": self.binary_path,
|
|
"repo_path": self.topsrcdir,
|
|
"raptor_path": self.raptor_dir,
|
|
"obj_path": self.topobjdir,
|
|
"log_name": "raptor",
|
|
"virtualenv_path": self.virtualenv_path,
|
|
"pypi_url": "http://pypi.org/simple",
|
|
"base_work_dir": self.mozharness_dir,
|
|
"exes": {
|
|
"python": self.python,
|
|
"virtualenv": [self.python, self.virtualenv_script],
|
|
},
|
|
"title": socket.gethostname(),
|
|
"default_actions": default_actions,
|
|
"raptor_cmd_line_args": self.raptor_args,
|
|
"host": self.host,
|
|
"live_sites": self.live_sites,
|
|
"disable_perf_tuning": self.disable_perf_tuning,
|
|
"conditioned_profile": self.conditioned_profile,
|
|
"is_release_build": self.is_release_build,
|
|
"device_name": self.device_name,
|
|
"enable_marionette_trace": self.enable_marionette_trace,
|
|
"browsertime_visualmetrics": self.browsertime_visualmetrics,
|
|
"browsertime_node": self.browsertime_node,
|
|
"mozbuild_path": get_state_dir(),
|
|
"clean": self.clean,
|
|
"screenshot_on_failure": self.screenshot_on_failure,
|
|
}
|
|
|
|
sys.path.insert(0, os.path.join(self.topsrcdir, "tools", "browsertime"))
|
|
try:
|
|
import platform
|
|
|
|
import mach_commands as browsertime
|
|
|
|
# We don't set `browsertime_{chromedriver,geckodriver} -- those will be found by
|
|
# browsertime in its `node_modules` directory, which is appropriate for local builds.
|
|
# We don't set `browsertime_ffmpeg` yet: it will need to be on the path. There is code
|
|
# to configure the environment including the path in
|
|
# `tools/browsertime/mach_commands.py` but integrating it here will take more effort.
|
|
self.config.update(
|
|
{
|
|
"browsertime_browsertimejs": browsertime.browsertime_path(),
|
|
"browsertime_vismet_script": browsertime.visualmetrics_path(),
|
|
}
|
|
)
|
|
|
|
def _get_browsertime_package():
|
|
with open(
|
|
os.path.join(
|
|
self.topsrcdir,
|
|
"tools",
|
|
"browsertime",
|
|
"node_modules",
|
|
"browsertime",
|
|
"package.json",
|
|
)
|
|
) as package:
|
|
return json.load(package)
|
|
|
|
def _get_browsertime_resolved():
|
|
try:
|
|
with open(
|
|
os.path.join(
|
|
self.topsrcdir,
|
|
"tools",
|
|
"browsertime",
|
|
"node_modules",
|
|
".package-lock.json",
|
|
)
|
|
) as package_lock:
|
|
return json.load(package_lock)["packages"][
|
|
"node_modules/browsertime"
|
|
]["resolved"]
|
|
except FileNotFoundError:
|
|
# Older versions of node/npm add this metadata to package.json
|
|
return _get_browsertime_package()["_from"]
|
|
|
|
def _should_install():
|
|
# If ffmpeg doesn't exist in the .mozbuild directory,
|
|
# then we should install
|
|
btime_cache = os.path.join(self.config["mozbuild_path"], "browsertime")
|
|
if not os.path.exists(btime_cache) or not any(
|
|
["ffmpeg" in cache_dir for cache_dir in os.listdir(btime_cache)]
|
|
):
|
|
return True
|
|
|
|
# If browsertime doesn't exist, install it
|
|
if not os.path.exists(
|
|
self.config["browsertime_browsertimejs"]
|
|
) or not os.path.exists(self.config["browsertime_vismet_script"]):
|
|
return True
|
|
|
|
# Browsertime exists, check if it's outdated
|
|
with open(
|
|
os.path.join(self.topsrcdir, "tools", "browsertime", "package.json")
|
|
) as new:
|
|
new_pkg = json.load(new)
|
|
|
|
return not _get_browsertime_resolved().endswith(
|
|
new_pkg["devDependencies"]["browsertime"]
|
|
)
|
|
|
|
def _get_browsertime_version():
|
|
# Returns the (version number, current commit) used
|
|
return (
|
|
_get_browsertime_package()["version"],
|
|
_get_browsertime_resolved(),
|
|
)
|
|
|
|
# Check if browsertime scripts exist and try to install them if
|
|
# they aren't
|
|
if _should_install():
|
|
# TODO: Make this "integration" nicer in the near future
|
|
print("Missing browsertime files...attempting to install")
|
|
subprocess.check_output(
|
|
[
|
|
os.path.join(self.topsrcdir, "mach"),
|
|
"browsertime",
|
|
"--setup",
|
|
"--clobber",
|
|
],
|
|
shell="windows" in platform.system().lower(),
|
|
)
|
|
if _should_install():
|
|
raise Exception(
|
|
"Failed installation attempt. Cannot find browsertime scripts. "
|
|
"Run `./mach browsertime --setup --clobber` to set it up."
|
|
)
|
|
|
|
# Bug 1766112 - For the time being, we need to trigger a
|
|
# clean build to upgrade browsertime. This should be disabled
|
|
# after some time.
|
|
print(
|
|
"Setting --clean to True to rebuild Python "
|
|
"environment for Browsertime upgrade..."
|
|
)
|
|
self.config["clean"] = True
|
|
|
|
print("Using browsertime version %s from %s" % _get_browsertime_version())
|
|
|
|
finally:
|
|
sys.path = sys.path[1:]
|
|
|
|
def make_args(self):
|
|
self.args = {
|
|
"config": {},
|
|
"initial_config_file": self.config_file_path,
|
|
}
|
|
|
|
def write_config(self):
|
|
try:
|
|
config_file = open(self.config_file_path, "w")
|
|
config_file.write(json.dumps(self.config))
|
|
config_file.close()
|
|
except IOError as e:
|
|
err_str = "Error writing to Raptor Mozharness config file {0}:{1}"
|
|
print(err_str.format(self.config_file_path, str(e)))
|
|
raise e
|
|
|
|
def run_mozharness(self):
|
|
sys.path.insert(0, self.mozharness_dir)
|
|
from mozharness.mozilla.testing.raptor import Raptor
|
|
|
|
raptor_mh = Raptor(
|
|
config=self.args["config"],
|
|
initial_config_file=self.args["initial_config_file"],
|
|
)
|
|
return raptor_mh.run()
|
|
|
|
|
|
def setup_node(command_context):
|
|
"""Fetch the latest node-18 binary and install it into the .mozbuild directory."""
|
|
import platform
|
|
|
|
from mozbuild.artifact_commands import artifact_toolchain
|
|
from mozbuild.nodeutil import find_node_executable
|
|
from packaging.version import Version
|
|
|
|
print("Setting up node for browsertime...")
|
|
state_dir = get_state_dir()
|
|
cache_path = os.path.join(state_dir, "browsertime", "node-18")
|
|
|
|
def __check_for_node():
|
|
# Check standard locations first
|
|
node_exe = find_node_executable(min_version=Version("18.0.0"))
|
|
if node_exe and (node_exe[0] is not None):
|
|
return node_exe[0]
|
|
if not os.path.exists(cache_path):
|
|
return None
|
|
|
|
# Check the browsertime-specific node location next
|
|
node_name = "node"
|
|
if platform.system() == "Windows":
|
|
node_name = "node.exe"
|
|
node_exe_path = os.path.join(
|
|
state_dir,
|
|
"browsertime",
|
|
"node-18",
|
|
"node",
|
|
)
|
|
else:
|
|
node_exe_path = os.path.join(
|
|
state_dir,
|
|
"browsertime",
|
|
"node-18",
|
|
"node",
|
|
"bin",
|
|
)
|
|
|
|
node_exe = os.path.join(node_exe_path, node_name)
|
|
if not os.path.exists(node_exe):
|
|
return None
|
|
|
|
return node_exe
|
|
|
|
node_exe = __check_for_node()
|
|
if node_exe is None:
|
|
toolchain_job = "{}-node-18"
|
|
plat = platform.system()
|
|
if plat == "Windows":
|
|
toolchain_job = toolchain_job.format("win64")
|
|
elif plat == "Darwin":
|
|
if platform.processor() == "arm":
|
|
toolchain_job = toolchain_job.format("macosx64-aarch64")
|
|
else:
|
|
toolchain_job = toolchain_job.format("macosx64")
|
|
else:
|
|
toolchain_job = toolchain_job.format("linux64")
|
|
|
|
print(
|
|
"Downloading Node v18 from Taskcluster toolchain {}...".format(
|
|
toolchain_job
|
|
)
|
|
)
|
|
|
|
if not os.path.exists(cache_path):
|
|
os.makedirs(cache_path, exist_ok=True)
|
|
|
|
# Change directories to where node should be installed
|
|
# before installing. Otherwise, it gets installed in the
|
|
# top level of the repo (or the current working directory).
|
|
cur_dir = os.getcwd()
|
|
os.chdir(cache_path)
|
|
artifact_toolchain(
|
|
command_context,
|
|
verbose=False,
|
|
from_build=[toolchain_job],
|
|
no_unpack=False,
|
|
retry=0,
|
|
cache_dir=cache_path,
|
|
)
|
|
os.chdir(cur_dir)
|
|
|
|
node_exe = __check_for_node()
|
|
if node_exe is None:
|
|
raise Exception("Could not find Node v18 binary for Raptor-Browsertime")
|
|
|
|
print("Finished downloading Node v18 from Taskcluster")
|
|
|
|
print("Node v18+ found at: %s" % node_exe)
|
|
return node_exe
|
|
|
|
|
|
def create_parser():
|
|
sys.path.insert(0, HERE) # allow to import the raptor package
|
|
from raptor.cmdline import create_parser
|
|
|
|
return create_parser(mach_interface=True)
|
|
|
|
|
|
@Command(
|
|
"raptor",
|
|
category="testing",
|
|
description="Run Raptor performance tests.",
|
|
parser=create_parser,
|
|
)
|
|
def run_raptor(command_context, **kwargs):
|
|
build_obj = command_context
|
|
|
|
# Setup node for browsertime
|
|
kwargs["browsertime_node"] = setup_node(command_context)
|
|
|
|
is_android = Conditions.is_android(build_obj) or kwargs["app"] in ANDROID_BROWSERS
|
|
|
|
if is_android:
|
|
from mozrunner.devices.android_device import (
|
|
InstallIntent,
|
|
verify_android_device,
|
|
)
|
|
|
|
install = (
|
|
InstallIntent.NO if kwargs.pop("noinstall", False) else InstallIntent.YES
|
|
)
|
|
verbose = False
|
|
if (
|
|
kwargs.get("log_mach_verbose")
|
|
or kwargs.get("log_tbpl_level") == "debug"
|
|
or kwargs.get("log_mach_level") == "debug"
|
|
or kwargs.get("log_raw_level") == "debug"
|
|
):
|
|
verbose = True
|
|
if not verify_android_device(
|
|
build_obj,
|
|
install=install,
|
|
app=kwargs["binary"],
|
|
verbose=verbose,
|
|
xre=True,
|
|
): # Equivalent to 'run_local' = True.
|
|
print(
|
|
"****************************************************************************"
|
|
)
|
|
print(
|
|
"Unable to verify device, please check your attached/connected android device"
|
|
)
|
|
print(
|
|
"****************************************************************************"
|
|
)
|
|
return 1
|
|
# Disable fission until geckoview supports fission by default.
|
|
# Need fission on Android? Use '--setpref fission.autostart=true'
|
|
kwargs["fission"] = False
|
|
|
|
# Remove mach global arguments from sys.argv to prevent them
|
|
# from being consumed by raptor. Treat any item in sys.argv
|
|
# occuring before "raptor" as a mach global argument.
|
|
argv = []
|
|
in_mach = True
|
|
for arg in sys.argv:
|
|
if not in_mach:
|
|
argv.append(arg)
|
|
if arg.startswith("raptor"):
|
|
in_mach = False
|
|
|
|
raptor = command_context._spawn(RaptorRunner)
|
|
|
|
try:
|
|
return raptor.run_test(argv, kwargs)
|
|
except BinaryNotFoundException as e:
|
|
command_context.log(
|
|
logging.ERROR, "raptor", {"error": str(e)}, "ERROR: {error}"
|
|
)
|
|
command_context.log(logging.INFO, "raptor", {"help": e.help()}, "{help}")
|
|
return 1
|
|
except Exception as e:
|
|
print(repr(e))
|
|
return 1
|
|
|
|
|
|
@Command(
|
|
"raptor-test",
|
|
category="testing",
|
|
description="Run Raptor performance tests.",
|
|
parser=create_parser,
|
|
)
|
|
def run_raptor_test(command_context, **kwargs):
|
|
return run_raptor(command_context, **kwargs)
|
|
|
|
|
|
class PythonVersionException(Exception):
|
|
pass
|