mirror of
https://github.com/mozilla/gecko-dev.git
synced 2025-11-08 04:09:03 +02:00
d4d66f19dc
This command is inteded to be usable as a git mergetool for the specific case of merging ini files in a way that should always succeed, produces reasonably likely output, but isn't reliably correct. The main use case is for the sync bot where we update metadata on branches and experience conflicts when we also have changes on master. We don't necessarily need to resolve these perfectly but we do need to provide a resolution automatically since otherwise it blocks syncs. Since the ini parser is in-tree we want to make this an in-tree command that the sync can use to provide the resolution. The general strategy is that we want to prefer the "new" metadata where possible. This won't always be correct e.g. if a test got fixed on master and simultaneously got edited to go from TIMEOUT to FAIL in a browser without the fix. But it's not a bad approximation (and generally a human will struggle to do the merge better by hand, so we have to assume later try jobs will fix things). Differential Revision: https://phabricator.services.mozilla.com/D40832 --HG-- extra : moz-landing-system : lando
389 lines
15 KiB
Python
389 lines
15 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
# Integrates the web-platform-tests test runner with mach.
|
|
|
|
from __future__ import absolute_import, unicode_literals, print_function
|
|
|
|
import os
|
|
import sys
|
|
|
|
from mozbuild.base import (
|
|
MachCommandBase,
|
|
MachCommandConditions as conditions,
|
|
MozbuildObject,
|
|
)
|
|
|
|
from mach.decorators import (
|
|
CommandProvider,
|
|
Command,
|
|
)
|
|
|
|
from mach_commands_base import WebPlatformTestsRunner, create_parser_wpt
|
|
|
|
|
|
class WebPlatformTestsRunnerSetup(MozbuildObject):
|
|
default_log_type = "mach"
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super(WebPlatformTestsRunnerSetup, self).__init__(*args, **kwargs)
|
|
self._here = os.path.join(self.topsrcdir, 'testing', 'web-platform')
|
|
kwargs["tests_root"] = os.path.join(self._here, "tests")
|
|
sys.path.insert(0, kwargs["tests_root"])
|
|
build_path = os.path.join(self.topobjdir, 'build')
|
|
if build_path not in sys.path:
|
|
sys.path.append(build_path)
|
|
|
|
def kwargs_common(self, kwargs):
|
|
tests_src_path = os.path.join(self._here, "tests")
|
|
if kwargs["product"] == "firefox_android":
|
|
# package_name may be different in the future
|
|
package_name = kwargs["package_name"]
|
|
if not package_name:
|
|
kwargs["package_name"] = package_name = "org.mozilla.geckoview.test"
|
|
|
|
# Note that this import may fail in non-firefox-for-android trees
|
|
from mozrunner.devices.android_device import verify_android_device
|
|
verify_android_device(self, install=True, verbose=False, xre=True, app=package_name)
|
|
|
|
if kwargs["certutil_binary"] is None:
|
|
kwargs["certutil_binary"] = os.path.join(os.environ.get('MOZ_HOST_BIN'), "certutil")
|
|
|
|
if kwargs["install_fonts"] is None:
|
|
kwargs["install_fonts"] = True
|
|
|
|
if kwargs["config"] is None:
|
|
kwargs["config"] = os.path.join(self.topobjdir, '_tests', 'web-platform', 'wptrunner.local.ini')
|
|
|
|
if kwargs["prefs_root"] is None:
|
|
kwargs["prefs_root"] = os.path.join(self.topsrcdir, 'testing', 'profiles')
|
|
|
|
if kwargs["stackfix_dir"] is None:
|
|
kwargs["stackfix_dir"] = self.bindir
|
|
|
|
if kwargs["exclude"] is None and kwargs["include"] is None and not sys.platform.startswith("linux"):
|
|
kwargs["exclude"] = ["css"]
|
|
|
|
if kwargs["ssl_type"] in (None, "pregenerated"):
|
|
cert_root = os.path.join(tests_src_path, "tools", "certs")
|
|
if kwargs["ca_cert_path"] is None:
|
|
kwargs["ca_cert_path"] = os.path.join(cert_root, "cacert.pem")
|
|
|
|
if kwargs["host_key_path"] is None:
|
|
kwargs["host_key_path"] = os.path.join(cert_root, "web-platform.test.key")
|
|
|
|
if kwargs["host_cert_path"] is None:
|
|
kwargs["host_cert_path"] = os.path.join(cert_root, "web-platform.test.pem")
|
|
|
|
if kwargs["lsan_dir"] is None:
|
|
kwargs["lsan_dir"] = os.path.join(self.topsrcdir, "build", "sanitizers")
|
|
|
|
if kwargs["reftest_screenshot"] is None:
|
|
kwargs["reftest_screenshot"] = "fail"
|
|
|
|
kwargs["capture_stdio"] = True
|
|
|
|
return kwargs
|
|
|
|
def kwargs_firefox(self, kwargs):
|
|
import mozinfo
|
|
from wptrunner import wptcommandline
|
|
kwargs = self.kwargs_common(kwargs)
|
|
|
|
if kwargs["binary"] is None:
|
|
kwargs["binary"] = self.get_binary_path()
|
|
|
|
if kwargs["certutil_binary"] is None:
|
|
kwargs["certutil_binary"] = self.get_binary_path('certutil')
|
|
|
|
if kwargs["webdriver_binary"] is None:
|
|
kwargs["webdriver_binary"] = self.get_binary_path("geckodriver", validate_exists=False)
|
|
|
|
|
|
if mozinfo.info["os"] == "win" and mozinfo.info["os_version"] == "6.1":
|
|
# On Windows 7 --install-fonts fails, so fall back to a Firefox-specific codepath
|
|
self.setup_fonts_firefox()
|
|
else:
|
|
kwargs["install_fonts"] = True
|
|
|
|
kwargs = wptcommandline.check_args(kwargs)
|
|
|
|
return kwargs
|
|
|
|
def kwargs_wptrun(self, kwargs):
|
|
from wptrunner import wptcommandline
|
|
|
|
if kwargs["metadata_root"] is None:
|
|
metadir = os.path.join(self._here, "products", kwargs["product"])
|
|
if not os.path.exists(metadir):
|
|
os.makedirs(metadir)
|
|
kwargs["metadata_root"] = metadir
|
|
|
|
src_manifest = os.path.join(self._here, "meta", "MANIFEST.json")
|
|
dest_manifest = os.path.join(kwargs["metadata_root"], "MANIFEST.json")
|
|
|
|
if not os.path.exists(dest_manifest) and os.path.exists(src_manifest):
|
|
with open(src_manifest) as src, open(dest_manifest, "w") as dest:
|
|
dest.write(src.read())
|
|
|
|
from tools.wpt import run
|
|
|
|
# Add additional kwargs consumed by the run frontend. Currently we don't
|
|
# have a way to set these through mach
|
|
kwargs["channel"] = None
|
|
kwargs["prompt"] = True
|
|
kwargs["install_browser"] = False
|
|
|
|
# Install the deps
|
|
# We do this explicitly to avoid calling pip with options that aren't
|
|
# supported in the in-tree version
|
|
wptrunner_path = os.path.join(self._here, "tests", "tools", "wptrunner")
|
|
browser_cls = run.product_setup[kwargs["product"]].browser_cls
|
|
requirements = ["requirements.txt"]
|
|
if hasattr(browser_cls, "requirements"):
|
|
requirements.append(browser_cls.requirements)
|
|
|
|
for filename in requirements:
|
|
path = os.path.join(wptrunner_path, filename)
|
|
if os.path.exists(path):
|
|
self.virtualenv_manager.install_pip_requirements(path, require_hashes=False)
|
|
|
|
venv = run.virtualenv.Virtualenv(self.virtualenv_manager.virtualenv_root,
|
|
skip_virtualenv_setup=True)
|
|
try:
|
|
kwargs = run.setup_wptrunner(venv, **kwargs)
|
|
except run.WptrunError as e:
|
|
print(e.message, file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
return kwargs
|
|
|
|
def setup_fonts_firefox(self):
|
|
# Ensure the Ahem font is available
|
|
if not sys.platform.startswith("darwin"):
|
|
font_path = os.path.join(os.path.dirname(self.get_binary_path()), "fonts")
|
|
else:
|
|
font_path = os.path.join(os.path.dirname(self.get_binary_path()), os.pardir, "Resources", "res", "fonts")
|
|
ahem_src = os.path.join(self.topsrcdir, "testing", "web-platform", "tests", "fonts", "Ahem.ttf")
|
|
ahem_dest = os.path.join(font_path, "Ahem.ttf")
|
|
if not os.path.exists(ahem_dest) and os.path.exists(ahem_src):
|
|
with open(ahem_src, "rb") as src, open(ahem_dest, "wb") as dest:
|
|
dest.write(src.read())
|
|
|
|
|
|
|
|
class WebPlatformTestsUpdater(MozbuildObject):
|
|
"""Update web platform tests."""
|
|
def setup_logging(self, **kwargs):
|
|
import update
|
|
return update.setup_logging(kwargs, {"mach": sys.stdout})
|
|
|
|
def update_manifest(self, logger, **kwargs):
|
|
import manifestupdate
|
|
return manifestupdate.run(logger=logger,
|
|
src_root=self.topsrcdir,
|
|
obj_root=self.topobjdir,
|
|
**kwargs)
|
|
|
|
def run_update(self, logger, **kwargs):
|
|
import update
|
|
from update import updatecommandline
|
|
|
|
self.update_manifest(logger, **kwargs)
|
|
|
|
if kwargs["config"] is None:
|
|
kwargs["config"] = os.path.join(self.topobjdir, '_tests', 'web-platform', 'wptrunner.local.ini')
|
|
if kwargs["product"] is None:
|
|
kwargs["product"] = "firefox"
|
|
|
|
kwargs["store_state"] = False
|
|
|
|
kwargs = updatecommandline.check_args(kwargs)
|
|
|
|
try:
|
|
update.run_update(logger, **kwargs)
|
|
except Exception:
|
|
import pdb
|
|
import traceback
|
|
traceback.print_exc()
|
|
# pdb.post_mortem()
|
|
|
|
|
|
class WebPlatformTestsUnittestRunner(MozbuildObject):
|
|
def run(self, **kwargs):
|
|
import unittestrunner
|
|
return unittestrunner.run(self.topsrcdir, **kwargs)
|
|
|
|
|
|
def create_parser_update():
|
|
from update import updatecommandline
|
|
return updatecommandline.create_parser()
|
|
|
|
|
|
def create_parser_create():
|
|
import argparse
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--no-editor", action="store_true",
|
|
help="Don't try to open the test in an editor")
|
|
p.add_argument("-e", "--editor", action="store", help="Editor to use")
|
|
p.add_argument("--long-timeout", action="store_true",
|
|
help="Test should be given a long timeout (typically 60s rather than 10s, but varies depending on environment)")
|
|
p.add_argument("--overwrite", action="store_true",
|
|
help="Allow overwriting an existing test file")
|
|
p.add_argument("-r", "--reftest", action="store_true",
|
|
help="Create a reftest rather than a testharness (js) test"),
|
|
p.add_argument("-m", "--reference", dest="ref", help="Path to the reference file")
|
|
p.add_argument("--mismatch", action="store_true",
|
|
help="Create a mismatch reftest")
|
|
p.add_argument("--wait", action="store_true",
|
|
help="Create a reftest that waits until takeScreenshot() is called")
|
|
p.add_argument("path", action="store", help="Path to the test file")
|
|
return p
|
|
|
|
|
|
def create_parser_manifest_update():
|
|
import manifestupdate
|
|
return manifestupdate.create_parser()
|
|
|
|
|
|
def create_parser_metadata_summary():
|
|
import metasummary
|
|
return metasummary.create_parser()
|
|
|
|
|
|
def create_parser_metadata_merge():
|
|
import metamerge
|
|
return metamerge.get_parser()
|
|
|
|
def create_parser_serve():
|
|
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__),
|
|
"tests", "tools")))
|
|
import serve
|
|
return serve.serve.get_parser()
|
|
|
|
|
|
def create_parser_unittest():
|
|
import unittestrunner
|
|
return unittestrunner.get_parser()
|
|
|
|
|
|
@CommandProvider
|
|
class MachCommands(MachCommandBase):
|
|
def setup(self):
|
|
self._activate_virtualenv()
|
|
|
|
@Command("web-platform-tests",
|
|
category="testing",
|
|
conditions=[conditions.is_firefox_or_android],
|
|
description="Run web-platform-tests.",
|
|
parser=create_parser_wpt)
|
|
def run_web_platform_tests(self, **params):
|
|
self.setup()
|
|
if conditions.is_android(self) and params["product"] != "firefox_android":
|
|
if params["product"] is None:
|
|
params["product"] = "firefox_android"
|
|
if "test_objects" in params:
|
|
for item in params["test_objects"]:
|
|
params["include"].append(item["name"])
|
|
del params["test_objects"]
|
|
if params.get('debugger', None):
|
|
import mozdebug
|
|
if not mozdebug.get_debugger_info(params.get('debugger')):
|
|
sys.exit(1)
|
|
|
|
wpt_setup = self._spawn(WebPlatformTestsRunnerSetup)
|
|
wpt_setup._mach_context = self._mach_context
|
|
wpt_runner = WebPlatformTestsRunner(wpt_setup)
|
|
|
|
if params["log_mach_screenshot"] is None:
|
|
params["log_mach_screenshot"] = True
|
|
|
|
logger = wpt_runner.setup_logging(**params)
|
|
|
|
if conditions.is_android(self) and params["product"] != "firefox_android":
|
|
logger.warning("Must specify --product=firefox_android in Android environment.")
|
|
|
|
return wpt_runner.run(logger, **params)
|
|
|
|
@Command("wpt",
|
|
category="testing",
|
|
conditions=[conditions.is_firefox_or_android],
|
|
description="Run web-platform-tests.",
|
|
parser=create_parser_wpt)
|
|
def run_wpt(self, **params):
|
|
return self.run_web_platform_tests(**params)
|
|
|
|
@Command("web-platform-tests-update",
|
|
category="testing",
|
|
description="Update web-platform-test metadata.",
|
|
parser=create_parser_update)
|
|
def update_web_platform_tests(self, **params):
|
|
self.setup()
|
|
self.virtualenv_manager.install_pip_package('html5lib==1.0.1')
|
|
self.virtualenv_manager.install_pip_package('ujson')
|
|
self.virtualenv_manager.install_pip_package('requests')
|
|
|
|
wpt_updater = self._spawn(WebPlatformTestsUpdater)
|
|
logger = wpt_updater.setup_logging(**params)
|
|
return wpt_updater.run_update(logger, **params)
|
|
|
|
@Command("wpt-update",
|
|
category="testing",
|
|
description="Update web-platform-test metadata.",
|
|
parser=create_parser_update)
|
|
def update_wpt(self, **params):
|
|
return self.update_web_platform_tests(**params)
|
|
|
|
@Command("wpt-manifest-update",
|
|
category="testing",
|
|
description="Update web-platform-test manifests.",
|
|
parser=create_parser_manifest_update)
|
|
def wpt_manifest_update(self, **params):
|
|
self.setup()
|
|
wpt_setup = self._spawn(WebPlatformTestsRunnerSetup)
|
|
wpt_runner = WebPlatformTestsRunner(wpt_setup)
|
|
logger = wpt_runner.setup_logging(**params)
|
|
logger.warning("The wpt manifest is now automatically updated, "
|
|
"so running this command is usually unnecessary")
|
|
return 0 if wpt_runner.update_manifest(logger, **params) else 1
|
|
|
|
@Command("wpt-serve",
|
|
category="testing",
|
|
description="Run the wpt server",
|
|
parser=create_parser_serve)
|
|
def wpt_manifest_serve(self, **params):
|
|
self.setup()
|
|
import logging
|
|
logger = logging.getLogger("web-platform-tests")
|
|
logger.addHandler(logging.StreamHandler(sys.stdout))
|
|
import serve
|
|
return 0 if serve.serve.run(**params) else 1
|
|
|
|
@Command("wpt-metadata-summary",
|
|
category="testing",
|
|
description="Create a json summary of the wpt metadata",
|
|
parser=create_parser_metadata_summary)
|
|
def wpt_summary(self, **params):
|
|
import metasummary
|
|
wpt_setup = self._spawn(WebPlatformTestsRunnerSetup)
|
|
return metasummary.run(wpt_setup.topsrcdir, wpt_setup.topobjdir, **params)
|
|
|
|
@Command("wpt-metadata-merge",
|
|
category="testing",
|
|
parser=create_parser_metadata_merge)
|
|
def wpt_meta_merge(self, **params):
|
|
import metamerge
|
|
if params["dest"] is None:
|
|
params["dest"] = params["current"]
|
|
return metamerge.run(**params)
|
|
|
|
@Command("wpt-unittest",
|
|
category="testing",
|
|
description="Run the wpt tools and wptrunner unit tests",
|
|
parser=create_parser_unittest)
|
|
def wpt_unittest(self, **params):
|
|
self.setup()
|
|
self.virtualenv_manager.install_pip_package('tox')
|
|
runner = self._spawn(WebPlatformTestsUnittestRunner)
|
|
return 0 if runner.run(**params) else 1
|