forked from mirrors/gecko-dev
577 lines
19 KiB
Python
577 lines
19 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
# flake8: noqa: E501
|
|
|
|
try:
|
|
import cPickle as pickle
|
|
except ImportError:
|
|
import pickle
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import tempfile
|
|
from collections import defaultdict
|
|
|
|
import manifestupdate
|
|
import mozpack.path as mozpath
|
|
import mozunit
|
|
import pytest
|
|
from mozbuild.base import MozbuildObject
|
|
from mozbuild.frontend.reader import BuildReader
|
|
from mozbuild.test.common import MockConfig
|
|
from mozfile import NamedTemporaryFile
|
|
from moztest.resolve import (
|
|
TEST_SUITES,
|
|
BuildBackendLoader,
|
|
TestManifestLoader,
|
|
TestResolver,
|
|
)
|
|
|
|
here = os.path.abspath(os.path.dirname(__file__))
|
|
data_path = os.path.join(here, "data")
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def topsrcdir():
|
|
return mozpath.join(data_path, "srcdir")
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def create_tests(topsrcdir):
|
|
def inner(*paths, **defaults):
|
|
tests = defaultdict(list)
|
|
for path in paths:
|
|
if isinstance(path, tuple):
|
|
path, kwargs = path
|
|
else:
|
|
kwargs = {}
|
|
|
|
path = mozpath.normpath(path)
|
|
manifest_name = kwargs.get("flavor", defaults.get("flavor", "manifest"))
|
|
manifest = kwargs.pop(
|
|
"manifest",
|
|
defaults.pop(
|
|
"manifest",
|
|
mozpath.join(mozpath.dirname(path), manifest_name + ".toml"),
|
|
),
|
|
)
|
|
|
|
manifest_abspath = mozpath.join(topsrcdir, manifest)
|
|
relpath = mozpath.relpath(path, mozpath.dirname(manifest))
|
|
test = {
|
|
"name": relpath,
|
|
"path": mozpath.join(topsrcdir, path),
|
|
"relpath": relpath,
|
|
"file_relpath": path,
|
|
"flavor": "faketest",
|
|
"dir_relpath": mozpath.dirname(path),
|
|
"here": mozpath.dirname(manifest_abspath),
|
|
"manifest": manifest_abspath,
|
|
"manifest_relpath": manifest,
|
|
}
|
|
test.update(**defaults)
|
|
test.update(**kwargs)
|
|
|
|
# Normalize paths to ensure that the fixture matches reality.
|
|
for k in [
|
|
"ancestor_manifest",
|
|
"manifest",
|
|
"manifest_relpath",
|
|
"path",
|
|
"relpath",
|
|
]:
|
|
p = test.get(k)
|
|
if p:
|
|
test[k] = p.replace("/", os.path.sep)
|
|
|
|
tests[path].append(test)
|
|
|
|
# dump tests to stdout for easier debugging on failure
|
|
print("The 'create_tests' fixture returned:")
|
|
print(json.dumps(dict(tests), indent=2, sort_keys=True))
|
|
return tests
|
|
|
|
return inner
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def all_tests(create_tests):
|
|
return create_tests(
|
|
*[
|
|
(
|
|
"apple/test_a11y.html",
|
|
{
|
|
"expected": "pass",
|
|
"manifest": "apple/a11y.toml",
|
|
"flavor": "a11y",
|
|
},
|
|
),
|
|
(
|
|
"banana/currant/test_xpcshell_A.js",
|
|
{
|
|
"firefox-appdir": "browser",
|
|
"flavor": "xpcshell",
|
|
"head": "head_global.js head_helpers.js head_http.js",
|
|
},
|
|
),
|
|
(
|
|
"banana/currant/test_xpcshell_B.js",
|
|
{
|
|
"firefox-appdir": "browser",
|
|
"flavor": "xpcshell",
|
|
"head": "head_global.js head_helpers.js head_http.js",
|
|
},
|
|
),
|
|
(
|
|
"carrot/test_included.js",
|
|
{
|
|
"ancestor_manifest": "carrot/xpcshell-one.toml",
|
|
"manifest": "carrot/xpcshell-shared.toml",
|
|
"flavor": "xpcshell",
|
|
"stick": "one",
|
|
},
|
|
),
|
|
(
|
|
"carrot/test_included.js",
|
|
{
|
|
"ancestor_manifest": "carrot/xpcshell-two.toml",
|
|
"manifest": "carrot/xpcshell-shared.toml",
|
|
"flavor": "xpcshell",
|
|
"stick": "two",
|
|
},
|
|
),
|
|
(
|
|
"dragonfruit/elderberry/test_xpcshell_C.js",
|
|
{
|
|
"flavor": "xpcshell",
|
|
"generated-files": "head_update.js",
|
|
"head": "head_update.js",
|
|
"manifest": "dragonfruit/xpcshell.toml",
|
|
"reason": "busted",
|
|
"run-sequentially": "Launches application.",
|
|
"skip-if": "os == 'android'",
|
|
},
|
|
),
|
|
(
|
|
"dragonfruit/elderberry/test_xpcshell_C.js",
|
|
{
|
|
"flavor": "xpcshell",
|
|
"generated-files": "head_update.js",
|
|
"head": "head_update.js head2.js",
|
|
"manifest": "dragonfruit/elderberry/xpcshell_updater.toml",
|
|
"reason": "don't work",
|
|
"run-sequentially": "Launches application.",
|
|
"skip-if": "os == 'android'",
|
|
},
|
|
),
|
|
(
|
|
"fig/grape/src/TestInstrumentationA.java",
|
|
{
|
|
"flavor": "instrumentation",
|
|
"manifest": "fig/grape/instrumentation.toml",
|
|
"subsuite": "background",
|
|
},
|
|
),
|
|
(
|
|
"fig/huckleberry/src/TestInstrumentationB.java",
|
|
{
|
|
"flavor": "instrumentation",
|
|
"manifest": "fig/huckleberry/instrumentation.toml",
|
|
"subsuite": "browser",
|
|
},
|
|
),
|
|
(
|
|
"juniper/browser_chrome.js",
|
|
{
|
|
"flavor": "browser-chrome",
|
|
"manifest": "juniper/browser.toml",
|
|
"skip-if": "e10s # broken",
|
|
},
|
|
),
|
|
(
|
|
"kiwi/browser_devtools.js",
|
|
{
|
|
"flavor": "browser-chrome",
|
|
"manifest": "kiwi/browser.toml",
|
|
"subsuite": "devtools",
|
|
"tags": "devtools",
|
|
},
|
|
),
|
|
]
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def defaults(topsrcdir):
|
|
def to_abspath(relpath):
|
|
# test-defaults.pkl uses absolute paths with platform-specific path separators.
|
|
# Use platform-specific separators if needed to avoid regressing on bug 1644223.
|
|
return os.path.normpath(os.path.join(topsrcdir, relpath))
|
|
|
|
return {
|
|
(to_abspath("dragonfruit/elderberry/xpcshell_updater.toml")): {
|
|
"support-files": "data/**\nxpcshell_updater.toml"
|
|
},
|
|
(
|
|
to_abspath("carrot/xpcshell-one.toml"),
|
|
to_abspath("carrot/xpcshell-shared.toml"),
|
|
): {
|
|
"head": "head_one.js",
|
|
},
|
|
(
|
|
to_abspath("carrot/xpcshell-two.toml"),
|
|
to_abspath("carrot/xpcshell-shared.toml"),
|
|
): {
|
|
"head": "head_two.js",
|
|
},
|
|
}
|
|
|
|
|
|
class WPTManifestNamespace(object):
|
|
"""Stand-in object for various WPT classes."""
|
|
|
|
def __init__(self, *args):
|
|
self.args = args
|
|
|
|
def __hash__(self):
|
|
return hash(str(self))
|
|
|
|
def __eq__(self, other):
|
|
return self.args == other.args
|
|
|
|
def __iter__(self):
|
|
yield tuple(self.args)
|
|
|
|
|
|
def fake_wpt_manifestupdate(topsrcdir, *args, **kwargs):
|
|
with open(os.path.join(topsrcdir, "wpt_manifest_data.json")) as fh:
|
|
data = json.load(fh)
|
|
|
|
items = {}
|
|
for tests_root, test_data in data.items():
|
|
kwargs = {"tests_path": os.path.join(topsrcdir, tests_root)}
|
|
|
|
for test_type, tests in test_data.items():
|
|
for test in tests:
|
|
obj = WPTManifestNamespace()
|
|
if "mozilla" in tests_root:
|
|
obj.id = "/_mozilla/" + test
|
|
else:
|
|
obj.id = "/" + test
|
|
|
|
items[WPTManifestNamespace(test_type, test, {obj})] = kwargs
|
|
return items
|
|
|
|
|
|
@pytest.fixture(params=[BuildBackendLoader, TestManifestLoader])
|
|
def resolver(request, tmpdir, monkeypatch, topsrcdir, all_tests, defaults):
|
|
topobjdir = tmpdir.mkdir("objdir").strpath
|
|
loader_cls = request.param
|
|
|
|
if loader_cls == BuildBackendLoader:
|
|
with open(os.path.join(topobjdir, "all-tests.pkl"), "wb") as fh:
|
|
pickle.dump(all_tests, fh)
|
|
with open(os.path.join(topobjdir, "test-defaults.pkl"), "wb") as fh:
|
|
pickle.dump(defaults, fh)
|
|
|
|
# The mock data already exists, so prevent BuildBackendLoader from regenerating
|
|
# the build information from the whole gecko tree...
|
|
class BuildBackendLoaderNeverOutOfDate(BuildBackendLoader):
|
|
def backend_out_of_date(self, backend_file):
|
|
return False
|
|
|
|
loader_cls = BuildBackendLoaderNeverOutOfDate
|
|
|
|
# Patch WPT's manifestupdate.run to return tests based on the contents of
|
|
# 'data/srcdir/wpt_manifest_data.json'.
|
|
monkeypatch.setattr(manifestupdate, "run", fake_wpt_manifestupdate)
|
|
|
|
resolver = TestResolver(
|
|
topsrcdir, None, None, topobjdir=topobjdir, loader_cls=loader_cls
|
|
)
|
|
resolver._puppeteer_loaded = True
|
|
|
|
if loader_cls == TestManifestLoader:
|
|
config = MockConfig(topsrcdir)
|
|
resolver.load_tests.reader = BuildReader(config)
|
|
return resolver
|
|
|
|
|
|
def test_load(resolver):
|
|
assert len(resolver.tests_by_path) == 9
|
|
|
|
assert len(resolver.tests_by_flavor["mochitest-plain"]) == 0
|
|
assert len(resolver.tests_by_flavor["xpcshell"]) == 4
|
|
assert len(resolver.tests_by_flavor["web-platform-tests"]) == 0
|
|
|
|
assert len(resolver.tests_by_manifest) == 9
|
|
|
|
resolver.add_wpt_manifest_data()
|
|
assert len(resolver.tests_by_path) == 11
|
|
assert len(resolver.tests_by_flavor["web-platform-tests"]) == 2
|
|
assert len(resolver.tests_by_manifest) == 11
|
|
assert "/html" in resolver.tests_by_manifest
|
|
assert "/_mozilla/html" in resolver.tests_by_manifest
|
|
|
|
|
|
def test_resolve_all(resolver):
|
|
assert len(list(resolver._resolve())) == 13
|
|
|
|
|
|
def test_resolve_filter_flavor(resolver):
|
|
assert len(list(resolver._resolve(flavor="xpcshell"))) == 6
|
|
|
|
|
|
def test_resolve_by_dir(resolver):
|
|
assert len(list(resolver._resolve(paths=["banana/currant"]))) == 2
|
|
|
|
|
|
def test_resolve_under_path(resolver):
|
|
assert len(list(resolver._resolve(under_path="banana"))) == 2
|
|
assert len(list(resolver._resolve(flavor="xpcshell", under_path="banana"))) == 2
|
|
|
|
|
|
def test_resolve_multiple_paths(resolver):
|
|
result = list(resolver.resolve_tests(paths=["banana", "dragonfruit"]))
|
|
assert len(result) == 4
|
|
|
|
|
|
def test_resolve_support_files(resolver):
|
|
expected_support_files = "data/**\nxpcshell_updater.toml"
|
|
tests = list(resolver.resolve_tests(paths=["dragonfruit"]))
|
|
assert len(tests) == 2
|
|
|
|
for test in tests:
|
|
if test["manifest"].endswith("xpcshell_updater.toml"):
|
|
assert test["support-files"] == expected_support_files
|
|
else:
|
|
assert "support-files" not in test
|
|
|
|
|
|
def test_resolve_path_prefix(resolver):
|
|
tests = list(resolver._resolve(paths=["juniper"]))
|
|
assert len(tests) == 1
|
|
|
|
# relative manifest
|
|
tests = list(resolver._resolve(paths=["apple/a11y.toml"]))
|
|
assert len(tests) == 1
|
|
assert tests[0]["name"] == "test_a11y.html"
|
|
|
|
# absolute manifest
|
|
tests = list(
|
|
resolver._resolve(paths=[os.path.join(resolver.topsrcdir, "apple/a11y.toml")])
|
|
)
|
|
assert len(tests) == 1
|
|
assert tests[0]["name"] == "test_a11y.html"
|
|
|
|
|
|
def test_cwd_children_only(resolver):
|
|
"""If cwd is defined, only resolve tests under the specified cwd."""
|
|
# Pretend we're under '/services' and ask for 'common'. This should
|
|
# pick up all tests from '/services/common'
|
|
tests = list(
|
|
resolver.resolve_tests(
|
|
paths=["currant"], cwd=os.path.join(resolver.topsrcdir, "banana")
|
|
)
|
|
)
|
|
|
|
assert len(tests) == 2
|
|
|
|
# Tests should be rewritten to objdir.
|
|
for t in tests:
|
|
assert t["here"] == mozpath.join(
|
|
resolver.topobjdir, "_tests/xpcshell/banana/currant"
|
|
)
|
|
|
|
|
|
def test_various_cwd(resolver):
|
|
"""Test various cwd conditions are all equal."""
|
|
expected = list(resolver.resolve_tests(paths=["banana"]))
|
|
actual = list(resolver.resolve_tests(paths=["banana"], cwd="/"))
|
|
assert actual == expected
|
|
|
|
actual = list(resolver.resolve_tests(paths=["banana"], cwd=resolver.topsrcdir))
|
|
assert actual == expected
|
|
|
|
actual = list(resolver.resolve_tests(paths=["banana"], cwd=resolver.topobjdir))
|
|
assert actual == expected
|
|
|
|
|
|
def test_subsuites(resolver):
|
|
"""Test filtering by subsuite."""
|
|
tests = list(resolver.resolve_tests(paths=["fig"]))
|
|
assert len(tests) == 2
|
|
|
|
tests = list(resolver.resolve_tests(paths=["fig"], subsuite="browser"))
|
|
assert len(tests) == 1
|
|
assert tests[0]["name"] == "src/TestInstrumentationB.java"
|
|
|
|
tests = list(resolver.resolve_tests(paths=["fig"], subsuite="background"))
|
|
assert len(tests) == 1
|
|
assert tests[0]["name"] == "src/TestInstrumentationA.java"
|
|
|
|
# Resolve tests *without* a subsuite.
|
|
tests = list(resolver.resolve_tests(flavor="browser-chrome", subsuite="undefined"))
|
|
assert len(tests) == 1
|
|
assert tests[0]["name"] == "browser_chrome.js"
|
|
|
|
|
|
def test_wildcard_patterns(resolver):
|
|
"""Test matching paths by wildcard."""
|
|
tests = list(resolver.resolve_tests(paths=["fig/**"]))
|
|
assert len(tests) == 2
|
|
for t in tests:
|
|
assert t["file_relpath"].startswith("fig")
|
|
|
|
tests = list(resolver.resolve_tests(paths=["**/**.js", "apple/**"]))
|
|
assert len(tests) == 9
|
|
for t in tests:
|
|
path = t["file_relpath"]
|
|
assert path.startswith("apple") or path.endswith(".js")
|
|
|
|
|
|
def test_resolve_metadata(resolver):
|
|
"""Test finding metadata from outgoing files."""
|
|
suites, tests = resolver.resolve_metadata(["bc"])
|
|
assert suites == {"mochitest-browser-chrome"}
|
|
assert tests == []
|
|
|
|
suites, tests = resolver.resolve_metadata(
|
|
["mochitest-a11y", "/browser", "xpcshell"]
|
|
)
|
|
assert suites == {"mochitest-a11y", "xpcshell"}
|
|
assert sorted(t["file_relpath"] for t in tests) == [
|
|
"juniper/browser_chrome.js",
|
|
"kiwi/browser_devtools.js",
|
|
]
|
|
|
|
|
|
def test_ancestor_manifest_defaults(resolver, topsrcdir, defaults):
|
|
"""Test that defaults from ancestor manifests are found."""
|
|
tests = list(resolver._resolve(paths=["carrot/test_included.js"]))
|
|
assert len(tests) == 2
|
|
|
|
if tests[0]["ancestor_manifest"] == os.path.join("carrot", "xpcshell-one.toml"):
|
|
[testOne, testTwo] = tests
|
|
else:
|
|
[testTwo, testOne] = tests
|
|
|
|
assert testOne["ancestor_manifest"] == os.path.join("carrot", "xpcshell-one.toml")
|
|
assert testOne["manifest_relpath"] == os.path.join("carrot", "xpcshell-shared.toml")
|
|
assert testOne["head"] == "head_one.js"
|
|
assert testOne["stick"] == "one"
|
|
|
|
assert testTwo["ancestor_manifest"] == os.path.join("carrot", "xpcshell-two.toml")
|
|
assert testTwo["manifest_relpath"] == os.path.join("carrot", "xpcshell-shared.toml")
|
|
assert testTwo["head"] == "head_two.js"
|
|
assert testTwo["stick"] == "two"
|
|
|
|
|
|
def test_task_regexes():
|
|
"""Test the task_regexes defined in TEST_SUITES."""
|
|
task_labels = [
|
|
"test-linux64/opt-marionette",
|
|
"test-linux64/opt-mochitest-plain",
|
|
"test-linux64/debug-mochitest-plain-e10s",
|
|
"test-linux64/opt-mochitest-a11y",
|
|
"test-linux64/opt-mochitest-browser",
|
|
"test-linux64/opt-mochitest-browser-chrome",
|
|
"test-linux64/opt-mochitest-browser-chrome-e10s",
|
|
"test-linux64/opt-mochitest-browser-chrome-e10s-11",
|
|
"test-linux64/opt-mochitest-chrome",
|
|
"test-linux64/opt-mochitest-devtools",
|
|
"test-linux64/opt-mochitest-devtools-chrome",
|
|
"test-linux64/opt-mochitest-gpu",
|
|
"test-linux64/opt-mochitest-gpu-e10s",
|
|
"test-linux64/opt-mochitest-media-e10s-1",
|
|
"test-linux64/opt-mochitest-media-e10s-11",
|
|
"test-linux64/opt-mochitest-browser-screenshots-1",
|
|
"test-linux64/opt-mochitest-browser-screenshots-e10s-1",
|
|
"test-linux64/opt-reftest",
|
|
"test-linux64/opt-geckoview-reftest",
|
|
"test-linux64/debug-reftest-e10s-1",
|
|
"test-linux64/debug-reftest-e10s-11",
|
|
"test-linux64/opt-robocop",
|
|
"test-linux64/opt-robocop-1",
|
|
"test-linux64/opt-robocop-e10s",
|
|
"test-linux64/opt-robocop-e10s-1",
|
|
"test-linux64/opt-robocop-e10s-11",
|
|
"test-linux64/opt-web-platform-tests-e10s-1",
|
|
"test-linux64/opt-web-platform-tests-reftest-e10s-1",
|
|
"test-linux64/opt-web-platform-tests-wdspec-e10s-1",
|
|
"test-linux64/opt-web-platform-tests-1",
|
|
"test-linux64/opt-web-platform-test-e10s-1",
|
|
"test-linux64/opt-xpcshell",
|
|
"test-linux64/opt-xpcshell-1",
|
|
"test-linux64/opt-xpcshell-2",
|
|
]
|
|
|
|
test_cases = {
|
|
"mochitest-browser-chrome": [
|
|
"test-linux64/opt-mochitest-browser-chrome",
|
|
"test-linux64/opt-mochitest-browser-chrome-e10s",
|
|
],
|
|
"mochitest-chrome": [
|
|
"test-linux64/opt-mochitest-chrome",
|
|
],
|
|
"mochitest-devtools-chrome": [
|
|
"test-linux64/opt-mochitest-devtools-chrome",
|
|
],
|
|
"mochitest-media": [
|
|
"test-linux64/opt-mochitest-media-e10s-1",
|
|
],
|
|
"mochitest-plain": [
|
|
"test-linux64/opt-mochitest-plain",
|
|
"test-linux64/debug-mochitest-plain-e10s",
|
|
],
|
|
"mochitest-plain-gpu": [
|
|
"test-linux64/opt-mochitest-gpu",
|
|
"test-linux64/opt-mochitest-gpu-e10s",
|
|
],
|
|
"mochitest-browser-screenshots": [
|
|
"test-linux64/opt-mochitest-browser-screenshots-1",
|
|
"test-linux64/opt-mochitest-browser-screenshots-e10s-1",
|
|
],
|
|
"reftest": [
|
|
"test-linux64/opt-reftest",
|
|
"test-linux64/opt-geckoview-reftest",
|
|
"test-linux64/debug-reftest-e10s-1",
|
|
],
|
|
"robocop": [
|
|
"test-linux64/opt-robocop",
|
|
"test-linux64/opt-robocop-1",
|
|
"test-linux64/opt-robocop-e10s",
|
|
"test-linux64/opt-robocop-e10s-1",
|
|
],
|
|
"web-platform-tests": [
|
|
"test-linux64/opt-web-platform-tests-e10s-1",
|
|
"test-linux64/opt-web-platform-tests-1",
|
|
],
|
|
"web-platform-tests-reftest": [
|
|
"test-linux64/opt-web-platform-tests-reftest-e10s-1",
|
|
],
|
|
"web-platform-tests-wdspec": [
|
|
"test-linux64/opt-web-platform-tests-wdspec-e10s-1",
|
|
],
|
|
"xpcshell": [
|
|
"test-linux64/opt-xpcshell",
|
|
"test-linux64/opt-xpcshell-1",
|
|
],
|
|
}
|
|
|
|
regexes = []
|
|
|
|
def match_task(task):
|
|
return any(re.search(pattern, task) for pattern in regexes)
|
|
|
|
for suite, expected in sorted(test_cases.items()):
|
|
print(suite)
|
|
regexes = TEST_SUITES[suite]["task_regex"]
|
|
assert set(filter(match_task, task_labels)) == set(expected)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
mozunit.main()
|