Bug 1762647 - [mochitest] Add support for custom Firefox command line arguments in manifests. r=jmaher

Differential Revision: https://phabricator.services.mozilla.com/D144337

testing/mochitest/runtests.py

@@ -942,6 +942,7 @@ class MochitestDesktop(object):
         self.sslTunnel = None
         self.manifest = None
         self.tests_by_manifest = defaultdict(list)
+        self.args_by_manifest = defaultdict(set)
         self.prefs_by_manifest = defaultdict(set)
         self.env_vars_by_manifest = defaultdict(set)
         self.tests_dirs_by_manifest = defaultdict(set)
@@ -954,6 +955,7 @@ class MochitestDesktop(object):
         self.start_script = None
         self.mozLogs = None
         self.start_script_kwargs = {}
+        self.extraArgs = []
         self.extraPrefs = {}
         self.extraEnv = {}
         self.extraTestsDirs = []
@@ -1671,11 +1673,12 @@ toolbar#nav-bar {
                 manifest_key = "{}:{}".format(test["ancestor_manifest"], manifest_key)
 
             self.tests_by_manifest[manifest_key.replace("\\", "/")].append(tp)
+            self.args_by_manifest[manifest_key].add(test.get("args"))
             self.prefs_by_manifest[manifest_key].add(test.get("prefs"))
             self.env_vars_by_manifest[manifest_key].add(test.get("environment"))
             self.tests_dirs_by_manifest[manifest_key].add(test.get("test-directories"))
 
-            for key in ["prefs", "environment", "test-directories"]:
+            for key in ["args", "prefs", "environment", "test-directories"]:
                 if key in test and not options.runByManifest and "disabled" not in test:
                     self.log.error(
                         "parsing {}: runByManifest mode must be enabled to "
@@ -1703,10 +1706,23 @@ toolbar#nav-bar {
                     testob["expected"] = patterns
             paths.append(testob)
 
-        # The 'prefs' key needs to be set in the DEFAULT section, unfortunately
+        # The 'args' key needs to be set in the DEFAULT section, unfortunately
         # we can't tell what comes from DEFAULT or not. So to validate this, we
-        # stash all prefs from tests in the same manifest into a set. If the
-        # length of the set > 1, then we know 'prefs' didn't come from DEFAULT.
+        # stash all args from tests in the same manifest into a set. If the
+        # length of the set > 1, then we know 'args' didn't come from DEFAULT.
+        args_not_default = [
+            m for m, p in six.iteritems(self.args_by_manifest) if len(p) > 1
+        ]
+        if args_not_default:
+            self.log.error(
+                "The 'args' key must be set in the DEFAULT section of a "
+                "manifest. Fix the following manifests: {}".format(
+                    "\n".join(args_not_default)
+                )
+            )
+            sys.exit(1)
+
+        # The 'prefs' key needs to be set in the DEFAULT section too.
         pref_not_default = [
             m for m, p in six.iteritems(self.prefs_by_manifest) if len(p) > 1
         ]
@@ -3077,6 +3093,20 @@ toolbar#nav-bar {
         for m in sorted(manifests):
             self.log.info("Running manifest: {}".format(m))
 
+            args = list(self.args_by_manifest[m])[0]
+            self.extraArgs = []
+            if args:
+                for arg in args.strip().split():
+                    # Split off the argument value if available so that both
+                    # name and value will be set individually
+                    self.extraArgs.extend(arg.split("="))
+
+                self.log.info(
+                    "The following arguments will be set:\n  {}".format(
+                        "\n  ".join(self.extraArgs)
+                    )
+                )
+
             prefs = list(self.prefs_by_manifest[m])[0]
             self.extraPrefs = origPrefs.copy()
             if prefs:
@@ -3365,7 +3395,7 @@ toolbar#nav-bar {
                     self.browserEnv,
                     options.app,
                     profile=self.profile,
-                    extraArgs=options.browserArgs,
+                    extraArgs=options.browserArgs + self.extraArgs,
                     utilityPath=options.utilityPath,
                     debuggerInfo=debuggerInfo,
                     valgrindPath=valgrindPath,

testing/mochitest/tests/python/conftest.py

@@ -18,6 +18,18 @@ here = os.path.abspath(os.path.dirname(__file__))
 setup_args = [os.path.join(here, "files"), "mochitest", "testing/mochitest"]
 
 
+@pytest.fixture
+def create_manifest(tmpdir, build_obj):
+    def inner(string, name="manifest.ini"):
+        manifest = tmpdir.join(name)
+        manifest.write(string, ensure=True)
+        # pylint --py3k: W1612
+        path = six.text_type(manifest)
+        return TestManifest(manifests=(path,), strict=False, rootdir=tmpdir.strpath)
+
+    return inner
+
+
 @pytest.fixture(scope="function")
 def parser(request):
     parser = pytest.importorskip("mochitest_options")

testing/mochitest/tests/python/files/browser-args.ini

@@ -0,0 +1,7 @@
+[DEFAULT]
+args =
+  --headless
+  --window-size=800,600
+  --new-tab http://example.org
+
+[browser_pass.js]

testing/mochitest/tests/python/files/mochitest-args.ini

@@ -0,0 +1,7 @@
+[DEFAULT]
+args =
+  --headless
+  --window-size=800,600
+  --new-tab http://example.org
+
+[test_pass.html]

testing/mochitest/tests/python/test_get_active_tests.py

@@ -54,6 +54,111 @@ def create_manifest(tmpdir, build_obj):
     return inner
 
 
+def test_args_validation(get_active_tests, create_manifest):
+    # Test args set in a single manifest.
+    manifest_relpath = "manifest.ini"
+    manifest = create_manifest(
+        dedent(
+            """
+    [DEFAULT]
+    args=
+      --cheese
+      --foo=bar
+      --foo1 bar1
+
+    [files/test_pass.html]
+    [files/test_fail.html]
+    """
+        )
+    )
+
+    options = {
+        "runByManifest": True,
+        "manifestFile": manifest,
+    }
+    md, tests = get_active_tests(**options)
+
+    assert len(tests) == 2
+    assert manifest_relpath in md.args_by_manifest
+
+    args = md.args_by_manifest[manifest_relpath]
+    assert len(args) == 1
+    assert args.pop() == "\n--cheese\n--foo=bar\n--foo1 bar1"
+
+    # Test args set with runByManifest disabled.
+    options["runByManifest"] = False
+    with pytest.raises(SystemExit):
+        get_active_tests(**options)
+
+    # Test args set in non-default section.
+    options["runByManifest"] = True
+    options["manifestFile"] = create_manifest(
+        dedent(
+            """
+    [files/test_pass.html]
+    args=--foo2=bar2
+    [files/test_fail.html]
+    """
+        )
+    )
+    with pytest.raises(SystemExit):
+        get_active_tests(**options)
+
+
+def test_args_validation_with_ancestor_manifest(get_active_tests, create_manifest):
+    # Test args set by an ancestor manifest.
+    create_manifest(
+        dedent(
+            """
+    [DEFAULT]
+    args=
+      --cheese
+
+    [files/test_pass.html]
+    [files/test_fail.html]
+    """
+        ),
+        name="subdir/manifest.ini",
+    )
+
+    manifest = create_manifest(
+        dedent(
+            """
+    [DEFAULT]
+    args =
+      --foo=bar
+
+    [include:manifest.ini]
+    [test_foo.html]
+    """
+        ),
+        name="subdir/ancestor-manifest.ini",
+    )
+
+    options = {
+        "runByManifest": True,
+        "manifestFile": manifest,
+    }
+
+    md, tests = get_active_tests(**options)
+    assert len(tests) == 3
+
+    key = os.path.join("subdir", "ancestor-manifest.ini")
+    assert key in md.args_by_manifest
+    args = md.args_by_manifest[key]
+    assert len(args) == 1
+    assert args.pop() == "\n--foo=bar"
+
+    key = "{}:{}".format(
+        os.path.join("subdir", "ancestor-manifest.ini"),
+        os.path.join("subdir", "manifest.ini"),
+    )
+    assert key in md.args_by_manifest
+    args = md.args_by_manifest[key]
+    assert len(args) == 1
+    assert args.pop() == "\n--foo=bar \n--cheese"
+
+
 def test_prefs_validation(get_active_tests, create_manifest):
     # Test prefs set in a single manifest.
     manifest_relpath = "manifest.ini"

testing/mochitest/tests/python/test_mochitest_integration.py

@@ -7,10 +7,9 @@ from __future__ import absolute_import
 import os
 from functools import partial
 
-from manifestparser import TestManifest
-
 import mozunit
 import pytest
+from manifestparser import TestManifest
 from moztest.selftest.output import get_mozharness_status, filter_action
 from conftest import setup_args
 
@@ -51,6 +50,43 @@ def test_manifest(setup_test_harness, request):
     return inner
 
 
+@pytest.mark.parametrize(
+    "flavor,manifest",
+    [
+        ("plain", "mochitest-args.ini"),
+        ("browser-chrome", "browser-args.ini"),
+    ],
+)
+def test_output_extra_args(flavor, manifest, runtests, test_manifest, test_name):
+    # Explicitly provide a manifestFile property that includes the
+    # manifest file that contains command line arguments.
+    extra_opts = {
+        "manifestFile": test_manifest([manifest]),
+        "runByManifest": True,
+    }
+
+    results = {
+        "status": 0,
+        "tbpl_status": TBPL_SUCCESS,
+        "log_level": (INFO, WARNING),
+    }
+
+    status, lines = runtests(test_name("pass"), **extra_opts)
+    assert status == results["status"]
+
+    tbpl_status, log_level, _ = get_mozharness_status(lines, status)
+    assert tbpl_status == results["tbpl_status"]
+    assert log_level in results["log_level"]
+
+    # Filter log entries for the application command including the used
+    # command line arguments.
+    lines = filter_action("log", lines)
+    command = next(
+        l["message"] for l in lines if l["message"].startswith("Application command")
+    )
+    assert "--headless --window-size 800,600 --new-tab http://example.org" in command
+
+
 @pytest.mark.parametrize("runFailures", ["selftest", ""])
 @pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])
 def test_output_pass(flavor, runFailures, runtests, test_name):

testing/mozbase/manifestparser/manifestparser/ini.py

@@ -167,6 +167,7 @@ def combine_fields(global_vars, local_vars):
     if not local_vars:
         return global_vars.copy()
     field_patterns = {
+        "args": "%s %s",
         "prefs": "%s %s",
         "skip-if": "%s\n%s",
         "support-files": "%s %s",
@@ -179,4 +180,5 @@ def combine_fields(global_vars, local_vars):
         global_value = global_vars[field_name]
         pattern = field_patterns[field_name]
         final_mapping[field_name] = pattern % (global_value, value)
+
     return final_mapping