diff --git a/taskcluster/gecko_taskgraph/actions/confirm_failure.py b/taskcluster/gecko_taskgraph/actions/confirm_failure.py index 0ccbd18eaecc..ee45d3312982 100644 --- a/taskcluster/gecko_taskgraph/actions/confirm_failure.py +++ b/taskcluster/gecko_taskgraph/actions/confirm_failure.py @@ -234,6 +234,8 @@ def confirm_failures(parameters, graph_config, input, task_group_id, task_id): suite = task_definition["extra"]["suite"] if "-coverage" in suite: suite = suite[: suite.index("-coverage")] + if "-qr" in suite: + suite = suite[: suite.index("-qr")] failures = get_failures(task_id, task_definition) if failures["dirs"] == [] and failures["tests"] == []: diff --git a/testing/mozharness/scripts/android_emulator_unittest.py b/testing/mozharness/scripts/android_emulator_unittest.py index 9f51f2ad4435..5fe172ace725 100644 --- a/testing/mozharness/scripts/android_emulator_unittest.py +++ b/testing/mozharness/scripts/android_emulator_unittest.py @@ -241,11 +241,24 @@ class AndroidEmulatorTest( return os.path.join(dirs["abs_test_install_dir"], test_dir) def _get_mozharness_test_paths(self, suite): - test_paths = os.environ.get("MOZHARNESS_TEST_PATHS") - if not test_paths: - return + test_paths = json.loads(os.environ.get("MOZHARNESS_TEST_PATHS", '""')) + confirm_paths = json.loads(os.environ.get("MOZHARNESS_CONFIRM_PATHS", '""')) - return json.loads(test_paths).get(suite) + if not test_paths or not test_paths.get(suite, []): + return None + + suite_test_paths = test_paths.get(suite, []) + if confirm_paths and confirm_paths.get(suite, []): + suite_test_paths = confirm_paths.get(suite, []) + + if suite in ("reftest", "crashtest"): + dirs = self.query_abs_dirs() + suite_test_paths = [ + os.path.join(dirs["abs_reftest_dir"], "tests", p) + for p in suite_test_paths + ] + + return suite_test_paths def _build_command(self): c = self.config @@ -335,7 +348,7 @@ class AndroidEmulatorTest( if not (self.verify_enabled or self.per_test_coverage): if user_paths: cmd.extend(user_paths) - elif not (self.verify_enabled or self.per_test_coverage): + else: if self.this_chunk is not None: cmd.extend(["--this-chunk", self.this_chunk]) if self.total_chunks is not None: @@ -352,7 +365,7 @@ class AndroidEmulatorTest( try_options, try_tests = self.try_args(self.test_suite) cmd.extend(try_options) - if not self.verify_enabled and not self.per_test_coverage: + if not self.verify_enabled and not self.per_test_coverage and not user_paths: cmd.extend( self.query_tests_args( self.config["suite_definitions"][self.test_suite].get("tests"), diff --git a/testing/mozharness/scripts/android_hardware_unittest.py b/testing/mozharness/scripts/android_hardware_unittest.py index 3abd79f0e24d..67bcf47bb57d 100644 --- a/testing/mozharness/scripts/android_hardware_unittest.py +++ b/testing/mozharness/scripts/android_hardware_unittest.py @@ -247,6 +247,7 @@ class AndroidHardwareTest( } user_paths = json.loads(os.environ.get("MOZHARNESS_TEST_PATHS", '""')) + confirm_paths = json.loads(os.environ.get("MOZHARNESS_CONFIRM_PATHS", '""')) for option in self.config["suite_definitions"][self.test_suite]["options"]: opt = option.split("=")[0] @@ -266,10 +267,7 @@ class AndroidHardwareTest( if option: cmd.extend([option]) - if user_paths: - if self.test_suite in user_paths: - cmd.extend(user_paths[self.test_suite]) - elif not self.verify_enabled: + if not self.verify_enabled and not user_paths: if self.this_chunk is not None: cmd.extend(["--this-chunk", self.this_chunk]) if self.total_chunks is not None: @@ -299,8 +297,25 @@ class AndroidHardwareTest( cmd.extend(["--setpref={}".format(p) for p in self.extra_prefs]) try_options, try_tests = self.try_args(self.test_suite) - cmd.extend(try_options) - if not self.verify_enabled and not self.per_test_coverage: + if try_options: + cmd.extend(try_options) + + if user_paths: + # reftest on android-hw uses a subset (reftest-qr) of tests, + # but scheduling only knows about 'reftest' + suite = self.test_suite + if suite == "reftest-qr": + suite = "reftest" + + if user_paths.get(suite, []): + suite_test_paths = user_paths.get(suite, []) + # NOTE: we do not want to prepend 'tests' if a single path + if confirm_paths and confirm_paths.get(suite, []): + suite_test_paths = confirm_paths.get(suite, []) + suite_test_paths = [os.path.join("tests", p) for p in suite_test_paths] + cmd.extend(suite_test_paths) + + elif not self.verify_enabled and not self.per_test_coverage: cmd.extend( self.query_tests_args( self.config["suite_definitions"][self.test_suite].get("tests"), diff --git a/testing/mozharness/scripts/desktop_unittest.py b/testing/mozharness/scripts/desktop_unittest.py index 38b807ccfe37..72c034989afa 100755 --- a/testing/mozharness/scripts/desktop_unittest.py +++ b/testing/mozharness/scripts/desktop_unittest.py @@ -586,7 +586,10 @@ class DesktopUnittest(TestingMixin, MercurialScript, MozbaseMixin, CodeCoverageM return self.symbols_url def _get_mozharness_test_paths(self, suite_category, suite): + # test_paths is the group name, confirm_paths can be the path+testname + # test_paths will always be the group name, unrelated to if confirm_paths is set or not. test_paths = json.loads(os.environ.get("MOZHARNESS_TEST_PATHS", '""')) + confirm_paths = json.loads(os.environ.get("MOZHARNESS_CONFIRM_PATHS", '""')) if "-coverage" in suite: suite = suite[: suite.index("-coverage")] @@ -595,6 +598,8 @@ class DesktopUnittest(TestingMixin, MercurialScript, MozbaseMixin, CodeCoverageM return None suite_test_paths = test_paths[suite] + if confirm_paths and suite in confirm_paths and confirm_paths[suite]: + suite_test_paths = confirm_paths[suite] if suite_category == "reftest": dirs = self.query_abs_dirs() diff --git a/testing/mozharness/scripts/marionette.py b/testing/mozharness/scripts/marionette.py index 8052927d2ac9..e027ca56a821 100755 --- a/testing/mozharness/scripts/marionette.py +++ b/testing/mozharness/scripts/marionette.py @@ -355,11 +355,17 @@ class MarionetteTest(TestingMixin, MercurialScript, TransferMixin, CodeCoverageM self.fatal("Could not create blobber upload directory") test_paths = json.loads(os.environ.get("MOZHARNESS_TEST_PATHS", '""')) + confirm_paths = json.loads(os.environ.get("MOZHARNESS_CONFIRM_PATHS", '""')) + + suite = "marionette" + if test_paths and suite in test_paths: + suite_test_paths = test_paths[suite] + if confirm_paths and suite in confirm_paths and confirm_paths[suite]: + suite_test_paths = confirm_paths[suite] - if test_paths and "marionette" in test_paths: paths = [ os.path.join(dirs["abs_test_install_dir"], "marionette", "tests", p) - for p in test_paths["marionette"] + for p in suite_test_paths ] cmd.extend(paths) else: diff --git a/testing/mozharness/scripts/web_platform_tests.py b/testing/mozharness/scripts/web_platform_tests.py index 2e3f4151ef00..83235633b7b7 100755 --- a/testing/mozharness/scripts/web_platform_tests.py +++ b/testing/mozharness/scripts/web_platform_tests.py @@ -446,10 +446,17 @@ class WebPlatformTest(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidM test_paths = set() if not (self.verify_enabled or self.per_test_coverage): + # mozharness_test_paths is a set of test groups (directories) to run + # if we have confirm_paths, this is a specific path we want to run and ignore the group mozharness_test_paths = json.loads( os.environ.get("MOZHARNESS_TEST_PATHS", '""') ) + confirm_paths = json.loads(os.environ.get("MOZHARNESS_CONFIRM_PATHS", '""')) + if mozharness_test_paths: + if confirm_paths: + mozharness_test_paths = confirm_paths + path = os.path.join(dirs["abs_fetches_dir"], "wpt_tests_by_group.json") if not os.path.exists(path):