Bug 1621095 - add reftest-qr to run focused webrender reftests on android phones. r=bc

add reftest-qr to run focused webrender reftests on android phones.

Differential Revision: https://phabricator.services.mozilla.com/D72312

layout/reftests/reftest-qr.list

@@ -0,0 +1,26 @@
+# tests that are more likely to be affected by WebRender changes
+# as per bug 1621095 this is created to reduce load on android phones
+
+# gfx
+include ../../gfx/tests/reftest/reftest.list
+
+include async-scrolling/reftest.list
+
+# bugs/
+include bugs/reftest.list
+
+# css animations
+include css-animations/reftest.list
+
+# invalidation
+include invalidation/reftest.list
+
+# sticky positioning
+include position-sticky/reftest.list
+
+# svg/
+include svg/reftest.list
+
+# 3d transforms
+include transform-3d/reftest.list
+

moz.build

@@ -78,6 +78,10 @@ with Files("**/reftest.list"):
     SCHEDULES.inclusive += ['test-verify', 'test-verify-gpu']
     SCHEDULES.exclusive = ['reftest']
 
+with Files("**/reftest-qr.list"):
+    SCHEDULES.inclusive += ['test-verify', 'test-verify-gpu']
+    SCHEDULES.exclusive = ['reftest']
+
 with Files("**/crashtest*.list"):
     SCHEDULES.inclusive += ['test-verify', 'test-verify-gpu']
     SCHEDULES.exclusive = ['crashtest']

python/mozbuild/mozbuild/action/test_archive.py

@@ -461,6 +461,7 @@ ARCHIVE_FILES = {
             'base': '',
             'manifests': [
                 'layout/reftests/reftest.list',
+                'layout/reftests/reftest-qr.list',
                 'testing/crashtest/crashtests.list',
             ],
             'dest': 'reftest/tests',

taskcluster/ci/test/reftest.yml

@@ -146,6 +146,18 @@ reftest:
             linux1804-64-qr/debug: ['trunk']
             default: []
 
+reftest-qr:
+    description: "Reftest webrender run"
+    treeherder-symbol: R(R)
+    chunks: 2
+    e10s: true
+    run-on-projects: built-projects
+    max-run-time: 2400
+    mozharness:
+        chunked: true
+    tier: 2
+
+
 reftest-gpu:
     description: "Reftest GPU run"
     treeherder-symbol: R(Rg)

taskcluster/ci/test/test-sets.yml

@@ -604,7 +604,7 @@ android-hw-aarch64-debug-unittests:
 
 android-hw-aarch64-qr:
     - crashtest
-    - reftest
+    - reftest-qr
 
 android-hw-arm7-raptor:
     - raptor-speedometer-geckoview

taskcluster/taskgraph/transforms/tests.py

@@ -1280,6 +1280,7 @@ CHUNK_SUITES_BLACKLIST = (
     'mochitest-webgl2-ext',
     'raptor',
     'reftest',
+    'reftest-qr',
     'reftest-gpu',
     'reftest-no-accel',
     'talos',

testing/mozbase/moztest/moztest/resolve.py

@@ -189,6 +189,14 @@ TEST_SUITES = {
         'task_regex': ['(opt|debug)-reftest($|.*(-1|[^0-9])$)',
                        'test-verify-gpu($|.*(-1|[^0-9])$)'],
     },
+    'reftest-qr': {
+        'aliases': ('rr',),
+        'build_flavor': 'reftest',
+        'mach_command': 'reftest',
+        'kwargs': {'tests': None},
+        'task_regex': ['(opt|debug)-reftest-qr($|.*(-1|[^0-9])$)',
+                       'test-verify-gpu($|.*(-1|[^0-9])$)'],
+    },
     'robocop': {
         'mach_command': 'robocop',
         'kwargs': {'test_paths': None},

testing/mozharness/configs/android/android_common.py

@@ -159,6 +159,31 @@ config = {
             ],
             "tests": ["tests/layout/reftests/reftest.list",],
         },
+        "reftest-qr": {
+            "run_filename": "remotereftest.py",
+            "testsdir": "reftest",
+            "options": [
+                "--app=%(app)s",
+                "--ignore-window-size",
+                "--remote-webserver=%(remote_webserver)s",
+                "--xre-path=%(xre_path)s",
+                "--utility-path=%(utility_path)s",
+                "--http-port=%(http_port)s",
+                "--ssl-port=%(ssl_port)s",
+                "--httpd-path", "%(modules_dir)s",
+                "--symbols-path=%(symbols_path)s",
+                "--extra-profile-file=fonts",
+                "--extra-profile-file=hyphenation",
+                "--suite=reftest",
+                "--log-raw=%(raw_log_file)s",
+                "--log-raw-level=%(log_raw_level)s",
+                "--log-errorsummary=%(error_summary_file)s",
+                "--log-tbpl-level=%(log_tbpl_level)s",
+                "--deviceSerial=%(device_serial)s",
+                "--topsrcdir=tests",
+            ],
+            "tests": ["tests/layout/reftests/reftest-qr.list",],
+        },
         "crashtest": {
             "run_filename": "remotereftest.py",
             "testsdir": "reftest",

testing/mozharness/mozharness/mozilla/testing/errors.py

@@ -25,6 +25,13 @@ _mochitest_summary = {
     'known_fail_group': "Todo",
 }
 
+_reftest_summary = {
+    'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''),  # NOQA: E501
+    'pass_group': "Successful",
+    'fail_group': "Unexpected",
+    'known_fail_group': "Known problems",
+}
+
 TinderBoxPrintRe = {
     "mochitest-chrome_summary": _mochitest_summary,
     "mochitest-webgl1-core_summary": _mochitest_summary,
@@ -42,30 +49,16 @@ TinderBoxPrintRe = {
         'fail_group': "failed",
         'known_fail_group': "todo",
     },
-    "reftest_summary": {
+    "reftest_summary": _reftest_summary,
-        'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''),  # NOQA: E501
+    "reftest-qr_summary": _reftest_summary,
-        'pass_group': "Successful",
+    "crashtest_summary": _reftest_summary,
-        'fail_group': "Unexpected",
-        'known_fail_group': "Known problems",
-    },
-    "crashtest_summary": {
-        'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''),  # NOQA: E501
-        'pass_group': "Successful",
-        'fail_group': "Unexpected",
-        'known_fail_group': "Known problems",
-    },
     "xpcshell_summary": {
         'regex': re.compile(r'''INFO \| (Passed|Failed|Todo): (\d+)'''),
         'pass_group': "Passed",
         'fail_group': "Failed",
         'known_fail_group': "Todo",
     },
-    "jsreftest_summary": {
+    "jsreftest_summary": _reftest_summary,
-        'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''),  # NOQA: E501
-        'pass_group': "Successful",
-        'fail_group': "Unexpected",
-        'known_fail_group': "Known problems",
-    },
     "instrumentation_summary": _mochitest_summary,
     "cppunittest_summary": {
         'regex': re.compile(r'''cppunittests INFO \| (Passed|Failed): (\d+)'''),

testing/mozharness/mozharness/mozilla/testing/testbase.py

@@ -320,6 +320,7 @@ You can set this by specifying --test-url URL
             'mochitest-webgpu': 'mochitest',
             'geckoview': 'mochitest',
             'geckoview-junit': 'mochitest',
+            'reftest-qr': 'reftest',
             'jsreftest': 'reftest',
             'crashtest': 'reftest',
             'reftest-debug': 'reftest',

tools/tryselect/test/test_mozharness_integration.py

@@ -25,6 +25,7 @@ MOZHARNESS_SCRIPTS = {
             'geckoview-junit',
             'jittest',
             'jsreftest',
+            'reftest-qr',
         ],
     },
     'desktop_unittest': {
@@ -45,7 +46,6 @@ MOZHARNESS_SCRIPTS = {
             'mochitest-valgrind-plain',
             'reftest-gpu',
             'reftest-no-accel',
-            'reftest-qr',
         ],
     },
 }