Bug 1638376 - Run recorded sites as live sites once a week. r=perftest-reviewers,alexandru.irimovici

This patch enables a new cron job that runs our recorded sites as live sites once a week through browsertime so we can continuously monitor how well our recordings match with what page the user is seeing.

Differential Revision: https://phabricator.services.mozilla.com/D75575

.cron.yml

@@ -107,6 +107,17 @@ jobs:
       when:
           - {weekday: 'Saturday', hour: 5, minute: 0}
 
+    - name: live-site-perf-testing
+      job:
+          type: decision-task
+          treeherder-symbol: lst
+          target-tasks-method: live_site_perf_testing
+          include-push-tasks: true
+      run-on-projects:
+          - mozilla-central
+      when:
+          - {weekday: 'Sunday', hour: 5, minute: 0}
+
     - name: general-perf-testing
       job:
           type: decision-task

taskcluster/ci/test/browsertime-mobile.yml

@@ -199,28 +199,46 @@ browsertime-tp6m-live:
     description: "Raptor (browsertime) tp6 live site page-load tests on android"
     raptor-test: tp6m
     raptor-subtests:
+        - allrecipes
+        - amazon
         - amazon-search
         - bbc
+        - bing
+        - bing-search-restaurants
+        - booking
         - booking-sf
+        - cnn
         - cnn-ampstories
         - discord
+        - ebay-kleinanzeigen
+        - ebay-kleinanzeigen-search
         - espn
         - expedia
+        - facebook
         - facebook-cristiano
         - fashionbeans
         - google
         - google-accounts
+        - google-maps
+        - google-search-restaurants
+        - instagram
+        - imdb
         - imdb-firefox
         - jianshu
         - medium-article
         - microsoft-support
         - nytimes
         - people-article
+        - reddit
         - reddit-thread
         - rumble-fox
+        - stackoverflow
         - stackoverflow-question
         - urbandictionary-define
+        - web-de
         - wikia-marvel
+        - wikipedia
+        - youtube
         - youtube-watch
     apps: ["fenix", "chrome-m"]
     run-on-projects: []

taskcluster/taskgraph/target_tasks.py

@@ -45,6 +45,34 @@ UNCOMMON_TRY_TASK_LABELS = [
 ]
 
 
+# These are live site performance tests we run three times a week
+LIVE_SITES = [
+    "amazon-search",
+    "bbc",
+    "booking-sf",
+    "cnn-ampstories",
+    "discord",
+    "espn",
+    "expedia",
+    "facebook-cristiano",
+    "fashionbeans",
+    "google",
+    "google-accounts",
+    "imdb-firefox",
+    "jianshu",
+    "medium-article",
+    "microsoft-support",
+    "nytimes",
+    "people-article",
+    "reddit-thread",
+    "rumble-fox",
+    "stackoverflow-question",
+    "urbandictionary-define",
+    "wikia-marvel",
+    "youtube-watch"
+]
+
+
 def _target_task(name):
     def wrap(func):
         _target_task_methods[name] = func
@@ -494,6 +522,40 @@ def target_tasks_fennec_v68(full_task_graph, parameters, graph_config):
     return [l for l, t in six.iteritems(full_task_graph.tasks) if filter(t)]
 
 
+@_target_task("live_site_perf_testing")
+def target_tasks_live_site_perf_testing(full_task_graph, parameters, graph_config):
+    """
+    Select browsertime live site tasks that should only run once a week.
+    """
+    def filter(task):
+        platform = task.attributes.get('build_platform')
+        attributes = task.attributes
+        vismet = attributes.get('kind') == 'visual-metrics-dep'
+        if attributes.get('unittest_suite') != 'raptor' and not vismet:
+            return False
+        try_name = attributes.get('raptor_try_name')
+        if vismet:
+            platform = task.task.get('extra').get('treeherder-platform')
+            try_name = task.label
+
+        if 'android' not in platform:
+            return False
+        if 'fenix' not in try_name:
+            return False
+        if ('browsertime' not in try_name or
+            'shippable' not in platform or
+            'live' not in try_name):
+            return False
+        for test in LIVE_SITES:
+            if try_name.endswith(test) or try_name.endswith(test + "-e10s"):
+                # These tests run 3 times a week, ignore them
+                return False
+
+        return True
+
+    return [l for l, t in six.iteritems(full_task_graph.tasks) if filter(t)]
+
+
 @_target_task('general_perf_testing')
 def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config):
     """
@@ -544,7 +606,11 @@ def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config)
             if 'speedometer' in try_name:
                 return True
             if '-live' in try_name:
-                return True
+                # We only want to select those which should run 3 times
+                # a week here, other live site tests should be removed
+                for test in LIVE_SITES:
+                    if try_name.endswith(test) or try_name.endswith(test + "-e10s"):
+                        return True
             return False
 
         # Run the following tests on android geckoview

taskcluster/taskgraph/transforms/visual_metrics_dep.py

@@ -44,6 +44,7 @@ def run_visual_metrics(config, jobs):
             # alongside the job it was triggered by to make it easier for
             # people to find it back.
             job['treeherder']['platform'] = platform
+            job['treeherder']['tier'] = treeherder_info['tier']
 
             # run-on-projects needs to be set based on the dependent task
             job['run-on-projects'] = attributes['run_on_projects']