diff --git a/.cron.yml b/.cron.yml index e39d6f2a0c67..3b82b2aaf147 100644 --- a/.cron.yml +++ b/.cron.yml @@ -107,6 +107,17 @@ jobs: when: - {weekday: 'Saturday', hour: 5, minute: 0} + - name: live-site-perf-testing + job: + type: decision-task + treeherder-symbol: lst + target-tasks-method: live_site_perf_testing + include-push-tasks: true + run-on-projects: + - mozilla-central + when: + - {weekday: 'Sunday', hour: 5, minute: 0} + - name: general-perf-testing job: type: decision-task diff --git a/taskcluster/ci/test/browsertime-mobile.yml b/taskcluster/ci/test/browsertime-mobile.yml index 056a20e14117..3afeaf92cac3 100644 --- a/taskcluster/ci/test/browsertime-mobile.yml +++ b/taskcluster/ci/test/browsertime-mobile.yml @@ -199,28 +199,46 @@ browsertime-tp6m-live: description: "Raptor (browsertime) tp6 live site page-load tests on android" raptor-test: tp6m raptor-subtests: + - allrecipes + - amazon - amazon-search - bbc + - bing + - bing-search-restaurants + - booking - booking-sf + - cnn - cnn-ampstories - discord + - ebay-kleinanzeigen + - ebay-kleinanzeigen-search - espn - expedia + - facebook - facebook-cristiano - fashionbeans - google - google-accounts + - google-maps + - google-search-restaurants + - instagram + - imdb - imdb-firefox - jianshu - medium-article - microsoft-support - nytimes - people-article + - reddit - reddit-thread - rumble-fox + - stackoverflow - stackoverflow-question - urbandictionary-define + - web-de - wikia-marvel + - wikipedia + - youtube - youtube-watch apps: ["fenix", "chrome-m"] run-on-projects: [] diff --git a/taskcluster/taskgraph/target_tasks.py b/taskcluster/taskgraph/target_tasks.py index 4fbf7e3adff3..7a4d327854ab 100644 --- a/taskcluster/taskgraph/target_tasks.py +++ b/taskcluster/taskgraph/target_tasks.py @@ -45,6 +45,34 @@ UNCOMMON_TRY_TASK_LABELS = [ ] +# These are live site performance tests we run three times a week +LIVE_SITES = [ + "amazon-search", + "bbc", + "booking-sf", + "cnn-ampstories", + "discord", + "espn", + "expedia", + "facebook-cristiano", + "fashionbeans", + "google", + "google-accounts", + "imdb-firefox", + "jianshu", + "medium-article", + "microsoft-support", + "nytimes", + "people-article", + "reddit-thread", + "rumble-fox", + "stackoverflow-question", + "urbandictionary-define", + "wikia-marvel", + "youtube-watch" +] + + def _target_task(name): def wrap(func): _target_task_methods[name] = func @@ -494,6 +522,40 @@ def target_tasks_fennec_v68(full_task_graph, parameters, graph_config): return [l for l, t in six.iteritems(full_task_graph.tasks) if filter(t)] +@_target_task("live_site_perf_testing") +def target_tasks_live_site_perf_testing(full_task_graph, parameters, graph_config): + """ + Select browsertime live site tasks that should only run once a week. + """ + def filter(task): + platform = task.attributes.get('build_platform') + attributes = task.attributes + vismet = attributes.get('kind') == 'visual-metrics-dep' + if attributes.get('unittest_suite') != 'raptor' and not vismet: + return False + try_name = attributes.get('raptor_try_name') + if vismet: + platform = task.task.get('extra').get('treeherder-platform') + try_name = task.label + + if 'android' not in platform: + return False + if 'fenix' not in try_name: + return False + if ('browsertime' not in try_name or + 'shippable' not in platform or + 'live' not in try_name): + return False + for test in LIVE_SITES: + if try_name.endswith(test) or try_name.endswith(test + "-e10s"): + # These tests run 3 times a week, ignore them + return False + + return True + + return [l for l, t in six.iteritems(full_task_graph.tasks) if filter(t)] + + @_target_task('general_perf_testing') def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config): """ @@ -544,7 +606,11 @@ def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config) if 'speedometer' in try_name: return True if '-live' in try_name: - return True + # We only want to select those which should run 3 times + # a week here, other live site tests should be removed + for test in LIVE_SITES: + if try_name.endswith(test) or try_name.endswith(test + "-e10s"): + return True return False # Run the following tests on android geckoview diff --git a/taskcluster/taskgraph/transforms/visual_metrics_dep.py b/taskcluster/taskgraph/transforms/visual_metrics_dep.py index b2ff318e9ef3..94510a9479a5 100644 --- a/taskcluster/taskgraph/transforms/visual_metrics_dep.py +++ b/taskcluster/taskgraph/transforms/visual_metrics_dep.py @@ -44,6 +44,7 @@ def run_visual_metrics(config, jobs): # alongside the job it was triggered by to make it easier for # people to find it back. job['treeherder']['platform'] = platform + job['treeherder']['tier'] = treeherder_info['tier'] # run-on-projects needs to be set based on the dependent task job['run-on-projects'] = attributes['run_on_projects']