From e27d2e2947ee70ac7dad6bfabbd49b66fc98bdaa Mon Sep 17 00:00:00 2001 From: KS Date: Tue, 14 May 2024 14:48:20 +0000 Subject: [PATCH] Bug 1840660 - Run performance tests against safari technology preview. r=perftest-reviewers,sparky This patch adds the ability to run Raptor performance tests (specifically just speedometer 3 for now) on Safari Technology Preview. Differential Revision: https://phabricator.services.mozilla.com/D209611 --- taskcluster/config.yml | 1 + .../gecko_taskgraph/transforms/test/raptor.py | 1 + taskcluster/gecko_taskgraph/util/perftest.py | 1 + .../kinds/test/browsertime-desktop.yml | 26 +++++++++++++++---- .../raptor/mac_external_browser_config.py | 1 + .../performance-artifact-schema.json | 1 + .../mozharness/mozilla/testing/raptor.py | 18 +++++++++++++ testing/perfdocs/generated/raptor.rst | 14 ++++++++-- testing/raptor/raptor/browsertime/desktop.py | 5 +++- testing/raptor/raptor/cmdline.py | 4 ++- testing/raptor/raptor/results.py | 2 +- .../tests/benchmarks/speedometer-desktop.toml | 6 ++--- 12 files changed, 67 insertions(+), 13 deletions(-) diff --git a/taskcluster/config.yml b/taskcluster/config.yml index 3c3ad2f5f405..15d811ca5033 100644 --- a/taskcluster/config.yml +++ b/taskcluster/config.yml @@ -43,6 +43,7 @@ treeherder: 'Btime-fenix': 'Browsertime performance tests on Fenix' 'Btime-refbrow': 'Browsertime performance tests on the reference browser' 'Btime-Saf': 'Browsertime performance tests on Safari' + 'Btime-STP': 'Browsertime performance tests on Safari Technology Preview' 'Btime-CaR': 'Browsertime performance tests on Chromium-as-Release' 'Btime-nofis-CaR': 'Browsertime performance tests on Chromium-as-Release without fission enabled' 'Fetch': 'Fetch and store content' diff --git a/taskcluster/gecko_taskgraph/transforms/test/raptor.py b/taskcluster/gecko_taskgraph/transforms/test/raptor.py index ca3574903717..68241bc585e5 100644 --- a/taskcluster/gecko_taskgraph/transforms/test/raptor.py +++ b/taskcluster/gecko_taskgraph/transforms/test/raptor.py @@ -79,6 +79,7 @@ def split_apps(config, tests): "fenix": "fenix", "refbrow": "refbrow", "safari": "Saf", + "safari-tp": "STP", "custom-car": "CaR", "cstm-car-m": "CaR", } diff --git a/taskcluster/gecko_taskgraph/util/perftest.py b/taskcluster/gecko_taskgraph/util/perftest.py index 01b153be378e..f06e978e9a03 100644 --- a/taskcluster/gecko_taskgraph/util/perftest.py +++ b/taskcluster/gecko_taskgraph/util/perftest.py @@ -8,6 +8,7 @@ def is_external_browser(label): external_browser in label for external_browser in ( "safari", + "safari-tp", "chrome", "custom-car", "chrome-m", diff --git a/taskcluster/kinds/test/browsertime-desktop.yml b/taskcluster/kinds/test/browsertime-desktop.yml index 90bf76201c27..87f427218dd1 100644 --- a/taskcluster/kinds/test/browsertime-desktop.yml +++ b/taskcluster/kinds/test/browsertime-desktop.yml @@ -18,6 +18,8 @@ job-defaults: - .*64.*shippable.* safari: - .*macos.*shippable.* + safari-tp: + - macosx1400.* custom-car: - .*64.*shippable.* default: [] @@ -60,6 +62,12 @@ job-defaults: - macosx64-aarch64-minidump-stackwalk - macosx64-aarch64-fix-stacks default: [] + safari-tp: + by-test-platform: + macosx1400.*: + - macosx64-aarch64-minidump-stackwalk + - macosx64-aarch64-fix-stacks + default: [] default: by-test-platform: windows.*-32.*.: @@ -339,14 +347,18 @@ browsertime-tp7: browsertime-benchmark: description: Raptor (browsertime) Benchmark tests raptor: - apps: [firefox, chrome, safari, custom-car] + apps: [firefox, chrome, safari, safari-tp, custom-car] subtests: by-test-platform: macosx1400.*shippable.*: - - [speedometer, sp] - - [speedometer3, sp3] - - [motionmark-1-3, mm-1.3] - - [motionmark-htmlsuite-1-3, mm-h-1.3] + by-app: + safari-tp: + - [speedometer3, sp3] + default: + - [speedometer, sp] + - [speedometer3, sp3] + - [motionmark-1-3, mm-1.3] + - [motionmark-htmlsuite-1-3, mm-h-1.3] default: by-app: firefox: @@ -474,6 +486,10 @@ browsertime-benchmark: by-test-platform: (linux|windows10-64).*shippable-qr.*: [mozilla-central] default: [] + safari-tp: + by-test-platform: + macosx1400.*: [mozilla-central] + default: [] default: [] unity-webgl: by-app: diff --git a/testing/mozharness/configs/raptor/mac_external_browser_config.py b/testing/mozharness/configs/raptor/mac_external_browser_config.py index 2e5a83e10e1b..b431208e21b3 100644 --- a/testing/mozharness/configs/raptor/mac_external_browser_config.py +++ b/testing/mozharness/configs/raptor/mac_external_browser_config.py @@ -17,6 +17,7 @@ config = { "populate-webroot", "create-virtualenv", "install-chromium-distribution", + "install-safari-technology-preview", "run-tests", ], "run_cmd_checks_enabled": True, diff --git a/testing/mozharness/external_tools/performance-artifact-schema.json b/testing/mozharness/external_tools/performance-artifact-schema.json index 2be6a22bf45a..048be33b8657 100644 --- a/testing/mozharness/external_tools/performance-artifact-schema.json +++ b/testing/mozharness/external_tools/performance-artifact-schema.json @@ -13,6 +13,7 @@ "refbrow", "fenix", "safari", + "safari-tp", "custom-car", "cstm-car-m" ], diff --git a/testing/mozharness/mozharness/mozilla/testing/raptor.py b/testing/mozharness/mozharness/mozilla/testing/raptor.py index 94df82294d66..0497ca5af5fa 100644 --- a/testing/mozharness/mozharness/mozilla/testing/raptor.py +++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py @@ -195,6 +195,7 @@ class Raptor( "refbrow", "fenix", "safari", + "safari-tp", "custom-car", "cstm-car-m", ], @@ -644,6 +645,7 @@ class Raptor( "create-virtualenv", "install-chrome-android", "install-chromium-distribution", + "install-safari-technology-preview", "install", "run-tests", ], @@ -844,6 +846,19 @@ class Raptor( self.device.shell_output("settings put global verifier_verify_adb_installs 1") rmtree(tmpdir) + def install_safari_technology_preview(self): + """Ensure latest version of Safari TP binary is running in CI""" + + if self.app != "safari-tp" or self.run_local: + return + + import mozprocess + + self.info("Checking for Safari Technology Preview updates...") + install_script = "/usr/local/bin/install_safari_softwareupdate_updates.py" + cmd = [sys.executable, install_script] + mozprocess.run_and_wait(cmd) + def install_chromium_distribution(self): """Install Google Chromium distribution in production""" linux, mac, win = "linux", "mac", "win" @@ -971,6 +986,9 @@ class Raptor( elif self.app == "safari" and not self.run_local: binary_path = "/Applications/Safari.app/Contents/MacOS/Safari" kw_options["binary"] = binary_path + elif self.app == "safari-tp" and not self.run_local: + binary_path = "/Applications/Safari Technology Preview.app/Contents/MacOS/Safari Technology Preview" + kw_options["binary"] = binary_path # Running on Chromium elif not self.run_local: # When running locally we already set the Chromium binary above, in init. diff --git a/testing/perfdocs/generated/raptor.rst b/testing/perfdocs/generated/raptor.rst index 8182f94f9003..0c18f04dcaa8 100644 --- a/testing/perfdocs/generated/raptor.rst +++ b/testing/perfdocs/generated/raptor.rst @@ -4614,7 +4614,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr **Owner**: SpiderMonkey Team * **alert threshold**: 2.0 - * **apps**: firefox, chrome, safari, custom-car + * **apps**: firefox, chrome, safari, safari-tp, custom-car * **expected**: pass * **expose browser profiler**: true * **gecko profile interval**: 1 @@ -5398,7 +5398,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr **Owner**: Performance Team * **alert threshold**: 2.0 - * **apps**: firefox, chrome, safari, custom-car + * **apps**: firefox, chrome, safari, safari-tp, custom-car * **browser cycles**: 5 * **cold**: true * **expected**: pass @@ -5695,6 +5695,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr - ❌ - ❌ - ❌ + * - **browsertime-benchmark-safari-tp-speedometer3** + - ✅ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -6087,6 +6092,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr - ❌ - ❌ - ❌ + * - **browsertime-benchmark-safari-tp-speedometer3** + - ✅ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** diff --git a/testing/raptor/raptor/browsertime/desktop.py b/testing/raptor/raptor/browsertime/desktop.py index fd9a938f2e41..b937112f768f 100644 --- a/testing/raptor/raptor/browsertime/desktop.py +++ b/testing/raptor/raptor/browsertime/desktop.py @@ -22,7 +22,6 @@ class BrowsertimeDesktop(PerftestDesktop, Browsertime): LOG.info("binary_path: {}".format(binary_path)) args_list = ["--viewPort", "1280x1024"] - if self.config["app"] in ( "chrome", "custom-car", @@ -33,6 +32,10 @@ class BrowsertimeDesktop(PerftestDesktop, Browsertime): "--chrome.binaryPath", binary_path, ] + + if self.config["app"] in ("safari-tp",): + return args_list + ["--browser", "safari", "--safari.useTechnologyPreview"] + return args_list + [ "--browser", self.config["app"], diff --git a/testing/raptor/raptor/cmdline.py b/testing/raptor/raptor/cmdline.py index 5542fdfdd89d..09cec118ba7b 100644 --- a/testing/raptor/raptor/cmdline.py +++ b/testing/raptor/raptor/cmdline.py @@ -7,10 +7,11 @@ import os import six from mozlog.commandline import add_logging_group -(FIREFOX, CHROME, SAFARI, CHROMIUM_RELEASE) = DESKTOP_APPS = [ +(FIREFOX, CHROME, SAFARI, SAFARI_TP, CHROMIUM_RELEASE) = DESKTOP_APPS = [ "firefox", "chrome", "safari", + "safari-tp", "custom-car", ] (GECKOVIEW, REFBROW, FENIX) = FIREFOX_ANDROID_APPS = [ @@ -29,6 +30,7 @@ APPS = { FIREFOX: {"long_name": "Firefox Desktop"}, CHROME: {"long_name": "Google Chrome Desktop"}, SAFARI: {"long_name": "Safari Desktop"}, + SAFARI_TP: {"long_name": "Safari Technology Preview Desktop"}, CHROMIUM_RELEASE: {"long_name": "Custom Chromium-as-Release desktop"}, GECKOVIEW: { "long_name": "Firefox GeckoView on Android", diff --git a/testing/raptor/raptor/results.py b/testing/raptor/raptor/results.py index 48ddfda25d23..0055f0935829 100644 --- a/testing/raptor/raptor/results.py +++ b/testing/raptor/raptor/results.py @@ -30,7 +30,7 @@ KNOWN_TEST_MODIFIERS = [ "bytecode-cached", ] NON_FIREFOX_OPTS = ("webrender", "bytecode-cached", "fission") -NON_FIREFOX_BROWSERS = ("chrome", "custom-car", "safari") +NON_FIREFOX_BROWSERS = ("chrome", "custom-car", "safari", "safari-tp") NON_FIREFOX_BROWSERS_MOBILE = ("chrome-m", "cstm-car-m") diff --git a/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.toml b/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.toml index f304493c2074..732377f24107 100644 --- a/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.toml +++ b/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.toml @@ -1,6 +1,6 @@ [DEFAULT] alert_threshold = 2.0 -apps = "firefox, chrome, safari, custom-car" +apps = "firefox, chrome, safari, safari-tp, custom-car" gecko_profile_interval = 1 expose_browser_profiler = true lower_is_better = false @@ -13,9 +13,9 @@ test_url = "http://:/Speedometer/index.html?raptor" type = "benchmark" unit = "score" -["speedometer"] +[speedometer] -["speedometer3"] +[speedometer3] browser_cycles = 5 owner = "Performance Team" cold = true