Bug 1840660 - Run performance tests against safari technology preview. r=perftest-reviewers,sparky

This patch adds the ability to run Raptor performance tests (specifically just speedometer 3 for now) on Safari Technology Preview. Differential Revision: https://phabricator.services.mozilla.com/D209611
2024-05-14 14:48:20 +00:00 · 2024-05-14 14:48:20 +00:00 · e27d2e2947
commit e27d2e2947
parent 9f4b4ab30a
12 changed files with 67 additions and 13 deletions
--- a/taskcluster/config.yml
+++ b/taskcluster/config.yml
@ -43,6 +43,7 @@ treeherder:
        'Btime-fenix': 'Browsertime performance tests on Fenix'
        'Btime-refbrow': 'Browsertime performance tests on the reference browser'
        'Btime-Saf': 'Browsertime performance tests on Safari'
+        'Btime-STP': 'Browsertime performance tests on Safari Technology Preview'
        'Btime-CaR': 'Browsertime performance tests on Chromium-as-Release'
        'Btime-nofis-CaR': 'Browsertime performance tests on Chromium-as-Release without fission enabled'
        'Fetch': 'Fetch and store content'
--- a/taskcluster/gecko_taskgraph/transforms/test/raptor.py
+++ b/taskcluster/gecko_taskgraph/transforms/test/raptor.py
@ -79,6 +79,7 @@ def split_apps(config, tests):
        "fenix": "fenix",
        "refbrow": "refbrow",
        "safari": "Saf",
+        "safari-tp": "STP",
        "custom-car": "CaR",
        "cstm-car-m": "CaR",
    }
--- a/taskcluster/gecko_taskgraph/util/perftest.py
+++ b/taskcluster/gecko_taskgraph/util/perftest.py
@ -8,6 +8,7 @@ def is_external_browser(label):
        external_browser in label
        for external_browser in (
            "safari",
+            "safari-tp",
            "chrome",
            "custom-car",
            "chrome-m",
--- a/taskcluster/kinds/test/browsertime-desktop.yml
+++ b/taskcluster/kinds/test/browsertime-desktop.yml
@ -18,6 +18,8 @@ job-defaults:
                - .*64.*shippable.*
            safari:
                - .*macos.*shippable.*
+            safari-tp:
+                - macosx1400.*
            custom-car:
                - .*64.*shippable.*
            default: []
@ -60,6 +62,12 @@ job-defaults:
                            - macosx64-aarch64-minidump-stackwalk
                            - macosx64-aarch64-fix-stacks
                        default: []
+                safari-tp:
+                    by-test-platform:
+                        macosx1400.*:
+                            - macosx64-aarch64-minidump-stackwalk
+                            - macosx64-aarch64-fix-stacks
+                        default: []
                default:
                    by-test-platform:
                        windows.*-32.*.:
@ -339,14 +347,18 @@ browsertime-tp7:
 browsertime-benchmark:
    description: Raptor (browsertime) Benchmark tests
    raptor:
-        apps: [firefox, chrome, safari, custom-car]
+        apps: [firefox, chrome, safari, safari-tp, custom-car]
        subtests:
            by-test-platform:
                macosx1400.*shippable.*:
-                    - [speedometer, sp]
-                    - [speedometer3, sp3]
-                    - [motionmark-1-3, mm-1.3]
-                    - [motionmark-htmlsuite-1-3, mm-h-1.3]
+                    by-app:
+                        safari-tp:
+                            - [speedometer3, sp3]
+                        default:
+                            - [speedometer, sp]
+                            - [speedometer3, sp3]
+                            - [motionmark-1-3, mm-1.3]
+                            - [motionmark-htmlsuite-1-3, mm-h-1.3]
                default:
                    by-app:
                        firefox:
@ -474,6 +486,10 @@ browsertime-benchmark:
                        by-test-platform:
                            (linux|windows10-64).*shippable-qr.*: [mozilla-central]
                            default: []
+                    safari-tp:
+                        by-test-platform:
+                            macosx1400.*: [mozilla-central]
+                            default: []
                    default: []
            unity-webgl:
                by-app:
--- a/testing/mozharness/configs/raptor/mac_external_browser_config.py
+++ b/testing/mozharness/configs/raptor/mac_external_browser_config.py
@ -17,6 +17,7 @@ config = {
        "populate-webroot",
        "create-virtualenv",
        "install-chromium-distribution",
+        "install-safari-technology-preview",
        "run-tests",
    ],
    "run_cmd_checks_enabled": True,
--- a/testing/mozharness/external_tools/performance-artifact-schema.json
+++ b/testing/mozharness/external_tools/performance-artifact-schema.json
@ -13,6 +13,7 @@
            "refbrow",
            "fenix",
            "safari",
+            "safari-tp",
            "custom-car",
            "cstm-car-m"
          ],
--- a/testing/mozharness/mozharness/mozilla/testing/raptor.py
+++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py
@ -195,6 +195,7 @@ class Raptor(
                        "refbrow",
                        "fenix",
                        "safari",
+                        "safari-tp",
                        "custom-car",
                        "cstm-car-m",
                    ],
@ -644,6 +645,7 @@ class Raptor(
                "create-virtualenv",
                "install-chrome-android",
                "install-chromium-distribution",
+                "install-safari-technology-preview",
                "install",
                "run-tests",
            ],
@ -844,6 +846,19 @@ class Raptor(
        self.device.shell_output("settings put global verifier_verify_adb_installs 1")
        rmtree(tmpdir)

+    def install_safari_technology_preview(self):
+        """Ensure latest version of Safari TP binary is running in CI"""
+
+        if self.app != "safari-tp" or self.run_local:
+            return
+
+        import mozprocess
+
+        self.info("Checking for Safari Technology Preview updates...")
+        install_script = "/usr/local/bin/install_safari_softwareupdate_updates.py"
+        cmd = [sys.executable, install_script]
+        mozprocess.run_and_wait(cmd)
+
    def install_chromium_distribution(self):
        """Install Google Chromium distribution in production"""
        linux, mac, win = "linux", "mac", "win"
@ -971,6 +986,9 @@ class Raptor(
        elif self.app == "safari" and not self.run_local:
            binary_path = "/Applications/Safari.app/Contents/MacOS/Safari"
            kw_options["binary"] = binary_path
+        elif self.app == "safari-tp" and not self.run_local:
+            binary_path = "/Applications/Safari Technology Preview.app/Contents/MacOS/Safari Technology Preview"
+            kw_options["binary"] = binary_path
        # Running on Chromium
        elif not self.run_local:
            # When running locally we already set the Chromium binary above, in init.
--- a/testing/perfdocs/generated/raptor.rst
+++ b/testing/perfdocs/generated/raptor.rst
@ -4614,7 +4614,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
   **Owner**: SpiderMonkey Team

   * **alert threshold**: 2.0
-   * **apps**: firefox, chrome, safari, custom-car
+   * **apps**: firefox, chrome, safari, safari-tp, custom-car
   * **expected**: pass
   * **expose browser profiler**: true
   * **gecko profile interval**: 1
@ -5398,7 +5398,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
   **Owner**: Performance Team

   * **alert threshold**: 2.0
-   * **apps**: firefox, chrome, safari, custom-car
+   * **apps**: firefox, chrome, safari, safari-tp, custom-car
   * **browser cycles**: 5
   * **cold**: true
   * **expected**: pass
@ -5695,6 +5695,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
        - ❌
        - ❌
        - ❌
+      * - **browsertime-benchmark-safari-tp-speedometer3**
+        - ✅
+        - ❌
+        - ❌
+        - ❌


   .. list-table:: **test-windows10-32-qr/opt**
@ -6087,6 +6092,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
        - ❌
        - ❌
        - ❌
+      * - **browsertime-benchmark-safari-tp-speedometer3**
+        - ✅
+        - ❌
+        - ❌
+        - ❌


   .. list-table:: **test-windows10-32-qr/opt**
--- a/testing/raptor/raptor/browsertime/desktop.py
+++ b/testing/raptor/raptor/browsertime/desktop.py
@ -22,7 +22,6 @@ class BrowsertimeDesktop(PerftestDesktop, Browsertime):
        LOG.info("binary_path: {}".format(binary_path))

        args_list = ["--viewPort", "1280x1024"]
-
        if self.config["app"] in (
            "chrome",
            "custom-car",
@ -33,6 +32,10 @@ class BrowsertimeDesktop(PerftestDesktop, Browsertime):
                "--chrome.binaryPath",
                binary_path,
            ]
+
+        if self.config["app"] in ("safari-tp",):
+            return args_list + ["--browser", "safari", "--safari.useTechnologyPreview"]
+
        return args_list + [
            "--browser",
            self.config["app"],
--- a/testing/raptor/raptor/cmdline.py
+++ b/testing/raptor/raptor/cmdline.py
@ -7,10 +7,11 @@ import os
 import six
 from mozlog.commandline import add_logging_group

-(FIREFOX, CHROME, SAFARI, CHROMIUM_RELEASE) = DESKTOP_APPS = [
+(FIREFOX, CHROME, SAFARI, SAFARI_TP, CHROMIUM_RELEASE) = DESKTOP_APPS = [
    "firefox",
    "chrome",
    "safari",
+    "safari-tp",
    "custom-car",
 ]
 (GECKOVIEW, REFBROW, FENIX) = FIREFOX_ANDROID_APPS = [
@ -29,6 +30,7 @@ APPS = {
    FIREFOX: {"long_name": "Firefox Desktop"},
    CHROME: {"long_name": "Google Chrome Desktop"},
    SAFARI: {"long_name": "Safari Desktop"},
+    SAFARI_TP: {"long_name": "Safari Technology Preview Desktop"},
    CHROMIUM_RELEASE: {"long_name": "Custom Chromium-as-Release desktop"},
    GECKOVIEW: {
        "long_name": "Firefox GeckoView on Android",
--- a/testing/raptor/raptor/results.py
+++ b/testing/raptor/raptor/results.py
@ -30,7 +30,7 @@ KNOWN_TEST_MODIFIERS = [
    "bytecode-cached",
 ]
 NON_FIREFOX_OPTS = ("webrender", "bytecode-cached", "fission")
-NON_FIREFOX_BROWSERS = ("chrome", "custom-car", "safari")
+NON_FIREFOX_BROWSERS = ("chrome", "custom-car", "safari", "safari-tp")
 NON_FIREFOX_BROWSERS_MOBILE = ("chrome-m", "cstm-car-m")


--- a/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.toml
+++ b/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.toml
@ -1,6 +1,6 @@
 [DEFAULT]
 alert_threshold = 2.0
-apps = "firefox, chrome, safari, custom-car"
+apps = "firefox, chrome, safari, safari-tp, custom-car"
 gecko_profile_interval = 1
 expose_browser_profiler = true
 lower_is_better = false
@ -13,9 +13,9 @@ test_url = "http://<host>:<port>/Speedometer/index.html?raptor"
 type = "benchmark"
 unit = "score"

-["speedometer"]
+[speedometer]

-["speedometer3"]
+[speedometer3]
 browser_cycles = 5
 owner = "Performance Team"
 cold = true