nagai/fune
3
0
Fork 0
forked from mirrors/gecko-dev
Code Pull requests Activity

Bug 1859911 - migrate raptor .ini to .toml. r=sparky,perftest-reviewers

Browse source 
Differential Revision: https://phabricator.services.mozilla.com/D197392
This commit is contained in:
Joel Maher 2024-01-09 20:43:41 +00:00
parent 57ffe819df
commit 23271e7f18
92 changed files with 1717 additions and 1839 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

2
docs/code-quality/lint/linters/perfdocs.rst
View file

@ -62,7 +62,7 @@ Here is an example of a configuration file for the Raptor framework:
.. parsed-literal::
name: raptor
manifest: testing/raptor/raptor/raptor.ini
manifest: testing/raptor/raptor/raptor.toml
suites:
desktop:
description: "Desktop tests."

7
testing/mozbase/manifestparser/manifestparser/util.py
View file

@ -43,4 +43,9 @@ def evaluate_list_from_string(list_string):
8,
```
"""
return ast.literal_eval("[" + "".join(list_string.strip(",").split("\n")) + "]")
parts = [
x.strip(",")
for x in list_string.strip(",").replace("\r", "").split("\n")
if x.strip()
]
return ast.literal_eval("[" + ",".join(parts) + "]")

70
testing/perfdocs/generated/raptor.rst
View file

@ -5322,13 +5322,13 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
**Owner**: :jgilbert and Graphics(gfx) Team
* **alert threshold**: 2.0
* **apps**: geckoview, refbrow, fenix, chrome-m
* **apps**: firefox, chrome, chromium, safari
* **expected**: pass
* **expose browser profiler**: true
* **gecko profile entries**: 8000000
* **gecko profile interval**: 1
* **lower is better**: false
* **page cycles**: 1
* **page cycles**: 5
* **page timeout**: 420000
* **repository**: https://github.com/mozilla/perf-automation
* **repository path**: benchmarks/unity-webgl
@ -5605,13 +5605,13 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
**Owner**: :jgilbert and Graphics(gfx) Team
* **alert threshold**: 2.0
* **apps**: firefox, chrome, chromium, safari
* **apps**: geckoview, refbrow, fenix, chrome-m
* **expected**: pass
* **expose browser profiler**: true
* **gecko profile entries**: 8000000
* **gecko profile interval**: 1
* **lower is better**: false
* **page cycles**: 5
* **page cycles**: 1
* **page timeout**: 420000
* **repository**: https://github.com/mozilla/perf-automation
* **repository path**: benchmarks/unity-webgl
@ -6139,7 +6139,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **newtab per cycle**: true
* **page cycles**: 5
* **page timeout**: 120000
* **preferences**: {"javascript.options.wasm_baselinejit": true, "javascript.options.wasm_optimizingjit": false}
* **preferences**: javascript.options.wasm_baselinejit=true javascript.options.wasm_optimizingjit=false
* **test url**: `<http://localhost:\<port\>/wasm-godot/index.html>`__
* **type**: benchmark
* **unit**: ms
@ -6353,7 +6353,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **newtab per cycle**: true
* **page cycles**: 5
* **page timeout**: 120000
* **preferences**: {"javascript.options.wasm_baselinejit": false, "javascript.options.wasm_optimizingjit": true}
* **preferences**: javascript.options.wasm_baselinejit=false javascript.options.wasm_optimizingjit=true
* **test url**: `<http://localhost:\<port\>/wasm-godot/index.html>`__
* **type**: benchmark
* **unit**: ms
@ -6801,7 +6801,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 5
* **page timeout**: 1200000
* **preferences**: {"javascript.options.wasm_baselinejit": true, "javascript.options.wasm_optimizingjit": false}
* **preferences**: javascript.options.wasm_baselinejit=true javascript.options.wasm_optimizingjit=false
* **repository**: https://github.com/mozilla/perf-automation
* **repository path**: benchmarks/wasm-misc
* **repository revision**: 61332db584026b73e37066d717a162825408c36b
@ -7017,7 +7017,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 5
* **page timeout**: 1200000
* **preferences**: {"javascript.options.wasm_baselinejit": false, "javascript.options.wasm_optimizingjit": true}
* **preferences**: javascript.options.wasm_baselinejit=false javascript.options.wasm_optimizingjit=true
* **repository**: https://github.com/mozilla/perf-automation
* **repository path**: benchmarks/wasm-misc
* **repository revision**: 61332db584026b73e37066d717a162825408c36b
@ -7233,7 +7233,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 5
* **page timeout**: 360000
* **preferences**: {"media.autoplay.default": 0, "media.autoplay.ask-permission": false, "media.autoplay.blocking_policy": 0, "media.allowed-to-play.enabled": true, "media.block-autoplay-until-in-foreground": false}
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false
* **test url**: `<http://\<host\>:\<port\>/webaudio/index.html?raptor>`__
* **type**: benchmark
* **unit**: score
@ -7462,9 +7462,9 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
**Owner**: PerfTest Team
* **alert on**: H264.1080p30@1X_dropped_frames, H264.1080p60@1X_dropped_frames, H264.1440p30@1X_dropped_frames, H264.144p15@1X_dropped_frames, H264.2160p30@1X_dropped_frames, H264.240p30@1X_dropped_frames, H264.360p30@1X_dropped_frames, H264.480p30@1X_dropped_frames, H264.720p30@1X_dropped_frames, H264.720p60@1X_dropped_frames, VP9.1080p30@1X_dropped_frames, VP9.1080p60@1X_dropped_frames, VP9.1440p30@1X_dropped_frames, VP9.1440p60@1X_dropped_frames, VP9.144p30@1X_dropped_frames, VP9.2160p30@1X_dropped_frames, VP9.2160p60@1X_dropped_frames, VP9.240p30@1X_dropped_frames, VP9.360p30@1X_dropped_frames, VP9.480p30@1X_dropped_frames, VP9.720p30@1X_dropped_frames, VP9.720p60@1X_dropped_frames
* **alert on**: H264.1080p30@1X_dropped_frames H264.1080p60@1X_dropped_frames H264.1440p30@1X_dropped_frames H264.144p15@1X_dropped_frames H264.2160p30@1X_dropped_frames H264.240p30@1X_dropped_frames H264.360p30@1X_dropped_frames H264.480p30@1X_dropped_frames H264.720p30@1X_dropped_frames H264.720p60@1X_dropped_frames VP9.1080p30@1X_dropped_frames VP9.1080p60@1X_dropped_frames VP9.1440p30@1X_dropped_frames VP9.1440p60@1X_dropped_frames VP9.144p30@1X_dropped_frames VP9.2160p30@1X_dropped_frames VP9.2160p60@1X_dropped_frames VP9.240p30@1X_dropped_frames VP9.360p30@1X_dropped_frames VP9.480p30@1X_dropped_frames VP9.720p30@1X_dropped_frames VP9.720p60@1X_dropped_frames
* **alert threshold**: 2.0
* **apps**: firefox, geckoview, fenix,refbrow, chrome
* **apps**: firefox, geckoview, fenix, refbrow, chrome
* **expected**: pass
* **expose browser profiler**: true
* **gecko profile entries**: 50000000
@ -7473,7 +7473,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 1
* **page timeout**: 2700000
* **preferences**: {"media.autoplay.default": 0, "media.autoplay.ask-permission": false, "media.autoplay.blocking_policy": 0, "media.allowed-to-play.enabled": true, "media.block-autoplay-until-in-foreground": false}
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false
* **subtest lower is better**: true
* **subtest unit**: score
* **test url**: `<http://yttest.prod.mozaws.net/2019/main.html?test_type=playbackperf-test&raptor=true&command=run&exclude=1,2&muted=true>`__
@ -7496,7 +7496,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 1
* **page timeout**: 2700000
* **preferences**: {"media.autoplay.default": 0, "media.autoplay.ask-permission": false, "media.autoplay.blocking_policy": 0, "media.allowed-to-play.enabled": true, "media.block-autoplay-until-in-foreground": false}
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false
* **subtest lower is better**: true
* **subtest unit**: score
* **test url**: `<https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-av1-test&raptor=true&exclude=1,2&muted=true&command=run>`__
@ -7538,7 +7538,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 20
* **page timeout**: 2700000
* **preferences**: {"media.autoplay.default": 0, "media.autoplay.ask-permission": false, "media.autoplay.blocking_policy": 0, "media.allowed-to-play.enabled": true, "media.block-autoplay-until-in-foreground": false}
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false
* **subtest lower is better**: true
* **subtest unit**: score
* **test url**: `<https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-h264-test&tests=18&raptor=true&muted=true&command=run&exclude=1,2>`__
@ -7740,7 +7740,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 20
* **page timeout**: 2700000
* **preferences**: {"media.autoplay.default": 0, "media.autoplay.ask-permission": false, "media.autoplay.blocking_policy": 0, "media.allowed-to-play.enabled": true, "media.block-autoplay-until-in-foreground": false}
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false
* **subtest lower is better**: true
* **subtest unit**: score
* **test url**: `<https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=46&muted=true&command=run&exclude=1,2>`__
@ -7942,7 +7942,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 20
* **page timeout**: 2700000
* **preferences**: {"media.autoplay.default": 0, "media.autoplay.ask-permission": false, "media.autoplay.blocking_policy": 0, "media.allowed-to-play.enabled": true, "media.block-autoplay-until-in-foreground": false, "full-screen-api.allow-trusted-requests-only": false, "full-screen-api.warning.timeout": 0}
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false full-screen-api.allow-trusted-requests-only=false full-screen-api.warning.timeout=0
* **subtest lower is better**: true
* **subtest unit**: score
* **test url**: `<https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-h264-test&tests=18&raptor=true&muted=true&command=run&fullscreen=true&exclude=1,2>`__
@ -8144,7 +8144,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 20
* **page timeout**: 2700000
* **preferences**: {"media.autoplay.default": 0, "media.autoplay.ask-permission": false, "media.autoplay.blocking_policy": 0, "media.allowed-to-play.enabled": true, "media.block-autoplay-until-in-foreground": false, "full-screen-api.allow-trusted-requests-only": false, "full-screen-api.warning.timeout": 0}
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false full-screen-api.allow-trusted-requests-only=false full-screen-api.warning.timeout=0
* **subtest lower is better**: true
* **subtest unit**: score
* **test url**: `<https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=46&muted=true&command=run&fullscreen=true&exclude=1,2>`__
@ -8345,7 +8345,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 1
* **page timeout**: 2700000
* **preferences**: {"media.autoplay.default": 0, "media.autoplay.ask-permission": false, "media.autoplay.blocking_policy": 0, "media.allowed-to-play.enabled": true, "media.block-autoplay-until-in-foreground": false}
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false
* **subtest lower is better**: true
* **subtest unit**: score
* **test url**: `<https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-h264-test&raptor=true&exclude=1,2&muted=true&command=run>`__
@ -8594,7 +8594,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
**Owner**: PerfTest Team
* **alert on**: H2641080p60fps@1X_dropped_frames, H264720p60fps@1X_dropped_frames
* **alert on**: H2641080p60fps@1X_dropped_frames H264720p60fps@1X_dropped_frames
* **alert threshold**: 2.0
* **apps**: firefox, geckoview, fenix, refbrow, chrome
* **expected**: pass
@ -8605,7 +8605,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 1
* **page timeout**: 2700000
* **preferences**: {"media.autoplay.default": 0, "media.autoplay.ask-permission": false, "media.autoplay.blocking_policy": 0, "media.allowed-to-play.enabled": true, "media.block-autoplay-until-in-foreground": false}
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false
* **subtest lower is better**: true
* **subtest unit**: score
* **test url**: `<https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&exclude=1,2&muted=true&command=run>`__
@ -8865,7 +8865,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 20
* **page timeout**: 2700000
* **preferences**: {"media.autoplay.default": 0, "media.autoplay.ask-permission": false, "media.autoplay.blocking_policy": 0, "media.allowed-to-play.enabled": true, "media.block-autoplay-until-in-foreground": false}
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false
* **subtest lower is better**: true
* **subtest unit**: score
* **test url**: `<https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&tests=18&muted=true&command=run&exclude=1,2>`__
@ -9067,7 +9067,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 20
* **page timeout**: 2700000
* **preferences**: {"media.autoplay.default": 0, "media.autoplay.ask-permission": false, "media.autoplay.blocking_policy": 0, "media.allowed-to-play.enabled": true, "media.block-autoplay-until-in-foreground": false}
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false
* **subtest lower is better**: true
* **subtest unit**: score
* **test url**: `<https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=14&muted=true&command=run&exclude=1,2>`__
@ -9269,7 +9269,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 20
* **page timeout**: 2700000
* **preferences**: {"media.autoplay.default": 0, "media.autoplay.ask-permission": false, "media.autoplay.blocking_policy": 0, "media.allowed-to-play.enabled": true, "media.block-autoplay-until-in-foreground": false, "full-screen-api.allow-trusted-requests-only": false, "full-screen-api.warning.timeout": 0}
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false full-screen-api.allow-trusted-requests-only=false full-screen-api.warning.timeout=0
* **subtest lower is better**: true
* **subtest unit**: score
* **test url**: `<https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&tests=18&muted=true&command=run&fullscreen=true&exclude=1,2>`__
@ -9471,7 +9471,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 20
* **page timeout**: 2700000
* **preferences**: {"media.autoplay.default": 0, "media.autoplay.ask-permission": false, "media.autoplay.blocking_policy": 0, "media.allowed-to-play.enabled": true, "media.block-autoplay-until-in-foreground": false, "full-screen-api.allow-trusted-requests-only": false, "full-screen-api.warning.timeout": 0}
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false full-screen-api.allow-trusted-requests-only=false full-screen-api.warning.timeout=0
* **subtest lower is better**: true
* **subtest unit**: score
* **test url**: `<https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=14&muted=true&command=run&fullscreen=true&exclude=1,2>`__
@ -9672,7 +9672,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 1
* **page timeout**: 2700000
* **preferences**: {"media.autoplay.default": 0, "media.autoplay.ask-permission": false, "media.autoplay.blocking_policy": 0, "media.allowed-to-play.enabled": true, "media.block-autoplay-until-in-foreground": false}
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false
* **subtest lower is better**: true
* **subtest unit**: score
* **test url**: `<https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&exclude=1,2&muted=true&command=run>`__
@ -9889,7 +9889,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 1
* **page timeout**: 2700000
* **preferences**: {"media.autoplay.default": 0, "media.autoplay.ask-permission": false, "media.autoplay.blocking_policy": 0, "media.allowed-to-play.enabled": true, "media.block-autoplay-until-in-foreground": false, "media.eme.enabled": true, "media.gmp-manager.updateEnabled": true, "media.eme.require-app-approval": false}
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false media.eme.enabled=true media.gmp-manager.updateEnabled=true media.eme.require-app-approval=false
* **subtest lower is better**: true
* **subtest unit**: score
* **test url**: `<https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-widevine-sfr-h264-test&raptor=true&exclude=1,2&muted=true&command=run>`__
@ -10106,7 +10106,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 1
* **page timeout**: 2700000
* **preferences**: {"media.autoplay.default": 0, "media.autoplay.ask-permission": false, "media.autoplay.blocking_policy": 0, "media.allowed-to-play.enabled": true, "media.block-autoplay-until-in-foreground": false, "media.eme.enabled": true, "media.gmp-manager.updateEnabled": true, "media.eme.require-app-approval": false}
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false media.eme.enabled=true media.gmp-manager.updateEnabled=true media.eme.require-app-approval=false
* **subtest lower is better**: true
* **subtest unit**: score
* **test url**: `<https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-widevine-hfr-test&raptor=true&exclude=1,2&muted=true&command=run>`__
@ -10323,7 +10323,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
* **lower is better**: true
* **page cycles**: 1
* **page timeout**: 2700000
* **preferences**: {"media.autoplay.default": 0, "media.autoplay.ask-permission": false, "media.autoplay.blocking_policy": 0, "media.allowed-to-play.enabled": true, "media.block-autoplay-until-in-foreground": false, "media.eme.enabled": true, "media.gmp-manager.updateEnabled": true, "media.eme.require-app-approval": false}
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false media.eme.enabled=true media.gmp-manager.updateEnabled=true media.eme.require-app-approval=false
* **subtest lower is better**: true
* **subtest unit**: score
* **test url**: `<https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-widevine-sfr-vp9-test&raptor=true&exclude=1,2&muted=true&command=run>`__
@ -20571,7 +20571,7 @@ Tests for page-load performance. The links direct to the actual websites that ar
* **playback**: mitmproxy
* **playback pageset manifest**: mitm8-linux-firefox-imgur.manifest
* **playback version**: 8.1.1
* **preferences**: {"media.autoplay.default": 5, "media.autoplay.ask-permission": true, "media.autoplay.blocking_policy": 1, "media.allowed-to-play.enabled": false, "media.block-autoplay-until-in-foreground": true}
* **preferences**: media.autoplay.default=5 media.autoplay.ask-permission=true media.autoplay.blocking_policy=1 media.allowed-to-play.enabled=false media.block-autoplay-until-in-foreground=true
* **secondary url**: `<https://imgur.com/gallery/WuOobqV>`__
* **test url**: `<https://imgur.com/gallery/uTI7fKy>`__
* **type**: pageload
@ -24578,7 +24578,7 @@ Tests for page-load performance. The links direct to the actual websites that ar
* **playback**: mitmproxy
* **playback pageset manifest**: mitm8-linux-firefox-twitch.manifest
* **playback version**: 8.1.1
* **preferences**: {"media.autoplay.default": 5, "media.autoplay.ask-permission": true, "media.autoplay.blocking_policy": 1, "media.allowed-to-play.enabled": false, "media.block-autoplay-until-in-foreground": true}
* **preferences**: media.autoplay.default=5 media.autoplay.ask-permission=true media.autoplay.blocking_policy=1 media.allowed-to-play.enabled=false media.block-autoplay-until-in-foreground=true
* **secondary url**: `<https://www.twitch.tv/gmashley>`__
* **test url**: `<https://www.twitch.tv/videos/894226211>`__
* **type**: pageload
@ -26871,7 +26871,7 @@ Browsertime tests that interact with the webpage. Includes responsiveness tests
* **playback**: mitmproxy
* **playback pageset manifest**: mitm7-linux-firefox-cnn-nav.manifest
* **playback version**: 8.1.1
* **test cmds**: ["measure.start", "landing"], ["navigate", "https://www.cnn.com"], ["wait.byTime", 4000], ["measure.stop", ""], ["measure.start", "world"], ["click.byXpathAndWait", "/html/body/div[5]/div/div/header/div/div[1]/div/div[2]/nav/ul/li[2]/a"], ["wait.byTime", 1000], ["measure.stop", ""],
* **test cmds**: ['measure.start', 'landing'] ['navigate', 'https://www.cnn.com'] ['wait.byTime', '4000'] ['measure.stop', ''] ['measure.start', 'world'] ['click.byXpathAndWait', '/html/body/div[5]/div/div/header/div/div[1]/div/div[2]/nav/ul/li[2]/a'] ['wait.byTime', '1000'] ['measure.stop', '']
* **test url**: `<https://www.cnn.com/>`__
* **type**: pageload
* **unit**: ms
@ -27111,7 +27111,7 @@ Browsertime tests that interact with the webpage. Includes responsiveness tests
* **playback**: mitmproxy
* **playback pageset manifest**: mitm6-windows-firefox-facebook-nav.manifest
* **playback version**: 8.1.1
* **test cmds**: ["measure.start", "landing"], ["navigate", "https://www.facebook.com/"], ["wait.byTime", 5000], ["measure.stop", ""], ["measure.start", "marketplace"], ["navigate", "https://www.facebook.com/marketplace"], ["wait.byTime", 5000], ["measure.stop", ""], ["measure.start", "groups"], ["navigate", "https://www.facebook.com/groups/discover/"], ["wait.byTime", 5000], ["measure.stop", ""], ["measure.start", "friends"], ["navigate", "https://www.facebook.com/friends/"], ["wait.byTime", 5000], ["measure.stop", ""],
* **test cmds**: ['measure.start', 'landing'] ['navigate', 'https://www.facebook.com/'] ['wait.byTime', '5000'] ['measure.stop', ''] ['measure.start', 'marketplace'] ['navigate', 'https://www.facebook.com/marketplace'] ['wait.byTime', '5000'] ['measure.stop', ''] ['measure.start', 'groups'] ['navigate', 'https://www.facebook.com/groups/discover/'] ['wait.byTime', '5000'] ['measure.stop', ''] ['measure.start', 'friends'] ['navigate', 'https://www.facebook.com/friends/'] ['wait.byTime', '5000'] ['measure.stop', '']
* **test url**: `<https://www.facebook.com>`__
* **type**: pageload
* **unit**: ms
@ -27351,7 +27351,7 @@ Browsertime tests that interact with the webpage. Includes responsiveness tests
* **playback**: mitmproxy
* **playback pageset manifest**: mitm6-windows-firefox-reddit-billgates-ama.manifest
* **playback version**: 8.1.1
* **test cmds**: ["measure.start", "billg-ama"], ["navigate", "https://www.reddit.com/r/IAmA/comments/m8n4vt/im_bill_gates_cochair_of_the_bill_and_melinda/"], ["wait.byTime", 5000], ["measure.stop", ""], ["measure.start", "members"], ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div[3]/div[2]/div/div[1]/div/div[4]/div[1]/div"], ["wait.byTime", 1000], ["measure.stop", ""],
* **test cmds**: ['measure.start', 'billg-ama'] ['navigate', 'https://www.reddit.com/r/IAmA/comments/m8n4vt/im_bill_gates_cochair_of_the_bill_and_melinda/'] ['wait.byTime', '5000'] ['measure.stop', ''] ['measure.start', 'members'] ['click.byXpathAndWait', '/html/body/div[1]/div/div[2]/div[2]/div/div[3]/div[2]/div/div[1]/div/div[4]/div[1]/div'] ['wait.byTime', '1000'] ['measure.stop', '']
* **test url**: `<https://www.reddit.com/>`__
* **type**: pageload
* **unit**: ms
@ -27591,7 +27591,7 @@ Browsertime tests that interact with the webpage. Includes responsiveness tests
* **playback**: mitmproxy
* **playback pageset manifest**: mitm6-windows-firefox-reddit-billgates-post.manifest
* **playback version**: 8.1.1
* **test cmds**: ["measure.start", "billg"], ["navigate", "https://www.reddit.com/user/thisisbillgates/"], ["wait.byTime", 500], ["measure.stop", ""], ["measure.start", "posts"], ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[2]"], ["wait.byTime", 500], ["measure.stop", ""], ["measure.start", "comments"], ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[3]"], ["wait.byTime", 500], ["measure.stop", ""], ["wait.byTime", 500],
* **test cmds**: ['measure.start', 'billg'] ['navigate', 'https://www.reddit.com/user/thisisbillgates/'] ['wait.byTime', '500'] ['measure.stop', ''] ['measure.start', 'posts'] ['click.byXpathAndWait', '/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[2]'] ['wait.byTime', '500'] ['measure.stop', ''] ['measure.start', 'comments'] ['click.byXpathAndWait', '/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[3]'] ['wait.byTime', '500'] ['measure.stop', ''] ['wait.byTime', '500']
* **test url**: `<https://www.reddit.com/user/thisisbillgates/>`__
* **type**: pageload
* **unit**: ms
@ -27831,7 +27831,7 @@ Browsertime tests that interact with the webpage. Includes responsiveness tests
* **playback**: mitmproxy
* **playback pageset manifest**: mitm6-windows-firefox-reddit-billgates-post.manifest
* **playback version**: 8.1.1
* **test cmds**: ["measure.start", "billg"], ["navigate", "https://www.reddit.com/user/thisisbillgates/"], ["wait.byTime", 500], ["measure.stop", ""], ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[3]"], ["wait.byTime", 500], ["measure.start", "hot"], ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[2]"], ["wait.byTime", 500], ["measure.stop", ""], ["measure.start", "top"], ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[3]"], ["wait.byTime", 500], ["measure.stop", ""], ["wait.byTime", 500],
* **test cmds**: ['measure.start', 'billg'] ['navigate', 'https://www.reddit.com/user/thisisbillgates/'] ['wait.byTime', '500'] ['measure.stop', ''] ['click.byXpathAndWait', '/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[3]'] ['wait.byTime', '500'] ['measure.start', 'hot'] ['click.byXpathAndWait', '/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[2]'] ['wait.byTime', '500'] ['measure.stop', ''] ['measure.start', 'top'] ['click.byXpathAndWait', '/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[3]'] ['wait.byTime', '500'] ['measure.stop', ''] ['wait.byTime', '500']
* **test url**: `<https://www.reddit.com/user/thisisbillgates/>`__
* **type**: pageload
* **unit**: ms

26
testing/raptor/raptor/browsertime/base.py
View file

@ -21,6 +21,7 @@ from logger.logger import RaptorLogger
from manifestparser.util import evaluate_list_from_string
from perftest import GECKO_PROFILER_APPS, TRACE_APPS, Perftest
from results import BrowsertimeResultsHandler
from utils import bool_from_str
LOG = RaptorLogger(component="raptor-browsertime")
@ -121,10 +122,33 @@ class Browsertime(Perftest):
def set_browser_test_prefs(self, raw_prefs):
# add test specific preferences
LOG.info("setting test-specific Firefox preferences")
self.profile.set_preferences(json.loads(raw_prefs))
self.profile.set_preferences(raw_prefs)
self.remove_mozprofile_delimiters_from_profile()
def _convert_prefs_to_dict(self, raw_prefs):
pref_dict = {}
prefparts = raw_prefs.split("\n")
for pref in prefparts:
if "=" not in pref:
continue
parts = pref.strip("\r").split("=")
try:
if "rue" in parts[-1] or "alse" in parts[-1]:
parts[-1] = bool_from_str(parts[-1])
else:
parts[-1] = int(parts[-1])
except ValueError:
pass
pref_dict[parts[0]] = parts[-1]
if len(parts) > 2:
pref_dict[parts[0]] = "=".join(parts[1:])
return pref_dict
def run_test_setup(self, test):
if test.get("preferences", ""):
test["preferences"] = self._convert_prefs_to_dict(test["preferences"])
super(Browsertime, self).run_test_setup(test)
if test.get("type") == "benchmark":

12
testing/raptor/raptor/cmdline.py
View file

@ -640,10 +640,10 @@ class _PrintTests(_StopAction):
from manifestparser import TestManifest
here = os.path.abspath(os.path.dirname(__file__))
raptor_ini = os.path.join(here, "raptor.ini")
raptor_toml = os.path.join(here, "raptor.toml")
for _app in self.integrated_apps:
test_manifest = TestManifest([raptor_ini], strict=False)
test_manifest = TestManifest([raptor_toml], strict=False)
info = {"app": _app}
available_tests = test_manifest.active_tests(
exists=False, disabled=False, filters=[self.filter_app], **info
@ -672,7 +672,9 @@ class _PrintTests(_StopAction):
# no test name; skip it
continue
suite = os.path.basename(next_test["manifest"])[:-4]
suite = ".".join(
os.path.basename(next_test["manifest"]).split(".")[:-1]
)
if suite not in test_list:
test_list[suite] = {"type": None, "subtests": []}
@ -684,7 +686,9 @@ class _PrintTests(_StopAction):
subtest = next_test["name"]
measure = next_test.get("measure")
if measure is not None:
subtest = "{0} ({1})".format(subtest, measure)
subtest = "{0} ({1})".format(
subtest, measure.replace("\n", ", ")
)
test_list[suite]["subtests"].append(subtest)
# print the list in a nice, readable format

63
testing/raptor/raptor/manifest.py
View file

@ -19,7 +19,7 @@ from utils import (
)
here = os.path.abspath(os.path.dirname(__file__))
raptor_ini = os.path.join(here, "raptor.ini")
raptor_toml = os.path.join(here, "raptor.toml")
tests_dir = os.path.join(here, "tests")
LOG = RaptorLogger(component="raptor-manifest")
@ -49,16 +49,16 @@ def filter_app(tests, values):
def get_browser_test_list(browser_app, run_local):
LOG.info(raptor_ini)
test_manifest = TestManifest([raptor_ini], strict=False)
LOG.info(raptor_toml)
test_manifest = TestManifest([raptor_toml], strict=False)
info = {"app": browser_app, "run_local": run_local}
return test_manifest.active_tests(
exists=False, disabled=False, filters=[filter_app], **info
)
def validate_test_ini(test_details):
# validate all required test details were found in the test INI
def validate_test_toml(test_details):
# validate all required test details were found in the test TOML
valid_settings = True
for setting in required_settings:
@ -93,13 +93,16 @@ def validate_test_ini(test_details):
)
# if 'alert-on' is specified, we need to make sure that the value given is valid
# i.e. any 'alert_on' values must be values that exist in the 'measure' ini setting
# i.e. any 'alert_on' values must be values that exist in the 'measure' toml setting
if test_details.get("alert_on") is not None:
# support with or without spaces, i.e. 'measure = fcp, loadtime' or '= fcp,loadtime'
# convert to a list; and remove any spaces
# when switching to .toml, values are \n separated, convert to a ',' and this will
# support scenarios where we have older .ini format as well as embedded commas
# this can also have regexes inside
test_details["alert_on"] = [
_item.strip() for _item in test_details["alert_on"].split(",")
_item.strip()
for _item in test_details["alert_on"].replace("\n", ",").split(",")
]
# this variable will store all the concrete values for alert_on elements
@ -202,7 +205,7 @@ def write_test_settings_json(args, test_details, oskey):
test_settings["raptor-options"]["measure"] = {}
# test_details['measure'] was already converted to a list in get_raptor_test_list below
# the 'hero=' line is still a raw string from the test INI
# the 'hero=' line is still a raw string from the test TOML
for m in test_details["measure"]:
test_settings["raptor-options"]["measure"][m] = True
if m == "hero":
@ -316,24 +319,24 @@ def write_test_settings_json(args, test_details, oskey):
def get_raptor_test_list(args, oskey):
"""
A test ini (i.e. raptor-firefox-tp6.ini) will have one or more subtests inside,
each with it's own name ([the-ini-file-test-section]).
A test toml (i.e. raptor-firefox-tp6.toml) will have one or more subtests inside,
each with it's own name ([the-toml-file-test-section]).
We want the ability to eiter:
- run * all * of the subtests listed inside the test ini; - or -
- just run a single one of those subtests that are inside the ini
- run * all * of the subtests listed inside the test toml; - or -
- just run a single one of those subtests that are inside the toml
A test name is received on the command line. This will either match the name
of a single subtest (within an ini) - or - if there's no matching single
of a single subtest (within an toml) - or - if there's no matching single
subtest with that name, then the test name provided might be the name of a
test ini itself (i.e. raptor-firefox-tp6) that contains multiple subtests.
test toml itself (i.e. raptor-firefox-tp6) that contains multiple subtests.
First look for a single matching subtest name in the list of all availble tests,
and if it's found we will just run that single subtest.
Then look at the list of all available tests - each available test has a manifest
name associated to it - and pull out all subtests whose manifest name matches
the test name provided on the command line i.e. run all subtests in a specified ini.
the test name provided on the command line i.e. run all subtests in a specified toml.
If no tests are found at all then the test name is invalid.
"""
@ -347,14 +350,14 @@ def get_raptor_test_list(args, oskey):
tests_to_run.append(next_test)
break
# no matches, so now look for all subtests that come from a test ini
# no matches, so now look for all subtests that come from a test toml
# manifest that matches the test name provided on the commmand line
if len(tests_to_run) == 0:
_ini = args.test + ".ini"
_toml = args.test + ".toml"
for next_test in available_tests:
head, tail = os.path.split(next_test["manifest"])
if tail == _ini:
# subtest comes from matching test ini file name, so add it
if tail == _toml:
# subtest comes from matching test toml file name, so add it
tests_to_run.append(next_test)
if args.collect_perfstats and args.app.lower() not in (
@ -374,7 +377,7 @@ def get_raptor_test_list(args, oskey):
next_test["playback"] = None
# go through each test and set the page-cycles and page-timeout, and some config flags
# the page-cycles value in the INI can be overriden when debug-mode enabled, when
# the page-cycles value in the TOML can be overriden when debug-mode enabled, when
# gecko-profiling enabled, or when --page-cycles cmd line arg was used (that overrides all)
for next_test in tests_to_run:
LOG.info("configuring settings for test %s" % next_test["name"])
@ -462,8 +465,8 @@ def get_raptor_test_list(args, oskey):
LOG.info("debug-mode enabled")
max_page_cycles = 2
# if --page-cycles was provided on the command line, use that instead of INI
# if just provided in the INI use that but cap at 3 if gecko-profiling is enabled
# if --page-cycles was provided on the command line, use that instead of TOML
# if just provided in the TOML use that but cap at 3 if gecko-profiling is enabled
if args.page_cycles is not None:
next_test["page_cycles"] = args.page_cycles
LOG.info(
@ -476,8 +479,8 @@ def get_raptor_test_list(args, oskey):
% next_test["page_cycles"]
)
# if --browser-cycles was provided on the command line, use that instead of INI
# if just provided in the INI use that but cap at 3 if gecko-profiling is enabled
# if --browser-cycles was provided on the command line, use that instead of TOML
# if just provided in the TOML use that but cap at 3 if gecko-profiling is enabled
if args.browser_cycles is not None:
next_test["browser_cycles"] = args.browser_cycles
LOG.info(
@ -491,7 +494,7 @@ def get_raptor_test_list(args, oskey):
% next_test["browser_cycles"]
)
# if --page-timeout was provided on the command line, use that instead of INI
# if --page-timeout was provided on the command line, use that instead of TOML
if args.page_timeout is not None:
LOG.info(
"setting page-timeout to %d as specified on cmd line"
@ -504,7 +507,7 @@ def get_raptor_test_list(args, oskey):
# check command line to see if we set cold page load from command line
if args.cold or next_test.get("cold") == "true":
# for raptor-webext jobs cold page-load is determined by the 'cold' key
# in test manifest INI
# in test manifest TOML
_running_cold = True
else:
# if it's a warm load test ignore browser_cycles if set
@ -533,7 +536,7 @@ def get_raptor_test_list(args, oskey):
next_test["browser_cycle"] = 1
# if --test-url-params was provided on the command line, add the params to the test_url
# provided in the INI
# provided in the TOML
if args.test_url_params is not None:
initial_test_url = next_test["test_url"]
next_test["test_url"] = add_test_url_params(
@ -568,7 +571,7 @@ def get_raptor_test_list(args, oskey):
% next_test.get("name", "Unknown")
)
# browsertime doesn't use the 'measure' test ini setting; however just for the sake
# browsertime doesn't use the 'measure' test toml setting; however just for the sake
# of supporting both webext and browsertime, just provide a dummy 'measure' setting
# here to prevent having to check in multiple places; it has no effect on what
# browsertime actually measures; remove this when eventually we remove webext support
@ -584,7 +587,7 @@ def get_raptor_test_list(args, oskey):
"largestContentfulPaint"
)
# convert 'measure =' test INI line to list
# convert 'measure =' test TOML line to list
if next_test.get("measure") is not None:
_measures = []
for measure in [m.strip() for m in next_test["measure"].split(",")]:
@ -630,7 +633,7 @@ def get_raptor_test_list(args, oskey):
# write out .json test setting files for the control server to read and send to web ext
if len(tests_to_run) != 0:
for test in tests_to_run:
if validate_test_ini(test):
if validate_test_toml(test):
write_test_settings_json(args, test, oskey)
else:
# test doesn't have valid settings, remove it from available list

2
testing/raptor/raptor/perfdocs/config.yml
View file

@ -3,7 +3,7 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
name: raptor
manifest: testing/raptor/raptor/raptor.ini
manifest: testing/raptor/raptor/raptor.toml
static-only: False
metrics:
"First Paint":

76
testing/raptor/raptor/raptor.ini
View file

@ -1,76 +0,0 @@
# raptor-browsertime desktop page-load tests
[include:tests/tp6/desktop/browsertime-tp6.ini]
# raptor-browsertime android page-load tests
[include:tests/tp6/mobile/browsertime-tp6m.ini]
# raptor-browsertime live page-load tests
[include:tests/tp6/live/browsertime-live.ini]
# raptor-browsertime benchmark tests
[include:tests/benchmarks/youtube-playback.ini]
[include:tests/benchmarks/unity-webgl-mobile.ini]
[include:tests/benchmarks/speedometer-desktop.ini]
[include:tests/benchmarks/speedometer-mobile.ini]
[include:tests/benchmarks/ares6.ini]
[include:tests/benchmarks/motionmark-animometer-desktop.ini]
[include:tests/benchmarks/motionmark-animometer-mobile.ini]
[include:tests/benchmarks/motionmark-htmlsuite-desktop.ini]
[include:tests/benchmarks/motionmark-htmlsuite-mobile.ini]
[include:tests/benchmarks/stylebench.ini]
[include:tests/benchmarks/wasm-misc.ini]
[include:tests/benchmarks/wasm-misc-baseline.ini]
[include:tests/benchmarks/wasm-misc-optimizing.ini]
[include:tests/benchmarks/webaudio.ini]
[include:tests/benchmarks/wasm-godot.ini]
[include:tests/benchmarks/wasm-godot-baseline.ini]
[include:tests/benchmarks/wasm-godot-optimizing.ini]
[include:tests/benchmarks/sunspider.ini]
[include:tests/benchmarks/assorted-dom.ini]
[include:tests/benchmarks/jetstream2.ini]
[include:tests/benchmarks/matrix-react-bench.ini]
[include:tests/benchmarks/unity-webgl-desktop.ini]
[include:tests/benchmarks/twitch-animation.ini]
# raptor-browsertime scenario tests
[include:tests/scenario/idle.ini]
# Fission process switch time test
[include:tests/custom/browsertime-process-switch.ini]
# First-install pageload test
[include:tests/custom/browsertime-welcome.ini]
# Interactive raptor-browsertime tests
[include:tests/interactive/browsertime-responsiveness.ini]
# Local custom browsertime tests
[include:tests/custom/browsertime-custom.ini]
# Custom grandprix benchmark test
[include:tests/custom/browsertime-grandprix.ini]
# Tests that are used for testing our regression system
[include:tests/custom/browsertime-regression-test.ini]
# raptor-browsertime unit tests
# this is required for the manifest unit tests
[include:tests/unittests/browsertime-tp6-unittest.ini]
# Custom upload speed test
[include:tests/custom/browsertime-upload.ini]
# Sample test for the Raptor python support file
[include:tests/custom/browsertime-sample-python-support.ini]
# Pageload benchmark test used for getting an overview of impact
[include:tests/custom/browsertime-tp6-bench.ini]
# Custom speculative connection test
[include:tests/custom/browsertime-speculative.ini]
# Custom throttled network pageload test
[include:tests/custom/browsertime-throttled-pageload.ini]
# Custom indexeddb test
[include:tests/custom/browsertime-indexeddb.ini]

101
testing/raptor/raptor/raptor.toml Normal file
View file

@ -0,0 +1,101 @@
[DEFAULT]
["include:tests/benchmarks/ares6.toml"]
# raptor-browsertime benchmark tests
["include:tests/benchmarks/assorted-dom.toml"]
["include:tests/benchmarks/jetstream2.toml"]
["include:tests/benchmarks/matrix-react-bench.toml"]
["include:tests/benchmarks/motionmark-animometer-desktop.toml"]
["include:tests/benchmarks/motionmark-animometer-mobile.toml"]
["include:tests/benchmarks/motionmark-htmlsuite-desktop.toml"]
["include:tests/benchmarks/motionmark-htmlsuite-mobile.toml"]
["include:tests/benchmarks/speedometer-desktop.toml"]
["include:tests/benchmarks/speedometer-mobile.toml"]
["include:tests/benchmarks/stylebench.toml"]
["include:tests/benchmarks/sunspider.toml"]
["include:tests/benchmarks/twitch-animation.toml"]
["include:tests/benchmarks/unity-webgl-desktop.toml"]
["include:tests/benchmarks/unity-webgl-mobile.toml"]
["include:tests/benchmarks/wasm-godot-baseline.toml"]
["include:tests/benchmarks/wasm-godot-optimizing.toml"]
["include:tests/benchmarks/wasm-godot.toml"]
["include:tests/benchmarks/wasm-misc-baseline.toml"]
["include:tests/benchmarks/wasm-misc-optimizing.toml"]
["include:tests/benchmarks/wasm-misc.toml"]
["include:tests/benchmarks/webaudio.toml"]
["include:tests/benchmarks/youtube-playback.toml"]
["include:tests/custom/browsertime-custom.toml"]
# Local custom browsertime tests
["include:tests/custom/browsertime-grandprix.toml"]
# Custom grandprix benchmark test
["include:tests/custom/browsertime-indexeddb.toml"]
# Custom indexeddb test
["include:tests/custom/browsertime-process-switch.toml"]
# Fission process switch time test
["include:tests/custom/browsertime-regression-test.toml"]
# Tests that are used for testing our regression system
["include:tests/custom/browsertime-sample-python-support.toml"]
# Sample test for the Raptor python support file
["include:tests/custom/browsertime-speculative.toml"]
# Custom speculative connection test
["include:tests/custom/browsertime-throttled-pageload.toml"]
# Custom throttled network pageload test
["include:tests/custom/browsertime-tp6-bench.toml"]
# Pageload benchmark test used for getting an overview of impact
["include:tests/custom/browsertime-upload.toml"]
# Custom upload speed test
["include:tests/custom/browsertime-welcome.toml"]
# First-install pageload test
["include:tests/interactive/browsertime-responsiveness.toml"]
# Interactive raptor-browsertime tests
["include:tests/scenario/idle.toml"]
# raptor-browsertime scenario tests
["include:tests/tp6/desktop/browsertime-tp6.toml"]
# raptor-browsertime desktop page-load tests
["include:tests/tp6/live/browsertime-live.toml"]
# raptor-browsertime live page-load tests
["include:tests/tp6/mobile/browsertime-tp6m.toml"]
# raptor-browsertime android page-load tests
["include:tests/unittests/browsertime-tp6-unittest.toml"]
# raptor-browsertime unit tests
# this is required for the manifest unit tests

23
testing/raptor/raptor/tests/benchmarks/ares6.ini
View file

@ -1,23 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# ARES-6 benchmark for firefox and chromium distributions
[DEFAULT]
alert_threshold = 2.0
apps = firefox, chrome, chromium, safari
gecko_profile_entries = 14000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
owner = :jandem and SpiderMonkey Team
page_cycles = 4
page_timeout = 270000
subtest_lower_is_better = true
subtest_unit = ms
test_url = http://<host>:<port>/ARES-6/index.html?raptor
type = benchmark
unit = ms
[ares6]

17
testing/raptor/raptor/tests/benchmarks/ares6.toml Normal file
View file

@ -0,0 +1,17 @@
[DEFAULT]
alert_threshold = 2.0
apps = "firefox, chrome, chromium, safari"
gecko_profile_entries = 14000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
owner = ":jandem and SpiderMonkey Team"
page_cycles = 4
page_timeout = 270000
subtest_lower_is_better = true
subtest_unit = "ms"
test_url = "http://<host>:<port>/ARES-6/index.html?raptor"
type = "benchmark"
unit = "ms"
["ares6"]

25
testing/raptor/raptor/tests/benchmarks/assorted-dom.ini
View file

@ -1,25 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# assorted-dom benchmark for firefox and chromium distributions
[DEFAULT]
alert_threshold = 2.0
apps = firefox, chrome, chromium, safari
gecko_profile_entries = 2000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
owner = PerfTest Team
page_cycles = 1
page_timeout = 60000
screen_capture = true
test_url = http://<host>:<port>/assorted/driver.html?raptor
type = benchmark
unit = ms
[assorted-dom]
repository = https://github.com/mozilla/perf-automation
repository_revision = 61332db584026b73e37066d717a162825408c36b
repository_path = benchmarks/assorted-dom

19
testing/raptor/raptor/tests/benchmarks/assorted-dom.toml Normal file
View file

@ -0,0 +1,19 @@
[DEFAULT]
alert_threshold = 2.0
apps = "firefox, chrome, chromium, safari"
gecko_profile_entries = 2000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
owner = "PerfTest Team"
page_cycles = 1
page_timeout = 60000
screen_capture = true
test_url = "http://<host>:<port>/assorted/driver.html?raptor"
type = "benchmark"
unit = "ms"
["assorted-dom"]
repository = "https://github.com/mozilla/perf-automation"
repository_revision = "61332db584026b73e37066d717a162825408c36b"
repository_path = "benchmarks/assorted-dom"

26
testing/raptor/raptor/tests/benchmarks/jetstream2.ini
View file

@ -1,26 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# JetStream-2 benchmark for firefox and chromium distributions
[DEFAULT]
alert_threshold = 2.0
apps = firefox, chrome, chromium, safari
gecko_profile_entries = 14000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = false
owner = :jandem and SpiderMonkey Team
page_cycles = 4
page_timeout = 2000000
subtest_lower_is_better = false
subtest_unit = score
test_url = http://<host>:<port>/index.html?raptor
type = benchmark
unit = score
[jetstream2]
repository = https://github.com/mozilla/perf-automation
repository_revision = 61332db584026b73e37066d717a162825408c36b
repository_path = benchmarks/JetStream2

20
testing/raptor/raptor/tests/benchmarks/jetstream2.toml Normal file
View file

@ -0,0 +1,20 @@
[DEFAULT]
alert_threshold = 2.0
apps = "firefox, chrome, chromium, safari"
gecko_profile_entries = 14000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = false
owner = ":jandem and SpiderMonkey Team"
page_cycles = 4
page_timeout = 2000000
subtest_lower_is_better = false
subtest_unit = "score"
test_url = "http://<host>:<port>/index.html?raptor"
type = "benchmark"
unit = "score"
["jetstream2"]
repository = "https://github.com/mozilla/perf-automation"
repository_revision = "61332db584026b73e37066d717a162825408c36b"
repository_path = "benchmarks/JetStream2"

26
testing/raptor/raptor/tests/benchmarks/matrix-react-bench.ini
View file

@ -1,26 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Matrix-react-bench benchmark for firefox and chromium distributions
[DEFAULT]
alert_threshold = 2.0
apps = firefox, chrome, chromium
gecko_profile_entries = 14000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
owner = :jandem and SpiderMonkey Team
page_cycles = 30
page_timeout = 2000000
subtest_lower_is_better = true
subtest_unit = ms
test_url = http://<host>:<port>/matrix_demo.html
type = benchmark
unit = ms
[matrix-react-bench]
repository = https://github.com/mozilla/perf-automation
repository_revision = 61332db584026b73e37066d717a162825408c36b
repository_path = benchmarks/matrix-react-bench

20
testing/raptor/raptor/tests/benchmarks/matrix-react-bench.toml Normal file
View file

@ -0,0 +1,20 @@
[DEFAULT]
alert_threshold = 2.0
apps = "firefox, chrome, chromium"
gecko_profile_entries = 14000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
owner = ":jandem and SpiderMonkey Team"
page_cycles = 30
page_timeout = 2000000
subtest_lower_is_better = true
subtest_unit = "ms"
test_url = "http://<host>:<port>/matrix_demo.html"
type = "benchmark"
unit = "ms"
["matrix-react-bench"]
repository = "https://github.com/mozilla/perf-automation"
repository_revision = "61332db584026b73e37066d717a162825408c36b"
repository_path = "benchmarks/matrix-react-bench"

25
testing/raptor/raptor/tests/benchmarks/motionmark-animometer-desktop.ini
View file

@ -1,25 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# motionmark benchmark for firefox and chromium distributions
[DEFAULT]
alert_threshold = 2.0
apps = firefox, chrome, chromium, safari
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = false
page_cycles = 1
page_timeout = 600000
owner = :jgilbert and Graphics(gfx) Team
test_url = http://<host>:<port>/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&raptor=true&oskey={platform}
type = benchmark
unit = score
[motionmark-animometer]
[motionmark-animometer-ramp]
apps = firefox, chrome, chromium, safari, custom-car
test_url = http://<host>:<port>/MotionMark/developer.html?test-interval=30&display=minimal&tiles=big&controller=ramp&frame-rate=60&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&raptor=true&oskey={platform}

19
testing/raptor/raptor/tests/benchmarks/motionmark-animometer-desktop.toml Normal file
View file

@ -0,0 +1,19 @@
[DEFAULT]
alert_threshold = 2.0
apps = "firefox, chrome, chromium, safari"
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = false
page_cycles = 1
page_timeout = 600000
owner = ":jgilbert and Graphics(gfx) Team"
test_url = "http://<host>:<port>/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&raptor=true&oskey={platform}"
type = "benchmark"
unit = "score"
["motionmark-animometer"]
["motionmark-animometer-ramp"]
apps = "firefox, chrome, chromium, safari, custom-car"
test_url = "http://<host>:<port>/MotionMark/developer.html?test-interval=30&display=minimal&tiles=big&controller=ramp&frame-rate=60&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&raptor=true&oskey={platform}"

24
testing/raptor/raptor/tests/benchmarks/motionmark-animometer-mobile.ini
View file

@ -1,24 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# motionmark benchmark for firefox and chromium distributions on mobile
[DEFAULT]
alert_threshold = 2.0
apps = fenix, chrome-m, geckoview
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = false
page_cycles = 1
page_timeout = 600000
owner = :jgilbert and Graphics(gfx) Team
test_url = http://<host>:<port>/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&raptor=true&oskey={platform}
type = benchmark
unit = score
[motionmark-animometer]
[motionmark-animometer-ramp]
test_url = http://<host>:<port>/MotionMark/developer.html?test-interval=30&display=minimal&tiles=big&controller=ramp&frame-rate=60&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&raptor=true&oskey={platform}

18
testing/raptor/raptor/tests/benchmarks/motionmark-animometer-mobile.toml Normal file
View file

@ -0,0 +1,18 @@
[DEFAULT]
alert_threshold = 2.0
apps = "fenix, chrome-m, geckoview"
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = false
page_cycles = 1
page_timeout = 600000
owner = ":jgilbert and Graphics(gfx) Team"
test_url = "http://<host>:<port>/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&raptor=true&oskey={platform}"
type = "benchmark"
unit = "score"
["motionmark-animometer"]
["motionmark-animometer-ramp"]
test_url = "http://<host>:<port>/MotionMark/developer.html?test-interval=30&display=minimal&tiles=big&controller=ramp&frame-rate=60&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&raptor=true&oskey={platform}"

26
testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-desktop.ini
View file

@ -1,26 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# motionmark benchmark for firefox and chromium distributions
[DEFAULT]
alert_threshold = 2.0
apps = firefox, chrome, chromium, safari
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_gecko_profiler = true
expose_chrome_trace = true
lower_is_better = false
owner = :jgilbert and Graphics(gfx) Team
page_cycles = 5
page_timeout = 600000
test_url = http://<host>:<port>/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&raptor=true&oskey={platform}
type = benchmark
unit = score
[motionmark-htmlsuite]
[motionmark-htmlsuite-ramp]
apps = firefox, chrome, chromium, safari, custom-car
test_url = http://<host>:<port>/MotionMark/developer.html?test-interval=30&display=minimal&tiles=big&controller=ramp&frame-rate=60&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&raptor=true&oskey={platform}

20
testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-desktop.toml Normal file
View file

@ -0,0 +1,20 @@
[DEFAULT]
alert_threshold = 2.0
apps = "firefox, chrome, chromium, safari"
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_gecko_profiler = true
expose_chrome_trace = true
lower_is_better = false
owner = ":jgilbert and Graphics(gfx) Team"
page_cycles = 5
page_timeout = 600000
test_url = "http://<host>:<port>/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&raptor=true&oskey={platform}"
type = "benchmark"
unit = "score"
["motionmark-htmlsuite"]
["motionmark-htmlsuite-ramp"]
apps = "firefox, chrome, chromium, safari, custom-car"
test_url = "http://<host>:<port>/MotionMark/developer.html?test-interval=30&display=minimal&tiles=big&controller=ramp&frame-rate=60&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&raptor=true&oskey={platform}"

25
testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-mobile.ini
View file

@ -1,25 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# motionmark benchmark for firefox and chromium distributions on mobile
[DEFAULT]
alert_threshold = 2.0
apps = fenix, chrome-m, geckoview
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_gecko_profiler = true
expose_chrome_trace = true
lower_is_better = false
owner = :jgilbert and Graphics(gfx) Team
page_cycles = 5
page_timeout = 600000
test_url = http://<host>:<port>/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&raptor=true&oskey={platform}
type = benchmark
unit = score
[motionmark-htmlsuite]
[motionmark-htmlsuite-ramp]
test_url = http://<host>:<port>/MotionMark/developer.html?test-interval=30&display=minimal&tiles=big&controller=ramp&frame-rate=60&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&raptor=true&oskey={platform}

19
testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite-mobile.toml Normal file
View file

@ -0,0 +1,19 @@
[DEFAULT]
alert_threshold = 2.0
apps = "fenix, chrome-m, geckoview"
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_gecko_profiler = true
expose_chrome_trace = true
lower_is_better = false
owner = ":jgilbert and Graphics(gfx) Team"
page_cycles = 5
page_timeout = 600000
test_url = "http://<host>:<port>/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&raptor=true&oskey={platform}"
type = "benchmark"
unit = "score"
["motionmark-htmlsuite"]
["motionmark-htmlsuite-ramp"]
test_url = "http://<host>:<port>/MotionMark/developer.html?test-interval=30&display=minimal&tiles=big&controller=ramp&frame-rate=60&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&raptor=true&oskey={platform}"

34
testing/raptor/raptor/tests/benchmarks/speedometer-desktop.ini
View file

@ -1,34 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# speedometer benchmark for desktop browsers
[DEFAULT]
alert_threshold = 2.0
apps = firefox, chrome, chromium, safari, custom-car
gecko_profile_entries = 14000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = false
owner = SpiderMonkey Team
page_cycles = 5
page_timeout = 600000
subtest_lower_is_better = true
subtest_unit = ms
test_url = http://<host>:<port>/Speedometer/index.html?raptor
type = benchmark
unit = score
[speedometer]
[speedometer3]
browser_cycles = 5
owner = Performance Team
cold = true
repository = https://github.com/WebKit/Speedometer
repository_revision = 4173e4d8a945ebe7080eb43188f0441a19d0c70a
host_from_parent = false
support_class = speedometer3.py
test_script = speedometer3.js
test_url = http://<host>:<port>/index.html

28
testing/raptor/raptor/tests/benchmarks/speedometer-desktop.toml Normal file
View file

@ -0,0 +1,28 @@
[DEFAULT]
alert_threshold = 2.0
apps = "firefox, chrome, chromium, safari, custom-car"
gecko_profile_entries = 14000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = false
owner = "SpiderMonkey Team"
page_cycles = 5
page_timeout = 600000
subtest_lower_is_better = true
subtest_unit = "ms"
test_url = "http://<host>:<port>/Speedometer/index.html?raptor"
type = "benchmark"
unit = "score"
["speedometer"]
["speedometer3"]
browser_cycles = 5
owner = "Performance Team"
cold = true
repository = "https://github.com/WebKit/Speedometer"
repository_revision = "4173e4d8a945ebe7080eb43188f0441a19d0c70a"
host_from_parent = false
support_class = "speedometer3.py"
test_script = "speedometer3.js"
test_url = "http://<host>:<port>/index.html"

34
testing/raptor/raptor/tests/benchmarks/speedometer-mobile.ini
View file

@ -1,34 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# speedometer benchmark for mobile browsers
[DEFAULT]
alert_threshold = 2.0
apps = fenix, geckoview, refbrow, chrome-m, cstm-car-m
gecko_profile_entries = 14000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = false
owner = SpiderMonkey Team
page_cycles = 5
page_timeout = 420000
subtest_lower_is_better = true
subtest_unit = ms
test_url = http://<host>:<port>/Speedometer/index.html?raptor
type = benchmark
unit = score
[speedometer]
[speedometer3]
browser_cycles = 5
owner = Performance Team
cold = true
repository = https://github.com/WebKit/Speedometer
repository_revision = 4173e4d8a945ebe7080eb43188f0441a19d0c70a
host_from_parent = false
support_class = speedometer3.py
test_script = speedometer3.js
test_url = http://<host>:<port>/index.html

28
testing/raptor/raptor/tests/benchmarks/speedometer-mobile.toml Normal file
View file

@ -0,0 +1,28 @@
[DEFAULT]
alert_threshold = 2.0
apps = "fenix, geckoview, refbrow, chrome-m, cstm-car-m"
gecko_profile_entries = 14000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = false
owner = "SpiderMonkey Team"
page_cycles = 5
page_timeout = 420000
subtest_lower_is_better = true
subtest_unit = "ms"
test_url = "http://<host>:<port>/Speedometer/index.html?raptor"
type = "benchmark"
unit = "score"
["speedometer"]
["speedometer3"]
browser_cycles = 5
owner = "Performance Team"
cold = true
repository = "https://github.com/WebKit/Speedometer"
repository_revision = "4173e4d8a945ebe7080eb43188f0441a19d0c70a"
host_from_parent = false
support_class = "speedometer3.py"
test_script = "speedometer3.js"
test_url = "http://<host>:<port>/index.html"

23
testing/raptor/raptor/tests/benchmarks/stylebench.ini
View file

@ -1,23 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# speedometer benchmark for firefox and chromium distributions
[DEFAULT]
alert_threshold = 2.0
apps = firefox, chrome, chromium, safari
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = false
owner = :emelio and Layout Team
page_cycles = 5
page_timeout = 140000
subtest_lower_is_better = true
subtest_unit = ms
test_url = http://<host>:<port>/StyleBench/index.html?raptor
type = benchmark
unit = score
[stylebench]

17
testing/raptor/raptor/tests/benchmarks/stylebench.toml Normal file
View file

@ -0,0 +1,17 @@
[DEFAULT]
alert_threshold = 2.0
apps = "firefox, chrome, chromium, safari"
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = false
owner = ":emelio and Layout Team"
page_cycles = 5
page_timeout = 140000
subtest_lower_is_better = true
subtest_unit = "ms"
test_url = "http://<host>:<port>/StyleBench/index.html?raptor"
type = "benchmark"
unit = "score"
["stylebench"]

21
testing/raptor/raptor/tests/benchmarks/sunspider.ini
View file

@ -1,21 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# sunspider benchmark for firefox and chromium distributions
[DEFAULT]
alert_threshold = 2.0
apps = firefox, chrome, chromium, safari
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
owner = :jandem and SpiderMonkey Team
page_cycles = 5
page_timeout = 55000
test_url = http://<host>:<port>/SunSpider/sunspider-1.0.1/sunspider-1.0.1/driver.html?raptor
type = benchmark
unit = ms
[sunspider]

15
testing/raptor/raptor/tests/benchmarks/sunspider.toml Normal file
View file

@ -0,0 +1,15 @@
[DEFAULT]
alert_threshold = 2.0
apps = "firefox, chrome, chromium, safari"
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
owner = ":jandem and SpiderMonkey Team"
page_cycles = 5
page_timeout = 55000
test_url = "http://<host>:<port>/SunSpider/sunspider-1.0.1/sunspider-1.0.1/driver.html?raptor"
type = "benchmark"
unit = "ms"
["sunspider"]

27
testing/raptor/raptor/tests/benchmarks/twitch-animation.ini
View file

@ -1,27 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Matrix-react-bench benchmark for firefox and chromium distributions
[DEFAULT]
alert_threshold = 2.0
apps = firefox
gecko_profile_entries = 14000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
owner = :jrmuizel
page_cycles = 1
page_timeout = 2000000
subtest_lower_is_better = true
subtest_unit = ms
test_url = http://<host>:<port>/index.html
type = benchmark
unit = ms
perfstats = true
[twitch-animation]
repository = https://github.com/mozilla/perf-automation
repository_revision = 61332db584026b73e37066d717a162825408c36b
repository_path = benchmarks/twitch-animation

21
testing/raptor/raptor/tests/benchmarks/twitch-animation.toml Normal file
View file

@ -0,0 +1,21 @@
[DEFAULT]
alert_threshold = 2.0
apps = "firefox"
gecko_profile_entries = 14000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
owner = ":jrmuizel"
page_cycles = 1
page_timeout = 2000000
subtest_lower_is_better = true
subtest_unit = "ms"
test_url = "http://<host>:<port>/index.html"
type = "benchmark"
unit = "ms"
perfstats = true
["twitch-animation"]
repository = "https://github.com/mozilla/perf-automation"
repository_revision = "61332db584026b73e37066d717a162825408c36b"
repository_path = "benchmarks/twitch-animation"

24
testing/raptor/raptor/tests/benchmarks/unity-webgl-desktop.ini
View file

@ -1,24 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# unity-webgl benchmark for firefox and chromium distributions
[DEFAULT]
alert_threshold = 2.0
apps = firefox, chrome, chromium, safari
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = false
owner = :jgilbert and Graphics(gfx) Team
page_cycles = 5
page_timeout = 420000 # 7 mins
test_url = http://<host>:<port>/index.html?raptor
type = benchmark
unit = score
[unity-webgl]
repository = https://github.com/mozilla/perf-automation
repository_revision = 61332db584026b73e37066d717a162825408c36b
repository_path = benchmarks/unity-webgl

18
testing/raptor/raptor/tests/benchmarks/unity-webgl-desktop.toml Normal file
View file

@ -0,0 +1,18 @@
[DEFAULT]
alert_threshold = 2.0
apps = "firefox, chrome, chromium, safari"
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = false
owner = ":jgilbert and Graphics(gfx) Team"
page_cycles = 5
page_timeout = 420000 # 7 mins
test_url = "http://<host>:<port>/index.html?raptor"
type = "benchmark"
unit = "score"
["unity-webgl"]
repository = "https://github.com/mozilla/perf-automation"
repository_revision = "61332db584026b73e37066d717a162825408c36b"
repository_path = "benchmarks/unity-webgl"

24
testing/raptor/raptor/tests/benchmarks/unity-webgl-mobile.ini
View file

@ -1,24 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# unity-webgl benchmark for mobile browsers
[DEFAULT]
alert_threshold = 2.0
apps = geckoview, refbrow, fenix, chrome-m
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = false
owner = :jgilbert and Graphics(gfx) Team
page_cycles = 1
page_timeout = 420000 # 7 mins
test_url = http://<host>:<port>/index.html?raptor
type = benchmark
unit = score
[unity-webgl]
repository = https://github.com/mozilla/perf-automation
repository_revision = 61332db584026b73e37066d717a162825408c36b
repository_path = benchmarks/unity-webgl

18
testing/raptor/raptor/tests/benchmarks/unity-webgl-mobile.toml Normal file
View file

@ -0,0 +1,18 @@
[DEFAULT]
alert_threshold = 2.0
apps = "geckoview, refbrow, fenix, chrome-m"
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = false
owner = ":jgilbert and Graphics(gfx) Team"
page_cycles = 1
page_timeout = 420000 # 7 mins
test_url = "http://<host>:<port>/index.html?raptor"
type = "benchmark"
unit = "score"
["unity-webgl"]
repository = "https://github.com/mozilla/perf-automation"
repository_revision = "61332db584026b73e37066d717a162825408c36b"
repository_path = "benchmarks/unity-webgl"

24
testing/raptor/raptor/tests/benchmarks/wasm-godot-baseline.ini
View file

@ -1,24 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Wasm-godot benchmark for firefox and chromium distributions
[DEFAULT]
alert_threshold = 2.0
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
newtab_per_cycle = true
owner = :lth and SpiderMonkey Team
page_cycles = 5
page_timeout = 120000
test_url = http://localhost:<port>/wasm-godot/index.html
type = benchmark
unit = ms
[wasm-godot-baseline]
apps = firefox
preferences = {"javascript.options.wasm_baselinejit": true,
"javascript.options.wasm_optimizingjit": false}

20
testing/raptor/raptor/tests/benchmarks/wasm-godot-baseline.toml Normal file
View file

@ -0,0 +1,20 @@
[DEFAULT]
alert_threshold = 2.0
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
newtab_per_cycle = true
owner = ":lth and SpiderMonkey Team"
page_cycles = 5
page_timeout = 120000
test_url = "http://localhost:<port>/wasm-godot/index.html"
type = "benchmark"
unit = "ms"
["wasm-godot-baseline"]
apps = "firefox"
preferences = [
"javascript.options.wasm_baselinejit=true",
"javascript.options.wasm_optimizingjit=false",
]

24
testing/raptor/raptor/tests/benchmarks/wasm-godot-optimizing.ini
View file

@ -1,24 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Wasm-godot benchmark for firefox and chromium distributions
[DEFAULT]
alert_threshold = 2.0
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
newtab_per_cycle = true
owner = :lth and SpiderMonkey Team
page_cycles = 5
page_timeout = 120000
test_url = http://localhost:<port>/wasm-godot/index.html
type = benchmark
unit = ms
[wasm-godot-optimizing]
apps = firefox
preferences = {"javascript.options.wasm_baselinejit": false,
"javascript.options.wasm_optimizingjit": true}

20
testing/raptor/raptor/tests/benchmarks/wasm-godot-optimizing.toml Normal file
View file

@ -0,0 +1,20 @@
[DEFAULT]
alert_threshold = 2.0
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
newtab_per_cycle = true
owner = ":lth and SpiderMonkey Team"
page_cycles = 5
page_timeout = 120000
test_url = "http://localhost:<port>/wasm-godot/index.html"
type = "benchmark"
unit = "ms"
["wasm-godot-optimizing"]
apps = "firefox"
preferences = [
"javascript.options.wasm_baselinejit=false",
"javascript.options.wasm_optimizingjit=true",
]

22
testing/raptor/raptor/tests/benchmarks/wasm-godot.ini
View file

@ -1,22 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Wasm-godot benchmark for firefox and chromium distributions
[DEFAULT]
alert_threshold = 2.0
apps = firefox, chrome, chromium, safari
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
newtab_per_cycle = true
owner = :lth and SpiderMonkey Team
page_cycles = 5
page_timeout = 120000
test_url = http://localhost:<port>/wasm-godot/index.html
type = benchmark
unit = ms
[wasm-godot]

16
testing/raptor/raptor/tests/benchmarks/wasm-godot.toml Normal file
View file

@ -0,0 +1,16 @@
[DEFAULT]
alert_threshold = 2.0
apps = "firefox, chrome, chromium, safari"
gecko_profile_entries = 8000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
newtab_per_cycle = true
owner = ":lth and SpiderMonkey Team"
page_cycles = 5
page_timeout = 120000
test_url = "http://localhost:<port>/wasm-godot/index.html"
type = "benchmark"
unit = "ms"
["wasm-godot"]

26
testing/raptor/raptor/tests/benchmarks/wasm-misc-baseline.ini
View file

@ -1,26 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Wasm-Misc benchmark for firefox and chromium distributions
[DEFAULT]
alert_threshold = 2.0
gecko_profile_entries = 4000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
owner = :lth and SpiderMonkey Team
page_cycles = 5
page_timeout = 1200000
test_url = http://<host>:<port>/index.html?raptor
type = benchmark
unit = ms
[wasm-misc-baseline]
apps = firefox
preferences = {"javascript.options.wasm_baselinejit": true,
"javascript.options.wasm_optimizingjit": false}
repository = https://github.com/mozilla/perf-automation
repository_revision = 61332db584026b73e37066d717a162825408c36b
repository_path = benchmarks/wasm-misc

22
testing/raptor/raptor/tests/benchmarks/wasm-misc-baseline.toml Normal file
View file

@ -0,0 +1,22 @@
[DEFAULT]
alert_threshold = 2.0
gecko_profile_entries = 4000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
owner = ":lth and SpiderMonkey Team"
page_cycles = 5
page_timeout = 1200000
test_url = "http://<host>:<port>/index.html?raptor"
type = "benchmark"
unit = "ms"
["wasm-misc-baseline"]
apps = "firefox"
preferences = [
"javascript.options.wasm_baselinejit=true",
"javascript.options.wasm_optimizingjit=false",
]
repository = "https://github.com/mozilla/perf-automation"
repository_revision = "61332db584026b73e37066d717a162825408c36b"
repository_path = "benchmarks/wasm-misc"

26
testing/raptor/raptor/tests/benchmarks/wasm-misc-optimizing.ini
View file

@ -1,26 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Wasm-Misc benchmark for firefox and chromium distributions
[DEFAULT]
alert_threshold = 2.0
gecko_profile_entries = 4000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
owner = :lth and SpiderMonkey Team
page_cycles = 5
page_timeout = 1200000
test_url = http://<host>:<port>/index.html?raptor
type = benchmark
unit = ms
[wasm-misc-optimizing]
apps = firefox
preferences = {"javascript.options.wasm_baselinejit": false,
"javascript.options.wasm_optimizingjit": true}
repository = https://github.com/mozilla/perf-automation
repository_revision = 61332db584026b73e37066d717a162825408c36b
repository_path = benchmarks/wasm-misc

22
testing/raptor/raptor/tests/benchmarks/wasm-misc-optimizing.toml Normal file
View file

@ -0,0 +1,22 @@
[DEFAULT]
alert_threshold = 2.0
gecko_profile_entries = 4000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
owner = ":lth and SpiderMonkey Team"
page_cycles = 5
page_timeout = 1200000
test_url = "http://<host>:<port>/index.html?raptor"
type = "benchmark"
unit = "ms"
["wasm-misc-optimizing"]
apps = "firefox"
preferences = [
"javascript.options.wasm_baselinejit=false",
"javascript.options.wasm_optimizingjit=true",
]
repository = "https://github.com/mozilla/perf-automation"
repository_revision = "61332db584026b73e37066d717a162825408c36b"
repository_path = "benchmarks/wasm-misc"

24
testing/raptor/raptor/tests/benchmarks/wasm-misc.ini
View file

@ -1,24 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Wasm-Misc benchmark for firefox and chromium distributions
[DEFAULT]
alert_threshold = 2.0
apps = firefox, chrome, chromium
gecko_profile_entries = 4000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
owner = :lth and SpiderMonkey Team
page_cycles = 5
page_timeout = 1200000
test_url = http://<host>:<port>/index.html?raptor
type = benchmark
unit = ms
[wasm-misc]
repository = https://github.com/mozilla/perf-automation
repository_revision = 61332db584026b73e37066d717a162825408c36b
repository_path = benchmarks/wasm-misc

18
testing/raptor/raptor/tests/benchmarks/wasm-misc.toml Normal file
View file

@ -0,0 +1,18 @@
[DEFAULT]
alert_threshold = 2.0
apps = "firefox, chrome, chromium"
gecko_profile_entries = 4000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
owner = ":lth and SpiderMonkey Team"
page_cycles = 5
page_timeout = 1200000
test_url = "http://<host>:<port>/index.html?raptor"
type = "benchmark"
unit = "ms"
["wasm-misc"]
repository = "https://github.com/mozilla/perf-automation"
repository_revision = "61332db584026b73e37066d717a162825408c36b"
repository_path = "benchmarks/wasm-misc"

26
testing/raptor/raptor/tests/benchmarks/webaudio.ini
View file

@ -1,26 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# speedometer benchmark for firefox and chromium distributions
[DEFAULT]
alert_threshold = 2.0
apps = firefox, chrome, chromium, safari
gecko_profile_entries = 4000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
owner = :padenot and Media Team
page_cycles = 5
page_timeout = 360000
test_url = http://<host>:<port>/webaudio/index.html?raptor
type = benchmark
unit = score
preferences = {"media.autoplay.default": 0,
"media.autoplay.ask-permission": false,
"media.autoplay.blocking_policy": 0,
"media.allowed-to-play.enabled": true,
"media.block-autoplay-until-in-foreground": false}
[webaudio]

22
testing/raptor/raptor/tests/benchmarks/webaudio.toml Normal file
View file

@ -0,0 +1,22 @@
[DEFAULT]
alert_threshold = 2.0
apps = "firefox, chrome, chromium, safari"
gecko_profile_entries = 4000000
gecko_profile_interval = 1
expose_browser_profiler = true
lower_is_better = true
owner = ":padenot and Media Team"
page_cycles = 5
page_timeout = 360000
test_url = "http://<host>:<port>/webaudio/index.html?raptor"
type = "benchmark"
unit = "score"
preferences = [
"media.autoplay.default=0",
"media.autoplay.ask-permission=false",
"media.autoplay.blocking_policy=0",
"media.allowed-to-play.enabled=true",
"media.block-autoplay-until-in-foreground=false",
]
["webaudio"]

192
testing/raptor/raptor/tests/benchmarks/youtube-playback.ini
View file

@ -1,192 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Youtube playback performance benchmark Browsertime tests
#
# Original location of source and media files:
# https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&exclude=1,2&muted=true&command=run
#
# Bug 1547717 - Cannot override autoplay preference due to GeckoRuntime Settings
# Bug 1554966 - With GeckoView there is no way yet to get around the problem that autoplay is blocked
# As such using muted playback everywhere the numbers across platforms should be closer
# Bug 1618566 - change test_url to production server, once all is done
# test_url = https://yttest.prod.mozaws.net/2019/main.html?test_type=playbackperf-hfr-test&raptor=true&exclude=1,2&muted=true&command=run
[DEFAULT]
alert_threshold = 2.0
gecko_profile_entries = 50000000
gecko_profile_interval = 1000
gecko_profile_threads = MediaPlayback
expose_browser_profiler = true
lower_is_better = true
owner = PerfTest Team
page_cycles = 1
# account for a page cycle duration of at maximum 45 minutes
page_timeout = 2700000
subtest_lower_is_better = true
subtest_unit = score
type = benchmark
unit = score
use_live_sites = true
preferences = {"media.autoplay.default": 0,
"media.autoplay.ask-permission": false,
"media.autoplay.blocking_policy": 0,
"media.allowed-to-play.enabled": true,
"media.block-autoplay-until-in-foreground": false}
[youtube-playback]
alert_on = H264.1080p30@1X_dropped_frames,
H264.1080p60@1X_dropped_frames,
H264.1440p30@1X_dropped_frames,
H264.144p15@1X_dropped_frames,
H264.2160p30@1X_dropped_frames,
H264.240p30@1X_dropped_frames,
H264.360p30@1X_dropped_frames,
H264.480p30@1X_dropped_frames,
H264.720p30@1X_dropped_frames,
H264.720p60@1X_dropped_frames,
VP9.1080p30@1X_dropped_frames,
VP9.1080p60@1X_dropped_frames,
VP9.1440p30@1X_dropped_frames,
VP9.1440p60@1X_dropped_frames,
VP9.144p30@1X_dropped_frames,
VP9.2160p30@1X_dropped_frames,
VP9.2160p60@1X_dropped_frames,
VP9.240p30@1X_dropped_frames,
VP9.360p30@1X_dropped_frames,
VP9.480p30@1X_dropped_frames,
VP9.720p30@1X_dropped_frames,
VP9.720p60@1X_dropped_frames
apps = firefox, geckoview, fenix,refbrow, chrome
test_url = http://yttest.prod.mozaws.net/2019/main.html?test_type=playbackperf-test&raptor=true&command=run&exclude=1,2&muted=true
[youtube-playback-av1-sfr]
apps = firefox, geckoview, fenix, refbrow, chrome
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-av1-test&raptor=true&exclude=1,2&muted=true&command=run
[youtube-playback-h264-1080p30]
apps = firefox
page_cycles = 20
gather_cpuTime = true
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-h264-test&tests=18&raptor=true&muted=true&command=run&exclude=1,2
[youtube-playback-h264-1080p60]
apps = firefox
page_cycles = 20
gather_cpuTime = true
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=46&muted=true&command=run&exclude=1,2
[youtube-playback-h264-full-1080p30]
apps = firefox
page_cycles = 20
gather_cpuTime = true
preferences = {"media.autoplay.default": 0,
"media.autoplay.ask-permission": false,
"media.autoplay.blocking_policy": 0,
"media.allowed-to-play.enabled": true,
"media.block-autoplay-until-in-foreground": false,
"full-screen-api.allow-trusted-requests-only": false,
"full-screen-api.warning.timeout": 0}
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-h264-test&tests=18&raptor=true&muted=true&command=run&fullscreen=true&exclude=1,2
[youtube-playback-h264-full-1080p60]
apps = firefox
page_cycles = 20
gather_cpuTime = true
preferences = {"media.autoplay.default": 0,
"media.autoplay.ask-permission": false,
"media.autoplay.blocking_policy": 0,
"media.allowed-to-play.enabled": true,
"media.block-autoplay-until-in-foreground": false,
"full-screen-api.allow-trusted-requests-only": false,
"full-screen-api.warning.timeout": 0}
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=46&muted=true&command=run&fullscreen=true&exclude=1,2
[youtube-playback-h264-sfr]
apps = firefox, geckoview, fenix, refbrow, chrome
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-h264-test&raptor=true&exclude=1,2&muted=true&command=run
[youtube-playback-hfr]
alert_on = H2641080p60fps@1X_dropped_frames,
H264720p60fps@1X_dropped_frames
apps = firefox, geckoview, fenix, refbrow, chrome
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&exclude=1,2&muted=true&command=run
[youtube-playback-v9-1080p30]
apps = firefox
page_cycles = 20
gather_cpuTime = true
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&tests=18&muted=true&command=run&exclude=1,2
[youtube-playback-v9-1080p60]
apps = firefox
page_cycles = 20
gather_cpuTime = true
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=14&muted=true&command=run&exclude=1,2
[youtube-playback-v9-full-1080p30]
apps = firefox
page_cycles = 20
gather_cpuTime = true
preferences = {"media.autoplay.default": 0,
"media.autoplay.ask-permission": false,
"media.autoplay.blocking_policy": 0,
"media.allowed-to-play.enabled": true,
"media.block-autoplay-until-in-foreground": false,
"full-screen-api.allow-trusted-requests-only": false,
"full-screen-api.warning.timeout": 0}
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&tests=18&muted=true&command=run&fullscreen=true&exclude=1,2
[youtube-playback-v9-full-1080p60]
apps = firefox
page_cycles = 20
gather_cpuTime = true
preferences = {"media.autoplay.default": 0,
"media.autoplay.ask-permission": false,
"media.autoplay.blocking_policy": 0,
"media.allowed-to-play.enabled": true,
"media.block-autoplay-until-in-foreground": false,
"full-screen-api.allow-trusted-requests-only": false,
"full-screen-api.warning.timeout": 0}
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=14&muted=true&command=run&fullscreen=true&exclude=1,2
[youtube-playback-vp9-sfr]
apps = firefox, geckoview, fenix, refbrow, chrome
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&exclude=1,2&muted=true&command=run
[youtube-playback-widevine-h264-sfr]
apps = firefox, geckoview, fenix, refbrow, chrome
preferences = {"media.autoplay.default": 0,
"media.autoplay.ask-permission": false,
"media.autoplay.blocking_policy": 0,
"media.allowed-to-play.enabled": true,
"media.block-autoplay-until-in-foreground": false,
"media.eme.enabled": true,
"media.gmp-manager.updateEnabled": true,
"media.eme.require-app-approval": false}
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-widevine-sfr-h264-test&raptor=true&exclude=1,2&muted=true&command=run
[youtube-playback-widevine-hfr]
apps = firefox, geckoview, fenix, refbrow, chrome
preferences = {"media.autoplay.default": 0,
"media.autoplay.ask-permission": false,
"media.autoplay.blocking_policy": 0,
"media.allowed-to-play.enabled": true,
"media.block-autoplay-until-in-foreground": false,
"media.eme.enabled": true,
"media.gmp-manager.updateEnabled": true,
"media.eme.require-app-approval": false}
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-widevine-hfr-test&raptor=true&exclude=1,2&muted=true&command=run
[youtube-playback-widevine-vp9-sfr]
apps = firefox, geckoview, fenix, refbrow, chrome
preferences = {"media.autoplay.default": 0,
"media.autoplay.ask-permission": false,
"media.autoplay.blocking_policy": 0,
"media.allowed-to-play.enabled": true,
"media.block-autoplay-until-in-foreground": false,
"media.eme.enabled": true,
"media.gmp-manager.updateEnabled": true,
"media.eme.require-app-approval": false}
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-widevine-sfr-vp9-test&raptor=true&exclude=1,2&muted=true&command=run

197
testing/raptor/raptor/tests/benchmarks/youtube-playback.toml Normal file
View file

@ -0,0 +1,197 @@
[DEFAULT]
alert_threshold = 2.0
gecko_profile_entries = 50000000
gecko_profile_interval = 1000
gecko_profile_threads = "MediaPlayback"
expose_browser_profiler = true
lower_is_better = true
owner = "PerfTest Team"
page_cycles = 1
# account for a page cycle duration of at maximum 45 minutes
page_timeout = 2700000
subtest_lower_is_better = true
subtest_unit = "score"
type = "benchmark"
unit = "score"
use_live_sites = true
preferences = [
"media.autoplay.default=0",
"media.autoplay.ask-permission=false",
"media.autoplay.blocking_policy=0",
"media.allowed-to-play.enabled=true",
"media.block-autoplay-until-in-foreground=false",
]
["youtube-playback"]
alert_on = [
"H264.1080p30@1X_dropped_frames",
"H264.1080p60@1X_dropped_frames",
"H264.1440p30@1X_dropped_frames",
"H264.144p15@1X_dropped_frames",
"H264.2160p30@1X_dropped_frames",
"H264.240p30@1X_dropped_frames",
"H264.360p30@1X_dropped_frames",
"H264.480p30@1X_dropped_frames",
"H264.720p30@1X_dropped_frames",
"H264.720p60@1X_dropped_frames",
"VP9.1080p30@1X_dropped_frames",
"VP9.1080p60@1X_dropped_frames",
"VP9.1440p30@1X_dropped_frames",
"VP9.1440p60@1X_dropped_frames",
"VP9.144p30@1X_dropped_frames",
"VP9.2160p30@1X_dropped_frames",
"VP9.2160p60@1X_dropped_frames",
"VP9.240p30@1X_dropped_frames",
"VP9.360p30@1X_dropped_frames",
"VP9.480p30@1X_dropped_frames",
"VP9.720p30@1X_dropped_frames",
"VP9.720p60@1X_dropped_frames",
]
apps = "firefox, geckoview, fenix, refbrow, chrome"
test_url = "http://yttest.prod.mozaws.net/2019/main.html?test_type=playbackperf-test&raptor=true&command=run&exclude=1,2&muted=true"
["youtube-playback-av1-sfr"]
apps = "firefox, geckoview, fenix, refbrow, chrome"
test_url = "https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-av1-test&raptor=true&exclude=1,2&muted=true&command=run"
["youtube-playback-h264-1080p30"]
apps = "firefox"
page_cycles = 20
gather_cpuTime = true
test_url = "https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-h264-test&tests=18&raptor=true&muted=true&command=run&exclude=1,2"
["youtube-playback-h264-1080p60"]
apps = "firefox"
page_cycles = 20
gather_cpuTime = true
test_url = "https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=46&muted=true&command=run&exclude=1,2"
["youtube-playback-h264-full-1080p30"]
apps = "firefox"
page_cycles = 20
gather_cpuTime = true
preferences = [
"media.autoplay.default=0",
"media.autoplay.ask-permission=false",
"media.autoplay.blocking_policy=0",
"media.allowed-to-play.enabled=true",
"media.block-autoplay-until-in-foreground=false",
"full-screen-api.allow-trusted-requests-only=false",
"full-screen-api.warning.timeout=0",
]
test_url = "https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-h264-test&tests=18&raptor=true&muted=true&command=run&fullscreen=true&exclude=1,2"
["youtube-playback-h264-full-1080p60"]
apps = "firefox"
page_cycles = 20
gather_cpuTime = true
preferences = [
"media.autoplay.default=0",
"media.autoplay.ask-permission=false",
"media.autoplay.blocking_policy=0",
"media.allowed-to-play.enabled=true",
"media.block-autoplay-until-in-foreground=false",
"full-screen-api.allow-trusted-requests-only=false",
"full-screen-api.warning.timeout=0",
]
test_url = "https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=46&muted=true&command=run&fullscreen=true&exclude=1,2"
["youtube-playback-h264-sfr"]
apps = "firefox, geckoview, fenix, refbrow, chrome"
test_url = "https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-h264-test&raptor=true&exclude=1,2&muted=true&command=run"
["youtube-playback-hfr"]
alert_on = [
"H2641080p60fps@1X_dropped_frames",
"H264720p60fps@1X_dropped_frames",
]
apps = "firefox, geckoview, fenix, refbrow, chrome"
test_url = "https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&exclude=1,2&muted=true&command=run"
["youtube-playback-v9-1080p30"]
apps = "firefox"
page_cycles = 20
gather_cpuTime = true
test_url = "https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&tests=18&muted=true&command=run&exclude=1,2"
["youtube-playback-v9-1080p60"]
apps = "firefox"
page_cycles = 20
gather_cpuTime = true
test_url = "https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=14&muted=true&command=run&exclude=1,2"
["youtube-playback-v9-full-1080p30"]
apps = "firefox"
page_cycles = 20
gather_cpuTime = true
preferences = [
"media.autoplay.default=0",
"media.autoplay.ask-permission=false",
"media.autoplay.blocking_policy=0",
"media.allowed-to-play.enabled=true",
"media.block-autoplay-until-in-foreground=false",
"full-screen-api.allow-trusted-requests-only=false",
"full-screen-api.warning.timeout=0",
]
test_url = "https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&tests=18&muted=true&command=run&fullscreen=true&exclude=1,2"
["youtube-playback-v9-full-1080p60"]
apps = "firefox"
page_cycles = 20
gather_cpuTime = true
preferences = [
"media.autoplay.default=0",
"media.autoplay.ask-permission=false",
"media.autoplay.blocking_policy=0",
"media.allowed-to-play.enabled=true",
"media.block-autoplay-until-in-foreground=false",
"full-screen-api.allow-trusted-requests-only=false",
"full-screen-api.warning.timeout=0",
]
test_url = "https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=14&muted=true&command=run&fullscreen=true&exclude=1,2"
["youtube-playback-vp9-sfr"]
apps = "firefox, geckoview, fenix, refbrow, chrome"
test_url = "https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&exclude=1,2&muted=true&command=run"
["youtube-playback-widevine-h264-sfr"]
apps = "firefox, geckoview, fenix, refbrow, chrome"
preferences = [
"media.autoplay.default=0",
"media.autoplay.ask-permission=false",
"media.autoplay.blocking_policy=0",
"media.allowed-to-play.enabled=true",
"media.block-autoplay-until-in-foreground=false",
"media.eme.enabled=true",
"media.gmp-manager.updateEnabled=true",
"media.eme.require-app-approval=false",
]
test_url = "https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-widevine-sfr-h264-test&raptor=true&exclude=1,2&muted=true&command=run"
["youtube-playback-widevine-hfr"]
apps = "firefox, geckoview, fenix, refbrow, chrome"
preferences = [
"media.autoplay.default=0",
"media.autoplay.ask-permission=false",
"media.autoplay.blocking_policy=0",
"media.allowed-to-play.enabled=true",
"media.block-autoplay-until-in-foreground=false",
"media.eme.enabled=true",
"media.gmp-manager.updateEnabled=true",
"media.eme.require-app-approval=false",
]
test_url = "https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-widevine-hfr-test&raptor=true&exclude=1,2&muted=true&command=run"
["youtube-playback-widevine-vp9-sfr"]
apps = "firefox, geckoview, fenix, refbrow, chrome"
preferences = [
"media.autoplay.default=0",
"media.autoplay.ask-permission=false",
"media.autoplay.blocking_policy=0",
"media.allowed-to-play.enabled=true",
"media.block-autoplay-until-in-foreground=false",
"media.eme.enabled=true",
"media.gmp-manager.updateEnabled=true",
"media.eme.require-app-approval=false",
]
test_url = "https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-widevine-sfr-vp9-test&raptor=true&exclude=1,2&muted=true&command=run"

29
testing/raptor/raptor/tests/custom/browsertime-custom.ini
View file

@ -1,29 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
[DEFAULT]
alert_on = fcp, loadtime
alert_threshold = 2.0
apps = firefox
browser_cycles = 1
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
measure = fnbpaint, fcp, dcf, loadtime
owner = PerfTest Team
page_cycles = 1
page_timeout = 60000
playback = mitmproxy
playback_version = 8.1.1
type = pageload
unit = ms
use_live_sites = true
# Use this to run a custom browsertime test locally (with a custom url).
# Essentially, this is "vanilla" browsertime in a raptor wrapper.
[browsertime]
playback_pageset_manifest = null.manifest
test_script = None
test_url = None

25
testing/raptor/raptor/tests/custom/browsertime-custom.toml Normal file
View file

@ -0,0 +1,25 @@
[DEFAULT]
alert_on = "fcp, loadtime"
alert_threshold = 2.0
apps = "firefox"
browser_cycles = 1
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
measure = "fnbpaint, fcp, dcf, loadtime"
owner = "PerfTest Team"
page_cycles = 1
page_timeout = 60000
playback = "mitmproxy"
playback_version = "8.1.1"
type = "pageload"
unit = "ms"
use_live_sites = true
# Use this to run a custom browsertime test locally (with a custom url).
# Essentially, this is "vanilla" browsertime in a raptor wrapper.
["browsertime"]
playback_pageset_manifest = "null.manifest"
test_script = "None"
test_url = "None"

29
testing/raptor/raptor/tests/custom/browsertime-grandprix.ini
View file

@ -1,29 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
[DEFAULT]
apps = firefox, chrome, chromium, safari
alert_threshold = 2.0
browser_cycles = 1
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
owner = PerfTest Team
page_cycles = 1
page_timeout = 1800000
output_timeout = 2000000
subtest_unit = ms
type = pageload
unit = score
use_live_sites = true
# grandprix custom benchmark test
[grandprix]
browsertime_args = --browsertime.grandprix_iterations=25
custom_data = true
subtest_name_filters = tests/,iterations/
test_script = grandprix.js
test_summary = flatten
test_url = None

25
testing/raptor/raptor/tests/custom/browsertime-grandprix.toml Normal file
View file

@ -0,0 +1,25 @@
[DEFAULT]
apps = "firefox, chrome, chromium, safari"
alert_threshold = 2.0
browser_cycles = 1
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
owner = "PerfTest Team"
page_cycles = 1
page_timeout = 1800000
output_timeout = 2000000
subtest_unit = "ms"
type = "pageload"
unit = "score"
use_live_sites = true
# grandprix custom benchmark test
["grandprix"]
browsertime_args = "--browsertime.grandprix_iterations=25"
custom_data = true
subtest_name_filters = "tests/,iterations/"
test_script = "grandprix.js"
test_summary = "flatten"
test_url = "None"

83
testing/raptor/raptor/tests/custom/browsertime-indexeddb.ini
View file

@ -1,83 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
[DEFAULT]
apps = firefox, chrome, chromium, safari
alert_threshold = 2.0
browser_cycles = 1
custom_data = true
gecko_profile_entries = 131072000
gecko_profile_features = js,stackwalk,cpu
gecko_profile_threads = GeckoMain,DOM Worker,IndexedDB
lower_is_better = true
measure = cpuTime
output_timeout = 2000000
owner = DOM Lifecycle, Workers and Storage Team
page_cycles = 1
page_timeout = 1800000
subtest_unit = ms
test_summary = flatten
type = pageload
unit = ms
playback = mitmproxy
playback_pageset_manifest = mitm8-linux-firefox-example.manifest
test_url = https://www.example.com
use_live_sites = false
# Meaning of the idb_write names:
# 1. Iterations - how many add/put requests are made? (k ~ few, M ~ many)
# 2. Buffer type - what kind of buffer is supplied to each add/put, Array (ar), ArrayBuffer (ab) or Blob (bl)?
# 3. Chunk size - how much data is stored in each add/put? (lower-case buffer type: small, capitalized buffer type: a lot)
# 4. Transactions - should each add/put share one big transaction (1 => atomic=1) or have its own transaction (N => atomic=0)?
[addMab1]
browsertime_args = --browsertime.iterations=16384 --browsertime.chunk_size=1024 --browsertime.buffer_type=ArrayBuffer --browsertime.atomic=1
test_script = indexeddb_write.js
[addMabN]
browsertime_args = --browsertime.iterations=16384 --browsertime.chunk_size=1024 --browsertime.buffer_type=ArrayBuffer --browsertime.atomic=0
test_script = indexeddb_write.js
[addMar1]
browsertime_args = --browsertime.iterations=16384 --browsertime.chunk_size=1024 --browsertime.buffer_type=Array --browsertime.atomic=1
test_script = indexeddb_write.js
[addMarN]
browsertime_args = --browsertime.iterations=16384 --browsertime.chunk_size=1024 --browsertime.buffer_type=Array --browsertime.atomic=0
test_script = indexeddb_write.js
[addMbl1]
browsertime_args = --browsertime.iterations=16384 --browsertime.chunk_size=1024 --browsertime.buffer_type=Blob --browsertime.atomic=1
test_script = indexeddb_write.js
[addMblN]
browsertime_args = --browsertime.iterations=16384 --browsertime.chunk_size=1024 --browsertime.buffer_type=Blob --browsertime.atomic=0
test_script = indexeddb_write.js
[addkAB1]
browsertime_args = --browsertime.iterations=128 --browsertime.chunk_size=1048576 --browsertime.buffer_type=ArrayBuffer --browsertime.atomic=1
test_script = indexeddb_write.js
[addkABN]
browsertime_args = --browsertime.iterations=128 --browsertime.chunk_size=1048576 --browsertime.buffer_type=ArrayBuffer --browsertime.atomic=0
test_script = indexeddb_write.js
[addkAR1]
browsertime_args = --browsertime.iterations=128 --browsertime.chunk_size=1048576 --browsertime.buffer_type=Array --browsertime.atomic=1
test_script = indexeddb_write.js
[addkARN]
browsertime_args = --browsertime.iterations=128 --browsertime.chunk_size=1048576 --browsertime.buffer_type=Array --browsertime.atomic=0
test_script = indexeddb_write.js
[addkBL1]
browsertime_args = --browsertime.iterations=128 --browsertime.chunk_size=1048576 --browsertime.buffer_type=Blob --browsertime.atomic=1
test_script = indexeddb_write.js
[addkBLN]
browsertime_args = --browsertime.iterations=128 --browsertime.chunk_size=1048576 --browsertime.buffer_type=Blob --browsertime.atomic=0
test_script = indexeddb_write.js
[getkeyrng]
test_script = indexeddb_getkeyrange.js

79
testing/raptor/raptor/tests/custom/browsertime-indexeddb.toml Normal file
View file

@ -0,0 +1,79 @@
[DEFAULT]
apps = "firefox, chrome, chromium, safari"
alert_threshold = 2.0
browser_cycles = 1
custom_data = true
gecko_profile_entries = 131072000
gecko_profile_features = "js,stackwalk,cpu"
gecko_profile_threads = "GeckoMain,DOM Worker,IndexedDB"
lower_is_better = true
measure = "cpuTime"
output_timeout = 2000000
owner = "DOM Lifecycle, Workers and Storage Team"
page_cycles = 1
page_timeout = 1800000
subtest_unit = "ms"
test_summary = "flatten"
type = "pageload"
unit = "ms"
playback = "mitmproxy"
playback_pageset_manifest = "mitm8-linux-firefox-example.manifest"
test_url = "https://www.example.com"
use_live_sites = false
# Meaning of the idb_write names:
# 1. Iterations - how many add/put requests are made? (k ~ few, M ~ many)
# 2. Buffer type - what kind of buffer is supplied to each add/put, Array (ar), ArrayBuffer (ab) or Blob (bl)?
# 3. Chunk size - how much data is stored in each add/put? (lower-case buffer type: small, capitalized buffer type: a lot)
# 4. Transactions - should each add/put share one big transaction (1 => atomic=1) or have its own transaction (N => atomic=0)?
["addMab1"]
browsertime_args = "--browsertime.iterations=16384 --browsertime.chunk_size=1024 --browsertime.buffer_type=ArrayBuffer --browsertime.atomic=1"
test_script = "indexeddb_write.js"
["addMabN"]
browsertime_args = "--browsertime.iterations=16384 --browsertime.chunk_size=1024 --browsertime.buffer_type=ArrayBuffer --browsertime.atomic=0"
test_script = "indexeddb_write.js"
["addMar1"]
browsertime_args = "--browsertime.iterations=16384 --browsertime.chunk_size=1024 --browsertime.buffer_type=Array --browsertime.atomic=1"
test_script = "indexeddb_write.js"
["addMarN"]
browsertime_args = "--browsertime.iterations=16384 --browsertime.chunk_size=1024 --browsertime.buffer_type=Array --browsertime.atomic=0"
test_script = "indexeddb_write.js"
["addMbl1"]
browsertime_args = "--browsertime.iterations=16384 --browsertime.chunk_size=1024 --browsertime.buffer_type=Blob --browsertime.atomic=1"
test_script = "indexeddb_write.js"
["addMblN"]
browsertime_args = "--browsertime.iterations=16384 --browsertime.chunk_size=1024 --browsertime.buffer_type=Blob --browsertime.atomic=0"
test_script = "indexeddb_write.js"
["addkAB1"]
browsertime_args = "--browsertime.iterations=128 --browsertime.chunk_size=1048576 --browsertime.buffer_type=ArrayBuffer --browsertime.atomic=1"
test_script = "indexeddb_write.js"
["addkABN"]
browsertime_args = "--browsertime.iterations=128 --browsertime.chunk_size=1048576 --browsertime.buffer_type=ArrayBuffer --browsertime.atomic=0"
test_script = "indexeddb_write.js"
["addkAR1"]
browsertime_args = "--browsertime.iterations=128 --browsertime.chunk_size=1048576 --browsertime.buffer_type=Array --browsertime.atomic=1"
test_script = "indexeddb_write.js"
["addkARN"]
browsertime_args = "--browsertime.iterations=128 --browsertime.chunk_size=1048576 --browsertime.buffer_type=Array --browsertime.atomic=0"
test_script = "indexeddb_write.js"
["addkBL1"]
browsertime_args = "--browsertime.iterations=128 --browsertime.chunk_size=1048576 --browsertime.buffer_type=Blob --browsertime.atomic=1"
test_script = "indexeddb_write.js"
["addkBLN"]
browsertime_args = "--browsertime.iterations=128 --browsertime.chunk_size=1048576 --browsertime.buffer_type=Blob --browsertime.atomic=0"
test_script = "indexeddb_write.js"
["getkeyrng"]
test_script = "indexeddb_getkeyrange.js"

32
testing/raptor/raptor/tests/custom/browsertime-process-switch.ini
View file

@ -1,32 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
[DEFAULT]
alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange
alert_threshold = 2.0
apps = firefox, chrome, chromium
browser_cycles = 25
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
owner = PerfTest Team
page_cycles = 25
page_timeout = 60000
playback = mitmproxy
playback_version = 5.1.1
type = pageload
unit = ms
use_live_sites = false
# raptor-browsertime fission process switch time test
[process-switch]
# Bug 1867507 - This test isn't producing a largestContentfulPaint
alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange
measure = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange
accept_zero_vismet = true
browsertime_args = --pageCompleteWaitTime=1000 --pageCompleteCheckInactivity=true
playback_pageset_manifest = mitm5-linux-firefox-proc-switch.manifest
test_script = process_switch.js
test_url = https://mozilla.seanfeng.dev/files/red.html,https://mozilla.pettay.fi/moztests/blue.html

28
testing/raptor/raptor/tests/custom/browsertime-process-switch.toml Normal file
View file

@ -0,0 +1,28 @@
[DEFAULT]
alert_on = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange"
alert_threshold = 2.0
apps = "firefox, chrome, chromium"
browser_cycles = 25
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
owner = "PerfTest Team"
page_cycles = 25
page_timeout = 60000
playback = "mitmproxy"
playback_version = "5.1.1"
type = "pageload"
unit = "ms"
use_live_sites = false
# raptor-browsertime fission process switch time test
["process-switch"]
# Bug 1867507 - This test isn't producing a largestContentfulPaint
alert_on = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange"
measure = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange"
accept_zero_vismet = true
browsertime_args = "--pageCompleteWaitTime=1000 --pageCompleteCheckInactivity=true"
playback_pageset_manifest = "mitm5-linux-firefox-proc-switch.manifest"
test_script = "process_switch.js"
test_url = "https://mozilla.seanfeng.dev/files/red.html,https://mozilla.pettay.fi/moztests/blue.html"

33
testing/raptor/raptor/tests/custom/browsertime-regression-test.ini
View file

@ -1,33 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
[DEFAULT]
apps = firefox, chrome, chromium, safari, geckoview, fenix
alert_threshold = 2.0
browser_cycles = 1
fore_window = 1
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
min_back_window = 2
max_back_window = 3
owner = PerfTest Team
page_cycles = 1
page_timeout = 1800000
output_timeout = 2000000
subtest_unit = ms
type = pageload
unit = score
use_live_sites = true
# Name these like `*-regression` where * represents
# the kind of data is being produced. This makes it clearer
# that it's not an actual performance test.
[constant-regression]
browsertime_args = --browsertime.constant_value=1500
custom_data = true
test_script = constant_regression_test.js
test_summary = flatten
test_url = None

29
testing/raptor/raptor/tests/custom/browsertime-regression-test.toml Normal file
View file

@ -0,0 +1,29 @@
[DEFAULT]
apps = "firefox, chrome, chromium, safari, geckoview, fenix"
alert_threshold = 2.0
browser_cycles = 1
fore_window = 1
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
min_back_window = 2
max_back_window = 3
owner = "PerfTest Team"
page_cycles = 1
page_timeout = 1800000
output_timeout = 2000000
subtest_unit = "ms"
type = "pageload"
unit = "score"
use_live_sites = true
# Name these like `*-regression` where * represents
# the kind of data is being produced. This makes it clearer
# that it's not an actual performance test.
["constant-regression"]
browsertime_args = "--browsertime.constant_value=1500"
custom_data = true
test_script = "constant_regression_test.js"
test_summary = "flatten"
test_url = "None"

33
testing/raptor/raptor/tests/custom/browsertime-sample-python-support.ini
View file

@ -1,33 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
[DEFAULT]
apps = firefox
alert_threshold = 2.0
browser_cycles = 1
fore_window = 1
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
min_back_window = 2
max_back_window = 3
owner = PerfTest Team
page_cycles = 1
page_timeout = 1800000
output_timeout = 2000000
subtest_unit = ms
type = pageload
unit = score
use_live_sites = true
# Uses the constant regression test since the test doesn't
# need to anything
[sample-python-support]
browsertime_args = --browsertime.constant_value={replace-with-constant-value}
custom_data = true
test_script = constant_regression_test.js
support_class = sample_python_support.py
test_summary = flatten
test_url = None

29
testing/raptor/raptor/tests/custom/browsertime-sample-python-support.toml Normal file
View file

@ -0,0 +1,29 @@
[DEFAULT]
apps = "firefox"
alert_threshold = 2.0
browser_cycles = 1
fore_window = 1
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
min_back_window = 2
max_back_window = 3
owner = "PerfTest Team"
page_cycles = 1
page_timeout = 1800000
output_timeout = 2000000
subtest_unit = "ms"
type = "pageload"
unit = "score"
use_live_sites = true
# Uses the constant regression test since the test doesn't
# need to anything
["sample-python-support"]
browsertime_args = "--browsertime.constant_value={replace-with-constant-value}"
custom_data = true
test_script = "constant_regression_test.js"
support_class = "sample_python_support.py"
test_summary = "flatten"
test_url = "None"

27
testing/raptor/raptor/tests/custom/browsertime-speculative.ini
View file

@ -1,27 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
[DEFAULT]
alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange
alert_change_type = absolute
alert_threshold = 10.0 # expected value is 0, allow for 10 ms within the runtime
apps = firefox
cold = true
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
owner = Network Team
page_cycles = 1 # this is a cold pageload test
page_timeout = 60000
type = pageload
unit = ms
use_live_sites = true
# Keep this list in alphabetical order
[connect]
browser_cycles = 3
test_script = speculative-connect.js
test_url = None
custom_data = true

23
testing/raptor/raptor/tests/custom/browsertime-speculative.toml Normal file
View file

@ -0,0 +1,23 @@
[DEFAULT]
alert_on = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange"
alert_change_type = "absolute"
alert_threshold = 10.0 # expected value is 0, allow for 10 ms within the runtime
apps = "firefox"
cold = true
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
owner = "Network Team"
page_cycles = 1 # this is a cold pageload test
page_timeout = 60000
type = "pageload"
unit = "ms"
use_live_sites = true
# Keep this list in alphabetical order
["connect"]
browser_cycles = 3
test_script = "speculative-connect.js"
test_url = "None"
custom_data = true

26
testing/raptor/raptor/tests/custom/browsertime-throttled-pageload.ini
View file

@ -1,26 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
[DEFAULT]
alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange
alert_threshold = 2.0
apps = firefox
cold = true
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
owner = Network Team
page_cycles = 1 # this is a cold pageload test
page_timeout = 60000
type = pageload
unit = ms
use_live_sites = true
# Keep this list in alphabetical order
[throttled]
browser_cycles = 3
test_script = throttled_pageload.js
test_url = None
custom_data = true

22
testing/raptor/raptor/tests/custom/browsertime-throttled-pageload.toml Normal file
View file

@ -0,0 +1,22 @@
[DEFAULT]
alert_on = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange"
alert_threshold = 2.0
apps = "firefox"
cold = true
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
owner = "Network Team"
page_cycles = 1 # this is a cold pageload test
page_timeout = 60000
type = "pageload"
unit = "ms"
use_live_sites = true
# Keep this list in alphabetical order
["throttled"]
browser_cycles = 3
test_script = "throttled_pageload.js"
test_url = "None"
custom_data = true

29
testing/raptor/raptor/tests/custom/browsertime-tp6-bench.ini
View file

@ -1,29 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
[DEFAULT]
apps = firefox, chrome, chromium, safari
alert_threshold = 2.0
browser_cycles = 1
cold = true
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
owner = PerfTest Team
page_cycles = 1
page_timeout = 1800000
output_timeout = 2000000
unit = ms
subtest_unit = ms
type = pageload
playback_version = 8.1.1
playback = mitmproxy
support_class = browsertime_tp6_bench.py
[tp6-bench]
custom_data = true
test_summary = flatten
test_script = browsertime_tp6_bench.js
test_url = None
playback_pageset_manifest = None

25
testing/raptor/raptor/tests/custom/browsertime-tp6-bench.toml Normal file
View file

@ -0,0 +1,25 @@
[DEFAULT]
apps = "firefox, chrome, chromium, safari"
alert_threshold = 2.0
browser_cycles = 1
cold = true
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
owner = "PerfTest Team"
page_cycles = 1
page_timeout = 1800000
output_timeout = 2000000
unit = "ms"
subtest_unit = "ms"
type = "pageload"
playback_version = "8.1.1"
playback = "mitmproxy"
support_class = "browsertime_tp6_bench.py"
["tp6-bench"]
custom_data = true
test_summary = "flatten"
test_script = "browsertime_tp6_bench.js"
test_url = "None"
playback_pageset_manifest = "None"

41
testing/raptor/raptor/tests/custom/browsertime-upload.ini
View file

@ -1,41 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
[DEFAULT]
apps = firefox, chrome, chromium, safari
alert_threshold = 2.0
browser_cycles = 1
gecko_profile_entries = 14000000
gecko_profile_interval = 1
subtest_lower_is_better = false
owner = Network Team
page_cycles = 1
page_timeout = 1800000
output_timeout = 2000000
subtest_unit = mbps
type = pageload
unit = mbps
use_live_sites = true
# upload performance custom tests
# http/2
[upload]
browsertime_args = --browsertime.upload_iterations=10 --firefox.preference=network.http.http3.enable:false --chrome.args disable-quic
custom_data = true
unit = mbps
lower_is_better = false
subtest_name_filters = tests/,iterations/
test_script = upload.js
test_url = None
# http/3
[upload-h3]
browsertime_args = --browsertime.upload_iterations=10 --firefox.preference=network.http.http3.enable:true
custom_data = true
unit = mbps
lower_is_better = false
subtest_name_filters = tests/,iterations/
test_script = upload.js
test_url = None

39
testing/raptor/raptor/tests/custom/browsertime-upload.toml Normal file
View file

@ -0,0 +1,39 @@
[DEFAULT]
apps = "firefox, chrome, chromium, safari"
alert_threshold = 2.0
browser_cycles = 1
gecko_profile_entries = 14000000
gecko_profile_interval = 1
subtest_lower_is_better = false
owner = "Network Team"
page_cycles = 1
page_timeout = 1800000
output_timeout = 2000000
subtest_unit = "mbps"
type = "pageload"
unit = "mbps"
use_live_sites = true
# upload performance custom tests
# http/2
["upload"]
browsertime_args = "--browsertime.upload_iterations=10 --firefox.preference=network.http.http3.enable:false --chrome.args disable-quic"
custom_data = true
unit = "mbps"
lower_is_better = false
subtest_name_filters = "tests/,iterations/"
test_script = "upload.js"
test_url = "None"
# http/3
["upload-h3"]
browsertime_args = "--browsertime.upload_iterations=10 --firefox.preference=network.http.http3.enable:true"
custom_data = true
unit = "mbps"
lower_is_better = false
subtest_name_filters = "tests/,iterations/"
test_script = "upload.js"
test_url = "None"

30
testing/raptor/raptor/tests/custom/browsertime-welcome.ini
View file

@ -1,30 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
[DEFAULT]
alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange
alert_threshold = 2.0
apps = firefox
browser_cycles = 25
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
owner = PerfTest Team
page_cycles = 25
page_timeout = 60000
playback = mitmproxy
playback_version = 8.1.1
type = pageload
unit = ms
use_live_sites = false
# raptor-browsertime first-install about:welcome pageload test
[welcome]
# Bug 1867507 - This test isn't producing a largestContentfulPaint
alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange
measure = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange
playback_pageset_manifest = mitm5-linux-firefox-welcome.manifest
test_script = welcome.js
test_url = about:welcome

26
testing/raptor/raptor/tests/custom/browsertime-welcome.toml Normal file
View file

@ -0,0 +1,26 @@
[DEFAULT]
alert_on = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange"
alert_threshold = 2.0
apps = "firefox"
browser_cycles = 25
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
owner = "PerfTest Team"
page_cycles = 25
page_timeout = 60000
playback = "mitmproxy"
playback_version = "8.1.1"
type = "pageload"
unit = "ms"
use_live_sites = false
# raptor-browsertime first-install about:welcome pageload test
["welcome"]
# Bug 1867507 - This test isn't producing a largestContentfulPaint
alert_on = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange"
measure = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange"
playback_pageset_manifest = "mitm5-linux-firefox-welcome.manifest"
test_script = "welcome.js"
test_url = "about:welcome"

123
testing/raptor/raptor/tests/interactive/browsertime-responsiveness.ini
View file

@ -1,123 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Raptor-browsertime interactive responsiveness tests
[DEFAULT]
accept_zero_vismet = true
alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange
alert_threshold = 2.0
apps = firefox, chrome, chromium, safari
browser_cycles = 25
gecko_profile_entries = 14000000
gecko_profile_interval = 1
interactive = true
lower_is_better = true
owner = PerfTest Team
page_cycles = 25
page_timeout = 60000
playback = mitmproxy
playback_version = 8.1.1
type = pageload
unit = ms
use_live_sites = false
# Keep this list in alphabetical order
# Do not use `measure.start(URL)` in interactive tests if they need to be recorded,
# see bug 1737822 for more information. Instead, use `navigate(URL)` after starting
# calling `measure.start(ALIAS)`.
[cnn-nav]
browser_cycles = 10 # used with --cold
playback_pageset_manifest = mitm7-linux-firefox-cnn-nav.manifest
test_cmds =
["measure.start", "landing"],
["navigate", "https://www.cnn.com"],
["wait.byTime", 4000],
["measure.stop", ""],
["measure.start", "world"],
["click.byXpathAndWait", "/html/body/div[5]/div/div/header/div/div[1]/div/div[2]/nav/ul/li[2]/a"],
["wait.byTime", 1000],
["measure.stop", ""],
test_url = https://www.cnn.com/
[facebook-nav]
browser_cycles = 10 # used with --cold
page_timeout = 90000
playback_pageset_manifest = mitm6-windows-firefox-facebook-nav.manifest
test_cmds =
["measure.start", "landing"],
["navigate", "https://www.facebook.com/"],
["wait.byTime", 5000],
["measure.stop", ""],
["measure.start", "marketplace"],
["navigate", "https://www.facebook.com/marketplace"],
["wait.byTime", 5000],
["measure.stop", ""],
["measure.start", "groups"],
["navigate", "https://www.facebook.com/groups/discover/"],
["wait.byTime", 5000],
["measure.stop", ""],
["measure.start", "friends"],
["navigate", "https://www.facebook.com/friends/"],
["wait.byTime", 5000],
["measure.stop", ""],
test_url = https://www.facebook.com
[reddit-billgates-ama]
page_timeout = 240000 # at most 4 minutes required per browser cycle
browser_cycles = 10 # used with --cold
playback_pageset_manifest = mitm6-windows-firefox-reddit-billgates-ama.manifest
test_cmds =
["measure.start", "billg-ama"],
["navigate", "https://www.reddit.com/r/IAmA/comments/m8n4vt/im_bill_gates_cochair_of_the_bill_and_melinda/"],
["wait.byTime", 5000],
["measure.stop", ""],
["measure.start", "members"],
["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div[3]/div[2]/div/div[1]/div/div[4]/div[1]/div"],
["wait.byTime", 1000],
["measure.stop", ""],
test_url = https://www.reddit.com/
[reddit-billgates-post-1]
browser_cycles = 10 # used with --cold
page_timeout = 90000
playback_pageset_manifest = mitm6-windows-firefox-reddit-billgates-post.manifest
test_cmds =
["measure.start", "billg"],
["navigate", "https://www.reddit.com/user/thisisbillgates/"],
["wait.byTime", 500],
["measure.stop", ""],
["measure.start", "posts"],
["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[2]"],
["wait.byTime", 500],
["measure.stop", ""],
["measure.start", "comments"],
["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[3]"],
["wait.byTime", 500],
["measure.stop", ""],
["wait.byTime", 500],
test_url = https://www.reddit.com/user/thisisbillgates/
[reddit-billgates-post-2]
browser_cycles = 10 # used with --cold
page_timeout = 90000
playback_pageset_manifest = mitm6-windows-firefox-reddit-billgates-post.manifest
test_cmds =
["measure.start", "billg"],
["navigate", "https://www.reddit.com/user/thisisbillgates/"],
["wait.byTime", 500],
["measure.stop", ""],
["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[3]"],
["wait.byTime", 500],
["measure.start", "hot"],
["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[2]"],
["wait.byTime", 500],
["measure.stop", ""],
["measure.start", "top"],
["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[3]"],
["wait.byTime", 500],
["measure.stop", ""],
["wait.byTime", 500],
test_url = https://www.reddit.com/user/thisisbillgates/

122
testing/raptor/raptor/tests/interactive/browsertime-responsiveness.toml Normal file
View file

@ -0,0 +1,122 @@
[DEFAULT]
accept_zero_vismet = true
alert_on = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange"
alert_threshold = 2.0
apps = "firefox, chrome, chromium, safari"
browser_cycles = 25
gecko_profile_entries = 14000000
gecko_profile_interval = 1
interactive = true
lower_is_better = true
owner = "PerfTest Team"
page_cycles = 25
page_timeout = 60000
playback = "mitmproxy"
playback_version = "8.1.1"
type = "pageload"
unit = "ms"
use_live_sites = false
# Keep this list in alphabetical order
# Do not use `measure.start(URL)` in interactive tests if they need to be recorded,
# see bug 1737822 for more information. Instead, use `navigate(URL)` after starting
# calling `measure.start(ALIAS)`.
["cnn-nav"]
browser_cycles = 10 # used with --cold
playback_pageset_manifest = "mitm7-linux-firefox-cnn-nav.manifest"
test_cmds = [
['measure.start', 'landing'],
['navigate', 'https://www.cnn.com'],
['wait.byTime', '4000'],
['measure.stop', ''],
['measure.start', 'world'],
['click.byXpathAndWait', '/html/body/div[5]/div/div/header/div/div[1]/div/div[2]/nav/ul/li[2]/a'],
['wait.byTime', '1000'],
['measure.stop', ''],
]
test_url = "https://www.cnn.com/"
["facebook-nav"]
browser_cycles = 10 # used with --cold
page_timeout = 90000
playback_pageset_manifest = "mitm6-windows-firefox-facebook-nav.manifest"
test_cmds = [
['measure.start', 'landing'],
['navigate', 'https://www.facebook.com/'],
['wait.byTime', '5000'],
['measure.stop', ''],
['measure.start', 'marketplace'],
['navigate', 'https://www.facebook.com/marketplace'],
['wait.byTime', '5000'],
['measure.stop', ''],
['measure.start', 'groups'],
['navigate', 'https://www.facebook.com/groups/discover/'],
['wait.byTime', '5000'],
['measure.stop', ''],
['measure.start', 'friends'],
['navigate', 'https://www.facebook.com/friends/'],
['wait.byTime', '5000'],
['measure.stop', ''],
]
test_url = "https://www.facebook.com"
["reddit-billgates-ama"]
page_timeout = 240000 # at most 4 minutes required per browser cycle
browser_cycles = 10 # used with --cold
playback_pageset_manifest = "mitm6-windows-firefox-reddit-billgates-ama.manifest"
test_cmds = [
['measure.start', 'billg-ama'],
['navigate', 'https://www.reddit.com/r/IAmA/comments/m8n4vt/im_bill_gates_cochair_of_the_bill_and_melinda/'],
['wait.byTime', '5000'],
['measure.stop', ''],
['measure.start', 'members'],
['click.byXpathAndWait', '/html/body/div[1]/div/div[2]/div[2]/div/div[3]/div[2]/div/div[1]/div/div[4]/div[1]/div'],
['wait.byTime', '1000'],
['measure.stop', ''],
]
test_url = "https://www.reddit.com/"
["reddit-billgates-post-1"]
browser_cycles = 10 # used with --cold
page_timeout = 90000
playback_pageset_manifest = "mitm6-windows-firefox-reddit-billgates-post.manifest"
test_cmds = [
['measure.start', 'billg'],
['navigate', 'https://www.reddit.com/user/thisisbillgates/'],
['wait.byTime', '500'],
['measure.stop', ''],
['measure.start', 'posts'],
['click.byXpathAndWait', '/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[2]'],
['wait.byTime', '500'],
['measure.stop', ''],
['measure.start', 'comments'],
['click.byXpathAndWait', '/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[3]'],
['wait.byTime', '500'],
['measure.stop', ''],
['wait.byTime', '500'],
]
test_url = "https://www.reddit.com/user/thisisbillgates/"
["reddit-billgates-post-2"]
browser_cycles = 10 # used with --cold
page_timeout = 90000
playback_pageset_manifest = "mitm6-windows-firefox-reddit-billgates-post.manifest"
test_cmds = [
['measure.start', 'billg'],
['navigate', 'https://www.reddit.com/user/thisisbillgates/'],
['wait.byTime', '500'],
['measure.stop', ''],
['click.byXpathAndWait', '/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[3]'],
['wait.byTime', '500'],
['measure.start', 'hot'],
['click.byXpathAndWait', '/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[2]'],
['wait.byTime', '500'],
['measure.stop', ''],
['measure.start', 'top'],
['click.byXpathAndWait', '/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[3]'],
['wait.byTime', '500'],
['measure.stop', ''],
['wait.byTime', '500'],
]
test_url = "https://www.reddit.com/user/thisisbillgates/"

16
testing/raptor/raptor/tests/scenario/idle.ini
View file

@ -1,16 +0,0 @@
[DEFAULT]
alert_threshold = 2.0
apps = fenix, geckoview, refbrow
lower_is_better = true
measure = fakeMeasure
owner = PerfTest Team
page_cycles = 1
page_timeout = 1320000
scenario_time = 1200000
test_url = about:blank
type = scenario
unit = scenarioComplete
[idle]
[idle-bg]
browsertime_args = --browsertime.scenario_time=60000 --browsertime.background_app=false

17
testing/raptor/raptor/tests/scenario/idle.toml Normal file
View file

@ -0,0 +1,17 @@
[DEFAULT]
alert_threshold = 2.0
apps = "fenix, geckoview, refbrow"
lower_is_better = true
measure = "fakeMeasure"
owner = "PerfTest Team"
page_cycles = 1
page_timeout = 1320000
scenario_time = 1200000
test_url = "about:blank"
type = "scenario"
unit = "scenarioComplete"
["idle"]
["idle-bg"]
browsertime_args = "--browsertime.scenario_time=60000 --browsertime.background_app=false"

178
testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.ini
View file

@ -1,178 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# raptor-browsertime tp6 desktop page-load test specification
[DEFAULT]
alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange, largestContentfulPaint
alert_threshold = 2.0
apps = firefox, chrome, chromium, safari, custom-car
browser_cycles = 25
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
owner = PerfTest Team
page_cycles = 25
page_timeout = 60000
playback = mitmproxy
playback_pageset_manifest = mitm7-linux-firefox-{subtest}.manifest
playback_version = 8.1.1
type = pageload
unit = ms
use_live_sites = false
# Keep this list in alphabetical order
[amazon]
test_url = https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1
secondary_url = https://www.amazon.com/Acer-A515-46-R14K-Quad-Core-Processor-Backlit/dp/B08VKNVDDR/ref=sr_1_3?dchild=1&keywords=laptop&qid=1627047187&sr=8-3
[bing-search]
playback_pageset_manifest = mitm5-linux-firefox-bing-search.manifest
test_url = https://www.bing.com/search?q=barack+obama
[buzzfeed]
secondary_url = https://www.buzzfeed.com/quizzes
test_url = https://www.buzzfeed.com/
[cnn]
secondary_url = https://www.cnn.com/weather
test_url = https://www.cnn.com/2021/03/22/weather/climate-change-warm-waters-lake-michigan/index.html
[ebay]
playback_pageset_manifest = mitm8-linux-firefox-ebay.manifest
secondary_url = https://www.ebay.com/deals
test_url = https://www.ebay.com/
[espn]
test_url = http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words
secondary_url = https://www.espn.com/nba/draft/news
[expedia]
secondary_url = https://groups.expedia.com/Group-Rate/?locale=en_US&ol=1
test_url = https://expedia.com/Hotel-Search?destination=New+York%2C+New+York&latLong=40.756680%2C-73.986470&regionId=178293&startDate=&endDate=&rooms=1&_xpid=11905%7C1&adults=2
[facebook]
playback_pageset_manifest = mitm6-linux-firefox-facebook.manifest
secondary_url = https://www.facebook.com/marketplace/?ref=bookmark
test_url = https://www.facebook.com
[fandom]
playback_pageset_manifest = mitm5-linux-firefox-fandom.manifest
test_url = https://www.fandom.com/articles/fallout-76-will-live-and-die-on-the-creativity-of-its-playerbase
[google-docs]
page_complete_wait_time = 8000
secondary_url = https://docs.google.com/document/d/1vUnn0ePU-ynArE1OdxyEHXR2G0sl74ja_st_4OOzlgE/preview
test_url = https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing
[google-mail]
playback_pageset_manifest = mitm5-linux-firefox-google-mail.manifest
test_url = https://mail.google.com/
[google-search]
playback_pageset_manifest = mitm5-linux-firefox-google-search.manifest
test_url = https://www.google.com/search?hl=en&q=barack+obama&cad=h
[google-slides]
playback_pageset_manifest = mitm6-linux-firefox-google-slides.manifest
playback_version = 6.0.2
secondary_url = https://docs.google.com/document/d/1vUnn0ePU-ynArE1OdxyEHXR2G0sl74ja_st_4OOzlgE/preview
test_url = https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing
[imdb]
test_url = https://www.imdb.com/title/tt0084967/?ref_=nv_sr_2
secondary_url = https://www.imdb.com/title/tt0084967/episodes/?ref_=tt_ov_epl
[imgur]
playback_pageset_manifest = mitm8-linux-firefox-imgur.manifest
preferences = {"media.autoplay.default": 5,
"media.autoplay.ask-permission": true,
"media.autoplay.blocking_policy": 1,
"media.allowed-to-play.enabled": false,
"media.block-autoplay-until-in-foreground": true}
secondary_url = https://imgur.com/gallery/WuOobqV
test_url = https://imgur.com/gallery/uTI7fKy
[instagram]
playback_pageset_manifest = mitm6-linux-firefox-instagram.manifest
secondary_url = https://www.instagram.com/nobelprize_org/
test_url = https://www.instagram.com/
[linkedin]
playback_pageset_manifest = mitm6-linux-firefox-linkedin.manifest
secondary_url = https://www.linkedin.com/in/thommy-harris-hk-385723106/
test_url = https://www.linkedin.com/feed/
[microsoft]
secondary_url = https://support.microsoft.com/en-us
test_url = https://www.microsoft.com/en-us/
[netflix]
playback_pageset_manifest = mitm6-linux-firefox-netflix.manifest
secondary_url = https://www.netflix.com/title/699257
test_url = https://www.netflix.com/title/80117263
[nytimes]
secondary_url = https://www.nytimes.com/section/opinion/columnists
test_url = https://www.nytimes.com/2020/02/19/opinion/surprise-medical-bill.html
[office]
secondary_url = https://www.office.com/
test_url = https://www.office.com/launch/powerpoint?auth=1
[outlook]
playback_pageset_manifest = mitm5-linux-firefox-live.manifest
test_url = https://outlook.live.com/mail/inbox
[paypal]
playback_pageset_manifest = mitm5-linux-firefox-paypal.manifest
test_url = https://www.paypal.com/myaccount/summary/
[pinterest]
playback_pageset_manifest = mitm6-linux-firefox-pinterest.manifest
secondary_url = https://www.pinterest.com/today/best/halloween-costumes-for-your-furry-friends/75787/
test_url = https://pinterest.com/
[reddit]
playback_pageset_manifest = mitm6-linux-firefox-reddit.manifest
secondary_url = https://www.reddit.com/r/technology/
test_url = https://www.reddit.com/r/technology/comments/9sqwyh/we_posed_as_100_senators_to_run_ads_on_facebook/
[tumblr]
playback_pageset_manifest = mitm6-linux-firefox-tumblr.manifest
secondary_url = https://www.tumblr.com/tagged/funny+cats?sort=top
test_url = https://www.tumblr.com/dashboard
[twitch]
measure = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange
alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange
playback_pageset_manifest = mitm8-linux-firefox-twitch.manifest
secondary_url = https://www.twitch.tv/gmashley
test_url = https://www.twitch.tv/videos/894226211
preferences = {"media.autoplay.default": 5,
"media.autoplay.ask-permission": true,
"media.autoplay.blocking_policy": 1,
"media.allowed-to-play.enabled": false,
"media.block-autoplay-until-in-foreground": true}
[twitter]
playback_pageset_manifest = mitm5-linux-firefox-twitter.manifest
test_url = https://twitter.com/BarackObama
[wikia]
secondary_url = https://marvel.fandom.com/wiki/Celestials
test_url = https://marvel.fandom.com/wiki/Black_Panther
[wikipedia]
secondary_url = https://en.wikipedia.org/wiki/Joe_Biden
test_url = https://en.wikipedia.org/wiki/Barack_Obama
[yahoo-mail]
playback_pageset_manifest = mitm5-linux-firefox-yahoo-mail.manifest
test_url = https://mail.yahoo.com/
[youtube]
secondary_url = https://www.youtube.com/watch?v=JrdEMERq8MA
test_url = https://www.youtube.com

174
testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.toml Normal file
View file

@ -0,0 +1,174 @@
[DEFAULT]
alert_on = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange, largestContentfulPaint"
alert_threshold = "2.0"
apps = "firefox, chrome, chromium, safari, custom-car"
browser_cycles = 25
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
owner = "PerfTest Team"
page_cycles = 25
page_timeout = 60000
playback = "mitmproxy"
playback_pageset_manifest = "mitm7-linux-firefox-{subtest}.manifest"
playback_version = "8.1.1"
type = "pageload"
unit = "ms"
use_live_sites = false
# Keep this list in alphabetical order
["amazon"]
test_url = "https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1"
secondary_url = "https://www.amazon.com/Acer-A515-46-R14K-Quad-Core-Processor-Backlit/dp/B08VKNVDDR/ref=sr_1_3?dchild=1&keywords=laptop&qid=1627047187&sr=8-3"
["bing-search"]
playback_pageset_manifest = "mitm5-linux-firefox-bing-search.manifest"
test_url = "https://www.bing.com/search?q=barack+obama"
["buzzfeed"]
secondary_url = "https://www.buzzfeed.com/quizzes"
test_url = "https://www.buzzfeed.com/"
["cnn"]
secondary_url = "https://www.cnn.com/weather"
test_url = "https://www.cnn.com/2021/03/22/weather/climate-change-warm-waters-lake-michigan/index.html"
["ebay"]
playback_pageset_manifest = "mitm8-linux-firefox-ebay.manifest"
secondary_url = "https://www.ebay.com/deals"
test_url = "https://www.ebay.com/"
["espn"]
test_url = "http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words"
secondary_url = "https://www.espn.com/nba/draft/news"
["expedia"]
secondary_url = "https://groups.expedia.com/Group-Rate/?locale=en_US&ol=1"
test_url = "https://expedia.com/Hotel-Search?destination=New+York%2C+New+York&latLong=40.756680%2C-73.986470&regionId=178293&startDate=&endDate=&rooms=1&_xpid=11905%7C1&adults=2"
["facebook"]
playback_pageset_manifest = "mitm6-linux-firefox-facebook.manifest"
secondary_url = "https://www.facebook.com/marketplace/?ref=bookmark"
test_url = "https://www.facebook.com"
["fandom"]
playback_pageset_manifest = "mitm5-linux-firefox-fandom.manifest"
test_url = "https://www.fandom.com/articles/fallout-76-will-live-and-die-on-the-creativity-of-its-playerbase"
["google-docs"]
page_complete_wait_time = 8000
secondary_url = "https://docs.google.com/document/d/1vUnn0ePU-ynArE1OdxyEHXR2G0sl74ja_st_4OOzlgE/preview"
test_url = "https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing"
["google-mail"]
playback_pageset_manifest = "mitm5-linux-firefox-google-mail.manifest"
test_url = "https://mail.google.com/"
["google-search"]
playback_pageset_manifest = "mitm5-linux-firefox-google-search.manifest"
test_url = "https://www.google.com/search?hl=en&q=barack+obama&cad=h"
["google-slides"]
playback_pageset_manifest = "mitm6-linux-firefox-google-slides.manifest"
playback_version = "6.0.2"
secondary_url = "https://docs.google.com/document/d/1vUnn0ePU-ynArE1OdxyEHXR2G0sl74ja_st_4OOzlgE/preview"
test_url = "https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing"
["imdb"]
test_url = "https://www.imdb.com/title/tt0084967/?ref_=nv_sr_2"
secondary_url = "https://www.imdb.com/title/tt0084967/episodes/?ref_=tt_ov_epl"
["imgur"]
playback_pageset_manifest = "mitm8-linux-firefox-imgur.manifest"
preferences = ['media.autoplay.default=5',
'media.autoplay.ask-permission=true',
'media.autoplay.blocking_policy=1',
'media.allowed-to-play.enabled=false',
'media.block-autoplay-until-in-foreground=true',
]
secondary_url = "https://imgur.com/gallery/WuOobqV"
test_url = "https://imgur.com/gallery/uTI7fKy"
["instagram"]
playback_pageset_manifest = "mitm6-linux-firefox-instagram.manifest"
secondary_url = "https://www.instagram.com/nobelprize_org/"
test_url = "https://www.instagram.com/"
["linkedin"]
playback_pageset_manifest = "mitm6-linux-firefox-linkedin.manifest"
secondary_url = "https://www.linkedin.com/in/thommy-harris-hk-385723106/"
test_url = "https://www.linkedin.com/feed/"
["microsoft"]
secondary_url = "https://support.microsoft.com/en-us"
test_url = "https://www.microsoft.com/en-us/"
["netflix"]
playback_pageset_manifest = "mitm6-linux-firefox-netflix.manifest"
secondary_url = "https://www.netflix.com/title/699257"
test_url = "https://www.netflix.com/title/80117263"
["nytimes"]
secondary_url = "https://www.nytimes.com/section/opinion/columnists"
test_url = "https://www.nytimes.com/2020/02/19/opinion/surprise-medical-bill.html"
["office"]
secondary_url = "https://www.office.com/"
test_url = "https://www.office.com/launch/powerpoint?auth=1"
["outlook"]
playback_pageset_manifest = "mitm5-linux-firefox-live.manifest"
test_url = "https://outlook.live.com/mail/inbox"
["paypal"]
playback_pageset_manifest = "mitm5-linux-firefox-paypal.manifest"
test_url = "https://www.paypal.com/myaccount/summary/"
["pinterest"]
playback_pageset_manifest = "mitm6-linux-firefox-pinterest.manifest"
secondary_url = "https://www.pinterest.com/today/best/halloween-costumes-for-your-furry-friends/75787/"
test_url = "https://pinterest.com/"
["reddit"]
playback_pageset_manifest = "mitm6-linux-firefox-reddit.manifest"
secondary_url = "https://www.reddit.com/r/technology/"
test_url = "https://www.reddit.com/r/technology/comments/9sqwyh/we_posed_as_100_senators_to_run_ads_on_facebook/"
["tumblr"]
playback_pageset_manifest = "mitm6-linux-firefox-tumblr.manifest"
secondary_url = "https://www.tumblr.com/tagged/funny+cats?sort=top"
test_url = "https://www.tumblr.com/dashboard"
["twitch"]
measure = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange"
alert_on = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange"
playback_pageset_manifest = "mitm8-linux-firefox-twitch.manifest"
secondary_url = "https://www.twitch.tv/gmashley"
test_url = "https://www.twitch.tv/videos/894226211"
preferences = ['media.autoplay.default=5',
'media.autoplay.ask-permission=true',
'media.autoplay.blocking_policy=1',
'media.allowed-to-play.enabled=false',
'media.block-autoplay-until-in-foreground=true',
]
["twitter"]
playback_pageset_manifest = "mitm5-linux-firefox-twitter.manifest"
test_url = "https://twitter.com/BarackObama"
["wikia"]
secondary_url = "https://marvel.fandom.com/wiki/Celestials"
test_url = "https://marvel.fandom.com/wiki/Black_Panther"
["wikipedia"]
secondary_url = "https://en.wikipedia.org/wiki/Joe_Biden"
test_url = "https://en.wikipedia.org/wiki/Barack_Obama"
["yahoo-mail"]
playback_pageset_manifest = "mitm5-linux-firefox-yahoo-mail.manifest"
test_url = "https://mail.yahoo.com/"
["youtube"]
secondary_url = "https://www.youtube.com/watch?v=JrdEMERq8MA"
test_url = "https://www.youtube.com"

70
testing/raptor/raptor/tests/tp6/live/browsertime-live.ini
View file

@ -1,70 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Raptor-Browsertime tp6 tests running on live sites only. There should
# not be any playback settings in this file. If there are, please move the
# tests to `mobile/browsertime-tp6m.ini` or `desktop/browsertime-tp6.ini`.
[DEFAULT]
alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange
alert_threshold = 2.0
apps = geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium
browser_cycles = 15
lower_is_better = true
owner = PerfTest Team
page_cycles = 25
page_timeout = 60000
type = pageload
unit = ms
use_live_sites = true
## Keep this test list in alphabetical order
[booking-sf]
test_url = https://www.booking.com/hotel/us/edwardian-san-francisco.html
[cnn]
test_url = https://www.cnn.com/2021/03/22/weather/climate-change-warm-waters-lake-michigan/index.html
[cnn-ampstories]
test_url = https://cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other
[discord]
test_url = https://discordapp.com/
[expedia]
test_url = https://expedia.com/Hotel-Search?destination=New+York%2C+New+York&latLong=40.756680%2C-73.986470&regionId=178293&startDate=&endDate=&rooms=1&_xpid=11905%7C1&adults=2 #noqa
[fashionbeans]
test_url = https://fashionbeans.com/article/coolest-menswear-stores-in-the-world
[google-accounts]
test_url = https://accounts.google.com
[imdb-firefox]
test_url = https://m.imdb.com/title/tt0083943/
[medium-article]
test_url = https://medium.com/s/coincidences-are-a-lie/could-america-have-also-been-the-birthplace-of-impressionism-cb3d31a2e22d #noqa
[nytimes]
test_url = https://www.nytimes.com/2020/02/19/opinion/surprise-medical-bill.html
[people-article]
test_url = https://people.com/amp-stories/royal-a-to-z/
[reddit-thread]
test_url = https://www.reddit.com/r/firefox/comments/7dkq03/its_been_a_while/
[rumble-fox]
test_url = https://rumble.com/v3c44t-foxes-jumping-on-my-trampoline.html
[stackoverflow-question]
test_url = https://stackoverflow.com/questions/927358/how-do-i-undo-the-most-recent-commits-in-git
[urbandictionary-define]
test_url = https://urbandictionary.com/define.php?term=awesome%20sauce
[wikia-marvel]
test_url = https://marvel.wikia.com/wiki/Black_Panther

62
testing/raptor/raptor/tests/tp6/live/browsertime-live.toml Normal file
View file

@ -0,0 +1,62 @@
[DEFAULT]
alert_on = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange"
alert_threshold = 2.0
apps = "geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium"
browser_cycles = 15
lower_is_better = true
owner = "PerfTest Team"
page_cycles = 25
page_timeout = 60000
type = "pageload"
unit = "ms"
use_live_sites = true
## Keep this test list in alphabetical order
["booking-sf"]
test_url = "https://www.booking.com/hotel/us/edwardian-san-francisco.html"
["cnn"]
test_url = "https://www.cnn.com/2021/03/22/weather/climate-change-warm-waters-lake-michigan/index.html"
["cnn-ampstories"]
test_url = "https://cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other"
["discord"]
test_url = "https://discordapp.com/"
["expedia"]
test_url = "https://expedia.com/Hotel-Search?destination=New+York%2C+New+York&latLong=40.756680%2C-73.986470&regionId=178293&startDate=&endDate=&rooms=1&_xpid=11905%7C1&adults=2" #noqa
["fashionbeans"]
test_url = "https://fashionbeans.com/article/coolest-menswear-stores-in-the-world"
["google-accounts"]
test_url = "https://accounts.google.com"
["imdb-firefox"]
test_url = "https://m.imdb.com/title/tt0083943/"
["medium-article"]
test_url = "https://medium.com/s/coincidences-are-a-lie/could-america-have-also-been-the-birthplace-of-impressionism-cb3d31a2e22d" #noqa
["nytimes"]
test_url = "https://www.nytimes.com/2020/02/19/opinion/surprise-medical-bill.html"
["people-article"]
test_url = "https://people.com/amp-stories/royal-a-to-z/"
["reddit-thread"]
test_url = "https://www.reddit.com/r/firefox/comments/7dkq03/its_been_a_while/"
["rumble-fox"]
test_url = "https://rumble.com/v3c44t-foxes-jumping-on-my-trampoline.html"
["stackoverflow-question"]
test_url = "https://stackoverflow.com/questions/927358/how-do-i-undo-the-most-recent-commits-in-git"
["urbandictionary-define"]
test_url = "https://urbandictionary.com/define.php?term=awesome%20sauce"
["wikia-marvel"]
test_url = "https://marvel.wikia.com/wiki/Black_Panther"

113
testing/raptor/raptor/tests/tp6/mobile/browsertime-tp6m.ini
View file

@ -1,113 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# raptor-browsertime tp6 tests on android
[DEFAULT]
alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange, largestContentfulPaint
alert_threshold = 2.0
apps = geckoview, fenix, refbrow, chrome-m
browser_cycles = 15
lower_is_better = true
owner = PerfTest Team
page_cycles = 25
page_timeout = 60000
playback = mitmproxy-android
playback_pageset_manifest = mitm6-android-fenix-{subtest}.manifest
playback_version = 8.1.1
type = pageload
unit = ms
use_live_sites = false
## Keep this test list in alphabetical order
[allrecipes]
test_url = https://www.allrecipes.com/
[amazon]
test_url = https://www.amazon.com
[amazon-search]
test_url = https://www.amazon.com/s/ref=nb_sb_noss_2/139-6317191-5622045?url=search-alias%3Daps&field-keywords=mobile+phone
[bing]
test_url = https://www.bing.com/
[bing-search-restaurants]
test_url = https://www.bing.com/search?q=restaurants+in+exton+pa+19341
[booking]
test_url = https://www.booking.com/
[cnn]
test_url = https://cnn.com
[cnn-ampstories]
test_url = https://cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other
[dailymail]
test_url = https://www.dailymail.co.uk/sciencetech/article-9749081/Experts-say-Hubble-repair-despite-NASA-insisting-multiple-options-fix.html
[ebay-kleinanzeigen]
test_url = https://m.ebay-kleinanzeigen.de
[ebay-kleinanzeigen-search]
test_url = https://m.ebay-kleinanzeigen.de/s-anzeigen/auf-zeit-wg-berlin/zimmer/c199-l3331
[espn]
test_url = http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words
[facebook]
login = true
playback_pageset_manifest = mitm6-g5-fenix-{subtest}.manifest
test_url = https://m.facebook.com
[facebook-cristiano]
test_url = https://m.facebook.com/Cristiano
[google]
login = true
playback_pageset_manifest = mitm6-g5-fenix-{subtest}.manifest
test_url = https://www.google.com
[google-maps]
test_url = https://www.google.com/maps?force=pwa
[google-search-restaurants]
login = true
playback_pageset_manifest = mitm6-g5-fenix-{subtest}.manifest
test_url = https://www.google.com/search?q=restaurants+near+me
[imdb]
test_url = https://m.imdb.com/
[instagram]
login = true
playback_pageset_manifest = mitm8-android-gve-{subtest}.manifest
test_url = https://www.instagram.com
[microsoft-support]
test_url = https://support.microsoft.com/en-us
[reddit]
test_url = https://www.reddit.com
[sina]
test_url = https://www.sina.com.cn/
[stackoverflow]
test_url = https://stackoverflow.com/
[web-de]
playback_pageset_manifest = mitm7-android-gve-p2-web-de.manifest
test_url = https://web.de/magazine/politik/politologe-glaubt-grossen-koalition-herbst-knallen-33563566
[wikipedia]
test_url = https://en.m.wikipedia.org/wiki/Main_Page
[youtube]
test_url = https://m.youtube.com
[youtube-watch]
test_url = https://www.youtube.com/watch?v=COU5T-Wafa4

107
testing/raptor/raptor/tests/tp6/mobile/browsertime-tp6m.toml Normal file
View file

@ -0,0 +1,107 @@
[DEFAULT]
alert_on = "fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange, largestContentfulPaint"
alert_threshold = 2.0
apps = "geckoview, fenix, refbrow, chrome-m"
browser_cycles = 15
lower_is_better = true
owner = "PerfTest Team"
page_cycles = 25
page_timeout = 60000
playback = "mitmproxy-android"
playback_pageset_manifest = "mitm6-android-fenix-{subtest}.manifest"
playback_version = "8.1.1"
type = "pageload"
unit = "ms"
use_live_sites = false
## Keep this test list in alphabetical order
["allrecipes"]
test_url = "https://www.allrecipes.com/"
["amazon"]
test_url = "https://www.amazon.com"
["amazon-search"]
test_url = "https://www.amazon.com/s/ref=nb_sb_noss_2/139-6317191-5622045?url=search-alias%3Daps&field-keywords=mobile+phone"
["bing"]
test_url = "https://www.bing.com/"
["bing-search-restaurants"]
test_url = "https://www.bing.com/search?q=restaurants+in+exton+pa+19341"
["booking"]
test_url = "https://www.booking.com/"
["cnn"]
test_url = "https://cnn.com"
["cnn-ampstories"]
test_url = "https://cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other"
["dailymail"]
test_url = "https://www.dailymail.co.uk/sciencetech/article-9749081/Experts-say-Hubble-repair-despite-NASA-insisting-multiple-options-fix.html"
["ebay-kleinanzeigen"]
test_url = "https://m.ebay-kleinanzeigen.de"
["ebay-kleinanzeigen-search"]
test_url = "https://m.ebay-kleinanzeigen.de/s-anzeigen/auf-zeit-wg-berlin/zimmer/c199-l3331"
["espn"]
test_url = "http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words"
["facebook"]
login = true
playback_pageset_manifest = "mitm6-g5-fenix-{subtest}.manifest"
test_url = "https://m.facebook.com"
["facebook-cristiano"]
test_url = "https://m.facebook.com/Cristiano"
["google"]
login = true
playback_pageset_manifest = "mitm6-g5-fenix-{subtest}.manifest"
test_url = "https://www.google.com"
["google-maps"]
test_url = "https://www.google.com/maps?force=pwa"
["google-search-restaurants"]
login = true
playback_pageset_manifest = "mitm6-g5-fenix-{subtest}.manifest"
test_url = "https://www.google.com/search?q=restaurants+near+me"
["imdb"]
test_url = "https://m.imdb.com/"
["instagram"]
login = true
playback_pageset_manifest = "mitm8-android-gve-{subtest}.manifest"
test_url = "https://www.instagram.com"
["microsoft-support"]
test_url = "https://support.microsoft.com/en-us"
["reddit"]
test_url = "https://www.reddit.com"
["sina"]
test_url = "https://www.sina.com.cn/"
["stackoverflow"]
test_url = "https://stackoverflow.com/"
["web-de"]
playback_pageset_manifest = "mitm7-android-gve-p2-web-de.manifest"
test_url = "https://web.de/magazine/politik/politologe-glaubt-grossen-koalition-herbst-knallen-33563566"
["wikipedia"]
test_url = "https://en.m.wikipedia.org/wiki/Main_Page"
["youtube"]
test_url = "https://m.youtube.com"
["youtube-watch"]
test_url = "https://www.youtube.com/watch?v=COU5T-Wafa4"

33
testing/raptor/raptor/tests/unittests/browsertime-tp6-unittest.ini
View file

@ -1,33 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# raptor tp6-unittest
[DEFAULT]
alert_on = fcp, loadtime
alert_threshold = 2.0
apps = firefox
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
owner = PerfTest Team
page_cycles = 25
page_timeout = 60000
playback = mitmproxy
playback_pageset_manifest = mitm7-linux-firefox-{subtest}.manifest
playback_version = 8.1.1
type = pageload
unit = ms
[amazon]
test_url = https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1
[facebook]
test_url = https://www.facebook.com
[google]
test_url = https://www.google.com/search?hl=en&q=barack+obama&cad=h
[youtube]
test_url = https://www.youtube.com

27
testing/raptor/raptor/tests/unittests/browsertime-tp6-unittest.toml Normal file
View file

@ -0,0 +1,27 @@
[DEFAULT]
alert_on = "fcp, loadtime"
alert_threshold = 2.0
apps = "firefox"
gecko_profile_entries = 14000000
gecko_profile_interval = 1
lower_is_better = true
owner = "PerfTest Team"
page_cycles = 25
page_timeout = 60000
playback = "mitmproxy"
playback_pageset_manifest = "mitm7-linux-firefox-{subtest}.manifest"
playback_version = "8.1.1"
type = "pageload"
unit = "ms"
["amazon"]
test_url = "https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1"
["facebook"]
test_url = "https://www.facebook.com"
["google"]
test_url = "https://www.google.com/search?hl=en&q=barack+obama&cad=h"
["youtube"]
test_url = "https://www.youtube.com"

12
testing/raptor/test/test_manifest.py
View file

@ -16,10 +16,10 @@ from manifest import (
add_test_url_params,
get_browser_test_list,
get_raptor_test_list,
validate_test_ini,
validate_test_toml,
)
# some test details (test INIs)
# some test details (test TOMLs)
VALID_MANIFESTS = [
{
# page load test with local playback
@ -153,13 +153,13 @@ def test_get_browser_test_list(app):
@pytest.mark.parametrize("test_details", VALID_MANIFESTS)
def test_validate_test_ini_valid(test_details):
assert validate_test_ini(test_details)
def test_validate_test_toml_valid(test_details):
assert validate_test_toml(test_details)
@pytest.mark.parametrize("test_details", INVALID_MANIFESTS)
def test_validate_test_ini_invalid(test_details):
assert not (validate_test_ini(test_details))
def test_validate_test_toml_invalid(test_details):
assert not (validate_test_toml(test_details))
def test_get_raptor_test_list_firefox(create_args):

14
testing/raptor/test/test_print_tests.py
View file

@ -9,24 +9,24 @@ from raptor import cmdline
def test_pageload_subtests(capsys, monkeypatch, tmpdir):
# cmdline.py is hard-coded to use raptor.ini from the same directory so we need
# cmdline.py is hard-coded to use raptor.toml from the same directory so we need
# to monkey patch os.dirname, which is not ideal. If we could make --print-tests
# respect the --test path that would be much better.
def mock(path):
return str(tmpdir)
monkeypatch.setattr(os.path, "dirname", mock)
manifest = tmpdir.join("raptor.ini")
manifest = tmpdir.join("raptor.toml")
manifest.write(
"""
[DEFAULT]
type = pageload
apps = firefox
type = "pageload"
apps = "firefox"
[raptor-subtest-1]
measure = foo, bar
["raptor-subtest-1"]
measure = ["foo", "bar"]
[raptor-subtest-2]
["raptor-subtest-2"]
"""
)
with pytest.raises(SystemExit):

2
tools/lint/perfdocs/verifier.py
View file

@ -18,7 +18,7 @@ Schema for the config.yml file.
Expecting a YAML file with a format such as this:
name: raptor
manifest: testing/raptor/raptor/raptor.ini
manifest: testing/raptor/raptor/raptor.toml
static-only: False
suites:
desktop: