Bug 1546323 Run cold page load tests against Fennec v64 in CI r=Bebe

Differential Revision: https://phabricator.services.mozilla.com/D31773

--HG--
extra : moz-landing-system : lando

taskcluster/ci/test/raptor.yml

@@ -2146,3 +2146,298 @@ raptor-tp6m-14-geckoview-cold:
             - --app=geckoview
             - --binary=org.mozilla.geckoview_example
             - --activity=org.mozilla.geckoview_example.GeckoViewActivity
+
+raptor-tp6m-1-fennec64-cold:
+    description: "Raptor tp6m-1 cold page-load on Fennec64"
+    max-run-time: 3600
+    try-name: raptor-tp6m-1-fennec64-cold
+    treeherder-symbol: Rap(tp6m-c-1-f64)
+    target:
+        by-test-platform:
+            android-hw-p2-8-0-android-aarch64.*:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-aarch64-opt
+                name: build/target.apk
+            default:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-api-16-opt
+                name: build/target.apk
+    run-on-projects: ['try']
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-1
+            - --app=fennec
+            - --binary=org.mozilla.firefox
+
+
+raptor-tp6m-2-fennec64-cold:
+    description: "Raptor tp6m-2 cold page-load on Fennec64"
+    max-run-time: 3600
+    try-name: raptor-tp6m-2-fennec64-cold
+    treeherder-symbol: Rap(tp6m-c-2-f64)
+    target:
+        by-test-platform:
+            android-hw-p2-8-0-android-aarch64.*:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-aarch64-opt
+                name: build/target.apk
+            default:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-api-16-opt
+                name: build/target.apk
+    run-on-projects: ['try']
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-2
+            - --app=fennec
+            - --binary=org.mozilla.firefox
+
+raptor-tp6m-3-fennec64-cold:
+    description: "Raptor tp6m-3 cold page-load on Fennec64"
+    max-run-time: 3600
+    try-name: raptor-tp6m-3-fennec64-cold
+    treeherder-symbol: Rap(tp6m-c-3-f64)
+    target:
+        by-test-platform:
+            android-hw-p2-8-0-android-aarch64.*:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-aarch64-opt
+                name: build/target.apk
+            default:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-api-16-opt
+                name: build/target.apk
+    run-on-projects: ['try']
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-3
+            - --app=fennec
+            - --binary=org.mozilla.firefox
+
+raptor-tp6m-4-fennec64-cold:
+    description: "Raptor tp6m-4 cold page-load on Fennec64"
+    max-run-time: 3600
+    try-name: raptor-tp6m-4-fennec64-cold
+    treeherder-symbol: Rap(tp6m-c-4-f64)
+    target:
+        by-test-platform:
+            android-hw-p2-8-0-android-aarch64.*:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-aarch64-opt
+                name: build/target.apk
+            default:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-api-16-opt
+                name: build/target.apk
+    run-on-projects: ['try']
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-4
+            - --app=fennec
+            - --binary=org.mozilla.firefox
+
+raptor-tp6m-5-fennec64-cold:
+    description: "Raptor tp6m-5 cold page-load on Fennec64"
+    max-run-time: 3600
+    try-name: raptor-tp6m-5-fennec64-cold
+    treeherder-symbol: Rap(tp6m-c-5-f64)
+    target:
+        by-test-platform:
+            android-hw-p2-8-0-android-aarch64.*:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-aarch64-opt
+                name: build/target.apk
+            default:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-api-16-opt
+                name: build/target.apk
+    run-on-projects: ['try']
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-1
+            - --app=fennec
+            - --binary=org.mozilla.firefox
+
+raptor-tp6m-6-fennec64-cold:
+    description: "Raptor tp6m-6 cold page-load on Fennec64"
+    max-run-time: 3600
+    try-name: raptor-tp6m-6-fennec64-cold
+    treeherder-symbol: Rap(tp6m-c-6-f64)
+    target:
+        by-test-platform:
+            android-hw-p2-8-0-android-aarch64.*:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-aarch64-opt
+                name: build/target.apk
+            default:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-api-16-opt
+                name: build/target.apk
+    run-on-projects: ['try']
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-6
+            - --app=fennec
+            - --binary=org.mozilla.firefox
+
+raptor-tp6m-7-fennec64-cold:
+    description: "Raptor tp6m-7 cold page-load on Fennec64"
+    max-run-time: 3600
+    try-name: raptor-tp6m-7-fennec64-cold
+    treeherder-symbol: Rap(tp6m-c-7-f64)
+    target:
+        by-test-platform:
+            android-hw-p2-8-0-android-aarch64.*:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-aarch64-opt
+                name: build/target.apk
+            default:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-api-16-opt
+                name: build/target.apk
+    run-on-projects: ['try']
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-7
+            - --app=fennec
+            - --binary=org.mozilla.firefox
+
+raptor-tp6m-8-fennec64-cold:
+    description: "Raptor tp6m-8 cold page-load on Fennec64"
+    max-run-time: 3600
+    try-name: raptor-tp6m-8-fennec64-cold
+    treeherder-symbol: Rap(tp6m-c-8-f64)
+    target:
+        by-test-platform:
+            android-hw-p2-8-0-android-aarch64.*:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-aarch64-opt
+                name: build/target.apk
+            default:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-api-16-opt
+                name: build/target.apk
+    run-on-projects: ['try']
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-8
+            - --app=fennec
+            - --binary=org.mozilla.firefox
+
+raptor-tp6m-9-fennec64-cold:
+    description: "Raptor tp6m-9 cold page-load on Fennec64"
+    max-run-time: 3600
+    try-name: raptor-tp6m-9-fennec64-cold
+    treeherder-symbol: Rap(tp6m-c-9-f64)
+    target:
+        by-test-platform:
+            android-hw-p2-8-0-android-aarch64.*:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-aarch64-opt
+                name: build/target.apk
+            default:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-api-16-opt
+                name: build/target.apk
+    run-on-projects: ['try']
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-9
+            - --app=fennec
+            - --binary=org.mozilla.firefox
+
+raptor-tp6m-10-fennec64-cold:
+    description: "Raptor tp6m-10 cold page-load on Fennec64"
+    max-run-time: 3600
+    try-name: raptor-tp6m-10-fennec64-cold
+    treeherder-symbol: Rap(tp6m-c-10-f64)
+    target:
+        by-test-platform:
+            android-hw-p2-8-0-android-aarch64.*:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-aarch64-opt
+                name: build/target.apk
+            default:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-api-16-opt
+                name: build/target.apk
+    run-on-projects: ['try']
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-10
+            - --app=fennec
+            - --binary=org.mozilla.firefox
+
+raptor-tp6m-11-fennec64-cold:
+    description: "Raptor tp6m-11 cold page-load on Fennec64"
+    max-run-time: 3600
+    try-name: raptor-tp6m-11-fennec64-cold
+    treeherder-symbol: Rap(tp6m-c-11-f64)
+    target:
+        by-test-platform:
+            android-hw-p2-8-0-android-aarch64.*:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-aarch64-opt
+                name: build/target.apk
+            default:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-api-16-opt
+                name: build/target.apk
+    run-on-projects: ['try']
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-11
+            - --app=fennec
+            - --binary=org.mozilla.firefox
+
+raptor-tp6m-12-fennec64-cold:
+    description: "Raptor tp6m-12 cold page-load on Fennec64"
+    max-run-time: 3600
+    try-name: raptor-tp6m-12-fennec64-cold
+    treeherder-symbol: Rap(tp6m-c-12-f64)
+    target:
+        by-test-platform:
+            android-hw-p2-8-0-android-aarch64.*:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-aarch64-opt
+                name: build/target.apk
+            default:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-api-16-opt
+                name: build/target.apk
+    run-on-projects: ['try']
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-12
+            - --app=fennec
+            - --binary=org.mozilla.firefox
+
+raptor-tp6m-13-fennec64-cold:
+    description: "Raptor tp6m-13 cold page-load on Fennec64"
+    max-run-time: 3600
+    try-name: raptor-tp6m-13-fennec64-cold
+    treeherder-symbol: Rap(tp6m-c-13-f64)
+    target:
+        by-test-platform:
+            android-hw-p2-8-0-android-aarch64.*:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-aarch64-opt
+                name: build/target.apk
+            default:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-api-16-opt
+                name: build/target.apk
+    run-on-projects: ['try']
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-13
+            - --app=fennec
+            - --binary=org.mozilla.firefox
+
+raptor-tp6m-14-fennec64-cold:
+    description: "Raptor tp6m-14 cold page-load on Fennec64"
+    max-run-time: 3600
+    try-name: raptor-tp6m-14-fennec64-cold
+    treeherder-symbol: Rap(tp6m-c-14-f64)
+    target:
+        by-test-platform:
+            android-hw-p2-8-0-android-aarch64.*:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-aarch64-opt
+                name: build/target.apk
+            default:
+                index: gecko.v2.mozilla-release.revision.65621d0fe1262af0643cec37c23b2d9ec42588ad.mobile.android-api-16-opt
+                name: build/target.apk
+    run-on-projects: ['try']
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-14
+            - --app=fennec
+            - --binary=org.mozilla.firefox

taskcluster/ci/test/test-sets.yml

@@ -534,6 +534,20 @@ android-hw-arm7-raptor:
     - raptor-tp6m-12-fennec-cold
     - raptor-tp6m-13-fennec-cold
     - raptor-tp6m-14-fennec-cold
+    - raptor-tp6m-1-fennec64-cold
+    - raptor-tp6m-2-fennec64-cold
+    - raptor-tp6m-3-fennec64-cold
+    - raptor-tp6m-4-fennec64-cold
+    - raptor-tp6m-5-fennec64-cold
+    - raptor-tp6m-6-fennec64-cold
+    - raptor-tp6m-7-fennec64-cold
+    - raptor-tp6m-8-fennec64-cold
+    - raptor-tp6m-9-fennec64-cold
+    - raptor-tp6m-10-fennec64-cold
+    - raptor-tp6m-11-fennec64-cold
+    - raptor-tp6m-12-fennec64-cold
+    - raptor-tp6m-13-fennec64-cold
+    - raptor-tp6m-14-fennec64-cold
 
 android-hw-aarch64-raptor:
     - raptor-speedometer-geckoview
@@ -600,6 +614,20 @@ android-hw-aarch64-raptor:
     - raptor-tp6m-12-fennec-cold
     - raptor-tp6m-13-fennec-cold
     - raptor-tp6m-14-fennec-cold
+    - raptor-tp6m-1-fennec64-cold
+    - raptor-tp6m-2-fennec64-cold
+    - raptor-tp6m-3-fennec64-cold
+    - raptor-tp6m-4-fennec64-cold
+    - raptor-tp6m-5-fennec64-cold
+    - raptor-tp6m-6-fennec64-cold
+    - raptor-tp6m-7-fennec64-cold
+    - raptor-tp6m-8-fennec64-cold
+    - raptor-tp6m-9-fennec64-cold
+    - raptor-tp6m-10-fennec64-cold
+    - raptor-tp6m-11-fennec64-cold
+    - raptor-tp6m-12-fennec64-cold
+    - raptor-tp6m-13-fennec64-cold
+    - raptor-tp6m-14-fennec64-cold
 
 android-hw-arm7-raptor-power:
     - raptor-speedometer-geckoview-power

testing/raptor/raptor/tests/raptor-tp6m-cold-1.ini

@@ -44,7 +44,7 @@ playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-facebook.manifest
 playback_recordings = android-facebook.mp
 measure = fnbpaint, fcp, dcf, loadtime
 
-[raptor-tp6m-cold-amazon-fennec]
+[raptor-tp6m-amazon-fennec-cold]
 apps = fennec
 test_url = https://www.amazon.com
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-amazon.manifest
@@ -52,7 +52,7 @@ playback_recordings = android-amazon.mp
 measure = fnbpaint, dcf, loadtime
 alert_on = loadtime
 
-[raptor-tp6m-cold-facebook-fennec]
+[raptor-tp6m-facebook-fennec-cold]
 apps = fennec
 test_url = https://m.facebook.com
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-facebook.manifest

testing/raptor/raptor/tests/raptor-tp6m-cold-10.ini

@@ -44,7 +44,7 @@ playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-stackoverflow.manif
 playback_recordings = mitmproxy-recordings-raptor-tp6m-stackoverflow.mp
 measure = fnbpaint, fcp, dcf, loadtime
 
-[raptor-tp6m-cold-bbc-fennec]
+[raptor-tp6m-bbc-fennec-cold]
 apps = fennec
 test_url = https://www.bbc.com/news/business-47245877
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-bbc.manifest
@@ -52,7 +52,7 @@ playback_recordings = mitmproxy-recordings-raptor-tp6m-bbc.mp
 measure = fnbpaint, dcf, loadtime
 alert_on = loadtime
 
-[raptor-tp6m-cold-stackoverflow-fennec]
+[raptor-tp6m-stackoverflow-fennec-cold]
 apps = fennec
 test_url = https://stackoverflow.com/
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-stackoverflow.manifest

testing/raptor/raptor/tests/raptor-tp6m-cold-11.ini

@@ -44,7 +44,7 @@ playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-jianshu.manifest
 playback_recordings = android-jianshu.mp
 measure = fnbpaint, fcp, dcf, loadtime
 
-[raptor-tp6m-cold-microsoft-support-fennec]
+[raptor-tp6m-microsoft-support-fennec-cold]
 apps = fennec
 test_url = https://support.microsoft.com/en-us
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-microsoft-support.manifest
@@ -52,7 +52,7 @@ playback_recordings = android-microsoft-support.mp
 measure = fnbpaint, dcf, loadtime
 alert_on = loadtime
 
-[raptor-tp6m-cold-jianshu-fennec]
+[raptor-tp6m-jianshu-fennec-cold]
 apps = fennec
 test_url = https://www.jianshu.com/
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-jianshu.manifest

testing/raptor/raptor/tests/raptor-tp6m-cold-12.ini

@@ -44,7 +44,7 @@ playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-allrecipes.manifest
 playback_recordings = android-allrecipes.mp
 measure = fnbpaint, fcp, dcf, loadtime
 
-[raptor-tp6m-cold-imdb-fennec]
+[raptor-tp6m-imdb-fennec-cold]
 apps = fennec
 test_url = https://m.imdb.com/
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-imdb.manifest
@@ -52,7 +52,7 @@ playback_recordings = android-imdb.mp
 measure = fnbpaint, dcf, loadtime
 alert_on = loadtime
 
-[raptor-tp6m-cold-allrecipes-fennec]
+[raptor-tp6m-allrecipes-fennec-cold]
 apps = fennec
 test_url = https://www.allrecipes.com/
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-allrecipes.manifest

testing/raptor/raptor/tests/raptor-tp6m-cold-13.ini

@@ -44,7 +44,7 @@ playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-web-de.manifest
 playback_recordings = android-web-de.mp
 measure = fnbpaint, fcp, dcf, loadtime
 
-[raptor-tp6m-cold-espn-fennec]
+[raptor-tp6m-espn-fennec-cold]
 apps = fennec
 test_url = http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-espn.manifest
@@ -52,7 +52,7 @@ playback_recordings = android-espn.mp
 measure = fnbpaint, dcf, loadtime
 alert_on = loadtime
 
-[raptor-tp6m-cold-web-de-fennec]
+[raptor-tp6m-web-de-fennec-cold]
 apps = fennec
 test_url = https://web.de/magazine/politik/politologe-glaubt-grossen-koalition-herbst-knallen-33563566
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-web-de.manifest

testing/raptor/raptor/tests/raptor-tp6m-cold-14.ini

@@ -44,7 +44,7 @@ playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-aframeio-animation.
 playback_recordings = android-aframeio-animation.mp
 measure = fnbpaint, fcp, dcf, loadtime
 
-[raptor-tp6m-cold-facebook-cristiano-fennec]
+[raptor-tp6m-facebook-cristiano-fennec-cold]
 apps = fennec
 test_url = https://m.facebook.com/Cristiano
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-facebook-cristiano.manifest
@@ -52,7 +52,7 @@ playback_recordings = android-facebook-cristiano.mp
 measure = fnbpaint, dcf, loadtime
 alert_on = loadtime
 
-[raptor-tp6m-cold-aframeio-animation-fennec]
+[raptor-tp6m-aframeio-animation-fennec-cold]
 apps = fennec
 test_url = https://aframe.io/examples/showcase/animation
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-aframeio-animation.manifest

testing/raptor/raptor/tests/raptor-tp6m-cold-2.ini

@@ -59,3 +59,4 @@ playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-youtube.manifest
 playback_recordings = android-youtube.mp
 measure = fnbpaint, dcf, loadtime
 alert_on = loadtime
+disabled = Intermittent fennec v64 - wont fix

testing/raptor/raptor/tests/raptor-tp6m-cold-3.ini

@@ -44,7 +44,7 @@ playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-bing-mobile.manifes
 playback_recordings = bing-mobile.mp
 measure = fnbpaint, fcp, dcf, loadtime
 
-[raptor-tp6m-instagram-fennec]
+[raptor-tp6m-instagram-fennec-cold]
 apps = fennec
 test_url = https://www.instagram.com
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-instagram-mobile.manifest
@@ -52,7 +52,7 @@ playback_recordings = instagram-mobile.mp
 measure = fnbpaint, dcf, loadtime
 alert_on = loadtime
 
-[raptor-tp6m-cold-bing-fennec]
+[raptor-tp6m-bing-fennec-cold]
 apps = fennec
 test_url = https://www.bing.com
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-bing-mobile.manifest

testing/raptor/raptor/tests/raptor-tp6m-cold-4.ini

@@ -44,7 +44,7 @@ playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-ebay-kleinanzeigen-
 playback_recordings = ebay-kleinanzeigen-mobile.mp
 measure = fnbpaint, fcp, dcf, loadtime
 
-[raptor-tp6m-cold-bing-restaurants-fennec]
+[raptor-tp6m-bing-restaurants-fennec-cold]
 apps = fennec
 test_url = https://www.bing.com/search?q=restaurants
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-bing-restaurants-mobile.manifest
@@ -52,7 +52,7 @@ playback_recordings = bing-restaurants-mobile.mp
 measure = fnbpaint, dcf, loadtime
 alert_on = loadtime
 
-[raptor-tp6m-ebay-kleinanzeigen-fennec]
+[raptor-tp6m-ebay-kleinanzeigen-fennec-cold]
 apps = fennec
 test_url = https://m.ebay-kleinanzeigen.de
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-ebay-kleinanzeigen-mobile.manifest

testing/raptor/raptor/tests/raptor-tp6m-cold-5.ini

@@ -44,7 +44,7 @@ playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-google-maps.manifes
 playback_recordings = google_maps_mobile.mp
 measure = fnbpaint, fcp, dcf, loadtime
 
-[raptor-tp6m-ebay-kleinanzeigen-search-fennec]
+[raptor-tp6m-ebay-kleinanzeigen-search-fennec-cold]
 apps = fennec
 test_url = https://m.ebay-kleinanzeigen.de/s-anzeigen/auf-zeit-wg-berlin/zimmer/c199-l3331
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-ebay-kleinanzeigen-search-mobile.manifest
@@ -52,7 +52,7 @@ playback_recordings = ebay-kleinanzeigen-search-mobile.mp
 measure = fnbpaint, dcf, loadtime
 alert_on = loadtime
 
-[raptor-tp6m-cold-google-maps-fennec]
+[raptor-tp6m-google-maps-fennec-cold]
 apps = fennec
 test_url = https://www.google.com/maps?force=pwa
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-google-maps.manifest

testing/raptor/raptor/tests/raptor-tp6m-cold-6.ini

@@ -46,7 +46,7 @@ playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-amazon-search.manif
 playback_recordings = android-amazon-search.mp
 measure = fnbpaint, fcp, dcf, loadtime
 
-[raptor-tp6m-cold-google-restaurants-fennec]
+[raptor-tp6m-google-restaurants-fennec-cold]
 apps = fennec
 test_url = https://www.google.com/search?q=restaurants+near+me
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-google-restaurants.manifest
@@ -55,7 +55,7 @@ measure = fnbpaint, dcf, loadtime
 alert_on = loadtime
 disabled = Bug 1541431 TEST-UNEXPECTED-FAIL: test 'raptor-tp6m-google-restaurants-fennec' timed out loading test page
 
-[raptor-tp6m-cold-amazon-search-fennec]
+[raptor-tp6m-cold-amazon-search-fennec-cold]
 apps = fennec
 test_url = https://www.amazon.com/s/ref=nb_sb_noss_2/139-6317191-5622045?url=search-alias%3Daps&field-keywords=mobile+phone
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-amazon-search.manifest

testing/raptor/raptor/tests/raptor-tp6m-cold-7.ini

@@ -44,7 +44,7 @@ playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-youtube-watch.manif
 playback_recordings = android-youtube-watch.mp
 measure = fnbpaint, fcp, dcf, loadtime
 
-[raptor-tp6m-cold-wikipedia-fennec]
+[raptor-tp6m-wikipedia-fennec-cold]
 apps = fennec
 test_url = https://en.m.wikipedia.org/wiki/Main_Page
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-wikipedia.manifest
@@ -52,7 +52,7 @@ playback_recordings = android-wikipedia.mp
 measure = fnbpaint, dcf, loadtime
 alert_on = loadtime
 
-[raptor-tp6m-cold-youtube-watch-fennec]
+[raptor-tp6m-youtube-watch-fennec-cold]
 apps = fennec
 test_url = https://www.youtube.com/watch?v=COU5T-Wafa4
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-youtube-watch.manifest

testing/raptor/raptor/tests/raptor-tp6m-cold-8.ini

@@ -46,7 +46,7 @@ playback_recordings = android-cnn.mp
 measure = fnbpaint, fcp, dcf, loadtime
 disabled = Bug 1533287 Intermittent timeouts running raptor-tp6m-cnn-geckoview
 
-[raptor-tp6m-cold-booking-fennec]
+[raptor-tp6m-booking-fennec-cold]
 apps = fennec
 test_url = https://www.booking.com/
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-booking.manifest
@@ -54,10 +54,11 @@ playback_recordings = android-booking.mp
 measure = fnbpaint, dcf, loadtime
 alert_on = loadtime
 
-[raptor-tp6m-cold-cnn-fennec]
+[raptor-tp6m-cnn-fennec-cold]
 apps = fennec
 test_url = https://edition.cnn.com/
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-cnn.manifest
 playback_recordings = android-cnn.mp
 measure = fnbpaint, dcf, loadtime
 alert_on = loadtime
+disabled = Intermittent on fennec v64 - wont fix

testing/raptor/raptor/tests/raptor-tp6m-cold-9.ini

@@ -44,7 +44,7 @@ playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-reddit.manifest
 playback_recordings = mitmproxy-recordings-raptor-tp6m-reddit.mp
 measure = fnbpaint, fcp, dcf, loadtime
 
-[raptor-tp6m-cold-cnn-ampstories-fennec]
+[raptor-tp6m-cnn-ampstories-fennec-cold]
 apps = fennec
 test_url = https://edition.cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-cnn-ampstories.manifest
@@ -52,7 +52,7 @@ playback_recordings = android-cnn-ampstories.mp
 measure = fnbpaint, dcf, loadtime
 alert_on = loadtime
 
-[raptor-tp6m-cold-reddit-fennec]
+[raptor-tp6m-reddit-fennec-cold]
 apps = fennec
 test_url = https://www.reddit.com
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-reddit.manifest