From ba5a0fdc374712bbb10898dad443d9101d344367 Mon Sep 17 00:00:00 2001 From: Aaron Train Date: Tue, 4 Jun 2024 13:07:52 +0000 Subject: [PATCH] Bug 1898010 - Add x86 and ARM v7a android-startup-test tasks for Fenix and Focus r=ohorvath,ahal,RyanVM Differential Revision: https://phabricator.services.mozilla.com/D211104 --- .../flank-armeabi-v7a-start-test.yml | 39 ++++++ .../androidTest/flank-x86-start-test.yml | 39 ++++++ .../flank-armeabi-v7a-start-test.yml | 36 ++++++ .../androidTest/flank-x86-start-test.yml | 36 ++++++ .../transforms/push_android_app.py | 16 --- .../transforms/startup_tests_deps.py | 28 ++++ .../kinds/android-startup-test/kind.yml | 122 ++++++++++++++++-- taskcluster/kinds/push-bundle/kind.yml | 2 + 8 files changed, 293 insertions(+), 25 deletions(-) create mode 100644 mobile/android/fenix/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml create mode 100644 mobile/android/fenix/automation/taskcluster/androidTest/flank-x86-start-test.yml create mode 100644 mobile/android/focus-android/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml create mode 100644 mobile/android/focus-android/automation/taskcluster/androidTest/flank-x86-start-test.yml create mode 100644 taskcluster/android_taskgraph/transforms/startup_tests_deps.py diff --git a/mobile/android/fenix/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml b/mobile/android/fenix/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml new file mode 100644 index 000000000000..99c7af4e0157 --- /dev/null +++ b/mobile/android/fenix/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml @@ -0,0 +1,39 @@ +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + timeout: 15m + async: false + num-flaky-test-attempts: 1 + + app: /app/path + test: /test/path + + auto-google-login: false + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - class org.mozilla.fenix.ui.HistoryTest#verifyHistoryMenuWithHistoryItemsTest + - class org.mozilla.fenix.ui.SettingsSearchTest#verifyShowSearchSuggestionsToggleTest + - class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest + - class org.mozilla.fenix.ui.HistoryTest#noHistoryInPrivateBrowsingTest + - class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest + + device: + - model: redfin + version: 30 + locale: en_US + +flank: + project: GOOGLE_PROJECT + max-test-shards: 2 + num-test-runs: 1 + output-style: compact + full-junit-result: true diff --git a/mobile/android/fenix/automation/taskcluster/androidTest/flank-x86-start-test.yml b/mobile/android/fenix/automation/taskcluster/androidTest/flank-x86-start-test.yml new file mode 100644 index 000000000000..41a4e108fe8a --- /dev/null +++ b/mobile/android/fenix/automation/taskcluster/androidTest/flank-x86-start-test.yml @@ -0,0 +1,39 @@ +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + timeout: 15m + async: false + num-flaky-test-attempts: 1 + + app: /app/path + test: /test/path + + auto-google-login: false + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - class org.mozilla.fenix.ui.HistoryTest#verifyHistoryMenuWithHistoryItemsTest + - class org.mozilla.fenix.ui.SettingsSearchTest#verifyShowSearchSuggestionsToggleTest + - class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest + - class org.mozilla.fenix.ui.HistoryTest#noHistoryInPrivateBrowsingTest + - class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest + + device: + - model: Pixel2 + version: 30 + locale: en_US + +flank: + project: GOOGLE_PROJECT + max-test-shards: 2 + num-test-runs: 1 + output-style: compact + full-junit-result: true diff --git a/mobile/android/focus-android/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml b/mobile/android/focus-android/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml new file mode 100644 index 000000000000..d6821c59ba32 --- /dev/null +++ b/mobile/android/focus-android/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml @@ -0,0 +1,36 @@ +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ +gcloud: + results-bucket: focus_android_test_artifacts + record-video: true + + timeout: 30m + async: false + num-flaky-test-attempts: 1 + + app: /app/path + test: /test/path + + auto-google-login: false + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - class org.mozilla.focus.activity.SearchTest#testBlankSearchDoesNothing + - class org.mozilla.focus.activity.EraseBrowsingDataTest#deleteHistoryOnRestartTest + + device: + - model: redfin + version: 30 + locale: en_US + +flank: + project: GOOGLE_PROJECT + max-test-shards: 1 + num-test-runs: 1 + output-style: compact + full-junit-result: true diff --git a/mobile/android/focus-android/automation/taskcluster/androidTest/flank-x86-start-test.yml b/mobile/android/focus-android/automation/taskcluster/androidTest/flank-x86-start-test.yml new file mode 100644 index 000000000000..3971ccdda137 --- /dev/null +++ b/mobile/android/focus-android/automation/taskcluster/androidTest/flank-x86-start-test.yml @@ -0,0 +1,36 @@ +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ +gcloud: + results-bucket: focus_android_test_artifacts + record-video: true + + timeout: 30m + async: false + num-flaky-test-attempts: 1 + + app: /app/path + test: /test/path + + auto-google-login: false + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - class org.mozilla.focus.activity.SearchTest#testBlankSearchDoesNothing + - class org.mozilla.focus.activity.EraseBrowsingDataTest#deleteHistoryOnRestartTest + + device: + - model: Pixel2 + version: 30 + locale: en_US + +flank: + project: GOOGLE_PROJECT + max-test-shards: 1 + num-test-runs: 1 + output-style: compact + full-junit-result: true diff --git a/taskcluster/android_taskgraph/transforms/push_android_app.py b/taskcluster/android_taskgraph/transforms/push_android_app.py index bca9b3c8dedc..8a651bbc4edb 100644 --- a/taskcluster/android_taskgraph/transforms/push_android_app.py +++ b/taskcluster/android_taskgraph/transforms/push_android_app.py @@ -34,19 +34,3 @@ def resolve_keys(config, tasks): } ) yield task - - -@transforms.add -def add_startup_test(config, tasks): - for task in tasks: - if "nightly" not in task["attributes"].get("build-type", ""): - yield task - continue - for dep_label, dep_task in config.kind_dependencies_tasks.items(): - if ( - dep_task.kind == "android-startup-test" - and dep_task.attributes["shipping-product"] - == task["attributes"]["shipping-product"] - ): - task["dependencies"]["android-startup-test"] = dep_label - yield task diff --git a/taskcluster/android_taskgraph/transforms/startup_tests_deps.py b/taskcluster/android_taskgraph/transforms/startup_tests_deps.py new file mode 100644 index 000000000000..1d14866b7083 --- /dev/null +++ b/taskcluster/android_taskgraph/transforms/startup_tests_deps.py @@ -0,0 +1,28 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def add_android_startup_test_dependencies(config, jobs): + scheduled_tasks = [ + t + for t in config.kind_dependencies_tasks.values() + if t.kind == "android-startup-test" + ] + for job in jobs: + job_build_type = job["attributes"].get("build-type", "") + matching_tasks = [ + t + for t in scheduled_tasks + if job_build_type in t.attributes.get("build-type", "") + ] + if matching_tasks: + job.setdefault("dependencies", {}).update( + {t.label: t.label for t in matching_tasks} + ) + yield job diff --git a/taskcluster/kinds/android-startup-test/kind.yml b/taskcluster/kinds/android-startup-test/kind.yml index d7fe550f9932..463a3aa0daf3 100644 --- a/taskcluster/kinds/android-startup-test/kind.yml +++ b/taskcluster/kinds/android-startup-test/kind.yml @@ -8,6 +8,9 @@ transforms: - gecko_taskgraph.transforms.job:transforms - gecko_taskgraph.transforms.task:transforms +kind-dependencies: + - signing-apk + job-defaults: description: Runs UI tests for sanity checking startup on Nightly treeherder: @@ -24,17 +27,18 @@ job-defaults: - name: public path: /builds/worker/artifacts type: directory + retry-exit-status: [20] run: using: run-commands use-caches: false run-on-projects: [] jobs: - fenix-arm-nightly-robo-opt: + fenix-arm64-v8a-nightly-robo-opt: attributes: build-type: fenix-nightly shipping-product: fenix - description: Fenix Nightly Opt Robo Test (ARM) + description: Fenix Nightly Opt Robo Test (arm64-v8a) dependencies: signing: signing-apk-fenix-nightly run: @@ -49,12 +53,12 @@ jobs: - [wget, {artifact-reference: ''}, '-O', app.apk] - [python3, ../../../taskcluster/scripts/tests/test-lab.py, arm-start-test-robo, app.apk] treeherder: - symbol: fenix-nightly(startup-arm-robo-opt) + symbol: fenix-nightly(startup-arm64-robo-opt) worker: env: GOOGLE_PROJECT: moz-fenix - fenix-arm-nightly: + fenix-arm64-v8a-nightly: attributes: build-type: fenix-nightly-firebase shipping-product: fenix @@ -74,12 +78,62 @@ jobs: - [wget, {artifact-reference: ''}, '-O', android-test.apk] - [python3, ../../../taskcluster/scripts/tests/test-lab.py, arm-start-test, app.apk, --apk_test, android-test.apk] treeherder: - symbol: fenix-nightly(startup-arm) + symbol: fenix-nightly(startup-arm64-v8a) worker: env: GOOGLE_PROJECT: moz-fenix - focus-arm-nightly: + fenix-armeabi-v7a-nightly: + attributes: + build-type: fenix-nightly-firebase + shipping-product: fenix + dependencies: + signing: signing-apk-fenix-nightly-firebase + signing-android-test: signing-apk-fenix-android-test-nightly + run: + secrets: + - name: project/mobile/firefox-android/fenix/firebase + key: firebaseToken + path: .firebase_token.json + json: true + pre-commands: + - ["cd", "mobile/android/fenix"] + commands: + - [wget, {artifact-reference: ''}, '-O', app.apk] + - [wget, {artifact-reference: ''}, '-O', android-test.apk] + - [python3, ../../../taskcluster/scripts/tests/test-lab.py, armeabi-v7a-start-test, app.apk, --apk_test, android-test.apk] + treeherder: + symbol: fenix-nightly(startup-armeabi-v7a) + worker: + env: + GOOGLE_PROJECT: moz-fenix + + fenix-x86-nightly: + attributes: + build-type: fenix-nightly-firebase + shipping-product: fenix + dependencies: + signing: signing-apk-fenix-nightly-firebase + signing-android-test: signing-apk-fenix-android-test-nightly + run: + secrets: + - name: project/mobile/firefox-android/fenix/firebase + key: firebaseToken + path: .firebase_token.json + json: true + pre-commands: + - ["cd", "mobile/android/fenix"] + commands: + - [wget, {artifact-reference: ''}, '-O', app.apk] + - [wget, {artifact-reference: ''}, '-O', android-test.apk] + - [python3, ../../../taskcluster/scripts/tests/test-lab.py, x86-start-test, app.apk, --apk_test, android-test.apk] + treeherder: + symbol: fenix-nightly(startup-x86) + worker: + env: + GOOGLE_PROJECT: moz-fenix + + focus-arm64-v8a-nightly: attributes: build-type: focus-nightly-firebase shipping-product: focus @@ -99,12 +153,62 @@ jobs: - [wget, {artifact-reference: ''}, '-O', android-test.apk] - [python3, ../../../taskcluster/scripts/tests/test-lab.py, arm-start-test, app.apk, --apk_test, android-test.apk] treeherder: - symbol: focus-nightly(startup-arm) + symbol: focus-nightly(startup-arm64-v8a) worker: env: GOOGLE_PROJECT: moz-focus-android - focus-arm-nightly-robo-opt: + focus-armeabi-v7a-nightly: + attributes: + build-type: focus-nightly-firebase + shipping-product: focus + dependencies: + signed-apk-debug-apk: signing-apk-focus-nightly-firebase + signed-apk-android-test: signing-apk-focus-android-test-nightly + run: + secrets: + - name: project/mobile/firefox-android/focus-android/firebase + key: firebaseToken + path: .firebase_token.json + json: true + pre-commands: + - ["cd", "mobile/android/focus-android"] + commands: + - [wget, {artifact-reference: ''}, '-O', app.apk] + - [wget, {artifact-reference: ''}, '-O', android-test.apk] + - [python3, ../../../taskcluster/scripts/tests/test-lab.py, armeabi-v7a-start-test, app.apk, --apk_test, android-test.apk] + treeherder: + symbol: focus-nightly(startup-armeabi-v7a) + worker: + env: + GOOGLE_PROJECT: moz-focus-android + + focus-x86-nightly: + attributes: + build-type: focus-nightly-firebase + shipping-product: focus + dependencies: + signed-apk-debug-apk: signing-apk-focus-nightly-firebase + signed-apk-android-test: signing-apk-focus-android-test-nightly + run: + secrets: + - name: project/mobile/firefox-android/focus-android/firebase + key: firebaseToken + path: .firebase_token.json + json: true + pre-commands: + - ["cd", "mobile/android/focus-android"] + commands: + - [wget, {artifact-reference: ''}, '-O', app.apk] + - [wget, {artifact-reference: ''}, '-O', android-test.apk] + - [python3, ../../../taskcluster/scripts/tests/test-lab.py, x86-start-test, app.apk, --apk_test, android-test.apk] + treeherder: + symbol: focus-nightly(startup-x86) + worker: + env: + GOOGLE_PROJECT: moz-focus-android + + focus-arm64-v8a-nightly-robo-opt: attributes: build-type: focus-nightly shipping-product: focus @@ -123,7 +227,7 @@ jobs: - [wget, {artifact-reference: ''}, '-O', app.apk] - [python3, ../../../taskcluster/scripts/tests/test-lab.py, arm-start-test-robo, app.apk] treeherder: - symbol: focus-nightly(startup-arm-robo-opt) + symbol: focus-nightly(startup-arm64-robo-opt) worker: env: GOOGLE_PROJECT: moz-focus-android diff --git a/taskcluster/kinds/push-bundle/kind.yml b/taskcluster/kinds/push-bundle/kind.yml index 2c48a1417776..de17e587580d 100644 --- a/taskcluster/kinds/push-bundle/kind.yml +++ b/taskcluster/kinds/push-bundle/kind.yml @@ -9,6 +9,7 @@ transforms: - android_taskgraph.transforms.upstream_artifacts - android_taskgraph.transforms.treeherder - android_taskgraph.transforms.push_android_app + - android_taskgraph.transforms.startup_tests_deps - gecko_taskgraph.transforms.task kind-dependencies: @@ -19,6 +20,7 @@ jobs: push-bundle: description: Publish Fenix and Focus AABs from-deps: + kinds: [signing-bundle] with-attributes: build-type: - fenix-nightly