Bug 1854553 - Introduce simple tests of Snap package r=releng-reviewers,taskgraph-reviewers,jmaher,jcristau

Differential Revision: https://phabricator.services.mozilla.com/D188932

taskcluster/ci/snap-upstream-test/kind.yml

@@ -0,0 +1,43 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: gecko_taskgraph.loader.transform:loader
+
+only-for-attributes:
+    - build_platform
+
+transforms:
+    - taskgraph.transforms.from_deps
+    - gecko_taskgraph.transforms.job
+    - gecko_taskgraph.transforms.task
+    - gecko_taskgraph.transforms.snap_test
+
+kind-dependencies:
+    - snap-upstream-build
+
+job-defaults:
+    treeherder:
+        kind: test
+        tier: 2
+    worker:
+        max-run-time: 900
+
+jobs:
+    basic:
+        from-deps:
+            group-by: single-with-filters
+        description: Runs basic tests on a Firefox Snap package
+        worker-type: t-linux-wayland
+        run:
+            using: run-task
+            checkout: false
+            command: >-
+              cd $MOZ_FETCHES_DIR/ && ./tests.sh
+        fetches:
+            snap-upstream-build:
+                - firefox.snap
+                - snap-tests.zip
+        treeherder:
+            platform: linux64/opt
+            symbol: Sel

taskcluster/docker/snap-coreXX-build/run.sh

@@ -81,3 +81,10 @@ SNAPCRAFT_BUILD_ENVIRONMENT_CPU=$(nproc) \
 CRAFT_PARTS_PACKAGE_REFRESH=0 \
   snapcraft --destructive-mode --verbose
 cp ./*.snap ./*.debug /builds/worker/artifacts/
+
+# Those are for fetches usage by the test task
+cp ./*.snap /builds/worker/artifacts/firefox.snap
+cp ./*.debug /builds/worker/artifacts/firefox.debug
+
+# Those are for running snap-upstream-test
+cd /builds/worker/checkouts/gecko/taskcluster/docker/snap-coreXX-build/snap-tests/ && zip -r9 /builds/worker/artifacts/snap-tests.zip ./*

taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2022 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import json
+import os
+import sys
+
+from selenium import webdriver
+from selenium.common.exceptions import TimeoutException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.firefox.options import Options
+from selenium.webdriver.firefox.service import Service
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+
+
+class SnapTests:
+    def __init__(self, exp):
+        driver_service = Service(
+            executable_path=r"/snap/firefox/current/usr/lib/firefox/geckodriver"
+        )
+        options = Options()
+        options.binary_location = r"/snap/firefox/current/usr/lib/firefox/firefox"
+        options.add_argument("--headless")
+        self._driver = webdriver.Firefox(service=driver_service, options=options)
+
+        object_methods = [
+            method_name
+            for method_name in dir(self)
+            if callable(getattr(self, method_name))
+        ]
+
+        self._wait = WebDriverWait(self._driver, self.get_timeout())
+
+        with open(exp, "r") as j:
+            self._expectations = json.load(j)
+
+        try:
+            for m in object_methods:
+                if m.startswith("test_"):
+                    print("Running: {}".format(m))
+                    getattr(self, m)(self._expectations[m])
+        finally:
+            self._driver.quit()
+
+    def get_timeout(self):
+        if "TEST_TIMEOUT" in os.environ.keys():
+            return int(os.getenv("TEST_TIMEOUT"))
+        else:
+            return 5
+
+    def open_tab(self, url):
+        opened_tabs = len(self._driver.window_handles)
+
+        self._driver.switch_to.new_window("tab")
+        self._wait.until(EC.number_of_windows_to_be(opened_tabs + 1))
+        self._driver.get(url)
+
+    def test_about_support(self, exp):
+        self.open_tab("about:support")
+
+        version_box = self._wait.until(
+            EC.visibility_of_element_located((By.ID, "version-box"))
+        )
+        self._wait.until(lambda d: len(version_box.text) > 0)
+        print("about:support version: {}".format(version_box.text))
+        assert version_box.text == exp["version_box"]
+
+        distributionid_box = self._wait.until(
+            EC.visibility_of_element_located((By.ID, "distributionid-box"))
+        )
+        self._wait.until(lambda d: len(distributionid_box.text) > 0)
+        print("about:support distribution ID: {}".format(distributionid_box.text))
+        assert distributionid_box.text == exp["distribution_id"]
+
+    def test_youtube(self, exp):
+        self.open_tab("https://www.youtube.com")
+
+        # Wait for the consent dialog and accept it
+        print("Wait for consent form")
+        try:
+            self._wait.until(
+                EC.visibility_of_element_located(
+                    (By.CSS_SELECTOR, "button[aria-label*=Accept]")
+                )
+            ).click()
+        except TimeoutException:
+            print("Wait for consent form: timed out, maybe it is not here")
+
+        # Find first video and click it
+        print("Wait for one video")
+        self._wait.until(
+            EC.visibility_of_element_located((By.ID, "video-title-link"))
+        ).click()
+
+        # Wait for duration to be set to something
+        print("Wait for video to start")
+        video = self._wait.until(
+            EC.visibility_of_element_located((By.CLASS_NAME, "html5-main-video"))
+        )
+        self._wait.until(lambda d: type(video.get_property("duration")) == float)
+        print("video duration: {}".format(video.get_property("duration")))
+        assert video.get_property("duration") > exp["duration"]
+
+        self._wait.until(lambda d: video.get_property("currentTime") > exp["playback"])
+        print("video played: {}".format(video.get_property("currentTime")))
+        assert video.get_property("currentTime") > exp["playback"]
+
+
+if __name__ == "__main__":
+    SnapTests(sys.argv[1])

taskcluster/docker/snap-coreXX-build/snap-tests/expectations.json.in

@@ -0,0 +1,10 @@
+{
+    "test_about_support": {
+        "version_box": "#RUNTIME_VERSION#",
+        "distribution_id": "canonical-002"
+    },
+    "test_youtube": {
+        "duration": 1,
+        "playback": 2
+    }
+}

taskcluster/docker/snap-coreXX-build/snap-tests/requirements.txt

@@ -0,0 +1,15 @@
+attrs==23.1.0
+certifi==2023.7.22
+exceptiongroup==1.1.3
+h11==0.14.0
+idna==3.4
+outcome==1.2.0
+PySocks==1.7.1
+PyYAML==6.0.1
+selenium==4.12.0
+sniffio==1.3.0
+sortedcontainers==2.4.0
+trio==0.22.2
+trio-websocket==0.10.4
+urllib3==2.0.5
+wsproto==1.2.0

taskcluster/docker/snap-coreXX-build/snap-tests/tests.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -ex
+
+pwd
+
+sudo snap refresh --hold=24h firefox
+
+sudo snap install --name firefox --dangerous ./firefox.snap
+
+RUNTIME_VERSION=$(snap run firefox --version | awk '{ print $3 }')
+
+python3 -m pip install --user -r requirements.txt
+
+sed -e "s/#RUNTIME_VERSION#/${RUNTIME_VERSION}/#" < expectations.json.in > expectations.json
+
+python3 basic_tests.py expectations.json

taskcluster/docs/kinds.rst

@@ -798,3 +798,7 @@ Injects attribution information into localized installers.
 snap-upstream-build
 -------------------
 Perform a Firefox Snap build using upstream tooling
+
+snap-upstream-test
+-------------------
+Test a Firefox Snap built using upstream tooling

taskcluster/gecko_taskgraph/transforms/snap_test.py

@@ -0,0 +1,42 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+"""
+
+
+import logging
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.dependencies import get_primary_dependency
+from taskgraph.util.treeherder import inherit_treeherder_from_dep
+
+logger = logging.getLogger(__name__)
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def fill_template(config, tasks):
+    for task in tasks:
+        dep = get_primary_dependency(config, task)
+        assert dep
+
+        inherit_treeherder_from_dep(task, dep)
+        task_platform = task["task"]["extra"]["treeherder"]["machine"]["platform"]
+
+        # Disambiguate the treeherder symbol.
+        full_platform_collection = (
+            task_platform + "-snap-" + task.get("label").split("-")[-1]
+        )
+        (platform, collection) = full_platform_collection.split("/")
+        task["task"]["extra"]["treeherder"]["collection"] = {collection: True}
+        task["task"]["extra"]["treeherder"]["machine"]["platform"] = platform
+        task["task"]["extra"]["treeherder-platform"] = full_platform_collection
+
+        timeout = 10
+        if collection != "opt":
+            timeout = 60
+        task["task"]["payload"]["env"]["TEST_TIMEOUT"] = "{}".format(timeout)
+
+        yield task