gecko-dev/testing/failedplatform.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from functools import reduce
from typing import Dict, List, Optional, Set


class FailedPlatform:
    """
    Stores all failures on different build types and test variants for a single platform.
    This allows us to detect when a platform failed on all build types or all test variants to
    generate a simpler skip-if condition.
    """

    def __init__(
        self,
        # Keys are build types, values are test variants for this build type
        # Tests variants can be composite by using the "+" character
        # eg: a11y_checks+swgl
        # each build_type[test_variant] has a {'pass': x, 'fail': y}
        # x and y represent number of times this was run in the last 30 days
        # See examples in
        # https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.latest.source.test-info-all/artifacts/public%2Ftest-info-testrun-matrix.json
        oop_permutations: Optional[
            Dict[
                str,  # Build type
                Dict[str, Dict[str, int]],  # Test Variant  # {'pass': x, 'fail': y}
            ]
        ],
    ) -> None:
        # Contains all test variants for each build type the task failed on
        self.failures: Dict[str, Set[str]] = {}
        self.oop_permutations = oop_permutations

    def get_possible_build_types(self) -> List[str]:
        return (
            list(self.oop_permutations.keys())
            if self.oop_permutations is not None
            else []
        )

    def get_possible_test_variants(self, build_type: str) -> List[str]:
        permutations = (
            self.oop_permutations.get(build_type, {})
            if self.oop_permutations is not None
            else []
        )
        return [tv for tv in permutations]

    def is_full_fail(self) -> bool:
        """
        Test if failed on every test variant of every build type
        """
        build_types = set(self.failures.keys())
        possible_build_types = self.get_possible_build_types()
        # If we do not have information on possible build types, do not consider it a full fail
        # This avoids creating a too broad skip-if condition
        if len(possible_build_types) == 0:
            return False
        return all(
            [
                bt in build_types and self.is_full_test_variants_fail(bt)
                for bt in possible_build_types
            ]
        )

    def is_full_test_variants_fail(self, build_type: str) -> bool:
        """
        Test if failed on every test variant of given build type
        """
        failed_variants = self.failures.get(build_type, [])
        possible_test_variants = self.get_possible_test_variants(build_type)
        # If we do not have information on possible test variants, do not consider it a full fail
        # This avoids creating a too broad skip-if condition
        if len(possible_test_variants) == 0:
            return False
        return all([t in failed_variants for t in possible_test_variants])

    def get_negated_variant(self, test_variant: str):
        if not test_variant.startswith("!"):
            return "!" + test_variant
        return test_variant.replace("!", "", 1)

    def get_no_variant_conditions(self, and_str: str, build_type: str):
        """
        The no_variant test variant does not really exist and is only internal.
        This function gets all available test variants for the given build type
        and negates them to create a skip-if that handle tasks without test variants
        """
        variants = [
            tv
            for tv in self.get_possible_test_variants(build_type)
            if tv != "no_variant"
        ]
        return_str = ""
        for tv in variants:
            return_str += and_str + self.get_negated_variant(tv)
        return return_str

    def get_test_variant_condition(
        self, and_str: str, build_type: str, test_variant: str
    ):
        """
        If the given test variant is part of another composite test variant, then add negations matching that composite
        variant to prevent overlapping in skips.
        eg: test variant "a11y_checks" is to be added while "a11y_checks+swgl" exists
        the resulting condition will be "a11y_checks && !swgl"
        """
        all_test_variants_parts = [
            tv.split("+")
            for tv in self.get_possible_test_variants(build_type)
            if tv not in ["no_variant", test_variant]
        ]
        test_variant_parts = test_variant.split("+")
        # List of composite test variants more specific than the current one
        matching_variants_parts = [
            tv_parts
            for tv_parts in all_test_variants_parts
            if all(x in tv_parts for x in test_variant_parts)
        ]
        variants_to_negate = [
            part
            for tv_parts in matching_variants_parts
            for part in tv_parts
            if part not in test_variant_parts
        ]

        return_str = reduce((lambda x, y: x + and_str + y), test_variant_parts, "")
        return_str = reduce(
            (lambda x, y: x + and_str + self.get_negated_variant(y)),
            variants_to_negate,
            return_str,
        )
        return return_str

    def get_test_variant_string(self, test_variant: str):
        """
        Some test variants strings need to be updated to match what is given in oop_permutations
        """
        if test_variant == "no-fission":
            return "!fission"
        if test_variant == "1proc":
            return "!e10s"
        return test_variant

    def get_skip_string(self, and_str: str, build_type: str, test_variant: str) -> str:
        if self.failures.get(build_type) is None:
            self.failures[build_type] = {test_variant}
        else:
            self.failures[build_type].add(test_variant)

        return_str = ""
        # If every test variant of every build type failed, do not add anything
        if not self.is_full_fail():
            return_str += and_str + build_type
            if not self.is_full_test_variants_fail(build_type):
                if test_variant == "no_variant":
                    return_str += self.get_no_variant_conditions(and_str, build_type)
                else:
                    return_str += self.get_test_variant_condition(
                        and_str, build_type, test_variant
                    )

        return return_str