# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import gzip
import io
import json
import logging
import os
import os.path
import pprint
import re
import sys
import tempfile
import urllib.parse
from enum import Enum
from pathlib import Path
from xmlrpc.client import Fault

from yaml import load

try:
    from yaml import CLoader as Loader
except ImportError:
    from yaml import Loader

import bugzilla
import mozci.push
import requests
from manifestparser import ManifestParser
from manifestparser.toml import add_skip_if, alphabetize_toml_str, sort_paths
from mozci.task import TestTask
from mozci.util.taskcluster import get_task

BUGZILLA_AUTHENTICATION_HELP = "Must create a Bugzilla API key per https://github.com/mozilla/mozci-tools/blob/main/citools/test_triage_bug_filer.py"
TASK_LOG = "live_backing.log"
TASK_ARTIFACT = "public/logs/" + TASK_LOG
ATTACHMENT_DESCRIPTION = "Compressed " + TASK_ARTIFACT + " for task "
ATTACHMENT_REGEX = (
    r".*Created attachment ([0-9]+)\n.*"
    + ATTACHMENT_DESCRIPTION
    + "([A-Za-z0-9_-]+)\n.*"
)


class MockResult(object):
    def __init__(self, result):
        self.result = result

    @property
    def group(self):
        return self.result["group"]

    @property
    def ok(self):
        _ok = self.result["ok"]
        return _ok


class MockTask(object):
    def __init__(self, task):
        self.task = task
        if "results" in self.task:
            self.task["results"] = [
                MockResult(result) for result in self.task["results"]
            ]
        else:
            self.task["results"] = []

    @property
    def failure_types(self):
        if "failure_types" in self.task:
            return self.task["failure_types"]
        else:  # note no failure_types in Task object
            return {}

    @property
    def id(self):
        return self.task["id"]

    @property
    def label(self):
        return self.task["label"]

    @property
    def results(self):
        return self.task["results"]


class Classification(object):
    "Classification of the failure (not the task result)"

    DISABLE_MANIFEST = "disable_manifest"  # crash found
    DISABLE_RECOMMENDED = "disable_recommended"  # disable first failing path
    INTERMITTENT = "intermittent"
    SECONDARY = "secondary"  # secondary failing path
    SUCCESS = "success"  # path always succeeds
    UNKNOWN = "unknown"


class Run(Enum):
    """
    constant indexes for attributes of a run
    """

    MANIFEST = 0
    TASK_ID = 1
    TASK_LABEL = 2
    RESULT = 3
    CLASSIFICATION = 4


class Skipfails(object):
    "mach manifest skip-fails implementation: Update manifests to skip failing tests"

    REPO = "repo"
    REVISION = "revision"
    TREEHERDER = "treeherder.mozilla.org"
    BUGZILLA_SERVER_DEFAULT = "bugzilla.allizom.org"

    def __init__(
        self,
        command_context=None,
        try_url="",
        verbose=False,
        bugzilla=None,
        dry_run=False,
        turbo=False,
    ):
        self.command_context = command_context
        if self.command_context is not None:
            self.topsrcdir = self.command_context.topsrcdir
        else:
            self.topsrcdir = Path(__file__).parent.parent
        self.topsrcdir = os.path.normpath(self.topsrcdir)
        if isinstance(try_url, list) and len(try_url) == 1:
            self.try_url = try_url[0]
        else:
            self.try_url = try_url
        self.dry_run = dry_run
        self.verbose = verbose
        self.turbo = turbo
        if bugzilla is not None:
            self.bugzilla = bugzilla
        else:
            if "BUGZILLA" in os.environ:
                self.bugzilla = os.environ["BUGZILLA"]
            else:
                self.bugzilla = Skipfails.BUGZILLA_SERVER_DEFAULT
        self.component = "skip-fails"
        self._bzapi = None
        self._attach_rx = None
        self.variants = {}
        self.tasks = {}
        self.pp = None
        self.headers = {}  # for Treeherder requests
        self.headers["Accept"] = "application/json"
        self.headers["User-Agent"] = "treeherder-pyclient"
        self.jobs_url = "https://treeherder.mozilla.org/api/jobs/"
        self.push_ids = {}
        self.job_ids = {}

    def _initialize_bzapi(self):
        """Lazily initializes the Bugzilla API"""
        if self._bzapi is None:
            self._bzapi = bugzilla.Bugzilla(self.bugzilla)
            self._attach_rx = re.compile(ATTACHMENT_REGEX, flags=re.M)

    def pprint(self, obj):
        if self.pp is None:
            self.pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)
        self.pp.pprint(obj)
        sys.stderr.flush()

    def error(self, e):
        if self.command_context is not None:
            self.command_context.log(
                logging.ERROR, self.component, {"error": str(e)}, "ERROR: {error}"
            )
        else:
            print(f"ERROR: {e}", file=sys.stderr, flush=True)

    def warning(self, e):
        if self.command_context is not None:
            self.command_context.log(
                logging.WARNING, self.component, {"error": str(e)}, "WARNING: {error}"
            )
        else:
            print(f"WARNING: {e}", file=sys.stderr, flush=True)

    def info(self, e):
        if self.command_context is not None:
            self.command_context.log(
                logging.INFO, self.component, {"error": str(e)}, "INFO: {error}"
            )
        else:
            print(f"INFO: {e}", file=sys.stderr, flush=True)

    def vinfo(self, e):
        if self.verbose:
            self.info(e)

    def run(
        self,
        meta_bug_id=None,
        save_tasks=None,
        use_tasks=None,
        save_failures=None,
        use_failures=None,
        max_failures=-1,
    ):
        "Run skip-fails on try_url, return True on success"

        try_url = self.try_url
        revision, repo = self.get_revision(try_url)

        if use_tasks is not None:
            if os.path.exists(use_tasks):
                self.vinfo(f"use tasks: {use_tasks}")
                tasks = self.read_json(use_tasks)
                tasks = [MockTask(task) for task in tasks]
            else:
                self.error(f"uses tasks JSON file does not exist: {use_tasks}")
                return False
        else:
            tasks = self.get_tasks(revision, repo)

        if use_failures is not None:
            if os.path.exists(use_failures):
                self.vinfo(f"use failures: {use_failures}")
                failures = self.read_json(use_failures)
            else:
                self.error(f"use failures JSON file does not exist: {use_failures}")
                return False
        else:
            failures = self.get_failures(tasks)
            if save_failures is not None:
                self.vinfo(f"save failures: {save_failures}")
                self.write_json(save_failures, failures)

        if save_tasks is not None:
            self.vinfo(f"save tasks: {save_tasks}")
            self.write_tasks(save_tasks, tasks)

        num_failures = 0
        for manifest in failures:
            if not manifest.endswith(".toml"):
                self.warning(f"cannot process skip-fails on INI manifests: {manifest}")
            else:
                for path in failures[manifest]["path"]:
                    for label in failures[manifest]["path"][path]:
                        classification = failures[manifest]["path"][path][label][
                            "classification"
                        ]
                        if classification.startswith("disable_") or (
                            self.turbo and classification == Classification.SECONDARY
                        ):
                            for task_id in failures[manifest]["path"][path][label][
                                "runs"
                            ].keys():
                                self.skip_failure(
                                    manifest,
                                    path,
                                    label,
                                    classification,
                                    task_id,
                                    try_url,
                                    revision,
                                    repo,
                                    meta_bug_id,
                                )
                                num_failures += 1
                                if max_failures >= 0 and num_failures >= max_failures:
                                    self.warning(
                                        f"max_failures={max_failures} threshold reached. stopping."
                                    )
                                    return True
                                break  # just use the first task_id
        return True

    def get_revision(self, url):
        parsed = urllib.parse.urlparse(url)
        if parsed.scheme != "https":
            raise ValueError("try_url scheme not https")
        if parsed.netloc != Skipfails.TREEHERDER:
            raise ValueError(f"try_url server not {Skipfails.TREEHERDER}")
        if len(parsed.query) == 0:
            raise ValueError("try_url query missing")
        query = urllib.parse.parse_qs(parsed.query)
        if Skipfails.REVISION not in query:
            raise ValueError("try_url query missing revision")
        revision = query[Skipfails.REVISION][0]
        if Skipfails.REPO in query:
            repo = query[Skipfails.REPO][0]
        else:
            repo = "try"
        self.vinfo(f"considering {repo} revision={revision}")
        return revision, repo

    def get_tasks(self, revision, repo):
        push = mozci.push.Push(revision, repo)
        return push.tasks

    def get_failures(self, tasks):
        """
        find failures and create structure comprised of runs by path:
           result:
            * False (failed)
            * True (passed)
           classification: Classification
            * unknown (default) < 3 runs
            * intermittent (not enough failures) >3 runs < 0.4 failure rate
            * disable_recommended (enough repeated failures) >3 runs >= 0.4
            * disable_manifest (disable DEFAULT if no other failures)
            * secondary (not first failure in group)
            * success
        """

        failures = {}
        manifest_paths = {}
        for task in tasks:
            try:
                if len(task.results) == 0:
                    continue  # ignore aborted tasks
                for manifest in task.failure_types:
                    if manifest not in failures:
                        failures[manifest] = {"sum_by_label": {}, "path": {}}
                    if manifest not in manifest_paths:
                        manifest_paths[manifest] = []
                    for path_type in task.failure_types[manifest]:
                        path, _type = path_type
                        if path == manifest:
                            path = "DEFAULT"
                        if path not in failures[manifest]["path"]:
                            failures[manifest]["path"][path] = {}
                        if path not in manifest_paths[manifest]:
                            manifest_paths[manifest].append(path)
                        if task.label not in failures[manifest]["sum_by_label"]:
                            failures[manifest]["sum_by_label"][task.label] = {
                                Classification.UNKNOWN: 0,
                                Classification.SECONDARY: 0,
                                Classification.INTERMITTENT: 0,
                                Classification.DISABLE_RECOMMENDED: 0,
                                Classification.DISABLE_MANIFEST: 0,
                                Classification.SUCCESS: 0,
                            }
                        if task.label not in failures[manifest]["path"][path]:
                            failures[manifest]["path"][path][task.label] = {
                                "total_runs": 0,
                                "failed_runs": 0,
                                "classification": Classification.UNKNOWN,
                                "runs": {task.id: False},
                            }
                        else:
                            failures[manifest]["path"][path][task.label]["runs"][
                                task.id
                            ] = False
            except AttributeError as ae:
                self.warning(f"unknown attribute in task: {ae}")

        # calculate success/failure for each known path
        for manifest in manifest_paths:
            manifest_paths[manifest] = sort_paths(manifest_paths[manifest])
        for task in tasks:
            try:
                if len(task.results) == 0:
                    continue  # ignore aborted tasks
                for result in task.results:
                    manifest = result.group
                    if manifest not in failures:
                        self.warning(
                            f"result for {manifest} not in any failures, ignored"
                        )
                        continue
                    for path in manifest_paths[manifest]:
                        if task.label not in failures[manifest]["sum_by_label"]:
                            failures[manifest]["sum_by_label"][task.label] = {
                                Classification.UNKNOWN: 0,
                                Classification.SECONDARY: 0,
                                Classification.INTERMITTENT: 0,
                                Classification.DISABLE_RECOMMENDED: 0,
                                Classification.DISABLE_MANIFEST: 0,
                                Classification.SUCCESS: 0,
                            }
                        if task.label not in failures[manifest]["path"][path]:
                            failures[manifest]["path"][path][task.label] = {
                                "total_runs": 0,
                                "failed_runs": 0,
                                "classification": Classification.UNKNOWN,
                                "runs": {},
                            }
                        if (
                            task.id
                            not in failures[manifest]["path"][path][task.label]["runs"]
                        ):
                            ok = True
                            failures[manifest]["path"][path][task.label]["runs"][
                                task.id
                            ] = ok
                        else:
                            ok = (
                                result.ok
                                or failures[manifest]["path"][path][task.label]["runs"][
                                    task.id
                                ]
                            )
                        failures[manifest]["path"][path][task.label]["total_runs"] += 1
                        if not ok:
                            failures[manifest]["path"][path][task.label][
                                "failed_runs"
                            ] += 1
            except AttributeError as ae:
                self.warning(f"unknown attribute in task: {ae}")

        # classify failures and roll up summary statistics
        for manifest in failures:
            for path in failures[manifest]["path"]:
                for label in failures[manifest]["path"][path]:
                    failed_runs = failures[manifest]["path"][path][label]["failed_runs"]
                    total_runs = failures[manifest]["path"][path][label]["total_runs"]
                    classification = failures[manifest]["path"][path][label][
                        "classification"
                    ]
                    if total_runs >= 3:
                        if failed_runs / total_runs < 0.4:
                            if failed_runs == 0:
                                classification = Classification.SUCCESS
                            else:
                                classification = Classification.INTERMITTENT
                        else:
                            classification = Classification.SECONDARY
                        failures[manifest]["path"][path][label][
                            "classification"
                        ] = classification
                    failures[manifest]["sum_by_label"][label][classification] += 1

        # Identify the first failure (for each test, in a manifest, by label)
        for manifest in failures:
            alpha_paths = sort_paths(failures[manifest]["path"].keys())
            for path in alpha_paths:
                for label in failures[manifest]["path"][path]:
                    primary = (
                        failures[manifest]["sum_by_label"][label][
                            Classification.DISABLE_RECOMMENDED
                        ]
                        == 0
                    )
                    if path == "DEFAULT":
                        classification = failures[manifest]["path"][path][label][
                            "classification"
                        ]
                        if (
                            classification == Classification.SECONDARY
                            and failures[manifest]["sum_by_label"][label][
                                classification
                            ]
                            == 1
                        ):
                            # ONLY failure in the manifest for this label => DISABLE
                            failures[manifest]["path"][path][label][
                                "classification"
                            ] = Classification.DISABLE_MANIFEST
                            failures[manifest]["sum_by_label"][label][
                                classification
                            ] -= 1
                            failures[manifest]["sum_by_label"][label][
                                Classification.DISABLE_MANIFEST
                            ] += 1

                    else:
                        if (
                            primary
                            and failures[manifest]["path"][path][label][
                                "classification"
                            ]
                            == Classification.SECONDARY
                        ):
                            # FIRST failure in the manifest for this label => DISABLE
                            failures[manifest]["path"][path][label][
                                "classification"
                            ] = Classification.DISABLE_RECOMMENDED
                            failures[manifest]["sum_by_label"][label][
                                Classification.SECONDARY
                            ] -= 1
                            failures[manifest]["sum_by_label"][label][
                                Classification.DISABLE_RECOMMENDED
                            ] += 1

        return failures

    def _get_os_version(self, os, platform):
        """Return the os_version given the label platform string"""
        i = platform.find(os)
        j = i + len(os)
        yy = platform[j : j + 2]
        mm = platform[j + 2 : j + 4]
        return yy + "." + mm

    def get_bug_by_id(self, id):
        """Get bug by bug id"""

        self._initialize_bzapi()
        bug = self._bzapi.getbug(id)
        return bug

    def get_bugs_by_summary(self, summary):
        """Get bug by bug summary"""

        self._initialize_bzapi()
        query = self._bzapi.build_query(short_desc=summary)
        query["include_fields"] = [
            "id",
            "product",
            "component",
            "status",
            "resolution",
            "summary",
            "blocks",
        ]
        bugs = self._bzapi.query(query)
        return bugs

    def create_bug(
        self,
        summary="Bug short description",
        description="Bug description",
        product="Testing",
        component="General",
        version="unspecified",
        bugtype="task",
    ):
        """Create a bug"""

        self._initialize_bzapi()
        if not self._bzapi.logged_in:
            self.error(
                "Must create a Bugzilla API key per https://github.com/mozilla/mozci-tools/blob/main/citools/test_triage_bug_filer.py"
            )
            raise PermissionError(f"Not authenticated for Bugzilla {self.bugzilla}")
        createinfo = self._bzapi.build_createbug(
            product=product,
            component=component,
            summary=summary,
            version=version,
            description=description,
        )
        createinfo["type"] = bugtype
        bug = self._bzapi.createbug(createinfo)
        return bug

    def add_bug_comment(self, id, comment, meta_bug_id=None):
        """Add a comment to an existing bug"""

        self._initialize_bzapi()
        if not self._bzapi.logged_in:
            self.error(BUGZILLA_AUTHENTICATION_HELP)
            raise PermissionError("Not authenticated for Bugzilla")
        if meta_bug_id is not None:
            blocks_add = [meta_bug_id]
        else:
            blocks_add = None
        updateinfo = self._bzapi.build_update(comment=comment, blocks_add=blocks_add)
        self._bzapi.update_bugs([id], updateinfo)

    def skip_failure(
        self,
        manifest,
        path,
        label,
        classification,
        task_id,
        try_url,
        revision,
        repo,
        meta_bug_id=None,
    ):
        """Skip a failure"""

        self.vinfo(f"===== Skip failure in manifest: {manifest} =====")
        skip_if = self.task_to_skip_if(task_id)
        if skip_if is None:
            self.warning(
                f"Unable to calculate skip-if condition from manifest={manifest} from failure label={label}"
            )
            return
        bug_reference = ""
        if classification == Classification.DISABLE_MANIFEST:
            filename = "DEFAULT"
            comment = "Disabled entire manifest due to crash result"
        else:
            filename = self.get_filename_in_manifest(manifest, path)
            comment = f'Disabled test due to failures: "{filename}"'
            if classification == Classification.SECONDARY:
                comment += " (secondary)"
                bug_reference = " (secondary)"
        comment += f"\nTry URL = {try_url}"
        comment += f"\nrevision = {revision}"
        comment += f"\nrepo = {repo}"
        comment += f"\nlabel = {label}"
        comment += f"\ntask_id = {task_id}"
        push_id = self.get_push_id(revision, repo)
        if push_id is not None:
            comment += f"\npush_id = {push_id}"
            job_id = self.get_job_id(push_id, task_id)
            if job_id is not None:
                comment += f"\njob_id = {job_id}"
                suggestions_url, line_number, line, log_url = self.get_bug_suggestions(
                    repo, job_id, path
                )
                if log_url is not None:
                    comment += f"\n\nBug suggestions: {suggestions_url}"
                    comment += f"\nSpecifically see at line {line_number} in the attached log: {log_url}"
                    comment += f'\n\n  "{line}"\n'
        platform, testname = self.label_to_platform_testname(label)
        if platform is not None:
            comment += "\n\nCommand line to reproduce:\n\n"
            comment += f"  \"mach try fuzzy -q '{platform}' {testname}\""
        bug_summary = f"MANIFEST {manifest}"
        attachments = {}
        bugs = self.get_bugs_by_summary(bug_summary)
        if len(bugs) == 0:
            description = (
                f"This bug covers excluded failing tests in the MANIFEST {manifest}"
            )
            description += "\n(generated by mach manifest skip-fails)"
            product, component = self.get_file_info(path)
            if self.dry_run:
                self.warning(
                    f'Dry-run NOT creating bug: {product}::{component} "{bug_summary}"'
                )
                bugid = "TBD"
            else:
                bug = self.create_bug(bug_summary, description, product, component)
                bugid = bug.id
                self.vinfo(
                    f'Created Bug {bugid} {product}::{component} : "{bug_summary}"'
                )
            bug_reference = f"Bug {bugid}" + bug_reference
        elif len(bugs) == 1:
            bugid = bugs[0].id
            bug_reference = f"Bug {bugid}" + bug_reference
            product = bugs[0].product
            component = bugs[0].component
            self.vinfo(f'Found Bug {bugid} {product}::{component} "{bug_summary}"')
            if meta_bug_id is not None:
                if meta_bug_id in bugs[0].blocks:
                    self.vinfo(f"  Bug {bugid} already blocks meta bug {meta_bug_id}")
                    meta_bug_id = None  # no need to add again
            comments = bugs[0].getcomments()
            for i in range(len(comments)):
                text = comments[i]["text"]
                m = self._attach_rx.findall(text)
                if len(m) == 1:
                    a_task_id = m[0][1]
                    attachments[a_task_id] = m[0][0]
                    if a_task_id == task_id:
                        self.vinfo(
                            f"  Bug {bugid} already has the compressed log attached for this task"
                        )
        else:
            self.error(f'More than one bug found for summary: "{bug_summary}"')
            return
        if self.dry_run:
            self.warning(f"Dry-run NOT adding comment to Bug {bugid}: {comment}")
            self.info(f'Dry-run NOT editing ["{filename}"] manifest: "{manifest}"')
            self.info(f'would add skip-if condition: "{skip_if}" # {bug_reference}')
            if task_id not in attachments:
                self.info("would add compressed log for this task")
            return
        self.add_bug_comment(bugid, comment, meta_bug_id)
        self.info(f"Added comment to Bug {bugid}: {comment}")
        if meta_bug_id is not None:
            self.info(f"  Bug {bugid} blocks meta Bug: {meta_bug_id}")
        if task_id not in attachments:
            self.add_attachment_log_for_task(bugid, task_id)
            self.info("Added compressed log for this task")
        mp = ManifestParser(use_toml=True, document=True)
        manifest_path = os.path.join(self.topsrcdir, os.path.normpath(manifest))
        mp.read(manifest_path)
        document = mp.source_documents[manifest_path]
        add_skip_if(document, filename, skip_if, bug_reference)
        manifest_str = alphabetize_toml_str(document)
        fp = io.open(manifest_path, "w", encoding="utf-8", newline="\n")
        fp.write(manifest_str)
        fp.close()
        self.info(f'Edited ["{filename}"] in manifest: "{manifest}"')
        self.info(f'added skip-if condition: "{skip_if}" # {bug_reference}')

    def get_variants(self):
        """Get mozinfo for each test variants"""

        if len(self.variants) == 0:
            variants_file = "taskcluster/ci/test/variants.yml"
            variants_path = os.path.join(
                self.topsrcdir, os.path.normpath(variants_file)
            )
            fp = io.open(variants_path, "r", encoding="utf-8")
            raw_variants = load(fp, Loader=Loader)
            fp.close()
            for k, v in raw_variants.items():
                mozinfo = k
                if "mozinfo" in v:
                    mozinfo = v["mozinfo"]
                self.variants[k] = mozinfo
        return self.variants

    def get_task(self, task_id):
        """Download details for task task_id"""

        if task_id in self.tasks:  # if cached
            task = self.tasks[task_id]
        else:
            task = get_task(task_id)
            self.tasks[task_id] = task
        return task

    def task_to_skip_if(self, task_id):
        """Calculate the skip-if condition for failing task task_id"""

        self.get_variants()
        task = self.get_task(task_id)
        os = None
        os_version = None
        bits = None
        display = None
        runtimes = []
        build_types = []
        test_setting = task.get("extra", {}).get("test-setting", {})
        platform = test_setting.get("platform", {})
        platform_os = platform.get("os", {})
        if "name" in platform_os:
            os = platform_os["name"]
            if os == "windows":
                os = "win"
            if os == "macosx":
                os = "mac"
        if "version" in platform_os:
            os_version = platform_os["version"]
            if len(os_version) == 4:
                os_version = os_version[0:2] + "." + os_version[2:4]
        if "arch" in platform:
            arch = platform["arch"]
            if arch == "x86" or arch.find("32") >= 0:
                bits = "32"
        if "display" in platform:
            display = platform["display"]
        if "runtime" in test_setting:
            for k in test_setting["runtime"]:
                if k in self.variants:
                    runtimes.append(self.variants[k])  # adds mozinfo
        if "build" in test_setting:
            tbuild = test_setting["build"]
            opt = False
            debug = False
            for k in tbuild:
                if k == "type":
                    if tbuild[k] == "opt":
                        opt = True
                    elif tbuild[k] == "debug":
                        debug = True
                else:
                    build_types.append(k)
            if len(build_types) == 0:
                if opt:
                    build_types.append("!debug")
                if debug:
                    build_types.append("debug")
        skip_if = None
        if os is not None:
            skip_if = "os == '" + os + "'"
            if os_version is not None:
                skip_if += " && "
                skip_if += "os_version == '" + os_version + "'"
            if bits is not None:
                skip_if += " && "
                skip_if += "bits == '" + bits + "'"
            if display is not None:
                skip_if += " && "
                skip_if += "display == '" + display + "'"
            for runtime in runtimes:
                skip_if += " && "
                skip_if += runtime
            for build_type in build_types:
                skip_if += " && "
                skip_if += build_type
        return skip_if

    def get_file_info(self, path, product="Testing", component="General"):
        """
        Get bugzilla product and component for the path.
        Provide defaults (in case command_context is not defined
        or there isn't file info available).
        """
        if path != "DEFAULT" and self.command_context is not None:
            reader = self.command_context.mozbuild_reader(config_mode="empty")
            info = reader.files_info([path])
            cp = info[path]["BUG_COMPONENT"]
            product = cp.product
            component = cp.component
        return product, component

    def get_filename_in_manifest(self, manifest, path):
        """return relative filename for path in manifest"""

        filename = os.path.basename(path)
        if filename == "DEFAULT":
            return filename
        manifest_dir = os.path.dirname(manifest)
        i = 0
        j = min(len(manifest_dir), len(path))
        while i < j and manifest_dir[i] == path[i]:
            i += 1
        if i < len(manifest_dir):
            for _ in range(manifest_dir.count("/", i) + 1):
                filename = "../" + filename
        elif i < len(path):
            filename = path[i + 1 :]
        return filename

    def get_push_id(self, revision, repo):
        """Return the push_id for revision and repo (or None)"""

        self.vinfo(f"Retrieving push_id for {repo} revision: {revision} ...")
        if revision in self.push_ids:  # if cached
            push_id = self.push_ids[revision]
        else:
            push_id = None
            push_url = f"https://treeherder.mozilla.org/api/project/{repo}/push/"
            params = {}
            params["full"] = "true"
            params["count"] = 10
            params["revision"] = revision
            r = requests.get(push_url, headers=self.headers, params=params)
            if r.status_code != 200:
                self.warning(f"FAILED to query Treeherder = {r} for {r.url}")
            else:
                response = r.json()
                if "results" in response:
                    results = response["results"]
                    if len(results) > 0:
                        r0 = results[0]
                        if "id" in r0:
                            push_id = r0["id"]
            self.push_ids[revision] = push_id
        return push_id

    def get_job_id(self, push_id, task_id):
        """Return the job_id for push_id, task_id (or None)"""

        self.vinfo(f"Retrieving job_id for push_id: {push_id}, task_id: {task_id} ...")
        if push_id in self.job_ids:  # if cached
            job_id = self.job_ids[push_id]
        else:
            job_id = None
            params = {}
            params["push_id"] = push_id
            r = requests.get(self.jobs_url, headers=self.headers, params=params)
            if r.status_code != 200:
                self.warning(f"FAILED to query Treeherder = {r} for {r.url}")
            else:
                response = r.json()
                if "results" in response:
                    results = response["results"]
                    if len(results) > 0:
                        for result in results:
                            if len(result) > 14:
                                if result[14] == task_id:
                                    job_id = result[1]
                                    break
            self.job_ids[push_id] = job_id
        return job_id

    def get_bug_suggestions(self, repo, job_id, path):
        """
        Return the (suggestions_url, line_number, line, log_url)
        for the given repo and job_id
        """
        self.vinfo(
            f"Retrieving bug_suggestions for {repo} job_id: {job_id}, path: {path} ..."
        )
        suggestions_url = f"https://treeherder.mozilla.org/api/project/{repo}/jobs/{job_id}/bug_suggestions/"
        line_number = None
        line = None
        log_url = None
        r = requests.get(suggestions_url, headers=self.headers)
        if r.status_code != 200:
            self.warning(f"FAILED to query Treeherder = {r} for {r.url}")
        else:
            response = r.json()
            if len(response) > 0:
                for sugg in response:
                    if sugg["path_end"] == path:
                        line_number = sugg["line_number"] + 1
                        line = sugg["search"]
                        log_url = f"https://treeherder.mozilla.org/logviewer?repo={repo}&job_id={job_id}&lineNumber={line_number}"
                        break
        rv = (suggestions_url, line_number, line, log_url)
        return rv

    def read_json(self, filename):
        """read data as JSON from filename"""
        fp = io.open(filename, "r", encoding="utf-8")
        data = json.load(fp)
        fp.close()
        return data

    def write_json(self, filename, data):
        """saves data as JSON to filename"""
        fp = io.open(filename, "w", encoding="utf-8")
        json.dump(data, fp, indent=2, sort_keys=True)
        fp.close()

    def write_tasks(self, save_tasks, tasks):
        """saves tasks as JSON to save_tasks"""
        jtasks = []
        for task in tasks:
            if not isinstance(task, TestTask):
                continue
            jtask = {}
            jtask["id"] = task.id
            jtask["label"] = task.label
            jtask["duration"] = task.duration
            jtask["result"] = task.result
            jtask["state"] = task.state
            jtags = {}
            for k, v in task.tags.items():
                if k == "createdForUser":
                    jtags[k] = "ci@mozilla.com"
                else:
                    jtags[k] = v
            jtask["tags"] = jtags
            jtask["tier"] = task.tier
            jtask["results"] = [
                {"group": r.group, "ok": r.ok, "duration": r.duration}
                for r in task.results
            ]
            jtask["errors"] = None  # Bug with task.errors property??
            jft = {}
            for k in task.failure_types:
                jft[k] = [[f[0], f[1].value] for f in task.failure_types[k]]
            jtask["failure_types"] = jft
            jtasks.append(jtask)
        self.write_json(save_tasks, jtasks)

    def label_to_platform_testname(self, label):
        """convert from label to platform, testname for mach command line"""
        platform = None
        testname = None
        platform_details = label.split("/")
        if len(platform_details) == 2:
            platform, details = platform_details
            words = details.split("-")
            if len(words) > 2:
                platform += "/" + words.pop(0)  # opt or debug
                try:
                    _chunk = int(words[-1])
                    words.pop()
                except ValueError:
                    pass
                words.pop()  # remove test suffix
                testname = "-".join(words)
            else:
                platform = None
        return platform, testname

    def add_attachment_log_for_task(self, bugid, task_id):
        """Adds compressed log for this task to bugid"""

        log_url = f"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/{task_id}/artifacts/public/logs/live_backing.log"
        r = requests.get(log_url, headers=self.headers)
        if r.status_code != 200:
            self.error(f"Unable get log for task: {task_id}")
            return
        attach_fp = tempfile.NamedTemporaryFile()
        fp = gzip.open(attach_fp, "wb")
        fp.write(r.text.encode("utf-8"))
        fp.close()
        self._initialize_bzapi()
        description = ATTACHMENT_DESCRIPTION + task_id
        file_name = TASK_LOG + ".gz"
        comment = "Added compressed log"
        content_type = "application/gzip"
        try:
            self._bzapi.attachfile(
                [bugid],
                attach_fp.name,
                description,
                file_name=file_name,
                comment=comment,
                content_type=content_type,
                is_private=False,
            )
        except Fault:
            pass  # Fault expected: Failed to fetch key 9372091 from network storage: The specified key does not exist.