fune/taskcluster/taskgraph/transforms/partner_attribution_beetmover.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the beetmover task into an actual task description.
"""

from __future__ import absolute_import, print_function, unicode_literals

from six import text_type
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.partners import (
    get_partner_config_by_kind,
    apply_partner_priority,
)
from taskgraph.util.schema import (
    optionally_keyed_by,
    resolve_keyed_by,
)
from taskgraph.util.scriptworker import (
    add_scope_prefix,
    get_beetmover_bucket_scope,
)
from taskgraph.util.taskcluster import get_artifact_prefix
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional

from collections import defaultdict
from copy import deepcopy


beetmover_description_schema = schema.extend(
    {
        # depname is used in taskref's to identify the taskID of the unsigned things
        Required("depname", default="build"): text_type,
        # unique label to describe this beetmover task, defaults to {dep.label}-beetmover
        Optional("label"): text_type,
        Required("partner-bucket-scope"): optionally_keyed_by(
            "release-level", text_type
        ),
        Required("partner-public-path"): Any(None, text_type),
        Required("partner-private-path"): Any(None, text_type),
        Optional("extra"): object,
        Required("shipping-phase"): task_description_schema["shipping-phase"],
        Optional("shipping-product"): task_description_schema["shipping-product"],
        Optional("priority"): task_description_schema["priority"],
    }
)

transforms = TransformSequence()
transforms.add_validate(beetmover_description_schema)
transforms.add(apply_partner_priority)


@transforms.add
def resolve_keys(config, jobs):
    for job in jobs:
        resolve_keyed_by(
            job,
            "partner-bucket-scope",
            item_name=job["label"],
            **{"release-level": config.params.release_level()}
        )
        yield job


@transforms.add
def split_public_and_private(config, jobs):
    # we need to separate private vs public destinations because beetmover supports one
    # in a single task. Only use a single task for each type though.
    partner_config = get_partner_config_by_kind(config, config.kind)
    for job in jobs:
        upstream_artifacts = job["primary-dependency"].release_artifacts
        attribution_task_ref = "<{}>".format(job["primary-dependency"].label)
        prefix = get_artifact_prefix(job["primary-dependency"])
        artifacts = defaultdict(list)
        for artifact in upstream_artifacts:
            partner, sub_partner, platform, locale, _ = artifact.replace(
                prefix + "/", ""
            ).split("/", 4)
            destination = "private"
            this_config = [
                p
                for p in partner_config["configs"]
                if (p["campaign"] == partner and p["content"] == sub_partner)
            ]
            if this_config[0].get("upload_to_candidates"):
                destination = "public"
            artifacts[destination].append(
                (artifact, partner, sub_partner, platform, locale)
            )

        action_scope = add_scope_prefix(config, "beetmover:action:push-to-partner")
        public_bucket_scope = get_beetmover_bucket_scope(config)
        partner_bucket_scope = add_scope_prefix(config, job["partner-bucket-scope"])
        repl_dict = {
            "build_number": config.params["build_number"],
            "release_partner_build_number": config.params[
                "release_partner_build_number"
            ],
            "version": config.params["version"],
            "partner": "{partner}",  # we'll replace these later, per artifact
            "subpartner": "{subpartner}",
            "platform": "{platform}",
            "locale": "{locale}",
        }
        for destination, destination_artifacts in artifacts.items():
            this_job = deepcopy(job)

            if destination == "public":
                this_job["scopes"] = [public_bucket_scope, action_scope]
                this_job["partner_public"] = True
            else:
                this_job["scopes"] = [partner_bucket_scope, action_scope]
                this_job["partner_public"] = False

            partner_path_key = "partner-{destination}-path".format(
                destination=destination
            )
            partner_path = this_job[partner_path_key].format(**repl_dict)
            this_job.setdefault("worker", {})[
                "upstream-artifacts"
            ] = generate_upstream_artifacts(
                attribution_task_ref, destination_artifacts, partner_path
            )

            yield this_job


@transforms.add
def make_task_description(config, jobs):
    for job in jobs:
        dep_job = job["primary-dependency"]

        attributes = dep_job.attributes
        build_platform = attributes.get("build_platform")
        if not build_platform:
            raise Exception("Cannot find build platform!")

        label = config.kind
        description = "Beetmover for partner attribution"
        if job["partner_public"]:
            label = "{}-public".format(label)
            description = "{} public".format(description)
        else:
            label = "{}-private".format(label)
            description = "{} private".format(description)
        attributes = copy_attributes_from_dependent_job(dep_job)

        task = {
            "label": label,
            "description": description,
            "dependencies": {dep_job.kind: dep_job.label},
            "attributes": attributes,
            "run-on-projects": dep_job.attributes.get("run_on_projects"),
            "shipping-phase": job["shipping-phase"],
            "shipping-product": job.get("shipping-product"),
            "partner_public": job["partner_public"],
            "worker": job["worker"],
            "scopes": job["scopes"],
        }
        # we may have reduced the priority for partner jobs, otherwise task.py will set it
        if job.get("priority"):
            task["priority"] = job["priority"]

        yield task


def generate_upstream_artifacts(attribution_task, artifacts, partner_path):
    upstream_artifacts = []
    for artifact, partner, subpartner, platform, locale in artifacts:
        upstream_artifacts.append(
            {
                "taskId": {"task-reference": attribution_task},
                "taskType": "repackage",
                "paths": [artifact],
                "locale": partner_path.format(
                    partner=partner,
                    subpartner=subpartner,
                    platform=platform,
                    locale=locale,
                ),
            }
        )

    if not upstream_artifacts:
        raise Exception("Couldn't find any upstream artifacts.")

    return upstream_artifacts


@transforms.add
def make_task_worker(config, jobs):
    for job in jobs:
        job["worker-type"] = "beetmover"
        worker = {
            "implementation": "beetmover",
            "release-properties": craft_release_properties(config, job),
            "partner-public": job["partner_public"],
        }
        job["worker"].update(worker)
        del job["partner_public"]

        yield job