fune/python/mozbuild/mozbuild/mach_commands.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, # You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import absolute_import, print_function, unicode_literals

import argparse
import itertools
import json
import logging
import operator
import os
import re
import shutil
import subprocess
import sys
import tempfile

import mozpack.path as mozpath

from mach.decorators import (
    CommandArgument,
    CommandArgumentGroup,
    CommandProvider,
    Command,
    SettingsProvider,
    SubCommand,
)

from mozbuild.base import (
    BinaryNotFoundException,
    BuildEnvironmentNotFoundException,
    MachCommandBase,
    MachCommandConditions as conditions,
    MozbuildObject,
)
from mozbuild.util import MOZBUILD_METRICS_PATH

here = os.path.abspath(os.path.dirname(__file__))

EXCESSIVE_SWAP_MESSAGE = """
===================
PERFORMANCE WARNING

Your machine experienced a lot of swap activity during the build. This is
possibly a sign that your machine doesn't have enough physical memory or
not enough available memory to perform the build. It's also possible some
other system activity during the build is to blame.

If you feel this message is not appropriate for your machine configuration,
please file a Firefox Build System :: General bug at
https://bugzilla.mozilla.org/enter_bug.cgi?product=Firefox%20Build%20System&component=General
and tell us about your machine and build configuration so we can adjust the
warning heuristic.
===================
"""


class StoreDebugParamsAndWarnAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
        sys.stderr.write(
            "The --debugparams argument is deprecated. Please "
            + "use --debugger-args instead.\n\n"
        )
        setattr(namespace, self.dest, values)


@CommandProvider
class Watch(MachCommandBase):
    """Interface to watch and re-build the tree."""

    @Command(
        "watch",
        category="post-build",
        description="Watch and re-build the tree.",
        conditions=[conditions.is_firefox],
    )
    @CommandArgument(
        "-v",
        "--verbose",
        action="store_true",
        help="Verbose output for what commands the watcher is running.",
    )
    def watch(self, verbose=False):
        """Watch and re-build the source tree."""

        if not conditions.is_artifact_build(self):
            print(
                "mach watch requires an artifact build. See "
                "https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Simple_Firefox_build"  # noqa
            )
            return 1

        if not self.substs.get("WATCHMAN", None):
            print(
                "mach watch requires watchman to be installed. See "
                "https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Incremental_builds_with_filesystem_watching"  # noqa
            )
            return 1

        self.activate_virtualenv()
        try:
            self.virtualenv_manager.install_pip_package("pywatchman==1.4.1")
        except Exception:
            print(
                "Could not install pywatchman from pip. See "
                "https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Incremental_builds_with_filesystem_watching"  # noqa
            )
            return 1

        from mozbuild.faster_daemon import Daemon

        daemon = Daemon(self.config_environment)

        try:
            return daemon.watch()
        except KeyboardInterrupt:
            # Suppress ugly stack trace when user hits Ctrl-C.
            sys.exit(3)


@CommandProvider
class CargoProvider(MachCommandBase):
    """Invoke cargo in useful ways."""

    @Command("cargo", category="build", description="Invoke cargo in useful ways.")
    def cargo(self):
        self._sub_mach(["help", "cargo"])
        return 1

    @SubCommand(
        "cargo",
        "check",
        description="Run `cargo check` on a given crate.  Defaults to gkrust.",
    )
    @CommandArgument(
        "--all-crates",
        default=None,
        action="store_true",
        help="Check all of the crates in the tree.",
    )
    @CommandArgument(
        "crates", default=None, nargs="*", help="The crate name(s) to check."
    )
    @CommandArgument(
        "--jobs",
        "-j",
        default="1",
        nargs="?",
        metavar="jobs",
        type=int,
        help="Run the tests in parallel using multiple processes.",
    )
    @CommandArgument("-v", "--verbose", action="store_true", help="Verbose output.")
    def check(self, all_crates=None, crates=None, jobs=0, verbose=False):
        # XXX duplication with `mach vendor rust`
        crates_and_roots = {
            "gkrust": "toolkit/library/rust",
            "gkrust-gtest": "toolkit/library/gtest/rust",
            "js": "js/rust",
            "mozjs_sys": "js/src",
            "baldrdash": "js/src/wasm/cranelift",
            "geckodriver": "testing/geckodriver",
        }

        if all_crates:
            crates = crates_and_roots.keys()
        elif crates is None or crates == []:
            crates = ["gkrust"]

        for crate in crates:
            root = crates_and_roots.get(crate, None)
            if not root:
                print(
                    "Cannot locate crate %s.  Please check your spelling or "
                    "add the crate information to the list." % crate
                )
                return 1

            check_targets = [
                "force-cargo-library-check",
                "force-cargo-host-library-check",
                "force-cargo-program-check",
                "force-cargo-host-program-check",
            ]

            ret = self._run_make(
                srcdir=False,
                directory=root,
                ensure_exit_code=0,
                silent=not verbose,
                print_directory=False,
                target=check_targets,
                num_jobs=jobs,
            )
            if ret != 0:
                return ret

        return 0


@CommandProvider
class Doctor(MachCommandBase):
    """Provide commands for diagnosing common build environment problems"""

    @Command("doctor", category="devenv", description="")
    @CommandArgument(
        "--fix",
        default=None,
        action="store_true",
        help="Attempt to fix found problems.",
    )
    def doctor(self, fix=None):
        self.activate_virtualenv()
        from mozbuild.doctor import Doctor

        doctor = Doctor(self.topsrcdir, self.topobjdir, fix)
        return doctor.check_all()


@CommandProvider(metrics_path=MOZBUILD_METRICS_PATH)
class Clobber(MachCommandBase):
    NO_AUTO_LOG = True
    CLOBBER_CHOICES = set(["objdir", "python", "gradle"])

    @Command(
        "clobber",
        category="build",
        description="Clobber the tree (delete the object directory).",
    )
    @CommandArgument(
        "what",
        default=["objdir", "python"],
        nargs="*",
        help="Target to clobber, must be one of {{{}}} (default "
        "objdir and python).".format(", ".join(CLOBBER_CHOICES)),
    )
    @CommandArgument("--full", action="store_true", help="Perform a full clobber")
    def clobber(self, what, full=False):
        """Clean up the source and object directories.

        Performing builds and running various commands generate various files.

        Sometimes it is necessary to clean up these files in order to make
        things work again. This command can be used to perform that cleanup.

        The `objdir` target removes most files in the current object directory
        (where build output is stored). Some files (like Visual Studio project
        files) are not removed by default. If you would like to remove the
        object directory in its entirety, run with `--full`.

        The `python` target will clean up various generated Python files from
        the source directory and will remove untracked files from well-known
        directories containing Python packages. Run this to remove .pyc files,
        compiled C extensions, etc. Note: all files not tracked or ignored by
        version control in third_party/python will be deleted. Run the `status`
        command of your VCS to see if any untracked files you haven't committed
        yet will be deleted.

        The `gradle` target will remove the "gradle" subdirectory of the object
        directory.

        By default, the command clobbers the `objdir` and `python` targets.
        """
        what = set(what)
        invalid = what - self.CLOBBER_CHOICES
        if invalid:
            print("Unknown clobber target(s): {}".format(", ".join(invalid)))
            return 1

        ret = 0
        if "objdir" in what:
            from mozbuild.controller.clobber import Clobberer

            try:
                Clobberer(self.topsrcdir, self.topobjdir, self.substs).remove_objdir(
                    full
                )
            except OSError as e:
                if sys.platform.startswith("win"):
                    if isinstance(e, WindowsError) and e.winerror in (5, 32):
                        self.log(
                            logging.ERROR,
                            "file_access_error",
                            {"error": e},
                            "Could not clobber because a file was in use. If the "
                            "application is running, try closing it. {error}",
                        )
                        return 1
                raise

        if "python" in what:
            if conditions.is_hg(self):
                cmd = [
                    "hg",
                    "--config",
                    "extensions.purge=",
                    "purge",
                    "--all",
                    "-I",
                    "glob:**.py[cdo]",
                    "-I",
                    "glob:**/__pycache__",
                    "-I",
                    "path:third_party/python/",
                ]
            elif conditions.is_git(self):
                cmd = [
                    "git",
                    "clean",
                    "-d",
                    "-f",
                    "-x",
                    "*.py[cdo]",
                    "*/__pycache__/*",
                    "third_party/python/",
                ]
            else:
                # We don't know what is tracked/untracked if we don't have VCS.
                # So we can't clean python/ and third_party/python/.
                cmd = ["find", ".", "-type", "f", "-name", "*.py[cdo]", "-delete"]
                subprocess.call(cmd, cwd=self.topsrcdir)
                cmd = [
                    "find",
                    ".",
                    "-type",
                    "d",
                    "-name",
                    "__pycache__",
                    "-empty",
                    "-delete",
                ]
            ret = subprocess.call(cmd, cwd=self.topsrcdir)

        if "gradle" in what:
            shutil.rmtree(mozpath.join(self.topobjdir, "gradle"))

        return ret

    @property
    def substs(self):
        try:
            return super(Clobber, self).substs
        except BuildEnvironmentNotFoundException:
            return {}


@CommandProvider
class Logs(MachCommandBase):
    """Provide commands to read mach logs."""

    NO_AUTO_LOG = True

    @Command("show-log", category="post-build", description="Display mach logs")
    @CommandArgument(
        "log_file",
        nargs="?",
        type=argparse.FileType("rb"),
        help="Filename to read log data from. Defaults to the log of the last "
        "mach command.",
    )
    def show_log(self, log_file=None):
        if not log_file:
            path = self._get_state_filename("last_log.json")
            log_file = open(path, "rb")

        if os.isatty(sys.stdout.fileno()):
            env = dict(os.environ)
            if "LESS" not in env:
                # Sensible default flags if none have been set in the user
                # environment.
                env[b"LESS"] = b"FRX"
            less = subprocess.Popen(["less"], stdin=subprocess.PIPE, env=env)
            # Various objects already have a reference to sys.stdout, so we
            # can't just change it, we need to change the file descriptor under
            # it to redirect to less's input.
            # First keep a copy of the sys.stdout file descriptor.
            output_fd = os.dup(sys.stdout.fileno())
            os.dup2(less.stdin.fileno(), sys.stdout.fileno())

        startTime = 0
        for line in log_file:
            created, action, params = json.loads(line)
            if not startTime:
                startTime = created
                self.log_manager.terminal_handler.formatter.start_time = created
            if "line" in params:
                record = logging.makeLogRecord(
                    {
                        "created": created,
                        "name": self._logger.name,
                        "levelno": logging.INFO,
                        "msg": "{line}",
                        "params": params,
                        "action": action,
                    }
                )
                self._logger.handle(record)

        if self.log_manager.terminal:
            # Close less's input so that it knows that we're done sending data.
            less.stdin.close()
            # Since the less's input file descriptor is now also the stdout
            # file descriptor, we still actually have a non-closed system file
            # descriptor for less's input. Replacing sys.stdout's file
            # descriptor with what it was before we replaced it will properly
            # close less's input.
            os.dup2(output_fd, sys.stdout.fileno())
            less.wait()


@CommandProvider
class Warnings(MachCommandBase):
    """Provide commands for inspecting warnings."""

    @property
    def database_path(self):
        return self._get_state_filename("warnings.json")

    @property
    def database(self):
        from mozbuild.compilation.warnings import WarningsDatabase

        path = self.database_path

        database = WarningsDatabase()

        if os.path.exists(path):
            database.load_from_file(path)

        return database

    @Command(
        "warnings-summary",
        category="post-build",
        description="Show a summary of compiler warnings.",
    )
    @CommandArgument(
        "-C",
        "--directory",
        default=None,
        help="Change to a subdirectory of the build directory first.",
    )
    @CommandArgument(
        "report",
        default=None,
        nargs="?",
        help="Warnings report to display. If not defined, show the most "
        "recent report.",
    )
    def summary(self, directory=None, report=None):
        database = self.database

        if directory:
            dirpath = self.join_ensure_dir(self.topsrcdir, directory)
            if not dirpath:
                return 1
        else:
            dirpath = None

        type_counts = database.type_counts(dirpath)
        sorted_counts = sorted(type_counts.items(), key=operator.itemgetter(1))

        total = 0
        for k, v in sorted_counts:
            print("%d\t%s" % (v, k))
            total += v

        print("%d\tTotal" % total)

    @Command(
        "warnings-list",
        category="post-build",
        description="Show a list of compiler warnings.",
    )
    @CommandArgument(
        "-C",
        "--directory",
        default=None,
        help="Change to a subdirectory of the build directory first.",
    )
    @CommandArgument(
        "--flags", default=None, nargs="+", help="Which warnings flags to match."
    )
    @CommandArgument(
        "report",
        default=None,
        nargs="?",
        help="Warnings report to display. If not defined, show the most "
        "recent report.",
    )
    def list(self, directory=None, flags=None, report=None):
        database = self.database

        by_name = sorted(database.warnings)

        topsrcdir = mozpath.normpath(self.topsrcdir)

        if directory:
            directory = mozpath.normsep(directory)
            dirpath = self.join_ensure_dir(topsrcdir, directory)
            if not dirpath:
                return 1

        if flags:
            # Flatten lists of flags.
            flags = set(itertools.chain(*[flaglist.split(",") for flaglist in flags]))

        for warning in by_name:
            filename = mozpath.normsep(warning["filename"])

            if filename.startswith(topsrcdir):
                filename = filename[len(topsrcdir) + 1 :]

            if directory and not filename.startswith(directory):
                continue

            if flags and warning["flag"] not in flags:
                continue

            if warning["column"] is not None:
                print(
                    "%s:%d:%d [%s] %s"
                    % (
                        filename,
                        warning["line"],
                        warning["column"],
                        warning["flag"],
                        warning["message"],
                    )
                )
            else:
                print(
                    "%s:%d [%s] %s"
                    % (filename, warning["line"], warning["flag"], warning["message"])
                )

    def join_ensure_dir(self, dir1, dir2):
        dir1 = mozpath.normpath(dir1)
        dir2 = mozpath.normsep(dir2)
        joined_path = mozpath.join(dir1, dir2)
        if os.path.isdir(joined_path):
            return joined_path
        print("Specified directory not found.")
        return None


@CommandProvider
class GTestCommands(MachCommandBase):
    @Command(
        "gtest", category="testing", description="Run GTest unit tests (C++ tests)."
    )
    @CommandArgument(
        "gtest_filter",
        default=b"*",
        nargs="?",
        metavar="gtest_filter",
        help="test_filter is a ':'-separated list of wildcard patterns "
        "(called the positive patterns), optionally followed by a '-' "
        "and another ':'-separated pattern list (called the negative patterns).",
    )
    @CommandArgument(
        "--jobs",
        "-j",
        default="1",
        nargs="?",
        metavar="jobs",
        type=int,
        help="Run the tests in parallel using multiple processes.",
    )
    @CommandArgument(
        "--tbpl-parser",
        "-t",
        action="store_true",
        help="Output test results in a format that can be parsed by TBPL.",
    )
    @CommandArgument(
        "--shuffle",
        "-s",
        action="store_true",
        help="Randomize the execution order of tests.",
    )
    @CommandArgument(
        "--enable-webrender",
        action="store_true",
        default=False,
        dest="enable_webrender",
        help="Enable the WebRender compositor in Gecko.",
    )
    @CommandArgumentGroup("Android")
    @CommandArgument(
        "--package",
        default="org.mozilla.geckoview.test",
        group="Android",
        help="Package name of test app.",
    )
    @CommandArgument(
        "--adbpath", dest="adb_path", group="Android", help="Path to adb binary."
    )
    @CommandArgument(
        "--deviceSerial",
        dest="device_serial",
        group="Android",
        help="adb serial number of remote device. "
        "Required when more than one device is connected to the host. "
        "Use 'adb devices' to see connected devices.",
    )
    @CommandArgument(
        "--remoteTestRoot",
        dest="remote_test_root",
        group="Android",
        help="Remote directory to use as test root " "(eg. /data/local/tmp/test_root).",
    )
    @CommandArgument(
        "--libxul", dest="libxul_path", group="Android", help="Path to gtest libxul.so."
    )
    @CommandArgument(
        "--no-install",
        action="store_true",
        default=False,
        group="Android",
        help="Skip the installation of the APK.",
    )
    @CommandArgumentGroup("debugging")
    @CommandArgument(
        "--debug",
        action="store_true",
        group="debugging",
        help="Enable the debugger. Not specifying a --debugger option will result in "
        "the default debugger being used.",
    )
    @CommandArgument(
        "--debugger",
        default=None,
        type=str,
        group="debugging",
        help="Name of debugger to use.",
    )
    @CommandArgument(
        "--debugger-args",
        default=None,
        metavar="params",
        type=str,
        group="debugging",
        help="Command-line arguments to pass to the debugger itself; "
        "split as the Bourne shell would.",
    )
    def gtest(
        self,
        shuffle,
        jobs,
        gtest_filter,
        tbpl_parser,
        enable_webrender,
        package,
        adb_path,
        device_serial,
        remote_test_root,
        libxul_path,
        no_install,
        debug,
        debugger,
        debugger_args,
    ):

        # We lazy build gtest because it's slow to link
        try:
            self.config_environment
        except Exception:
            print("Please run |./mach build| before |./mach gtest|.")
            return 1

        res = self._mach_context.commands.dispatch(
            "build", self._mach_context, what=["recurse_gtest"]
        )
        if res:
            print("Could not build xul-gtest")
            return res

        if self.substs.get("MOZ_WIDGET_TOOLKIT") == "cocoa":
            self._run_make(
                directory="browser/app", target="repackage", ensure_exit_code=True
            )

        cwd = os.path.join(self.topobjdir, "_tests", "gtest")

        if not os.path.isdir(cwd):
            os.makedirs(cwd)

        if conditions.is_android(self):
            if jobs != 1:
                print("--jobs is not supported on Android and will be ignored")
            if debug or debugger or debugger_args:
                print(
                    "--debug options are not supported on Android and will be ignored"
                )
            from mozrunner.devices.android_device import InstallIntent

            return self.android_gtest(
                cwd,
                shuffle,
                gtest_filter,
                package,
                adb_path,
                device_serial,
                remote_test_root,
                libxul_path,
                enable_webrender,
                InstallIntent.NO if no_install else InstallIntent.YES,
            )

        if (
            package
            or adb_path
            or device_serial
            or remote_test_root
            or libxul_path
            or no_install
        ):
            print("One or more Android-only options will be ignored")

        app_path = self.get_binary_path("app")
        args = [app_path, "-unittest", "--gtest_death_test_style=threadsafe"]

        if sys.platform.startswith("win") and "MOZ_LAUNCHER_PROCESS" in self.defines:
            args.append("--wait-for-browser")

        if debug or debugger or debugger_args:
            args = self.prepend_debugger_args(args, debugger, debugger_args)
            if not args:
                return 1

        # Use GTest environment variable to control test execution
        # For details see:
        # https://code.google.com/p/googletest/wiki/AdvancedGuide#Running_Test_Programs:_Advanced_Options
        gtest_env = {b"GTEST_FILTER": gtest_filter}

        # Note: we must normalize the path here so that gtest on Windows sees
        # a MOZ_GMP_PATH which has only Windows dir seperators, because
        # nsIFile cannot open the paths with non-Windows dir seperators.
        xre_path = os.path.join(os.path.normpath(self.topobjdir), "dist", "bin")
        gtest_env["MOZ_XRE_DIR"] = xre_path
        gtest_env["MOZ_GMP_PATH"] = os.pathsep.join(
            os.path.join(xre_path, p, "1.0") for p in ("gmp-fake", "gmp-fakeopenh264")
        )

        gtest_env[b"MOZ_RUN_GTEST"] = b"True"

        if shuffle:
            gtest_env[b"GTEST_SHUFFLE"] = b"True"

        if tbpl_parser:
            gtest_env[b"MOZ_TBPL_PARSER"] = b"True"

        if enable_webrender:
            gtest_env[b"MOZ_WEBRENDER"] = b"1"
            gtest_env[b"MOZ_ACCELERATED"] = b"1"
        else:
            gtest_env[b"MOZ_WEBRENDER"] = b"0"

        if jobs == 1:
            return self.run_process(
                args=args,
                append_env=gtest_env,
                cwd=cwd,
                ensure_exit_code=False,
                pass_thru=True,
            )

        from mozprocess import ProcessHandlerMixin
        import functools

        def handle_line(job_id, line):
            # Prepend the jobId
            line = "[%d] %s" % (job_id + 1, line.strip())
            self.log(logging.INFO, "GTest", {"line": line}, "{line}")

        gtest_env["GTEST_TOTAL_SHARDS"] = str(jobs)
        processes = {}
        for i in range(0, jobs):
            gtest_env["GTEST_SHARD_INDEX"] = str(i)
            processes[i] = ProcessHandlerMixin(
                [app_path, "-unittest"],
                cwd=cwd,
                env=gtest_env,
                processOutputLine=[functools.partial(handle_line, i)],
                universal_newlines=True,
            )
            processes[i].run()

        exit_code = 0
        for process in processes.values():
            status = process.wait()
            if status:
                exit_code = status

        # Clamp error code to 255 to prevent overflowing multiple of
        # 256 into 0
        if exit_code > 255:
            exit_code = 255

        return exit_code

    def android_gtest(
        self,
        test_dir,
        shuffle,
        gtest_filter,
        package,
        adb_path,
        device_serial,
        remote_test_root,
        libxul_path,
        enable_webrender,
        install,
    ):
        # setup logging for mozrunner
        from mozlog.commandline import setup_logging

        format_args = {"level": self._mach_context.settings["test"]["level"]}
        default_format = self._mach_context.settings["test"]["format"]
        setup_logging("mach-gtest", {}, {default_format: sys.stdout}, format_args)

        # ensure that a device is available and test app is installed
        from mozrunner.devices.android_device import verify_android_device, get_adb_path

        verify_android_device(
            self, install=install, app=package, device_serial=device_serial
        )

        if not adb_path:
            adb_path = get_adb_path(self)
        if not libxul_path:
            libxul_path = os.path.join(
                self.topobjdir, "dist", "bin", "gtest", "libxul.so"
            )

        # run gtest via remotegtests.py
        exit_code = 0
        import imp

        path = os.path.join("testing", "gtest", "remotegtests.py")
        with open(path, "r") as fh:
            imp.load_module("remotegtests", fh, path, (".py", "r", imp.PY_SOURCE))
        import remotegtests

        tester = remotegtests.RemoteGTests()
        if not tester.run_gtest(
            test_dir,
            shuffle,
            gtest_filter,
            package,
            adb_path,
            device_serial,
            remote_test_root,
            libxul_path,
            None,
            enable_webrender,
        ):
            exit_code = 1
        tester.cleanup()

        return exit_code

    def prepend_debugger_args(self, args, debugger, debugger_args):
        """
        Given an array with program arguments, prepend arguments to run it under a
        debugger.

        :param args: The executable and arguments used to run the process normally.
        :param debugger: The debugger to use, or empty to use the default debugger.
        :param debugger_args: Any additional parameters to pass to the debugger.
        """

        import mozdebug

        if not debugger:
            # No debugger name was provided. Look for the default ones on
            # current OS.
            debugger = mozdebug.get_default_debugger_name(
                mozdebug.DebuggerSearch.KeepLooking
            )

        if debugger:
            debuggerInfo = mozdebug.get_debugger_info(debugger, debugger_args)

        if not debugger or not debuggerInfo:
            print("Could not find a suitable debugger in your PATH.")
            return None

        # Parameters come from the CLI. We need to convert them before
        # their use.
        if debugger_args:
            from mozbuild import shellutil

            try:
                debugger_args = shellutil.split(debugger_args)
            except shellutil.MetaCharacterException as e:
                print(
                    "The --debugger_args you passed require a real shell to parse them."
                )
                print("(We can't handle the %r character.)" % e.char)
                return None

        # Prepend the debugger args.
        args = [debuggerInfo.path] + debuggerInfo.args + args
        return args


@CommandProvider
class Package(MachCommandBase):
    """Package the built product for distribution."""

    @Command(
        "package",
        category="post-build",
        description="Package the built product for distribution as an APK, DMG, etc.",
    )
    @CommandArgument(
        "-v",
        "--verbose",
        action="store_true",
        help="Verbose output for what commands the packaging process is running.",
    )
    def package(self, verbose=False):
        ret = self._run_make(
            directory=".", target="package", silent=not verbose, ensure_exit_code=False
        )
        if ret == 0:
            self.notify("Packaging complete")
        return ret


def _get_android_install_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--app",
        default="org.mozilla.geckoview_example",
        help="Android package to install " "(default: org.mozilla.geckoview_example)",
    )
    parser.add_argument(
        "--verbose",
        "-v",
        action="store_true",
        help="Print verbose output when installing.",
    )
    return parser


def setup_install_parser():
    build = MozbuildObject.from_environment(cwd=here)
    if conditions.is_android(build):
        return _get_android_install_parser()
    return argparse.ArgumentParser()


@CommandProvider
class Install(MachCommandBase):
    """Install a package."""

    @Command(
        "install",
        category="post-build",
        conditions=[conditions.has_build],
        parser=setup_install_parser,
        description="Install the package on the machine (or device in the case of Android).",
    )
    def install(self, **kwargs):
        if conditions.is_android(self):
            from mozrunner.devices.android_device import (
                verify_android_device,
                InstallIntent,
            )

            ret = verify_android_device(self, install=InstallIntent.YES, **kwargs) == 0
        else:
            ret = self._run_make(
                directory=".", target="install", ensure_exit_code=False
            )

        if ret == 0:
            self.notify("Install complete")
        return ret


@SettingsProvider
class RunSettings:
    config_settings = [
        (
            "runprefs.*",
            "string",
            """
Pass a pref into Firefox when using `mach run`, of the form `foo.bar=value`.
Prefs will automatically be cast into the appropriate type. Integers can be
single quoted to force them to be strings.
""".strip(),
        ),
    ]


def _get_android_run_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--app",
        default="org.mozilla.geckoview_example",
        help="Android package to run " "(default: org.mozilla.geckoview_example)",
    )
    parser.add_argument(
        "--intent",
        default="android.intent.action.VIEW",
        help="Android intent action to launch with "
        "(default: android.intent.action.VIEW)",
    )
    parser.add_argument(
        "--setenv",
        dest="env",
        action="append",
        default=[],
        help="Set target environment variable, like FOO=BAR",
    )
    parser.add_argument(
        "--profile",
        "-P",
        default=None,
        help="Path to Gecko profile, like /path/to/host/profile "
        "or /path/to/target/profile",
    )
    parser.add_argument("--url", default=None, help="URL to open")
    parser.add_argument(
        "--no-install",
        action="store_true",
        default=False,
        help="Do not try to install application on device before running "
        "(default: False)",
    )
    parser.add_argument(
        "--no-wait",
        action="store_true",
        default=False,
        help="Do not wait for application to start before returning "
        "(default: False)",
    )
    parser.add_argument(
        "--enable-fission",
        action="store_true",
        help="Run the program with Fission (site isolation) enabled.",
    )
    parser.add_argument(
        "--fail-if-running",
        action="store_true",
        default=False,
        help="Fail if application is already running (default: False)",
    )
    parser.add_argument(
        "--restart",
        action="store_true",
        default=False,
        help="Stop the application if it is already running (default: False)",
    )
    return parser


def _get_jsshell_run_parser():
    parser = argparse.ArgumentParser()
    group = parser.add_argument_group("the compiled program")
    group.add_argument(
        "params",
        nargs="...",
        default=[],
        help="Command-line arguments to be passed through to the program. Not "
        "specifying a --profile or -P option will result in a temporary profile "
        "being used.",
    )

    group = parser.add_argument_group("debugging")
    group.add_argument(
        "--debug",
        action="store_true",
        help="Enable the debugger. Not specifying a --debugger option will result "
        "in the default debugger being used.",
    )
    group.add_argument(
        "--debugger", default=None, type=str, help="Name of debugger to use."
    )
    group.add_argument(
        "--debugger-args",
        default=None,
        metavar="params",
        type=str,
        help="Command-line arguments to pass to the debugger itself; "
        "split as the Bourne shell would.",
    )
    group.add_argument(
        "--debugparams",
        action=StoreDebugParamsAndWarnAction,
        default=None,
        type=str,
        dest="debugger_args",
        help=argparse.SUPPRESS,
    )

    return parser


def _get_desktop_run_parser():
    parser = argparse.ArgumentParser()
    group = parser.add_argument_group("the compiled program")
    group.add_argument(
        "params",
        nargs="...",
        default=[],
        help="Command-line arguments to be passed through to the program. Not "
        "specifying a --profile or -P option will result in a temporary profile "
        "being used.",
    )
    group.add_argument("--packaged", action="store_true", help="Run a packaged build.")
    group.add_argument(
        "--app", help="Path to executable to run (default: output of ./mach build)"
    )
    group.add_argument(
        "--remote",
        "-r",
        action="store_true",
        help="Do not pass the --no-remote argument by default.",
    )
    group.add_argument(
        "--background",
        "-b",
        action="store_true",
        help="Do not pass the --foreground argument by default on Mac.",
    )
    group.add_argument(
        "--noprofile",
        "-n",
        action="store_true",
        help="Do not pass the --profile argument by default.",
    )
    group.add_argument(
        "--disable-e10s",
        action="store_true",
        help="Run the program with electrolysis disabled.",
    )
    group.add_argument(
        "--enable-crash-reporter",
        action="store_true",
        help="Run the program with the crash reporter enabled.",
    )
    group.add_argument(
        "--enable-fission",
        action="store_true",
        help="Run the program with Fission (site isolation) enabled.",
    )
    group.add_argument(
        "--setpref",
        action="append",
        default=[],
        help="Set the specified pref before starting the program. Can be set "
        "multiple times. Prefs can also be set in ~/.mozbuild/machrc in the "
        "[runprefs] section - see `./mach settings` for more information.",
    )
    group.add_argument(
        "--temp-profile",
        action="store_true",
        help="Run the program using a new temporary profile created inside "
        "the objdir.",
    )
    group.add_argument(
        "--macos-open",
        action="store_true",
        help="On macOS, run the program using the open(1) command. Per open(1), "
        "the browser is launched \"just as if you had double-clicked the file's "
        'icon". The browser can not be launched under a debugger with this '
        "option.",
    )

    group = parser.add_argument_group("debugging")
    group.add_argument(
        "--debug",
        action="store_true",
        help="Enable the debugger. Not specifying a --debugger option will result "
        "in the default debugger being used.",
    )
    group.add_argument(
        "--debugger", default=None, type=str, help="Name of debugger to use."
    )
    group.add_argument(
        "--debugger-args",
        default=None,
        metavar="params",
        type=str,
        help="Command-line arguments to pass to the debugger itself; "
        "split as the Bourne shell would.",
    )
    group.add_argument(
        "--debugparams",
        action=StoreDebugParamsAndWarnAction,
        default=None,
        type=str,
        dest="debugger_args",
        help=argparse.SUPPRESS,
    )

    group = parser.add_argument_group("DMD")
    group.add_argument(
        "--dmd",
        action="store_true",
        help="Enable DMD. The following arguments have no effect without this.",
    )
    group.add_argument(
        "--mode",
        choices=["live", "dark-matter", "cumulative", "scan"],
        help="Profiling mode. The default is 'dark-matter'.",
    )
    group.add_argument(
        "--stacks",
        choices=["partial", "full"],
        help="Allocation stack trace coverage. The default is 'partial'.",
    )
    group.add_argument(
        "--show-dump-stats", action="store_true", help="Show stats when doing dumps."
    )

    return parser


def setup_run_parser():
    build = MozbuildObject.from_environment(cwd=here)
    if conditions.is_android(build):
        return _get_android_run_parser()
    if conditions.is_jsshell(build):
        return _get_jsshell_run_parser()
    return _get_desktop_run_parser()


@CommandProvider
class RunProgram(MachCommandBase):
    """Run the compiled program."""

    @Command(
        "run",
        category="post-build",
        conditions=[conditions.has_build_or_shell],
        parser=setup_run_parser,
        description="Run the compiled program, possibly under a debugger or DMD.",
    )
    def run(self, **kwargs):
        if conditions.is_android(self):
            return self._run_android(**kwargs)
        if conditions.is_jsshell(self):
            return self._run_jsshell(**kwargs)
        return self._run_desktop(**kwargs)

    def _run_android(
        self,
        app="org.mozilla.geckoview_example",
        intent=None,
        env=[],
        profile=None,
        url=None,
        no_install=None,
        no_wait=None,
        fail_if_running=None,
        restart=None,
        enable_fission=False,
    ):
        from mozrunner.devices.android_device import (
            verify_android_device,
            _get_device,
            InstallIntent,
        )
        from six.moves import shlex_quote

        if app == "org.mozilla.geckoview_example":
            activity_name = "org.mozilla.geckoview_example.GeckoViewActivity"
        elif app == "org.mozilla.geckoview.test":
            activity_name = "org.mozilla.geckoview.test.TestRunnerActivity"
        elif "fennec" in app or "firefox" in app:
            activity_name = "org.mozilla.gecko.BrowserApp"
        else:
            raise RuntimeError("Application not recognized: {}".format(app))

        # `verify_android_device` respects `DEVICE_SERIAL` if it is set and sets it otherwise.
        verify_android_device(
            self, app=app, install=InstallIntent.NO if no_install else InstallIntent.YES
        )
        device_serial = os.environ.get("DEVICE_SERIAL")
        if not device_serial:
            print("No ADB devices connected.")
            return 1

        device = _get_device(self.substs, device_serial=device_serial)

        args = []
        if profile:
            if os.path.isdir(profile):
                host_profile = profile
                # Always /data/local/tmp, rather than `device.test_root`, because GeckoView only
                # takes its configuration file from /data/local/tmp, and we want to follow suit.
                target_profile = "/data/local/tmp/{}-profile".format(app)
                device.rm(target_profile, recursive=True, force=True)
                device.push(host_profile, target_profile)
                self.log(
                    logging.INFO,
                    "run",
                    {"host_profile": host_profile, "target_profile": target_profile},
                    'Pushed profile from host "{host_profile}" to target "{target_profile}"',
                )
            else:
                target_profile = profile
                self.log(
                    logging.INFO,
                    "run",
                    {"target_profile": target_profile},
                    'Using profile from target "{target_profile}"',
                )

            args = ["--profile", shlex_quote(target_profile)]

        if enable_fission:
            env.append("MOZ_FORCE_ENABLE_FISSION=1")

        extras = {}
        for i, e in enumerate(env):
            extras["env{}".format(i)] = e
        if args:
            extras["args"] = " ".join(args)

        if env or args:
            restart = True

        if restart:
            fail_if_running = False
            self.log(
                logging.INFO,
                "run",
                {"app": app},
                "Stopping {app} to ensure clean restart.",
            )
            device.stop_application(app)

        # We'd prefer to log the actual `am start ...` command, but it's not trivial to wire the
        # device's logger to mach's logger.
        self.log(
            logging.INFO,
            "run",
            {"app": app, "activity_name": activity_name},
            "Starting {app}/{activity_name}.",
        )

        device.launch_application(
            app_name=app,
            activity_name=activity_name,
            intent=intent,
            extras=extras,
            url=url,
            wait=not no_wait,
            fail_if_running=fail_if_running,
        )

        return 0

    def _run_jsshell(self, params, debug, debugger, debugger_args):
        try:
            binpath = self.get_binary_path("app")
        except BinaryNotFoundException as e:
            self.log(logging.ERROR, "run", {"error": str(e)}, "ERROR: {error}")
            self.log(logging.INFO, "run", {"help": e.help()}, "{help}")
            return 1

        args = [binpath]

        if params:
            args.extend(params)

        extra_env = {
            "RUST_BACKTRACE": "full",
        }

        if debug or debugger or debugger_args:
            if "INSIDE_EMACS" in os.environ:
                self.log_manager.terminal_handler.setLevel(logging.WARNING)

            import mozdebug

            if not debugger:
                # No debugger name was provided. Look for the default ones on
                # current OS.
                debugger = mozdebug.get_default_debugger_name(
                    mozdebug.DebuggerSearch.KeepLooking
                )

            if debugger:
                self.debuggerInfo = mozdebug.get_debugger_info(debugger, debugger_args)

            if not debugger or not self.debuggerInfo:
                print("Could not find a suitable debugger in your PATH.")
                return 1

            # Parameters come from the CLI. We need to convert them before
            # their use.
            if debugger_args:
                from mozbuild import shellutil

                try:
                    debugger_args = shellutil.split(debugger_args)
                except shellutil.MetaCharacterException as e:
                    print(
                        "The --debugger-args you passed require a real shell to parse them."
                    )
                    print("(We can't handle the %r character.)" % e.char)
                    return 1

            # Prepend the debugger args.
            args = [self.debuggerInfo.path] + self.debuggerInfo.args + args

        return self.run_process(
            args=args, ensure_exit_code=False, pass_thru=True, append_env=extra_env
        )

    def _run_desktop(
        self,
        params,
        packaged,
        app,
        remote,
        background,
        noprofile,
        disable_e10s,
        enable_crash_reporter,
        enable_fission,
        setpref,
        temp_profile,
        macos_open,
        debug,
        debugger,
        debugger_args,
        dmd,
        mode,
        stacks,
        show_dump_stats,
    ):
        from mozprofile import Profile, Preferences

        try:
            if packaged:
                binpath = self.get_binary_path(where="staged-package")
            else:
                binpath = app or self.get_binary_path("app")
        except BinaryNotFoundException as e:
            self.log(logging.ERROR, "run", {"error": str(e)}, "ERROR: {error}")
            if packaged:
                self.log(
                    logging.INFO,
                    "run",
                    {
                        "help": "It looks like your build isn't packaged. "
                        "You can run |./mach package| to package it."
                    },
                    "{help}",
                )
            else:
                self.log(logging.INFO, "run", {"help": e.help()}, "{help}")
            return 1

        args = []
        if macos_open:
            if debug:
                print(
                    "The browser can not be launched in the debugger "
                    "when using the macOS open command."
                )
                return 1
            try:
                m = re.search(r"^.+\.app", binpath)
                apppath = m.group(0)
                args = ["open", apppath, "--args"]
            except Exception as e:
                print(
                    "Couldn't get the .app path from the binary path. "
                    "The macOS open option can only be used on macOS"
                )
                print(e)
                return 1
        else:
            args = [binpath]

        if params:
            args.extend(params)

        if not remote:
            args.append("-no-remote")

        if not background and sys.platform == "darwin":
            args.append("-foreground")

        if sys.platform.startswith("win") and "MOZ_LAUNCHER_PROCESS" in self.defines:
            args.append("-wait-for-browser")

        no_profile_option_given = all(
            p not in params for p in ["-profile", "--profile", "-P"]
        )
        if no_profile_option_given and not noprofile:
            prefs = {
                "browser.aboutConfig.showWarning": False,
                "browser.shell.checkDefaultBrowser": False,
                "general.warnOnAboutConfig": False,
            }
            prefs.update(self._mach_context.settings.runprefs)
            prefs.update([p.split("=", 1) for p in setpref])
            for pref in prefs:
                prefs[pref] = Preferences.cast(prefs[pref])

            tmpdir = os.path.join(self.topobjdir, "tmp")
            if not os.path.exists(tmpdir):
                os.makedirs(tmpdir)

            if temp_profile:
                path = tempfile.mkdtemp(dir=tmpdir, prefix="profile-")
            else:
                path = os.path.join(tmpdir, "profile-default")

            profile = Profile(path, preferences=prefs)
            args.append("-profile")
            args.append(profile.profile)

        if not no_profile_option_given and setpref:
            print("setpref is only supported if a profile is not specified")
            return 1

            if not no_profile_option_given:
                # The profile name may be non-ascii, but come from the
                # commandline as str, so convert here with a better guess at
                # an encoding than the default.
                encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
                args = [
                    unicode(a, encoding) if not isinstance(a, unicode) else a
                    for a in args
                ]

        some_debugging_option = debug or debugger or debugger_args

        # By default, because Firefox is a GUI app, on Windows it will not
        # 'create' a console to which stdout/stderr is printed. This means
        # printf/dump debugging is invisible. We default to adding the
        # -attach-console argument to fix this. We avoid this if we're launched
        # under a debugger (which can do its own picking up of stdout/stderr).
        # We also check for both the -console and -attach-console flags:
        # -console causes Firefox to create a separate window;
        # -attach-console just ends us up with output that gets relayed via mach.
        # We shouldn't override the user using -console. For more info, see
        # https://bugzilla.mozilla.org/show_bug.cgi?id=1257155
        if (
            sys.platform.startswith("win")
            and not some_debugging_option
            and "-console" not in args
            and "--console" not in args
            and "-attach-console" not in args
            and "--attach-console" not in args
        ):
            args.append("-attach-console")

        extra_env = {
            "MOZ_DEVELOPER_REPO_DIR": self.topsrcdir,
            "MOZ_DEVELOPER_OBJ_DIR": self.topobjdir,
            "RUST_BACKTRACE": "full",
        }

        if not enable_crash_reporter:
            extra_env["MOZ_CRASHREPORTER_DISABLE"] = "1"
        else:
            extra_env["MOZ_CRASHREPORTER"] = "1"

        if disable_e10s:
            version_file = os.path.join(
                self.topsrcdir, "browser", "config", "version.txt"
            )
            f = open(version_file, "r")
            extra_env["MOZ_FORCE_DISABLE_E10S"] = f.read().strip()

        if enable_fission:
            extra_env["MOZ_FORCE_ENABLE_FISSION"] = "1"

        if some_debugging_option:
            if "INSIDE_EMACS" in os.environ:
                self.log_manager.terminal_handler.setLevel(logging.WARNING)

            import mozdebug

            if not debugger:
                # No debugger name was provided. Look for the default ones on
                # current OS.
                debugger = mozdebug.get_default_debugger_name(
                    mozdebug.DebuggerSearch.KeepLooking
                )

            if debugger:
                self.debuggerInfo = mozdebug.get_debugger_info(debugger, debugger_args)

            if not debugger or not self.debuggerInfo:
                print("Could not find a suitable debugger in your PATH.")
                return 1

            # Parameters come from the CLI. We need to convert them before
            # their use.
            if debugger_args:
                from mozbuild import shellutil

                try:
                    debugger_args = shellutil.split(debugger_args)
                except shellutil.MetaCharacterException as e:
                    print(
                        "The --debugger-args you passed require a real shell to parse them."
                    )
                    print("(We can't handle the %r character.)" % e.char)
                    return 1

            # Prepend the debugger args.
            args = [self.debuggerInfo.path] + self.debuggerInfo.args + args

        if dmd:
            dmd_params = []

            if mode:
                dmd_params.append("--mode=" + mode)
            if stacks:
                dmd_params.append("--stacks=" + stacks)
            if show_dump_stats:
                dmd_params.append("--show-dump-stats=yes")

            if dmd_params:
                extra_env["DMD"] = " ".join(dmd_params)
            else:
                extra_env["DMD"] = "1"

        return self.run_process(
            args=args, ensure_exit_code=False, pass_thru=True, append_env=extra_env
        )


@CommandProvider
class Buildsymbols(MachCommandBase):
    """Produce a package of debug symbols suitable for use with Breakpad."""

    @Command(
        "buildsymbols",
        category="post-build",
        description="Produce a package of Breakpad-format symbols.",
    )
    def buildsymbols(self):
        return self._run_make(
            directory=".", target="buildsymbols", ensure_exit_code=False
        )


@CommandProvider
class MachDebug(MachCommandBase):
    @Command(
        "environment",
        category="build-dev",
        description="Show info about the mach and build environment.",
    )
    @CommandArgument(
        "--format",
        default="pretty",
        choices=["pretty", "json"],
        help="Print data in the given format.",
    )
    @CommandArgument("--output", "-o", type=str, help="Output to the given file.")
    @CommandArgument(
        "--verbose", "-v", action="store_true", help="Print verbose output."
    )
    def environment(self, format, output=None, verbose=False):
        func = getattr(self, "_environment_%s" % format.replace(".", "_"))

        if output:
            # We want to preserve mtimes if the output file already exists
            # and the content hasn't changed.
            from mozbuild.util import FileAvoidWrite

            with FileAvoidWrite(output) as out:
                return func(out, verbose)
        return func(sys.stdout, verbose)

    def _environment_pretty(self, out, verbose):
        state_dir = self._mach_context.state_dir
        import platform

        print("platform:\n\t%s" % platform.platform(), file=out)
        print("python version:\n\t%s" % sys.version, file=out)
        print("python prefix:\n\t%s" % sys.prefix, file=out)
        print("mach cwd:\n\t%s" % self._mach_context.cwd, file=out)
        print("os cwd:\n\t%s" % os.getcwd(), file=out)
        print("mach directory:\n\t%s" % self._mach_context.topdir, file=out)
        print("state directory:\n\t%s" % state_dir, file=out)

        print("object directory:\n\t%s" % self.topobjdir, file=out)

        if self.mozconfig["path"]:
            print("mozconfig path:\n\t%s" % self.mozconfig["path"], file=out)
            if self.mozconfig["configure_args"]:
                print("mozconfig configure args:", file=out)
                for arg in self.mozconfig["configure_args"]:
                    print("\t%s" % arg, file=out)

            if self.mozconfig["make_extra"]:
                print("mozconfig extra make args:", file=out)
                for arg in self.mozconfig["make_extra"]:
                    print("\t%s" % arg, file=out)

            if self.mozconfig["make_flags"]:
                print("mozconfig make flags:", file=out)
                for arg in self.mozconfig["make_flags"]:
                    print("\t%s" % arg, file=out)

        config = None

        try:
            config = self.config_environment

        except Exception:
            pass

        if config:
            print("config topsrcdir:\n\t%s" % config.topsrcdir, file=out)
            print("config topobjdir:\n\t%s" % config.topobjdir, file=out)

            if verbose:
                print("config substitutions:", file=out)
                for k in sorted(config.substs):
                    print("\t%s: %s" % (k, config.substs[k]), file=out)

                print("config defines:", file=out)
                for k in sorted(config.defines):
                    print("\t%s" % k, file=out)

    def _environment_json(self, out, verbose):
        import json

        class EnvironmentEncoder(json.JSONEncoder):
            def default(self, obj):
                if isinstance(obj, MozbuildObject):
                    result = {
                        "topsrcdir": obj.topsrcdir,
                        "topobjdir": obj.topobjdir,
                        "mozconfig": obj.mozconfig,
                    }
                    if verbose:
                        result["substs"] = obj.substs
                        result["defines"] = obj.defines
                    return result
                elif isinstance(obj, set):
                    return list(obj)
                return json.JSONEncoder.default(self, obj)

        json.dump(self, cls=EnvironmentEncoder, sort_keys=True, fp=out)


@CommandProvider
class Repackage(MachCommandBase):
    """Repackages artifacts into different formats.

    This is generally used after packages are signed by the signing
    scriptworkers in order to bundle things up into shippable formats, such as a
    .dmg on OSX or an installer exe on Windows.
    """

    @Command(
        "repackage",
        category="misc",
        description="Repackage artifacts into different formats.",
    )
    def repackage(self):
        print("Usage: ./mach repackage [dmg|installer|mar] [args...]")

    @SubCommand(
        "repackage", "dmg", description="Repackage a tar file into a .dmg for OSX"
    )
    @CommandArgument("--input", "-i", type=str, required=True, help="Input filename")
    @CommandArgument("--output", "-o", type=str, required=True, help="Output filename")
    def repackage_dmg(self, input, output):
        if not os.path.exists(input):
            print("Input file does not exist: %s" % input)
            return 1

        if not os.path.exists(os.path.join(self.topobjdir, "config.status")):
            print(
                "config.status not found.  Please run |mach configure| "
                "prior to |mach repackage|."
            )
            return 1

        from mozbuild.repackaging.dmg import repackage_dmg

        repackage_dmg(input, output)

    @SubCommand(
        "repackage", "installer", description="Repackage into a Windows installer exe"
    )
    @CommandArgument(
        "--tag",
        type=str,
        required=True,
        help="The .tag file used to build the installer",
    )
    @CommandArgument(
        "--setupexe",
        type=str,
        required=True,
        help="setup.exe file inside the installer",
    )
    @CommandArgument(
        "--package",
        type=str,
        required=False,
        help="Optional package .zip for building a full installer",
    )
    @CommandArgument("--output", "-o", type=str, required=True, help="Output filename")
    @CommandArgument(
        "--package-name",
        type=str,
        required=False,
        help="Name of the package being rebuilt",
    )
    @CommandArgument(
        "--sfx-stub", type=str, required=True, help="Path to the self-extraction stub."
    )
    @CommandArgument(
        "--use-upx",
        required=False,
        action="store_true",
        help="Run UPX on the self-extraction stub.",
    )
    def repackage_installer(
        self, tag, setupexe, package, output, package_name, sfx_stub, use_upx
    ):
        from mozbuild.repackaging.installer import repackage_installer

        repackage_installer(
            topsrcdir=self.topsrcdir,
            tag=tag,
            setupexe=setupexe,
            package=package,
            output=output,
            package_name=package_name,
            sfx_stub=sfx_stub,
            use_upx=use_upx,
        )

    @SubCommand("repackage", "msi", description="Repackage into a MSI")
    @CommandArgument(
        "--wsx",
        type=str,
        required=True,
        help="The wsx file used to build the installer",
    )
    @CommandArgument(
        "--version",
        type=str,
        required=True,
        help="The Firefox version used to create the installer",
    )
    @CommandArgument(
        "--locale", type=str, required=True, help="The locale of the installer"
    )
    @CommandArgument(
        "--arch", type=str, required=True, help="The archtecture you are building."
    )
    @CommandArgument("--setupexe", type=str, required=True, help="setup.exe installer")
    @CommandArgument(
        "--candle", type=str, required=False, help="location of candle binary"
    )
    @CommandArgument(
        "--light", type=str, required=False, help="location of light binary"
    )
    @CommandArgument("--output", "-o", type=str, required=True, help="Output filename")
    def repackage_msi(
        self, wsx, version, locale, arch, setupexe, candle, light, output
    ):
        from mozbuild.repackaging.msi import repackage_msi

        repackage_msi(
            topsrcdir=self.topsrcdir,
            wsx=wsx,
            version=version,
            locale=locale,
            arch=arch,
            setupexe=setupexe,
            candle=candle,
            light=light,
            output=output,
        )

    @SubCommand("repackage", "mar", description="Repackage into complete MAR file")
    @CommandArgument("--input", "-i", type=str, required=True, help="Input filename")
    @CommandArgument("--mar", type=str, required=True, help="Mar binary path")
    @CommandArgument("--output", "-o", type=str, required=True, help="Output filename")
    @CommandArgument(
        "--arch", type=str, required=True, help="The archtecture you are building."
    )
    @CommandArgument("--mar-channel-id", type=str, help="Mar channel id")
    def repackage_mar(self, input, mar, output, arch, mar_channel_id):
        from mozbuild.repackaging.mar import repackage_mar

        repackage_mar(
            self.topsrcdir,
            input,
            mar,
            output,
            arch=arch,
            mar_channel_id=mar_channel_id,
        )


@SettingsProvider
class TelemetrySettings:
    config_settings = [
        (
            "build.telemetry",
            "boolean",
            """
Enable submission of build system telemetry.
        """.strip(),
            False,
        ),
    ]


@CommandProvider
class L10NCommands(MachCommandBase):
    @Command(
        "package-multi-locale",
        category="post-build",
        description="Package a multi-locale version of the built product "
        "for distribution as an APK, DMG, etc.",
    )
    @CommandArgument(
        "--locales",
        metavar="LOCALES",
        nargs="+",
        required=True,
        help='List of locales to package, including "en-US"',
    )
    @CommandArgument(
        "--verbose", action="store_true", help="Log informative status messages."
    )
    def package_l10n(self, verbose=False, locales=[]):
        if "RecursiveMake" not in self.substs["BUILD_BACKENDS"]:
            print(
                "Artifact builds do not support localization. "
                "If you know what you are doing, you can use:\n"
                "ac_add_options --disable-compile-environment\n"
                "export BUILD_BACKENDS=FasterMake,RecursiveMake\n"
                "in your mozconfig."
            )
            return 1

        if "en-US" not in locales:
            self.log(
                logging.WARN,
                "package-multi-locale",
                {"locales": locales},
                'List of locales does not include default locale "en-US": '
                '{locales}; adding "en-US"',
            )
            locales.append("en-US")
        locales = list(sorted(locales))

        append_env = {
            # We are only (re-)packaging, we don't want to (re-)build
            # anything inside Gradle.
            "GRADLE_INVOKED_WITHIN_MACH_BUILD": "1",
            "MOZ_CHROME_MULTILOCALE": " ".join(locales),
        }

        for locale in locales:
            if locale == "en-US":
                self.log(
                    logging.INFO,
                    "package-multi-locale",
                    {"locale": locale},
                    "Skipping default locale {locale}",
                )
                continue

            self.log(
                logging.INFO,
                "package-multi-locale",
                {"locale": locale},
                "Processing chrome Gecko resources for locale {locale}",
            )
            self.run_process(
                [
                    mozpath.join(self.topsrcdir, "mach"),
                    "build",
                    "chrome-{}".format(locale),
                ],
                append_env=append_env,
                pass_thru=True,
                ensure_exit_code=True,
                cwd=mozpath.join(self.topsrcdir),
            )

        if self.substs["MOZ_BUILD_APP"] == "mobile/android":
            self.log(
                logging.INFO,
                "package-multi-locale",
                {},
                "Invoking `mach android assemble-app`",
            )
            self.run_process(
                [mozpath.join(self.topsrcdir, "mach"), "android", "assemble-app"],
                append_env=append_env,
                pass_thru=True,
                ensure_exit_code=True,
                cwd=mozpath.join(self.topsrcdir),
            )

        self.log(
            logging.INFO,
            "package-multi-locale",
            {},
            "Invoking multi-locale `mach package`",
        )
        self._run_make(
            directory=self.topobjdir,
            target=["package", "AB_CD=multi"],
            append_env=append_env,
            pass_thru=True,
            ensure_exit_code=True,
        )

        if self.substs["MOZ_BUILD_APP"] == "mobile/android":
            self.log(
                logging.INFO,
                "package-multi-locale",
                {},
                "Invoking `mach android archive-geckoview`",
            )
            self.run_process(
                [mozpath.join(self.topsrcdir, "mach"), "android", "archive-geckoview"],
                append_env=append_env,
                pass_thru=True,
                ensure_exit_code=True,
                cwd=mozpath.join(self.topsrcdir),
            )

        return 0


@CommandProvider
class CreateMachEnvironment(MachCommandBase):
    """Create the mach virtualenvs."""

    @Command(
        "create-mach-environment",
        category="devenv",
        description=(
            "Create the `mach` virtualenvs. If executed with python3 (the "
            "default when entering from `mach`), create both a python3 "
            "and python2.7 virtualenv. If executed with python2, only "
            "create the python2.7 virtualenv."
        ),
    )
    @CommandArgument(
        "-f",
        "--force",
        action="store_true",
        help=("Force re-creating the virtualenv even if it is already " "up-to-date."),
    )
    def create_mach_environment(self, force=False):
        from mozboot.util import get_mach_virtualenv_root
        from mozbuild.pythonutil import find_python2_executable
        from mozbuild.virtualenv import VirtualenvManager
        from six import PY2

        virtualenv_path = get_mach_virtualenv_root(py2=PY2)
        if sys.executable.startswith(virtualenv_path):
            print(
                "You can only create a mach environment with the system "
                "Python. Re-run this `mach` command with the system Python.",
                file=sys.stderr,
            )
            return 1

        manager = VirtualenvManager(
            self.topsrcdir,
            virtualenv_path,
            sys.stdout,
            os.path.join(self.topsrcdir, "build", "mach_virtualenv_packages.txt"),
            populate_local_paths=False,
        )

        if manager.up_to_date(sys.executable) and not force:
            print("virtualenv at %s is already up to date." % virtualenv_path)
        else:
            manager.build(sys.executable)

        manager.install_pip_package("zstandard>=0.9.0,<=0.13.0")

        if not PY2:
            # This can fail on some platforms. See
            # https://bugzilla.mozilla.org/show_bug.cgi?id=1660120
            try:
                manager.install_pip_package("glean_sdk~=32.3.1")
            except subprocess.CalledProcessError:
                print(
                    "Could not install glean_sdk, so telemetry will not be "
                    "collected. Continuing."
                )
            print("Python 3 mach environment created.")
            python2, _ = find_python2_executable()
            if not python2:
                print(
                    "WARNING! Could not find a Python 2 executable to create "
                    "a Python 2 virtualenv",
                    file=sys.stderr,
                )
                return 0
            args = [
                python2,
                os.path.join(self.topsrcdir, "mach"),
                "create-mach-environment",
            ]
            if force:
                args.append("-f")
            ret = subprocess.call(args)
            if ret:
                print(
                    "WARNING! Failed to create a Python 2 mach environment.",
                    file=sys.stderr,
                )
        else:
            print("Python 2 mach environment created.")