Bug 1623321 - Add perftest r=sparky,perftest-reviewers,marionette-reviewers,whimboo,ahal

mach perftest

Differential Revision: https://phabricator.services.mozilla.com/D67342

--HG--
extra : moz-landing-system : lando

build/mach_bootstrap.py

@@ -53,6 +53,7 @@ MACH_MODULES = [
     'python/mozbuild/mozbuild/compilation/codecomplete.py',
     'python/mozbuild/mozbuild/frontend/mach_commands.py',
     'python/mozbuild/mozbuild/mach_commands.py',
+    'python/mozperftest/mozperftest/mach_commands.py',
     'python/mozrelease/mozrelease/mach_commands.py',
     'python/safety/mach_commands.py',
     'remote/mach_commands.py',

build/virtualenv_packages.txt

@@ -2,6 +2,7 @@ mozilla.pth:python/mach
 mozilla.pth:python/mozboot
 mozilla.pth:python/mozbuild
 mozilla.pth:python/mozlint
+mozilla.pth:python/mozperftest
 mozilla.pth:python/mozrelease
 mozilla.pth:python/mozterm
 mozilla.pth:python/mozversioncontrol

python/moz.build

@@ -38,6 +38,10 @@ with Files('mozrelease/**'):
 with Files('mach_commands.py'):
     BUG_COMPONENT = ('Testing', 'Python Test')
 
+with Files('mozperftest/**'):
+    BUG_COMPONENT = ('Testing', 'Performance')
+
+
 SPHINX_PYTHON_PACKAGE_DIRS += [
     'mach',
     'mozbuild/mozbuild',
@@ -63,6 +67,7 @@ PYTHON_UNITTEST_MANIFESTS += [
     'mozbuild/mozbuild/test/python2.ini',
     'mozbuild/mozpack/test/python.ini',
     'mozlint/test/python.ini',
+    'mozperftest/mozperftest/tests/python.ini',
     'mozrelease/test/python.ini',
     'mozterm/test/python.ini',
     'mozversioncontrol/test/python.ini',

python/mozbuild/mozbuild/base.py

@@ -1017,6 +1017,15 @@ class MachCommandConditions(object):
             return not MachCommandConditions.is_artifact_build(cls)
         return False
 
+    @staticmethod
+    def is_buildapp_in(cls, apps):
+        """Must have a build for one of the given app"""
+        for app in apps:
+            attr = getattr(MachCommandConditions, 'is_{}'.format(app), None)
+            if attr and attr(cls):
+                return True
+        return False
+
 
 class PathArgument(object):
     """Parse a filesystem path argument and transform it in various ways."""

python/mozperftest/mozperftest/__init__.py

@@ -0,0 +1,42 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import mozlog
+
+from mozperftest.browser import pick_browser
+from mozperftest.system import pick_system
+from mozperftest.metrics import pick_metrics
+from mozperftest.argparser import PerftestArgumentParser
+
+
+logger = mozlog.commandline.setup_logging("mozperftest", {})
+
+
+def get_parser():
+    return PerftestArgumentParser()
+
+
+# XXX todo, turn the dict into a class that controls
+# what gets in and out of it
+# we want a structure with :
+#  - results
+#  - browser prefs
+#  - ??
+def get_metadata(mach_cmd, flavor, **kwargs):
+    res = {
+        "mach_cmd": mach_cmd,
+        "mach_args": kwargs,
+        "flavor": flavor,
+        "browser": {"prefs": {}},
+    }
+    return res
+
+
+def get_env(mach_cmd, flavor="script", test_objects=None, resolve_tests=True, **kwargs):
+    # XXX do something with flavors, etc
+    if flavor != "script":
+        raise NotImplementedError(flavor)
+
+    return [
+        layer(flavor, mach_cmd) for layer in (pick_system, pick_browser, pick_metrics)
+    ]

python/mozperftest/mozperftest/argparser.py

@@ -0,0 +1,110 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from argparse import ArgumentParser, SUPPRESS
+import os
+import mozlog
+
+
+here = os.path.abspath(os.path.dirname(__file__))
+try:
+    from mozbuild.base import MozbuildObject, MachCommandConditions as conditions
+
+    build_obj = MozbuildObject.from_environment(cwd=here)
+except ImportError:
+    build_obj = None
+    conditions = None
+
+SUPPORTED_APPS = ("generic", "android")
+
+
+class GenericGroup:
+    """ Generic options
+    """
+
+    name = "Generic"
+
+    args = [
+        [
+            ["tests"],
+            {
+                "nargs": "*",
+                "metavar": "TEST",
+                "default": [],
+                "help": "Test to run. Can be a single test file or a directory of tests "
+                "(to run recursively). If omitted, the entire suite is run.",
+            },
+        ],
+        [
+            # XXX this should live in mozperftest.metrics
+            ["--perfherder"],
+            {
+                "action": "store_true",
+                "default": False,
+                "help": "Output data in the perfherder format.",
+            },
+        ],
+        [
+            # XXX this should live in mozperftest.metrics
+            ["--output"],
+            {
+                "type": str,
+                "default": "artifacts",
+                "help": "Path to where data will be stored, defaults to a top-level "
+                "`artifacts` folder.",
+            },
+        ],
+        [
+            # XXX this should live in mozperftest.metrics
+            ["--prefix"],
+            {
+                "type": str,
+                "default": "",
+                "help": "Prefix the output files with this string.",
+            },
+        ],
+    ]
+    defaults = {}
+
+
+class PerftestArgumentParser(ArgumentParser):
+    """%(prog)s [options] [test paths]"""
+
+    def __init__(self, app=None, **kwargs):
+        ArgumentParser.__init__(
+            self, usage=self.__doc__, conflict_handler="resolve", **kwargs
+        )
+        # XXX see if this list will vary depending on the flavor & app
+        self.groups = [GenericGroup]
+        self.oldcwd = os.getcwd()
+        self.app = app
+        if not self.app and build_obj:
+            if conditions.is_android(build_obj):
+                self.app = "android"
+        if not self.app:
+            self.app = "generic"
+
+        if self.app not in SUPPORTED_APPS:
+            self.error(
+                "Unrecognized app '{}'! Must be one of: {}".format(
+                    self.app, ", ".join(SUPPORTED_APPS)
+                )
+            )
+
+        defaults = {}
+        for klass in self.groups:
+            defaults.update(klass.defaults)
+            group = self.add_argument_group(klass.name, klass.__doc__)
+
+            for cli, kwargs in klass.args:
+                if "default" in kwargs and isinstance(kwargs["default"], list):
+                    kwargs["default"] = []
+
+                if "suppress" in kwargs:
+                    if kwargs["suppress"]:
+                        kwargs["help"] = SUPPRESS
+                    del kwargs["suppress"]
+                group.add_argument(*cli, **kwargs)
+
+        self.set_defaults(**defaults)
+        mozlog.commandline.add_logging_group(self)

python/mozperftest/mozperftest/base.py

@@ -0,0 +1,70 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import logging
+
+
+class MachEnvironment:
+    def __init__(self, mach_command):
+        self.return_code = 0
+        self.mach_cmd = mach_command
+        self.log = mach_command.log
+        self.run_process = mach_command.run_process
+
+    def info(self, msg, name="mozperftest", **kwargs):
+        self.log(logging.INFO, name, kwargs, msg)
+
+    def debug(self, msg, name="mozperftest", **kwargs):
+        self.log(logging.DEBUG, name, kwargs, msg)
+
+    def warning(self, msg, name="mozperftest", **kwargs):
+        self.log(logging.WARNING, name, kwargs, msg)
+
+    def __enter__(self):
+        self.setup()
+        return self
+
+    def __exit__(self, type, value, traceback):
+        # XXX deal with errors here
+        self.teardown()
+
+    def __call__(self, metadata):
+        pass
+
+    def setup(self):
+        pass
+
+    def teardown(self):
+        pass
+
+
+class MultipleMachEnvironment(MachEnvironment):
+    def __init__(self, mach_command, factories):
+        super(MultipleMachEnvironment, self).__init__(mach_command)
+        self.envs = [factory(mach_command) for factory in factories]
+
+    def __enter__(self):
+        for env in self.envs:
+            env.setup()
+        return self
+
+    def __exit__(self, type, value, traceback):
+        # XXX deal with errors here
+        for env in self.envs:
+            env.teardown()
+
+    def __call__(self, metadata):
+        for env in self.envs:
+            metadata = env(metadata)
+        return metadata
+
+    # XXX not needed?
+    def _call_env(self, name):
+        def _call(*args, **kw):
+            return [getattr(env, name)(*args, **kw) for env in self.envs]
+
+        return _call
+
+    # XXX not needed?
+    def __getattr__(self, name):
+        return self._call_env(name)

python/mozperftest/mozperftest/browser/__init__.py

@@ -0,0 +1,10 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from mozperftest.browser.browsertime import BrowsertimeRunner
+
+
+def pick_browser(flavor, mach_cmd):
+    if flavor == "script":
+        return BrowsertimeRunner(mach_cmd)
+    raise NotImplementedError(flavor)

python/mozperftest/mozperftest/browser/browsertime.py

@@ -0,0 +1,444 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import collections
+import json
+import os
+import stat
+import sys
+import re
+import shutil
+
+from mozbuild.util import mkdir
+import mozpack.path as mozpath
+
+from mozperftest.browser.noderunner import NodeRunner
+from mozperftest.utils import host_platform
+
+
+AUTOMATION = "MOZ_AUTOMATION" in os.environ
+BROWSERTIME_SRC_ROOT = os.path.dirname(__file__)
+PILLOW_VERSION = "6.0.0"
+PYSSIM_VERSION = "0.4"
+
+# Map from `host_platform()` to a `fetch`-like syntax.
+host_fetches = {
+    "darwin": {
+        "ffmpeg": {
+            "type": "static-url",
+            "url": "https://github.com/ncalexan/geckodriver/releases/download/v0.24.0-android/ffmpeg-4.1.1-macos64-static.zip",  # noqa
+            # An extension to `fetch` syntax.
+            "path": "ffmpeg-4.1.1-macos64-static",
+        }
+    },
+    "linux64": {
+        "ffmpeg": {
+            "type": "static-url",
+            "url": "https://github.com/ncalexan/geckodriver/releases/download/v0.24.0-android/ffmpeg-4.1.4-i686-static.tar.xz",  # noqa
+            # An extension to `fetch` syntax.
+            "path": "ffmpeg-4.1.4-i686-static",
+        },
+        # TODO: install a static ImageMagick.  All easily available binaries are
+        # not statically linked, so they will (mostly) fail at runtime due to
+        # missing dependencies.  For now we require folks to install ImageMagick
+        # globally with their package manager of choice.
+    },
+    "win64": {
+        "ffmpeg": {
+            "type": "static-url",
+            "url": "https://github.com/ncalexan/geckodriver/releases/download/v0.24.0-android/ffmpeg-4.1.1-win64-static.zip",  # noqa
+            # An extension to `fetch` syntax.
+            "path": "ffmpeg-4.1.1-win64-static",
+        },
+        "ImageMagick": {
+            "type": "static-url",
+            # 'url': 'https://imagemagick.org/download/binaries/ImageMagick-7.0.8-39-portable-Q16-x64.zip',  # noqa
+            # imagemagick.org doesn't keep old versions; the mirror below does.
+            "url": "https://ftp.icm.edu.pl/packages/ImageMagick/binaries/ImageMagick-7.0.8-39-portable-Q16-x64.zip",  # noqa
+            # An extension to `fetch` syntax.
+            "path": "ImageMagick-7.0.8",
+        },
+    },
+}
+
+
+class BrowsertimeRunner(NodeRunner):
+    def __init__(self, mach_cmd):
+        super(BrowsertimeRunner, self).__init__(mach_cmd)
+        self.topsrcdir = mach_cmd.topsrcdir
+        self._mach_context = mach_cmd._mach_context
+        self.virtualenv_manager = mach_cmd.virtualenv_manager
+        self.proxy = None
+        self._created_dirs = []
+
+    @property
+    def artifact_cache_path(self):
+        """Downloaded artifacts will be kept here."""
+        # The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE.
+        return mozpath.join(self._mach_context.state_dir, "cache", "browsertime")
+
+    @property
+    def state_path(self):
+        """Unpacked artifacts will be kept here."""
+        # The convention is $MOZBUILD_STATE_PATH/$FEATURE.
+        res = mozpath.join(self._mach_context.state_dir, "browsertime")
+        mkdir(res)
+        return res
+
+    @property
+    def browsertime_js(self):
+        root = os.environ.get("BROWSERTIME", self.state_path)
+        return os.path.join(
+            root, "node_modules", "browsertime", "bin", "browsertime.js"
+        )
+
+    def setup_prerequisites(self):
+        """Install browsertime and visualmetrics.py prerequisites.
+        """
+
+        from mozbuild.action.tooltool import unpack_file
+        from mozbuild.artifact_cache import ArtifactCache
+
+        if not AUTOMATION and host_platform().startswith("linux"):
+            # On Linux ImageMagick needs to be installed manually, and `mach bootstrap` doesn't
+            # do that (yet).  Provide some guidance.
+            try:
+                from shutil import which
+            except ImportError:
+                from shutil_which import which
+
+            im_programs = ("compare", "convert", "mogrify")
+            for im_program in im_programs:
+                prog = which(im_program)
+                if not prog:
+                    print(
+                        "Error: On Linux, ImageMagick must be on the PATH. "
+                        "Install ImageMagick manually and try again (or update PATH). "
+                        "On Ubuntu and Debian, try `sudo apt-get install imagemagick`. "
+                        "On Fedora, try `sudo dnf install imagemagick`. "
+                        "On CentOS, try `sudo yum install imagemagick`."
+                    )
+                    return 1
+
+        # Download the visualmetrics.py requirements.
+        artifact_cache = ArtifactCache(
+            self.artifact_cache_path, log=self.log, skip_cache=False
+        )
+
+        fetches = host_fetches[host_platform()]
+        for tool, fetch in sorted(fetches.items()):
+            archive = artifact_cache.fetch(fetch["url"])
+            # TODO: assert type, verify sha256 (and size?).
+
+            if fetch.get("unpack", True):
+                cwd = os.getcwd()
+                try:
+                    mkdir(self.state_path)
+                    os.chdir(self.state_path)
+                    self.info("Unpacking temporary location {path}", path=archive)
+
+                    if "win64" in host_platform() and "imagemagick" in tool.lower():
+                        # Windows archive does not contain a subfolder
+                        # so we make one for it here
+                        mkdir(fetch.get("path"))
+                        os.chdir(os.path.join(self.state_path, fetch.get("path")))
+                        unpack_file(archive)
+                        os.chdir(self.state_path)
+                    else:
+                        unpack_file(archive)
+
+                    # Make sure the expected path exists after extraction
+                    path = os.path.join(self.state_path, fetch.get("path"))
+                    if not os.path.exists(path):
+                        raise Exception("Cannot find an extracted directory: %s" % path)
+
+                    try:
+                        # Some archives provide binaries that don't have the
+                        # executable bit set so we need to set it here
+                        for root, dirs, files in os.walk(path):
+                            for edir in dirs:
+                                loc_to_change = os.path.join(root, edir)
+                                st = os.stat(loc_to_change)
+                                os.chmod(loc_to_change, st.st_mode | stat.S_IEXEC)
+                            for efile in files:
+                                loc_to_change = os.path.join(root, efile)
+                                st = os.stat(loc_to_change)
+                                os.chmod(loc_to_change, st.st_mode | stat.S_IEXEC)
+                    except Exception as e:
+                        raise Exception(
+                            "Could not set executable bit in %s, error: %s"
+                            % (path, str(e))
+                        )
+                finally:
+                    os.chdir(cwd)
+
+    def _need_install(self, package):
+        from pip._internal.req.constructors import install_req_from_line
+
+        req = install_req_from_line("Pillow")
+        req.check_if_exists(use_user_site=False)
+        if req.satisfied_by is None:
+            return True
+        venv_site_lib = os.path.abspath(
+            os.path.join(self.virtualenv_manager.bin_path, "..", "lib")
+        )
+        site_packages = os.path.abspath(req.satisfied_by.location)
+        return not site_packages.startswith(venv_site_lib)
+
+    def setup(self, should_clobber=False, new_upstream_url=""):
+        """Install browsertime and visualmetrics.py prerequisites and the Node.js package.
+        """
+        super(BrowsertimeRunner, self).setup()
+
+        # installing Python deps on the fly
+        for dep in ("Pillow==%s" % PILLOW_VERSION, "pyssim==%s" % PYSSIM_VERSION):
+            if self._need_install(dep):
+                self.virtualenv_manager._run_pip(["install", dep])
+
+        # check if the browsertime package has been deployed correctly
+        # for this we just check for the browsertime directory presence
+        if os.path.exists(self.browsertime_js):
+            return
+
+        sys.path.append(mozpath.join(self.topsrcdir, "tools", "lint", "eslint"))
+        import setup_helper
+
+        if not new_upstream_url:
+            self.setup_prerequisites()
+
+        # preparing ~/.mozbuild/browsertime
+        for file in ("package.json", "package-lock.json"):
+            src = mozpath.join(BROWSERTIME_SRC_ROOT, file)
+            target = mozpath.join(self.state_path, file)
+            if not os.path.exists(target):
+                shutil.copyfile(src, target)
+
+        package_json_path = mozpath.join(self.state_path, "package.json")
+
+        if new_upstream_url:
+            self.info(
+                "Updating browsertime node module version in {package_json_path} "
+                "to {new_upstream_url}",
+                new_upstream_url=new_upstream_url,
+                package_json_path=package_json_path,
+            )
+
+            if not re.search("/tarball/[a-f0-9]{40}$", new_upstream_url):
+                raise ValueError(
+                    "New upstream URL does not end with /tarball/[a-f0-9]{40}: '{}'".format(
+                        new_upstream_url
+                    )
+                )
+
+            with open(package_json_path) as f:
+                existing_body = json.loads(
+                    f.read(), object_pairs_hook=collections.OrderedDict
+                )
+
+            existing_body["devDependencies"]["browsertime"] = new_upstream_url
+
+            updated_body = json.dumps(existing_body)
+
+            with open(package_json_path, "w") as f:
+                f.write(updated_body)
+
+        # Install the browsertime Node.js requirements.
+        if not setup_helper.check_node_executables_valid():
+            return
+
+        # To use a custom `geckodriver`, set
+        # os.environ[b"GECKODRIVER_BASE_URL"] = bytes(url)
+        # to an endpoint with binaries named like
+        # https://github.com/sitespeedio/geckodriver/blob/master/install.js#L31.
+        if AUTOMATION:
+            os.environ["CHROMEDRIVER_SKIP_DOWNLOAD"] = "true"
+            os.environ["GECKODRIVER_SKIP_DOWNLOAD"] = "true"
+
+        self.info(
+            "Installing browsertime node module from {package_json}",
+            package_json=package_json_path,
+        )
+        setup_helper.package_setup(
+            self.state_path,
+            "browsertime",
+            should_update=new_upstream_url != "",
+            should_clobber=should_clobber,
+            no_optional=new_upstream_url or AUTOMATION,
+        )
+
+    def append_env(self, append_path=True):
+        env = super(BrowsertimeRunner, self).append_env(append_path)
+        fetches = host_fetches[host_platform()]
+
+        # Ensure that bare `ffmpeg` and ImageMagick commands
+        # {`convert`,`compare`,`mogrify`} are found.  The `visualmetrics.py`
+        # script doesn't take these as configuration, so we do this (for now).
+        # We should update the script itself to accept this configuration.
+        path = env.get("PATH", "").split(os.pathsep)
+        path_to_ffmpeg = mozpath.join(self.state_path, fetches["ffmpeg"]["path"])
+
+        path_to_imagemagick = None
+        if "ImageMagick" in fetches:
+            path_to_imagemagick = mozpath.join(
+                self.state_path, fetches["ImageMagick"]["path"]
+            )
+
+        if path_to_imagemagick:
+            # ImageMagick ships ffmpeg (on Windows, at least) so we
+            # want to ensure that our ffmpeg goes first, just in case.
+            path.insert(
+                0,
+                self.state_path
+                if host_platform().startswith("win")
+                else mozpath.join(path_to_imagemagick, "bin"),
+            )  # noqa
+        path.insert(
+            0,
+            path_to_ffmpeg
+            if host_platform().startswith("linux")
+            else mozpath.join(path_to_ffmpeg, "bin"),
+        )  # noqa
+
+        # On windows, we need to add the ImageMagick directory to the path
+        # otherwise compare won't be found, and the built-in OS convert
+        # method will be used instead of the ImageMagick one.
+        if "win64" in host_platform() and path_to_imagemagick:
+            # Bug 1596237 - In the windows ImageMagick distribution, the ffmpeg
+            # binary is directly located in the root directory, so here we
+            # insert in the 3rd position to avoid taking precedence over ffmpeg
+            path.insert(2, path_to_imagemagick)
+
+        # On macOs, we can't install our own ImageMagick because the
+        # System Integrity Protection (SIP) won't let us set DYLD_LIBRARY_PATH
+        # unless we deactivate SIP with "csrutil disable".
+        # So we're asking the user to install it.
+        #
+        # if ImageMagick was installed via brew, we want to make sure we
+        # include the PATH
+        if host_platform() == "darwin":
+            for p in os.environ["PATH"].split(os.pathsep):
+                p = p.strip()
+                if not p or p in path:
+                    continue
+                path.append(p)
+
+        if path_to_imagemagick:
+            env.update(
+                {
+                    # See https://imagemagick.org/script/download.php.
+                    # Harmless on other platforms.
+                    "LD_LIBRARY_PATH": mozpath.join(path_to_imagemagick, "lib"),
+                    "DYLD_LIBRARY_PATH": mozpath.join(path_to_imagemagick, "lib"),
+                    "MAGICK_HOME": path_to_imagemagick,
+                }
+            )
+
+        return env
+
+    def extra_default_args(self, args=[]):
+        # Add Mozilla-specific default arguments.  This is tricky because browsertime is quite
+        # loose about arguments; repeat arguments are generally accepted but then produce
+        # difficult to interpret type errors.
+
+        def extract_browser_name(args):
+            "Extracts the browser name if any"
+            # These are BT arguments, it's BT job to check them
+            # here we just want to extract the browser name
+            res = re.findall("(--browser|-b)[= ]([\w]+)", " ".join(args))
+            if res == []:
+                return None
+            return res[0][-1]
+
+        def matches(args, *flags):
+            "Return True if any argument matches any of the given flags (maybe with an argument)."
+            for flag in flags:
+                if flag in args or any(arg.startswith(flag + "=") for arg in args):
+                    return True
+            return False
+
+        extra_args = []
+
+        # Default to Firefox.  Override with `-b ...` or `--browser=...`.
+        specifies_browser = matches(args, "-b", "--browser")
+        if not specifies_browser:
+            extra_args.extend(("-b", "firefox"))
+
+        # Default to not collect HAR.  Override with `--skipHar=false`.
+        specifies_har = matches(args, "--har", "--skipHar", "--gzipHar")
+        if not specifies_har:
+            extra_args.append("--skipHar")
+
+        if not matches(args, "--android"):
+            # If --firefox.binaryPath is not specified, default to the objdir binary
+            # Note: --firefox.release is not a real browsertime option, but it will
+            #       silently ignore it instead and default to a release installation.
+            specifies_binaryPath = matches(
+                args,
+                "--firefox.binaryPath",
+                "--firefox.release",
+                "--firefox.nightly",
+                "--firefox.beta",
+                "--firefox.developer",
+            )
+
+            if not specifies_binaryPath:
+                specifies_binaryPath = extract_browser_name(args) == "chrome"
+
+            if not specifies_binaryPath:
+                try:
+                    extra_args.extend(("--firefox.binaryPath", self.get_binary_path()))
+                except Exception:
+                    print(
+                        "Please run |./mach build| "
+                        "or specify a Firefox binary with --firefox.binaryPath."
+                    )
+                    return 1
+
+        if extra_args:
+            self.debug(
+                "Running browsertime with extra default arguments: {extra_args}",
+                extra_args=extra_args,
+            )
+
+        return extra_args
+
+    def get_profile(self, metadata):
+        # XXX we'll use conditioned profiles
+        from mozprofile import create_profile
+
+        profile = create_profile(app="firefox")
+        prefs = metadata["browser"]["prefs"]
+        profile.set_preferences(prefs)
+        self.info("Created profile at %s" % profile.profile)
+        self._created_dirs.append(profile.profile)
+        return profile
+
+    def __call__(self, metadata):
+        # keep the object around
+        # see https://bugzilla.mozilla.org/show_bug.cgi?id=1625118
+        profile = self.get_profile(metadata)
+
+        test_script = metadata["mach_args"]["tests"][0]
+        result_dir = os.path.join(os.path.dirname(__file__), "browsertime-results")
+        args = [
+            "--resultDir",
+            result_dir,
+            "--firefox.profileTemplate",
+            profile.profile,
+            "--iterations",
+            "1",
+            test_script,
+        ]
+
+        firefox_args = ["--firefox.developer"]
+        extra = self.extra_default_args(args=firefox_args)
+        command = [self.browsertime_js] + extra + args
+        self.info("Running browsertime with this command %s" % " ".join(command))
+        self.node(command)
+        metadata["results"] = result_dir
+        return metadata
+
+    def teardown(self):
+        for dir in self._created_dirs:
+            if os.path.exists(dir):
+                shutil.rmtree(dir)

python/mozperftest/mozperftest/browser/noderunner.py

@@ -0,0 +1,74 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import os
+import sys
+
+import mozpack.path as mozpath
+
+from mozperftest.base import MachEnvironment
+from mozperftest.utils import silence
+
+
+class NodeRunner(MachEnvironment):
+    def __init__(self, mach_cmd):
+        super(NodeRunner, self).__init__(mach_cmd)
+        self.topsrcdir = mach_cmd.topsrcdir
+        self._mach_context = mach_cmd._mach_context
+        self.python_path = mach_cmd.virtualenv_manager.python_path
+
+        from mozbuild.nodeutil import find_node_executable
+
+        self.node_path = os.path.abspath(find_node_executable()[0])
+
+    def setup(self):
+        """Install the Node.js package.
+        """
+        self.mach_cmd._activate_virtualenv()
+        self.verify_node_install()
+
+    def node(self, args):
+        """Invoke node (interactively) with the given arguments."""
+        return self.run_process(
+            [self.node_path] + args,
+            append_env=self.append_env(),
+            pass_thru=True,  # Allow user to run Node interactively.
+            ensure_exit_code=False,  # Don't throw on non-zero exit code.
+            cwd=mozpath.join(self.topsrcdir),
+        )
+
+    def append_env(self, append_path=True):
+        # Ensure that bare `node` and `npm` in scripts, including post-install
+        # scripts, finds the binary we're invoking with.  Without this, it's
+        # easy for compiled extensions to get mismatched versions of the Node.js
+        # extension API.
+        path = os.environ.get("PATH", "").split(os.pathsep) if append_path else []
+        node_dir = os.path.dirname(self.node_path)
+        path = [node_dir] + path
+
+        return {
+            "PATH": os.pathsep.join(path),
+            # Bug 1560193: The JS library browsertime uses to execute commands
+            # (execa) will muck up the PATH variable and put the directory that
+            # node is in first in path. If this is globally-installed node,
+            # that means `/usr/bin` will be inserted first which means that we
+            # will get `/usr/bin/python` for `python`.
+            #
+            # Our fork of browsertime supports a `PYTHON` environment variable
+            # that points to the exact python executable to use.
+            "PYTHON": self.python_path,
+        }
+
+    def verify_node_install(self):
+        # check if Node is installed
+        sys.path.append(mozpath.join(self.topsrcdir, "tools", "lint", "eslint"))
+        import setup_helper
+
+        with silence():
+            node_valid = setup_helper.check_node_executables_valid()
+        if not node_valid:
+            # running again to get details printed out
+            setup_helper.check_node_executables_valid()
+            raise ValueError("Can't find Node. did you run ./mach bootstrap ?")
+
+        return True

python/mozperftest/mozperftest/browser/package.json

@@ -0,0 +1,12 @@
+{
+    "name": "mozilla-central-tools-browsertime",
+    "description": "This package file is for node modules used in mozilla-central/tools/browsertime",
+    "repository": {},
+    "license": "MPL-2.0",
+    "dependencies": {},
+    "devDependencies": {
+        "browsertime": "https://github.com/sitespeedio/browsertime/tarball/f4294047e3ee72d284022ed8141cad5a07120c47"
+    },
+    "notes(private)": "We don't want to publish to npm, so this is marked as private",
+    "private": true
+}

python/mozperftest/mozperftest/mach_commands.py

@@ -0,0 +1,34 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from functools import partial
+
+from mach.decorators import CommandProvider, Command
+from mozbuild.base import MachCommandBase, MachCommandConditions as conditions
+
+
+def get_perftest_parser():
+    from mozperftest import get_parser
+
+    return get_parser
+
+
+@CommandProvider
+class Perftest(MachCommandBase):
+    @Command(
+        "perftest",
+        category="testing",
+        conditions=[partial(conditions.is_buildapp_in, apps=["firefox", "android"])],
+        description="Run any flavor of perftest",
+        parser=get_perftest_parser,
+    )
+    def run_perftest(
+        self, flavor="script", test_objects=None, resolve_tests=True, **kwargs
+    ):
+        from mozperftest import get_env, get_metadata
+
+        system, browser, metrics = get_env(self, flavor, test_objects, resolve_tests)
+        metadata = get_metadata(self, flavor, **kwargs)
+
+        with system, browser, metrics:
+            metrics(browser(system(metadata)))

python/mozperftest/mozperftest/metrics/__init__.py

@@ -0,0 +1,11 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from mozperftest.base import MultipleMachEnvironment
+from mozperftest.metrics.perfherder import Perfherder
+
+
+def pick_metrics(flavor, mach_cmd):
+    if flavor == "script":
+        return MultipleMachEnvironment(mach_cmd, (Perfherder,))
+    raise NotImplementedError(flavor)

python/mozperftest/mozperftest/metrics/browsertime.py

@@ -0,0 +1,195 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from statistics import mean
+
+
+class MissingResultsError(Exception):
+    pass
+
+
+def format_metrics(
+    name,
+    type,
+    value,
+    replicates=None,
+    unit="ms",
+    lower_is_better=True,
+    alert_threshold=2,
+    **kw
+):
+    res = {
+        "alertThreshold": alert_threshold,
+        "type": type,
+        "unit": unit,
+        "lowerIsBetter": lower_is_better,
+        "name": name,
+        "value": value,
+    }
+    res.update(kw)
+    return res
+
+
+def process(results, log, app="firefox", cold=True):
+    """Takes a list of results and processes them.
+
+    Assumes that the data being given is coming from browsertime. Each result in
+    the list is treated as a new subtest in the suite. In other words, if you
+    return 3 browsertime.json files, each of them will be its own subtest and
+    they will not be combined together.
+
+    :param results list: A list containing the data to process, each entry
+        must be a single subtest. The entries are not combined.
+
+    :return dict: A perfherder-formatted data blob.
+    """
+    allresults = []
+    for c, result in enumerate(results):
+        log("Results {}: parsing results from browsertime json".format(c))
+        allresults.append(parse(result, log, app=app, cold=cold))
+
+    # Create a subtest entry per result entry
+    suites = []
+    perfherder = {
+        "suites": suites,
+        "framework": {"name": "browsertime"},
+        "application": {"name": app},
+    }
+
+    for res in allresults:
+        res = res[0]
+        measurements = res["measurements"]
+        subtests = []
+
+        values = [measurements[key][0] for key in measurements]
+        suite = format_metrics(
+            "btime-testing",
+            "perftest-script",
+            mean(values),
+            extraOptions=[],
+            subtests=subtests,
+        )
+
+        for measure in measurements:
+            vals = measurements[measure]
+            subtests.append(
+                format_metrics(measure, "perftest-script", mean(vals), replicates=vals)
+            )
+
+        suites.append(suite)
+
+    print(perfherder)
+    return perfherder
+
+
+def parse(results, log, app, cold):
+    # bt to raptor names
+    measure = ["fnbpaint", "fcp", "dcf", "loadtime"]
+    conversion = (
+        ("fnbpaint", "firstPaint"),
+        ("fcp", "timeToContentfulPaint"),
+        ("dcf", "timeToDomContentFlushed"),
+        ("loadtime", "loadEventEnd"),
+    )
+
+    chrome_raptor_conversion = {
+        "timeToContentfulPaint": ["paintTiming", "first-contentful-paint"]
+    }
+
+    def _get_raptor_val(mdict, mname, retval=False):
+        if type(mname) != list:
+            if mname in mdict:
+                return mdict[mname]
+            return retval
+        target = mname[-1]
+        tmpdict = mdict
+        for name in mname[:-1]:
+            tmpdict = tmpdict.get(name, {})
+        if target in tmpdict:
+            return tmpdict[target]
+
+        return retval
+
+    res = []
+
+    # Do some preliminary results validation. When running cold page-load, the results will
+    # be all in one entry already, as browsertime groups all cold page-load iterations in
+    # one results entry with all replicates within. When running warm page-load, there will
+    # be one results entry for every warm page-load iteration; with one single replicate
+    # inside each.
+
+    # XXX added this because it was not defined
+    page_cycles = 1
+
+    if cold:
+        if len(results) == 0:
+            raise MissingResultsError("Missing results for all cold browser cycles.")
+    else:
+        if len(results) != int(page_cycles):
+            raise MissingResultsError("Missing results for at least 1 warm page-cycle.")
+
+    # now parse out the values
+    for raw_result in results:
+        if not raw_result["browserScripts"]:
+            raise MissingResultsError("Browsertime cycle produced no measurements.")
+
+        if raw_result["browserScripts"][0].get("timings") is None:
+            raise MissingResultsError("Browsertime cycle is missing all timings")
+
+        # Desktop chrome doesn't have `browser` scripts data available for now
+        bt_browser = raw_result["browserScripts"][0].get("browser", None)
+        bt_ver = raw_result["info"]["browsertime"]["version"]
+        bt_url = (raw_result["info"]["url"],)
+        bt_result = {
+            "bt_ver": bt_ver,
+            "browser": bt_browser,
+            "url": bt_url,
+            "measurements": {},
+            "statistics": {},
+        }
+
+        custom_types = raw_result["browserScripts"][0].get("custom")
+        if custom_types:
+            for custom_type in custom_types:
+                bt_result["measurements"].update(
+                    {k: [v] for k, v in custom_types[custom_type].items()}
+                )
+        else:
+            # extracting values from browserScripts and statistics
+            for bt, raptor in conversion:
+                if measure is not None and bt not in measure:
+                    continue
+                # chrome we just measure fcp and loadtime; skip fnbpaint and dcf
+                if app and "chrome" in app.lower() and bt in ("fnbpaint", "dcf"):
+                    continue
+                # fennec doesn't support 'fcp'
+                if app and "fennec" in app.lower() and bt == "fcp":
+                    continue
+
+                # chrome currently uses different names (and locations) for some metrics
+                if raptor in chrome_raptor_conversion and _get_raptor_val(
+                    raw_result["browserScripts"][0]["timings"],
+                    chrome_raptor_conversion[raptor],
+                ):
+                    raptor = chrome_raptor_conversion[raptor]
+
+                # XXX looping several times in the list, could do better
+                for cycle in raw_result["browserScripts"]:
+                    if bt not in bt_result["measurements"]:
+                        bt_result["measurements"][bt] = []
+                    val = _get_raptor_val(cycle["timings"], raptor)
+                    if not val:
+                        raise MissingResultsError(
+                            "Browsertime cycle missing {} measurement".format(raptor)
+                        )
+                    bt_result["measurements"][bt].append(val)
+
+                # let's add the browsertime statistics; we'll use those for overall values
+                # instead of calculating our own based on the replicates
+                bt_result["statistics"][bt] = _get_raptor_val(
+                    raw_result["statistics"]["timings"], raptor, retval={}
+                )
+
+        res.append(bt_result)
+
+    return res

python/mozperftest/mozperftest/metrics/common.py

@@ -0,0 +1,67 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from pathlib import Path
+from mozperftest.metrics.utils import open_file
+
+
+class CommonMetrics(object):
+    """CommonMetrics is a metrics class that contains code that is
+    commonly used across all metrics classes.
+
+    The metrics classes will be composed of this objcet, rather than inherit from it.
+    """
+
+    def __init__(self, results, output="artifacts", prefix="", **kwargs):
+        """Initialize CommonMetrics object.
+
+        :param results list/dict/str: Can be a single path to a result, a
+            list of paths, or a dict containing the data itself.
+        :param output str: Path of where the data will be stored.
+        :param prefix str: Prefix the output files with this string.
+        """
+        self.prefix = prefix
+        self.output = output
+
+        p = Path(output)
+        p.mkdir(parents=True, exist_ok=True)
+
+        self.results = self.parse_results(results)
+        if not self.results:
+            self.return_code = 1
+            raise Exception("Could not find any results to process.")
+
+    def parse_results(self, results):
+        """This function determines the type of results, and processes
+        it accordingly.
+
+        If a single file path is given, the file is opened
+        and the data is returned. If a list is given, then all the files
+        in that list (can include directories) are opened and returned.
+        If a dictionary is returned, then nothing will be done to the
+        results, but it will be returned within a list to keep the
+        `self.results` variable type consistent.
+
+        :param results list/dict/str: Path, or list of paths to the data (
+            or the data itself in a dict) of the data to be processed.
+
+        :return list: List of data objects to be processed.
+        """
+        res = []
+        if isinstance(results, dict):
+            res.append(results)
+        elif isinstance(results, str) or isinstance(results, Path):
+            # Expecting a single path or a directory
+            p = Path(results)
+            if not p.exists():
+                self.warning("Given path does not exist: {}".format(results))
+            elif p.is_dir():
+                files = [f for f in p.glob("**/*") if not f.is_dir()]
+                res.extend(self.parse_results(files))
+            else:
+                res.append(open_file(p.as_posix()))
+        elif isinstance(results, list):
+            # Expecting a list of paths
+            for path in results:
+                res.extend(self.parse_results(path))
+        return res

python/mozperftest/mozperftest/metrics/perfherder.py

@@ -0,0 +1,63 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import os
+
+from mozperftest.base import MachEnvironment
+from mozperftest.metrics.common import CommonMetrics
+from mozperftest.metrics.utils import write_json
+from mozperftest.metrics.browsertime import process
+
+
+class MissingResultsError(Exception):
+    pass
+
+
+KNOWN_FLAVORS = ["script"]
+FLAVOR_TO_PROCESSOR = {"script": process, "default": process}
+
+
+class Perfherder(MachEnvironment):
+    def __call__(self, metadata):
+        """Processes the given results into a perfherder-formatted data blob.
+
+        If the `--perfherder` flag isn't providec, then the
+        results won't be processed into a perfherder-data blob. If the
+        flavor is unknown to us, then we assume that it comes from
+        browsertime.
+
+        :param results list/dict/str: Results to process.
+        :param perfherder bool: True if results should be processed
+            into a perfherder-data blob.
+        :param flavor str: The flavor that is being processed.
+        """
+        # XXX work is happening in cwd, we need to define where
+        # the artifacts are uploaded?
+        # if not perfherder:
+        #    return
+        flavor = metadata.get("flavor")
+        if not flavor or flavor not in KNOWN_FLAVORS:
+            flavor = "default"
+            self.warning(
+                "Unknown flavor {} was given; we don't know how to process "
+                "its results. Attempting with default browsertime processing...".format(
+                    flavor
+                )
+            )
+
+        # Get the common requirements for metrics (i.e. output path,
+        # results to process)
+        cm = CommonMetrics(metadata["results"], **metadata["mach_args"])
+
+        # Process the results and save them
+        # TODO: Get app/browser name from metadata/kwargs
+        proc = FLAVOR_TO_PROCESSOR[flavor](cm.results, self.info, app="firefox")
+
+        file = "perfherder-data.json"
+        if cm.prefix:
+            file = "{}-{}".format(cm.prefix, file)
+        self.info(
+            "Writing perfherder results to {}".format(os.path.join(cm.output, file))
+        )
+        metadata["output"] = write_json(proc, cm.output, file)
+        return metadata

python/mozperftest/mozperftest/metrics/utils.py

@@ -0,0 +1,36 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import os
+import json
+
+
+def open_file(path):
+    """Opens a file and returns its contents.
+
+    :param path str: Path to the file, if it's a
+        JSON, then a dict will be returned, otherwise,
+        the raw contents (not split by line) will be
+        returned.
+    :return dict/str: Returns a dict for JSON data, and
+        a str for any other type.
+    """
+    with open(path) as f:
+        if os.path.splitext(path)[-1] == ".json":
+            return json.load(f)
+        return f.read()
+
+
+def write_json(data, path, file):
+    """Writes data to a JSON file.
+
+    :param data dict: Data to write.
+    :param path str: Directory of where the data will be stored.
+    :param file str: Name of the JSON file.
+
+    Returns the path of the file.
+    """
+    path = os.path.join(path, file)
+    with open(path, "w+") as f:
+        json.dump(data, f)
+    return path

python/mozperftest/mozperftest/system/__init__.py

@@ -0,0 +1,11 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from mozperftest.base import MultipleMachEnvironment
+from mozperftest.system.proxy import ProxyRunner
+
+
+def pick_system(flavor, mach_cmd):
+    if flavor == "script":
+        return MultipleMachEnvironment(mach_cmd, (ProxyRunner,))
+    raise NotImplementedError(flavor)

python/mozperftest/mozperftest/system/proxy.py

@@ -0,0 +1,52 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import os
+import mozinfo
+from mozproxy import get_playback
+from mozperftest.base import MachEnvironment
+
+
+HERE = os.path.dirname(__file__)
+
+
+class ProxyRunner(MachEnvironment):
+    def __init__(self, mach_cmd):
+        super(ProxyRunner, self).__init__(mach_cmd)
+        self.proxy = None
+
+    def setup(self):
+        pass
+
+    def __call__(self, metadata):
+        self.metadata = metadata
+        # replace with artifacts
+        config = {
+            "run_local": True,
+            "playback_tool": "mitmproxy",
+            "host": "localhost",
+            "binary": self.mach_cmd.get_binary_path(),
+            "obj_path": self.mach_cmd.topobjdir,
+            "platform": mozinfo.os,
+            "playback_files": [os.path.join(HERE, "example.dump")],
+            "app": "firefox",
+        }
+        self.info("setting up the proxy")
+        self.proxy = get_playback(config)
+        if self.proxy is not None:
+            self.proxy.start()
+            port = str(self.proxy.port)
+            prefs = {}
+            prefs["network.proxy.type"] = 1
+            prefs["network.proxy.http"] = "localhost"
+            prefs["network.proxy.http_port"] = port
+            prefs["network.proxy.ssl"] = "localhost"
+            prefs["network.proxy.ssl_port"] = port
+            prefs["network.proxy.no_proxies_on"] = "localhost"
+            metadata["browser"]["prefs"].update(prefs)
+        return metadata
+
+    def teardown(self):
+        if self.proxy is not None:
+            self.proxy.stop()
+            self.proxy = None

python/mozperftest/mozperftest/tests/__init__.py

@@ -0,0 +1 @@
+#

python/mozperftest/mozperftest/tests/example.js

@@ -0,0 +1,19 @@
+async function setUp(context) {
+  context.log.info("setUp example!");
+}
+
+async function test(context, commands) {
+  context.log.info("Test with setUp/tearDown example!");
+  await commands.measure.start("https://www.sitespeed.io/");
+  await commands.measure.start("https://www.mozilla.org/en-US/");
+}
+
+async function tearDown(context) {
+  context.log.info("tearDown example!");
+}
+
+module.exports = {   // eslint-disable-line
+  setUp,
+  tearDown,
+  test,
+};

python/mozperftest/mozperftest/tests/python.ini

@@ -0,0 +1,10 @@
+[DEFAULT]
+subsuite = mozbase
+skip-if = python == 2
+
+[test_mach_commands.py]
+[test_utils.py]
+[test_base.py]
+[test_browsertime.py]
+[test_argparser.py]
+[test_proxy.py]

python/mozperftest/mozperftest/tests/support.py

@@ -0,0 +1,17 @@
+import tempfile
+from mock import MagicMock
+
+
+def get_running_env():
+    from mozbuild.base import MozbuildObject
+
+    config = MozbuildObject.from_environment()
+    mach_cmd = MagicMock()
+    mach_cmd.get_binary_path = lambda: ""
+    mach_cmd.topsrcdir = config.topsrcdir
+    mach_cmd.topobjdir = config.topobjdir
+    mach_cmd._mach_context = MagicMock()
+    mach_cmd._mach_context.state_dir = tempfile.mkdtemp()
+    metadata = {"mach_cmd": mach_cmd, "browser": {"prefs": {}}}
+
+    return mach_cmd, metadata

python/mozperftest/mozperftest/tests/test_argparser.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import mozunit
+from mozperftest.argparser import PerftestArgumentParser
+
+
+def test_argparser():
+    parser = PerftestArgumentParser()
+    args = ["test_one.js"]
+    res = parser.parse_args(args)
+    assert res.tests == ["test_one.js"]
+
+
+if __name__ == "__main__":
+    mozunit.main()

python/mozperftest/mozperftest/tests/test_base.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+import mozunit
+from mozperftest.base import MachEnvironment, MultipleMachEnvironment
+from mock import MagicMock
+
+
+class Env(MachEnvironment):
+    called = 0
+
+    def setup(self):
+        self.called += 1
+
+    def teardown(self):
+        self.called += 1
+
+
+def test_mach_environement():
+    mach = MagicMock()
+
+    with Env(mach) as env:
+        env.info("info")
+        env.debug("debug")
+
+    assert env.called == 2
+
+
+def test_multiple_mach_environement():
+    mach = MagicMock()
+    factories = [Env, Env, Env]
+
+    with MultipleMachEnvironment(mach, factories) as env:
+        env.info("info")
+        env.debug("debug")
+
+    for env in env.envs:
+        assert env.called == 2
+
+
+if __name__ == "__main__":
+    mozunit.main()

python/mozperftest/mozperftest/tests/test_browsertime.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+import os
+import mozunit
+import mock
+import shutil
+
+from mozperftest.browser import pick_browser
+from mozperftest.tests.support import get_running_env
+
+HERE = os.path.dirname(__file__)
+
+
+def fetch(self, url):
+    return os.path.join(HERE, "fetched_artifact.zip")
+
+
+@mock.patch(
+    "mozperftest.browser.noderunner.NodeRunner.verify_node_install", new=lambda x: True
+)
+@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch)
+def test_browser():
+    mach_cmd, metadata = get_running_env()
+    runs = []
+
+    def _run_process(*args, **kw):
+        runs.append((args, kw))
+
+    mach_cmd.run_process = _run_process
+    metadata["mach_args"] = {"tests": [os.path.join(HERE, "example.js")]}
+
+    try:
+        with pick_browser("script", mach_cmd) as env:
+            env(metadata)
+    finally:
+        shutil.rmtree(mach_cmd._mach_context.state_dir)
+
+    args = runs[0][0]
+    # kwargs = runs[0][1]
+    # XXX more checks
+    assert args[-1][-1] == os.path.join(HERE, "example.js")
+
+
+if __name__ == "__main__":
+    mozunit.main()

python/mozperftest/mozperftest/tests/test_mach_commands.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import mozunit
+import os
+import mock
+
+from mach.registrar import Registrar
+
+Registrar.categories = {"testing": []}
+Registrar.commands_by_category = {"testing": set()}
+
+
+from mozperftest.mach_commands import Perftest
+
+
+def get_env(*args):
+    class FakeModule:
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *args, **kw):
+            pass
+
+        def __call__(self, metadata):
+            return metadata
+
+    return FakeModule(), FakeModule(), FakeModule()
+
+
+@mock.patch("mozperftest.get_env", new=get_env)
+def test_command():
+    from mozbuild.base import MozbuildObject
+
+    config = MozbuildObject.from_environment()
+
+    class context:
+        topdir = config.topobjdir
+        cwd = os.getcwd()
+        settings = {}
+        log_manager = {}
+
+    test = Perftest(context())
+    test.run_perftest()
+
+
+if __name__ == "__main__":
+    mozunit.main()

python/mozperftest/mozperftest/tests/test_perfherder.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+import os
+import mozunit
+import json
+
+from mozperftest.metrics import pick_metrics
+from mozperftest.tests.support import get_running_env
+
+HERE = os.path.dirname(__file__)
+
+
+def test_metrics():
+    mach_cmd, metadata = get_running_env()
+    runs = []
+
+    def _run_process(*args, **kw):
+        runs.append((args, kw))
+
+    mach_cmd.run_process = _run_process
+    metadata["mach_args"] = {"tests": [os.path.join(HERE, "example.js")]}
+    metadata["results"] = os.path.join(HERE, "browsertime-results")
+
+    with pick_metrics("script", mach_cmd) as env:
+        env(metadata)
+
+    with open(metadata["output"]) as f:
+        output = json.loads(f.read())
+
+    os.remove(metadata["output"])
+    # XXX more checks
+    assert output["suites"][0]["value"] > 0
+
+
+if __name__ == "__main__":
+    mozunit.main()

python/mozperftest/mozperftest/tests/test_proxy.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+import mozunit
+
+from mozperftest.system import pick_system
+from mozperftest.tests.support import get_running_env
+
+
+def test_proxy():
+    mach_cmd, metadata = get_running_env()
+
+    # XXX this will run for real, we need to mock HTTP calls
+    with pick_system("script", mach_cmd) as proxy:
+        proxy(metadata)
+
+
+if __name__ == "__main__":
+    mozunit.main()

python/mozperftest/mozperftest/tests/test_utils.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import sys
+import mozunit
+from mozperftest.utils import host_platform, silence
+
+
+def test_silence():
+    with silence():
+        print("HIDDEN")
+
+
+def test_host_platform():
+    plat = host_platform()
+
+    # a bit useless... :)
+    if sys.platform.startswith("darwin"):
+        assert plat == "darwin"
+    else:
+        if sys.maxsize > 2 ** 32:
+            assert "64" in plat
+        else:
+            assert "64" not in plat
+
+
+if __name__ == "__main__":
+    mozunit.main()

python/mozperftest/mozperftest/utils.py

@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import contextlib
+import sys
+from six import StringIO
+
+
+@contextlib.contextmanager
+def silence():
+    oldout, olderr = sys.stdout, sys.stderr
+    try:
+        sys.stdout, sys.stderr = StringIO(), StringIO()
+        yield
+    finally:
+        sys.stdout, sys.stderr = oldout, olderr
+
+
+def host_platform():
+    is_64bits = sys.maxsize > 2 ** 32
+
+    if sys.platform.startswith("win"):
+        if is_64bits:
+            return "win64"
+    elif sys.platform.startswith("linux"):
+        if is_64bits:
+            return "linux64"
+    elif sys.platform.startswith("darwin"):
+        return "darwin"
+
+    raise ValueError("sys.platform is not yet supported: {}".format(sys.platform))

python/mozperftest/setup.cfg

@@ -0,0 +1,2 @@
+[bdist_wheel]
+universal = 1

python/mozperftest/setup.py

@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+from setuptools import setup
+
+PACKAGE_NAME = "mozperftest"
+PACKAGE_VERSION = "0.1"
+
+deps = ["mozlog >= 6.0", "mozdevice >= 3.0.2", "mozproxy", "mozinfo"]
+
+setup(
+    name=PACKAGE_NAME,
+    version=PACKAGE_VERSION,
+    description="Mozilla's mach perftest command",
+    classifiers=[
+        "Programming Language :: Python :: 3.6",
+    ],
+    # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
+    keywords="",
+    author="Mozilla Performance Test Engineering Team",
+    author_email="tools@lists.mozilla.org",
+    url="https://hg.mozilla.org/mozilla-central/file/tip/python/mozperftest",
+    license="MPL",
+    packages=["mozperftest"],
+    include_package_data=True,
+    zip_safe=False,
+    install_requires=deps
+)

taskcluster/ci/source-test/python.yml

@@ -312,11 +312,11 @@ mozbuild:
         toolchain:
             by-platform:
                 linux1804-64/opt:
-                    - linux64-node
+                    - linux64-node-10
                 macosx1014-64/opt:
-                    - macosx64-node
+                    - macosx64-node-10
                 windows10-64/opt:
-                    - win64-node
+                    - win64-node-10
     when:
         files-changed:
             - '**/moz.configure'
@@ -325,6 +325,7 @@ mozbuild:
             - 'python/mach/**'
             - 'python/mozboot/**'
             - 'python/mozbuild/**'
+            - 'python/mozperftest/**'
             - 'python/mozterm/**'
             - 'python/mozversioncontrol/**'
             - 'testing/mozbase/**'

testing/marionette/mach_commands.py

@@ -5,6 +5,7 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
+import functools
 import os
 import sys
 
@@ -62,25 +63,12 @@ def run_marionette(tests, binary=None, topsrcdir=None, **kwargs):
         return 0
 
 
-def is_buildapp_in(*apps):
-    def is_buildapp_supported(cls):
-        for a in apps:
-            c = getattr(conditions, 'is_{}'.format(a), None)
-            if c and c(cls):
-                return True
-        return False
-
-    is_buildapp_supported.__doc__ = 'Must have a {} build.'.format(
-        ' or '.join(apps))
-    return is_buildapp_supported
-
-
 @CommandProvider
 class MarionetteTest(MachCommandBase):
     @Command("marionette-test",
              category="testing",
              description="Remote control protocol to Gecko, used for browser automation.",
-             conditions=[is_buildapp_in(*SUPPORTED_APPS)],
+             conditions=[functools.partial(conditions.is_buildapp_in, apps=SUPPORTED_APPS)],
              parser=create_parser_tests,
              )
     def marionette_test(self, tests, **kwargs):

testing/mochitest/mach_commands.py

@@ -6,6 +6,7 @@ from __future__ import absolute_import, unicode_literals
 
 from argparse import Namespace
 from collections import defaultdict
+import functools
 import logging
 import os
 import sys
@@ -255,21 +256,6 @@ def setup_junit_argument_parser():
     return parser
 
 
-# condition filters
-
-def is_buildapp_in(*apps):
-    def is_buildapp_supported(cls):
-        for a in apps:
-            c = getattr(conditions, 'is_{}'.format(a), None)
-            if c and c(cls):
-                return True
-        return False
-
-    is_buildapp_supported.__doc__ = 'Must have a {} build.'.format(
-        ' or '.join(apps))
-    return is_buildapp_supported
-
-
 def verify_host_bin():
     # validate MOZ_HOST_BIN environment variables for Android tests
     MOZ_HOST_BIN = os.environ.get('MOZ_HOST_BIN')
@@ -289,7 +275,7 @@ def verify_host_bin():
 @CommandProvider
 class MachCommands(MachCommandBase):
     @Command('mochitest', category='testing',
-             conditions=[is_buildapp_in(*SUPPORTED_APPS)],
+             conditions=[functools.partial(conditions.is_buildapp_in, apps=SUPPORTED_APPS)],
              description='Run any flavor of mochitest (integration test).',
              parser=setup_argument_parser)
     def run_mochitest_general(self, flavor=None, test_objects=None, resolve_tests=True, **kwargs):
@@ -300,7 +286,7 @@ class MachCommands(MachCommandBase):
 
         buildapp = None
         for app in SUPPORTED_APPS:
-            if is_buildapp_in(app)(self):
+            if conditions.is_buildapp_in(self, apps=[app]):
                 buildapp = app
                 break
 

tools/lint/py2.yml

@@ -33,6 +33,7 @@ py2:
         # These paths are intentionally excluded (Python 3 only)
         - config/printconfigsetting.py
         - python/mozlint
+        - python/mozperftest
         - tools/crashreporter/system-symbols/win/symsrv-fetch.py
         - tools/github-sync
         - tools/lint