forked from mirrors/gecko-dev
e0656d7178
The description in the docstring will be shown for `mach help pastebin`, and the shorter description in the command decorator will be shown in the list of useful mach commands. Differential Revision: https://phabricator.services.mozilla.com/D168466
594 lines
18 KiB
Python
594 lines
18 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
import argparse
|
|
import logging
|
|
import subprocess
|
|
import sys
|
|
from datetime import datetime, timedelta
|
|
from operator import itemgetter
|
|
|
|
from mach.decorators import Command, CommandArgument, SubCommand
|
|
from mozbuild.base import MozbuildObject
|
|
|
|
|
|
def _get_busted_bugs(payload):
|
|
import requests
|
|
|
|
payload = dict(payload)
|
|
payload["include_fields"] = "id,summary,last_change_time,resolution"
|
|
payload["blocks"] = 1543241
|
|
response = requests.get("https://bugzilla.mozilla.org/rest/bug", payload)
|
|
response.raise_for_status()
|
|
return response.json().get("bugs", [])
|
|
|
|
|
|
@Command(
|
|
"busted",
|
|
category="misc",
|
|
description="Query known bugs in our tooling, and file new ones.",
|
|
)
|
|
def busted_default(command_context):
|
|
unresolved = _get_busted_bugs({"resolution": "---"})
|
|
creation_time = datetime.now() - timedelta(days=15)
|
|
creation_time = creation_time.strftime("%Y-%m-%dT%H-%M-%SZ")
|
|
resolved = _get_busted_bugs({"creation_time": creation_time})
|
|
resolved = [bug for bug in resolved if bug["resolution"]]
|
|
all_bugs = sorted(
|
|
unresolved + resolved, key=itemgetter("last_change_time"), reverse=True
|
|
)
|
|
if all_bugs:
|
|
for bug in all_bugs:
|
|
print(
|
|
"[%s] Bug %s - %s"
|
|
% (
|
|
"UNRESOLVED"
|
|
if not bug["resolution"]
|
|
else "RESOLVED - %s" % bug["resolution"],
|
|
bug["id"],
|
|
bug["summary"],
|
|
)
|
|
)
|
|
else:
|
|
print("No known tooling issues found.")
|
|
|
|
|
|
@SubCommand("busted", "file", description="File a bug for busted tooling.")
|
|
@CommandArgument(
|
|
"against",
|
|
help=(
|
|
"The specific mach command that is busted (i.e. if you encountered "
|
|
"an error with `mach build`, run `mach busted file build`). If "
|
|
"the issue is not connected to any particular mach command, you "
|
|
"can also run `mach busted file general`."
|
|
),
|
|
)
|
|
def busted_file(command_context, against):
|
|
import webbrowser
|
|
|
|
if (
|
|
against != "general"
|
|
and against not in command_context._mach_context.commands.command_handlers
|
|
):
|
|
print(
|
|
"%s is not a valid value for `against`. `against` must be "
|
|
"the name of a `mach` command, or else the string "
|
|
'"general".' % against
|
|
)
|
|
return 1
|
|
|
|
if against == "general":
|
|
product = "Firefox Build System"
|
|
component = "General"
|
|
else:
|
|
import inspect
|
|
|
|
import mozpack.path as mozpath
|
|
|
|
# Look up the file implementing that command, then cross-refernce
|
|
# moz.build files to get the product/component.
|
|
handler = command_context._mach_context.commands.command_handlers[against]
|
|
sourcefile = mozpath.relpath(
|
|
inspect.getsourcefile(handler.func), command_context.topsrcdir
|
|
)
|
|
reader = command_context.mozbuild_reader(config_mode="empty")
|
|
try:
|
|
res = reader.files_info([sourcefile])[sourcefile]["BUG_COMPONENT"]
|
|
product, component = res.product, res.component
|
|
except TypeError:
|
|
# The file might not have a bug set.
|
|
product = "Firefox Build System"
|
|
component = "General"
|
|
|
|
uri = (
|
|
"https://bugzilla.mozilla.org/enter_bug.cgi?"
|
|
"product=%s&component=%s&blocked=1543241" % (product, component)
|
|
)
|
|
webbrowser.open_new_tab(uri)
|
|
|
|
|
|
MACH_PASTEBIN_DURATIONS = {
|
|
"onetime": "onetime",
|
|
"hour": "3600",
|
|
"day": "86400",
|
|
"week": "604800",
|
|
"month": "2073600",
|
|
}
|
|
|
|
EXTENSION_TO_HIGHLIGHTER = {
|
|
".hgrc": "ini",
|
|
"Dockerfile": "docker",
|
|
"Makefile": "make",
|
|
"applescript": "applescript",
|
|
"arduino": "arduino",
|
|
"bash": "bash",
|
|
"bat": "bat",
|
|
"c": "c",
|
|
"clojure": "clojure",
|
|
"cmake": "cmake",
|
|
"coffee": "coffee-script",
|
|
"console": "console",
|
|
"cpp": "cpp",
|
|
"cs": "csharp",
|
|
"css": "css",
|
|
"cu": "cuda",
|
|
"cuda": "cuda",
|
|
"dart": "dart",
|
|
"delphi": "delphi",
|
|
"diff": "diff",
|
|
"django": "django",
|
|
"docker": "docker",
|
|
"elixir": "elixir",
|
|
"erlang": "erlang",
|
|
"go": "go",
|
|
"h": "c",
|
|
"handlebars": "handlebars",
|
|
"haskell": "haskell",
|
|
"hs": "haskell",
|
|
"html": "html",
|
|
"ini": "ini",
|
|
"ipy": "ipythonconsole",
|
|
"ipynb": "ipythonconsole",
|
|
"irc": "irc",
|
|
"j2": "django",
|
|
"java": "java",
|
|
"js": "js",
|
|
"json": "json",
|
|
"jsx": "jsx",
|
|
"kt": "kotlin",
|
|
"less": "less",
|
|
"lisp": "common-lisp",
|
|
"lsp": "common-lisp",
|
|
"lua": "lua",
|
|
"m": "objective-c",
|
|
"make": "make",
|
|
"matlab": "matlab",
|
|
"md": "_markdown",
|
|
"nginx": "nginx",
|
|
"numpy": "numpy",
|
|
"patch": "diff",
|
|
"perl": "perl",
|
|
"php": "php",
|
|
"pm": "perl",
|
|
"postgresql": "postgresql",
|
|
"py": "python",
|
|
"rb": "rb",
|
|
"rs": "rust",
|
|
"rst": "rst",
|
|
"sass": "sass",
|
|
"scss": "scss",
|
|
"sh": "bash",
|
|
"sol": "sol",
|
|
"sql": "sql",
|
|
"swift": "swift",
|
|
"tex": "tex",
|
|
"typoscript": "typoscript",
|
|
"vim": "vim",
|
|
"xml": "xml",
|
|
"xslt": "xslt",
|
|
"yaml": "yaml",
|
|
"yml": "yaml",
|
|
}
|
|
|
|
|
|
def guess_highlighter_from_path(path):
|
|
"""Return a known highlighter from a given path
|
|
|
|
Attempt to select a highlighter by checking the file extension in the mapping
|
|
of extensions to highlighter. If that fails, attempt to pass the basename of
|
|
the file. Return `_code` as the default highlighter if that fails.
|
|
"""
|
|
import os
|
|
|
|
_name, ext = os.path.splitext(path)
|
|
|
|
if ext.startswith("."):
|
|
ext = ext[1:]
|
|
|
|
if ext in EXTENSION_TO_HIGHLIGHTER:
|
|
return EXTENSION_TO_HIGHLIGHTER[ext]
|
|
|
|
basename = os.path.basename(path)
|
|
|
|
return EXTENSION_TO_HIGHLIGHTER.get(basename, "_code")
|
|
|
|
|
|
PASTEMO_MAX_CONTENT_LENGTH = 250 * 1024 * 1024
|
|
|
|
PASTEMO_URL = "https://paste.mozilla.org/api/"
|
|
|
|
|
|
@Command(
|
|
"pastebin",
|
|
category="misc",
|
|
description="Command line interface to paste.mozilla.org.",
|
|
)
|
|
@CommandArgument(
|
|
"--list-highlighters",
|
|
action="store_true",
|
|
help="List known highlighters and exit",
|
|
)
|
|
@CommandArgument(
|
|
"--highlighter", default=None, help="Syntax highlighting to use for paste"
|
|
)
|
|
@CommandArgument(
|
|
"--expires",
|
|
default="week",
|
|
choices=sorted(MACH_PASTEBIN_DURATIONS.keys()),
|
|
help="Expire paste after given time duration (default: %(default)s)",
|
|
)
|
|
@CommandArgument(
|
|
"--verbose",
|
|
action="store_true",
|
|
help="Print extra info such as selected syntax highlighter",
|
|
)
|
|
@CommandArgument(
|
|
"path",
|
|
nargs="?",
|
|
default=None,
|
|
help="Path to file for upload to paste.mozilla.org",
|
|
)
|
|
def pastebin(command_context, list_highlighters, highlighter, expires, verbose, path):
|
|
"""Command line interface to `paste.mozilla.org`.
|
|
|
|
Takes either a filename whose content should be pasted, or reads
|
|
content from standard input. If a highlighter is specified it will
|
|
be used, otherwise the file name will be used to determine an
|
|
appropriate highlighter.
|
|
"""
|
|
|
|
import requests
|
|
|
|
def verbose_print(*args, **kwargs):
|
|
"""Print a string if `--verbose` flag is set"""
|
|
if verbose:
|
|
print(*args, **kwargs)
|
|
|
|
# Show known highlighters and exit.
|
|
if list_highlighters:
|
|
lexers = set(EXTENSION_TO_HIGHLIGHTER.values())
|
|
print("Available lexers:\n - %s" % "\n - ".join(sorted(lexers)))
|
|
return 0
|
|
|
|
# Get a correct expiry value.
|
|
try:
|
|
verbose_print("Setting expiry from %s" % expires)
|
|
expires = MACH_PASTEBIN_DURATIONS[expires]
|
|
verbose_print("Using %s as expiry" % expires)
|
|
except KeyError:
|
|
print(
|
|
"%s is not a valid duration.\n"
|
|
"(hint: try one of %s)"
|
|
% (expires, ", ".join(MACH_PASTEBIN_DURATIONS.keys()))
|
|
)
|
|
return 1
|
|
|
|
data = {
|
|
"format": "json",
|
|
"expires": expires,
|
|
}
|
|
|
|
# Get content to be pasted.
|
|
if path:
|
|
verbose_print("Reading content from %s" % path)
|
|
try:
|
|
with open(path, "r") as f:
|
|
content = f.read()
|
|
except IOError:
|
|
print("ERROR. No such file %s" % path)
|
|
return 1
|
|
|
|
lexer = guess_highlighter_from_path(path)
|
|
if lexer:
|
|
data["lexer"] = lexer
|
|
else:
|
|
verbose_print("Reading content from stdin")
|
|
content = sys.stdin.read()
|
|
|
|
# Assert the length of content to be posted does not exceed the maximum.
|
|
content_length = len(content)
|
|
verbose_print("Checking size of content is okay (%d)" % content_length)
|
|
if content_length > PASTEMO_MAX_CONTENT_LENGTH:
|
|
print(
|
|
"Paste content is too large (%d, maximum %d)"
|
|
% (content_length, PASTEMO_MAX_CONTENT_LENGTH)
|
|
)
|
|
return 1
|
|
|
|
data["content"] = content
|
|
|
|
# Highlight as specified language, overwriting value set from filename.
|
|
if highlighter:
|
|
verbose_print("Setting %s as highlighter" % highlighter)
|
|
data["lexer"] = highlighter
|
|
|
|
try:
|
|
verbose_print("Sending request to %s" % PASTEMO_URL)
|
|
resp = requests.post(PASTEMO_URL, data=data)
|
|
|
|
# Error code should always be 400.
|
|
# Response content will include a helpful error message,
|
|
# so print it here (for example, if an invalid highlighter is
|
|
# provided, it will return a list of valid highlighters).
|
|
if resp.status_code >= 400:
|
|
print("Error code %d: %s" % (resp.status_code, resp.content))
|
|
return 1
|
|
|
|
verbose_print("Pasted successfully")
|
|
|
|
response_json = resp.json()
|
|
|
|
verbose_print("Paste highlighted as %s" % response_json["lexer"])
|
|
print(response_json["url"])
|
|
|
|
return 0
|
|
except Exception as e:
|
|
print("ERROR. Paste failed.")
|
|
print("%s" % e)
|
|
return 1
|
|
|
|
|
|
class PypiBasedTool:
|
|
"""
|
|
Helper for loading a tool that is hosted on pypi. The package is expected
|
|
to expose a `mach_interface` module which has `new_release_on_pypi`,
|
|
`parser`, and `run` functions.
|
|
"""
|
|
|
|
def __init__(self, module_name, pypi_name=None):
|
|
self.name = module_name
|
|
self.pypi_name = pypi_name or module_name
|
|
|
|
def _import(self):
|
|
# Lazy loading of the tools mach interface.
|
|
# Note that only the mach_interface module should be used from this file.
|
|
import importlib
|
|
|
|
try:
|
|
return importlib.import_module("%s.mach_interface" % self.name)
|
|
except ImportError:
|
|
return None
|
|
|
|
def create_parser(self, subcommand=None):
|
|
# Create the command line parser.
|
|
# If the tool is not installed, or not up to date, it will
|
|
# first be installed.
|
|
cmd = MozbuildObject.from_environment()
|
|
cmd.activate_virtualenv()
|
|
tool = self._import()
|
|
if not tool:
|
|
# The tool is not here at all, install it
|
|
cmd.virtualenv_manager.install_pip_package(self.pypi_name)
|
|
print(
|
|
"%s was installed. please re-run your"
|
|
" command. If you keep getting this message please "
|
|
" manually run: 'pip install -U %s'." % (self.pypi_name, self.pypi_name)
|
|
)
|
|
else:
|
|
# Check if there is a new release available
|
|
release = tool.new_release_on_pypi()
|
|
if release:
|
|
print(release)
|
|
# there is one, so install it. Note that install_pip_package
|
|
# does not work here, so just run pip directly.
|
|
subprocess.check_call(
|
|
[
|
|
cmd.virtualenv_manager.python_path,
|
|
"-m",
|
|
"pip",
|
|
"install",
|
|
f"{self.pypi_name}=={release}",
|
|
]
|
|
)
|
|
print(
|
|
"%s was updated to version %s. please"
|
|
" re-run your command." % (self.pypi_name, release)
|
|
)
|
|
else:
|
|
# Tool is up to date, return the parser.
|
|
if subcommand:
|
|
return tool.parser(subcommand)
|
|
else:
|
|
return tool.parser()
|
|
# exit if we updated or installed mozregression because
|
|
# we may have already imported mozregression and running it
|
|
# as this may cause issues.
|
|
sys.exit(0)
|
|
|
|
def run(self, **options):
|
|
tool = self._import()
|
|
tool.run(options)
|
|
|
|
|
|
def mozregression_create_parser():
|
|
# Create the mozregression command line parser.
|
|
# if mozregression is not installed, or not up to date, it will
|
|
# first be installed.
|
|
loader = PypiBasedTool("mozregression")
|
|
return loader.create_parser()
|
|
|
|
|
|
@Command(
|
|
"mozregression",
|
|
category="misc",
|
|
description="Regression range finder for nightly and inbound builds.",
|
|
parser=mozregression_create_parser,
|
|
)
|
|
def run(command_context, **options):
|
|
command_context.activate_virtualenv()
|
|
mozregression = PypiBasedTool("mozregression")
|
|
mozregression.run(**options)
|
|
|
|
|
|
@Command(
|
|
"node",
|
|
category="devenv",
|
|
description="Run the NodeJS interpreter used for building.",
|
|
)
|
|
@CommandArgument("args", nargs=argparse.REMAINDER)
|
|
def node(command_context, args):
|
|
from mozbuild.nodeutil import find_node_executable
|
|
|
|
# Avoid logging the command
|
|
command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)
|
|
|
|
node_path, _ = find_node_executable()
|
|
|
|
return command_context.run_process(
|
|
[node_path] + args,
|
|
pass_thru=True, # Allow user to run Node interactively.
|
|
ensure_exit_code=False, # Don't throw on non-zero exit code.
|
|
)
|
|
|
|
|
|
@Command(
|
|
"npm",
|
|
category="devenv",
|
|
description="Run the npm executable from the NodeJS used for building.",
|
|
)
|
|
@CommandArgument("args", nargs=argparse.REMAINDER)
|
|
def npm(command_context, args):
|
|
from mozbuild.nodeutil import find_npm_executable
|
|
|
|
# Avoid logging the command
|
|
command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)
|
|
|
|
import os
|
|
|
|
# Add node and npm from mozbuild to front of system path
|
|
#
|
|
# This isn't pretty, but npm currently executes itself with
|
|
# `#!/usr/bin/env node`, which means it just uses the node in the
|
|
# current PATH. As a result, stuff gets built wrong and installed
|
|
# in the wrong places and probably other badness too without this:
|
|
npm_path, _ = find_npm_executable()
|
|
if not npm_path:
|
|
exit(-1, "could not find npm executable")
|
|
path = os.path.abspath(os.path.dirname(npm_path))
|
|
os.environ["PATH"] = "{}{}{}".format(path, os.pathsep, os.environ["PATH"])
|
|
|
|
# karma-firefox-launcher needs the path to firefox binary.
|
|
firefox_bin = command_context.get_binary_path(validate_exists=False)
|
|
if os.path.exists(firefox_bin):
|
|
os.environ["FIREFOX_BIN"] = firefox_bin
|
|
|
|
return command_context.run_process(
|
|
[npm_path, "--scripts-prepend-node-path=auto"] + args,
|
|
pass_thru=True, # Avoid eating npm output/error messages
|
|
ensure_exit_code=False, # Don't throw on non-zero exit code.
|
|
)
|
|
|
|
|
|
def logspam_create_parser(subcommand):
|
|
# Create the logspam command line parser.
|
|
# if logspam is not installed, or not up to date, it will
|
|
# first be installed.
|
|
loader = PypiBasedTool("logspam", "mozilla-log-spam")
|
|
return loader.create_parser(subcommand)
|
|
|
|
|
|
from functools import partial
|
|
|
|
|
|
@Command(
|
|
"logspam",
|
|
category="misc",
|
|
description="Warning categorizer for treeherder test runs.",
|
|
)
|
|
def logspam(command_context):
|
|
pass
|
|
|
|
|
|
@SubCommand("logspam", "report", parser=partial(logspam_create_parser, "report"))
|
|
def report(command_context, **options):
|
|
command_context.activate_virtualenv()
|
|
logspam = PypiBasedTool("logspam")
|
|
logspam.run(command="report", **options)
|
|
|
|
|
|
@SubCommand("logspam", "bisect", parser=partial(logspam_create_parser, "bisect"))
|
|
def bisect(command_context, **options):
|
|
command_context.activate_virtualenv()
|
|
logspam = PypiBasedTool("logspam")
|
|
logspam.run(command="bisect", **options)
|
|
|
|
|
|
@SubCommand("logspam", "file", parser=partial(logspam_create_parser, "file"))
|
|
def create(command_context, **options):
|
|
command_context.activate_virtualenv()
|
|
logspam = PypiBasedTool("logspam")
|
|
logspam.run(command="file", **options)
|
|
|
|
|
|
# mots_loader will be used when running commands and subcommands, as well as
|
|
# when creating the parsers.
|
|
mots_loader = PypiBasedTool("mots")
|
|
|
|
|
|
def mots_create_parser(subcommand=None):
|
|
return mots_loader.create_parser(subcommand)
|
|
|
|
|
|
def mots_run_subcommand(command, command_context, **options):
|
|
command_context.activate_virtualenv()
|
|
mots_loader.run(command=command, **options)
|
|
|
|
|
|
class motsSubCommand(SubCommand):
|
|
"""A helper subclass that reduces repitition when defining subcommands."""
|
|
|
|
def __init__(self, subcommand):
|
|
super().__init__(
|
|
"mots",
|
|
subcommand,
|
|
parser=partial(mots_create_parser, subcommand),
|
|
)
|
|
|
|
|
|
@Command(
|
|
"mots",
|
|
category="misc",
|
|
description="Manage module information in-tree using the mots CLI.",
|
|
parser=mots_create_parser,
|
|
)
|
|
def mots(command_context, **options):
|
|
"""The main mots command call."""
|
|
command_context.activate_virtualenv()
|
|
mots_loader.run(**options)
|
|
|
|
|
|
# Define subcommands that will be proxied through mach.
|
|
for sc in (
|
|
"clean",
|
|
"check-hashes",
|
|
"export",
|
|
"export-and-clean",
|
|
"module",
|
|
"query",
|
|
"settings",
|
|
"user",
|
|
"validate",
|
|
):
|
|
# Pass through args and kwargs, but add the subcommand string as the first argument.
|
|
motsSubCommand(sc)(lambda *a, **kw: mots_run_subcommand(sc, *a, **kw))
|