nagai/fune
3
0
Fork 0
forked from mirrors/gecko-dev
Code Pull requests Activity

Backed out changeset e1921c5112d8 (bug 1696251) for causing bustages complaining about 'CommandContext'. CLOSED TREE

Browse source
This commit is contained in:
Butkovits Atila 2021-07-16 20:35:55 +03:00
parent d021d948ec
commit a07f790e42
53 changed files with 1425 additions and 1813 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

24
build/valgrind/mach_commands.py
View file

@ -59,7 +59,7 @@ class MachCommands(MachCommandBase):
from six import string_types from six import string_types
from valgrind.output_handler import OutputHandler from valgrind.output_handler import OutputHandler
build_dir = os.path.join(command_context.topsrcdir, "build") build_dir = os.path.join(self.topsrcdir, "build")
# XXX: currently we just use the PGO inputs for Valgrind runs. This may # XXX: currently we just use the PGO inputs for Valgrind runs. This may
# change in the future. # change in the future.
@ -68,9 +68,7 @@ class MachCommands(MachCommandBase):
with TemporaryDirectory() as profilePath: with TemporaryDirectory() as profilePath:
# TODO: refactor this into mozprofile # TODO: refactor this into mozprofile
profile_data_dir = os.path.join( profile_data_dir = os.path.join(self.topsrcdir, "testing", "profiles")
command_context.topsrcdir, "testing", "profiles"
)
with open(os.path.join(profile_data_dir, "profiles.json"), "r") as fh: with open(os.path.join(profile_data_dir, "profiles.json"), "r") as fh:
base_profiles = json.load(fh)["valgrind"] base_profiles = json.load(fh)["valgrind"]
@ -91,7 +89,7 @@ class MachCommands(MachCommandBase):
prefs[k] = Preferences.cast(v) prefs[k] = Preferences.cast(v)
quitter = os.path.join( quitter = os.path.join(
command_context.topsrcdir, "tools", "quitter", "quitter@mozilla.org.xpi" self.topsrcdir, "tools", "quitter", "quitter@mozilla.org.xpi"
) )
locations = ServerLocations() locations = ServerLocations()
@ -115,7 +113,7 @@ class MachCommands(MachCommandBase):
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
env["XPCOM_DEBUG_BREAK"] = "warn" env["XPCOM_DEBUG_BREAK"] = "warn"
outputHandler = OutputHandler(command_context.log) outputHandler = OutputHandler(self.log)
kp_kwargs = { kp_kwargs = {
"processOutputLine": [outputHandler], "processOutputLine": [outputHandler],
"universal_newlines": True, "universal_newlines": True,
@ -173,7 +171,7 @@ class MachCommands(MachCommandBase):
try: try:
runner = FirefoxRunner( runner = FirefoxRunner(
profile=profile, profile=profile,
binary=command_context.get_binary_path(), binary=self.get_binary_path(),
cmdargs=firefox_args, cmdargs=firefox_args,
env=env, env=env,
process_args=kp_kwargs, process_args=kp_kwargs,
@ -187,7 +185,7 @@ class MachCommands(MachCommandBase):
supps = outputHandler.suppression_count supps = outputHandler.suppression_count
if errs != supps: if errs != supps:
status = 1 # turns the TBPL job orange status = 1 # turns the TBPL job orange
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"valgrind-fail-parsing", "valgrind-fail-parsing",
{"errs": errs, "supps": supps}, {"errs": errs, "supps": supps},
@ -197,7 +195,7 @@ class MachCommands(MachCommandBase):
elif errs == 0: elif errs == 0:
status = 0 status = 0
command_context.log( self.log(
logging.INFO, logging.INFO,
"valgrind-pass", "valgrind-pass",
{}, {},
@ -209,13 +207,13 @@ class MachCommands(MachCommandBase):
if binary_not_found_exception: if binary_not_found_exception:
status = 2 # turns the TBPL job red status = 2 # turns the TBPL job red
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"valgrind-fail-errors", "valgrind-fail-errors",
{"error": str(binary_not_found_exception)}, {"error": str(binary_not_found_exception)},
"TEST-UNEXPECTED-FAIL | valgrind-test | {error}", "TEST-UNEXPECTED-FAIL | valgrind-test | {error}",
) )
command_context.log( self.log(
logging.INFO, logging.INFO,
"valgrind-fail-errors", "valgrind-fail-errors",
{"help": binary_not_found_exception.help()}, {"help": binary_not_found_exception.help()},
@ -223,7 +221,7 @@ class MachCommands(MachCommandBase):
) )
elif exitcode is None: elif exitcode is None:
status = 2 # turns the TBPL job red status = 2 # turns the TBPL job red
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"valgrind-fail-timeout", "valgrind-fail-timeout",
{"timeout": timeout}, {"timeout": timeout},
@ -232,7 +230,7 @@ class MachCommands(MachCommandBase):
) )
elif exitcode != 0: elif exitcode != 0:
status = 2 # turns the TBPL job red status = 2 # turns the TBPL job red
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"valgrind-fail-errors", "valgrind-fail-errors",
{"exitcode": exitcode}, {"exitcode": exitcode},

27
devtools/shared/css/generated/mach_commands.py
View file

@ -49,39 +49,35 @@ class MachCommands(MachCommandBase):
"""Generate the static css properties database for devtools and write it to file.""" """Generate the static css properties database for devtools and write it to file."""
print("Re-generating the css properties database...") print("Re-generating the css properties database...")
db = self.get_properties_db_from_xpcshell(command_context) db = self.get_properties_db_from_xpcshell()
if not db: if not db:
return 1 return 1
self.output_template( self.output_template(
command_context,
{ {
"preferences": stringify(db["preferences"]), "preferences": stringify(db["preferences"]),
"cssProperties": stringify(db["cssProperties"]), "cssProperties": stringify(db["cssProperties"]),
"pseudoElements": stringify(db["pseudoElements"]), "pseudoElements": stringify(db["pseudoElements"]),
}, }
) )
def get_properties_db_from_xpcshell(self, command_context): def get_properties_db_from_xpcshell(self):
"""Generate the static css properties db for devtools from an xpcshell script.""" """Generate the static css properties db for devtools from an xpcshell script."""
build = MozbuildObject.from_environment() build = MozbuildObject.from_environment()
# Get the paths # Get the paths
script_path = resolve_path( script_path = resolve_path(
command_context.topsrcdir, self.topsrcdir, "devtools/shared/css/generated/generate-properties-db.js"
"devtools/shared/css/generated/generate-properties-db.js",
) )
gre_path = resolve_path(command_context.topobjdir, "dist/bin") gre_path = resolve_path(self.topobjdir, "dist/bin")
browser_path = resolve_path(command_context.topobjdir, "dist/bin/browser") browser_path = resolve_path(self.topobjdir, "dist/bin/browser")
try: try:
xpcshell_path = build.get_binary_path(what="xpcshell") xpcshell_path = build.get_binary_path(what="xpcshell")
except BinaryNotFoundException as e: except BinaryNotFoundException as e:
command_context.log( self.log(
logging.ERROR, "devtools-css-db", {"error": str(e)}, "ERROR: {error}" logging.ERROR, "devtools-css-db", {"error": str(e)}, "ERROR: {error}"
) )
command_context.log( self.log(logging.INFO, "devtools-css-db", {"help": e.help()}, "{help}")
logging.INFO, "devtools-css-db", {"help": e.help()}, "{help}"
)
return None return None
print(browser_path) print(browser_path)
@ -102,14 +98,13 @@ class MachCommands(MachCommandBase):
return json.loads(contents) return json.loads(contents)
def output_template(self, command_context, substitutions): def output_template(self, substitutions):
"""Output a the properties-db.js from a template.""" """Output a the properties-db.js from a template."""
js_template_path = resolve_path( js_template_path = resolve_path(
command_context.topsrcdir, self.topsrcdir, "devtools/shared/css/generated/properties-db.js.in"
"devtools/shared/css/generated/properties-db.js.in",
) )
destination_path = resolve_path( destination_path = resolve_path(
command_context.topsrcdir, "devtools/shared/css/generated/properties-db.js" self.topsrcdir, "devtools/shared/css/generated/properties-db.js"
) )
with open(js_template_path, "rb") as handle: with open(js_template_path, "rb") as handle:

12
dom/bindings/mach_commands.py
View file

@ -36,7 +36,7 @@ class WebIDLProvider(MachCommandBase):
def webidl_example(self, command_context, interface): def webidl_example(self, command_context, interface):
from mozwebidlcodegen import BuildSystemWebIDL from mozwebidlcodegen import BuildSystemWebIDL
manager = command_context._spawn(BuildSystemWebIDL).manager manager = self._spawn(BuildSystemWebIDL).manager
for i in interface: for i in interface:
manager.generate_example_files(i) manager.generate_example_files(i)
@ -47,17 +47,15 @@ class WebIDLProvider(MachCommandBase):
description="Run WebIDL tests (Interface Browser parser).", description="Run WebIDL tests (Interface Browser parser).",
) )
def webidl_test(self, command_context, **kwargs): def webidl_test(self, command_context, **kwargs):
sys.path.insert( sys.path.insert(0, os.path.join(self.topsrcdir, "other-licenses", "ply"))
0, os.path.join(command_context.topsrcdir, "other-licenses", "ply")
)
# Ensure the topobjdir exists. On a Taskcluster test run there won't be # Ensure the topobjdir exists. On a Taskcluster test run there won't be
# an objdir yet. # an objdir yet.
mkdir(command_context.topobjdir) mkdir(self.topobjdir)
# Make sure we drop our cached grammar bits in the objdir, not # Make sure we drop our cached grammar bits in the objdir, not
# wherever we happen to be running from. # wherever we happen to be running from.
os.chdir(command_context.topobjdir) os.chdir(self.topobjdir)
if kwargs["verbose"] is None: if kwargs["verbose"] is None:
kwargs["verbose"] = False kwargs["verbose"] = False
@ -66,7 +64,7 @@ class WebIDLProvider(MachCommandBase):
# objdir. But we're going to try loading it as a python # objdir. But we're going to try loading it as a python
# module, so we need to make sure the objdir is in our search # module, so we need to make sure the objdir is in our search
# path. # path.
sys.path.insert(0, command_context.topobjdir) sys.path.insert(0, self.topobjdir)
import runtests import runtests

69
js/src/devtools/rootAnalysis/mach_commands.py
View file

@ -79,13 +79,13 @@ class MachCommands(MachCommandBase):
def gcc_dir(self): def gcc_dir(self):
return os.path.join(self.tools_dir(), "gcc") return os.path.join(self.tools_dir(), "gcc")
def script_dir(self, command_context): def script_dir(self):
return os.path.join(command_context.topsrcdir, "js/src/devtools/rootAnalysis") return os.path.join(self.topsrcdir, "js/src/devtools/rootAnalysis")
def work_dir(self, command_context, application, given): def work_dir(self, application, given):
if given is not None: if given is not None:
return given return given
return os.path.join(command_context.topsrcdir, "haz-" + application) return os.path.join(self.topsrcdir, "haz-" + application)
def ensure_dir_exists(self, dir): def ensure_dir_exists(self, dir):
os.makedirs(dir, exist_ok=True) os.makedirs(dir, exist_ok=True)
@ -122,11 +122,8 @@ class MachCommands(MachCommandBase):
os.chdir(self.ensure_dir_exists(self.tools_dir())) os.chdir(self.ensure_dir_exists(self.tools_dir()))
try: try:
kwargs["from_build"] = ("linux64-gcc-sixgill", "linux64-gcc-9") kwargs["from_build"] = ("linux64-gcc-sixgill", "linux64-gcc-9")
command_context._mach_context.commands.dispatch( self._mach_context.commands.dispatch(
"artifact", "artifact", self._mach_context, subcommand="toolchain", **kwargs
command_context._mach_context,
subcommand="toolchain",
**kwargs
) )
finally: finally:
os.chdir(orig_dir) os.chdir(orig_dir)
@ -155,8 +152,8 @@ class MachCommands(MachCommandBase):
or os.environ.get("MOZCONFIG") or os.environ.get("MOZCONFIG")
or default_mozconfig or default_mozconfig
) )
mozconfig_path = os.path.join(command_context.topsrcdir, mozconfig_path) mozconfig_path = os.path.join(self.topsrcdir, mozconfig_path)
loader = MozconfigLoader(command_context.topsrcdir) loader = MozconfigLoader(self.topsrcdir)
mozconfig = loader.read_mozconfig(mozconfig_path) mozconfig = loader.read_mozconfig(mozconfig_path)
# Validate the mozconfig settings in case the user overrode the default. # Validate the mozconfig settings in case the user overrode the default.
@ -173,20 +170,20 @@ class MachCommands(MachCommandBase):
# Set a default objdir for the shell, for developer builds. # Set a default objdir for the shell, for developer builds.
os.environ.setdefault( os.environ.setdefault(
"MOZ_OBJDIR", os.path.join(command_context.topsrcdir, "obj-haz-shell") "MOZ_OBJDIR", os.path.join(self.topsrcdir, "obj-haz-shell")
) )
return command_context._mach_context.commands.dispatch( return self._mach_context.commands.dispatch(
"build", command_context._mach_context, **kwargs "build", self._mach_context, **kwargs
) )
def read_json_file(self, filename): def read_json_file(self, filename):
with open(filename) as fh: with open(filename) as fh:
return json.load(fh) return json.load(fh)
def ensure_shell(self, command_context, objdir): def ensure_shell(self, objdir):
if objdir is None: if objdir is None:
objdir = os.path.join(command_context.topsrcdir, "obj-haz-shell") objdir = os.path.join(self.topsrcdir, "obj-haz-shell")
try: try:
binaries = self.read_json_file(os.path.join(objdir, "binaries.json")) binaries = self.read_json_file(os.path.join(objdir, "binaries.json"))
@ -221,11 +218,9 @@ no shell found in %s -- must build the JS shell with `mach hazards build-shell`
if objdir is None: if objdir is None:
objdir = os.environ.get("HAZ_OBJDIR") objdir = os.environ.get("HAZ_OBJDIR")
if objdir is None: if objdir is None:
objdir = os.path.join( objdir = os.path.join(self.topsrcdir, "obj-analyzed-" + application)
command_context.topsrcdir, "obj-analyzed-" + application
)
work_dir = self.work_dir(command_context, application, kwargs["work_dir"]) work_dir = self.work_dir(application, kwargs["work_dir"])
self.ensure_dir_exists(work_dir) self.ensure_dir_exists(work_dir)
with open(os.path.join(work_dir, "defaults.py"), "wt") as fh: with open(os.path.join(work_dir, "defaults.py"), "wt") as fh:
data = textwrap.dedent( data = textwrap.dedent(
@ -238,9 +233,9 @@ no shell found in %s -- must build the JS shell with `mach hazards build-shell`
gcc_bin = "{gcc_dir}/bin" gcc_bin = "{gcc_dir}/bin"
""" """
).format( ).format(
script_dir=self.script_dir(command_context), script_dir=self.script_dir(),
objdir=objdir, objdir=objdir,
srcdir=command_context.topsrcdir, srcdir=self.topsrcdir,
sixgill_dir=self.sixgill_dir(), sixgill_dir=self.sixgill_dir(),
gcc_dir=self.gcc_dir(), gcc_dir=self.gcc_dir(),
) )
@ -248,14 +243,14 @@ no shell found in %s -- must build the JS shell with `mach hazards build-shell`
buildscript = " ".join( buildscript = " ".join(
[ [
command_context.topsrcdir + "/mach hazards compile", self.topsrcdir + "/mach hazards compile",
"--application=" + application, "--application=" + application,
"--haz-objdir=" + objdir, "--haz-objdir=" + objdir,
] ]
) )
args = [ args = [
sys.executable, sys.executable,
os.path.join(self.script_dir(command_context), "analyze.py"), os.path.join(self.script_dir(), "analyze.py"),
"dbs", "dbs",
"--upto", "--upto",
"dbs", "dbs",
@ -263,7 +258,7 @@ no shell found in %s -- must build the JS shell with `mach hazards build-shell`
"--buildcommand=" + buildscript, "--buildcommand=" + buildscript,
] ]
return command_context.run_process(args=args, cwd=work_dir, pass_thru=True) return self.run_process(args=args, cwd=work_dir, pass_thru=True)
@inherit_command_args("build") @inherit_command_args("build")
@SubCommand("hazards", "compile", description=argparse.SUPPRESS) @SubCommand("hazards", "compile", description=argparse.SUPPRESS)
@ -300,13 +295,13 @@ no shell found in %s -- must build the JS shell with `mach hazards build-shell`
mozconfig_path = ( mozconfig_path = (
kwargs.pop("mozconfig", None) or env.get("MOZCONFIG") or default_mozconfig kwargs.pop("mozconfig", None) or env.get("MOZCONFIG") or default_mozconfig
) )
mozconfig_path = os.path.join(command_context.topsrcdir, mozconfig_path) mozconfig_path = os.path.join(self.topsrcdir, mozconfig_path)
# Validate the mozconfig. # Validate the mozconfig.
# Require an explicit --enable-application=APP (even if you just # Require an explicit --enable-application=APP (even if you just
# want to build the default browser application.) # want to build the default browser application.)
loader = MozconfigLoader(command_context.topsrcdir) loader = MozconfigLoader(self.topsrcdir)
mozconfig = loader.read_mozconfig(mozconfig_path) mozconfig = loader.read_mozconfig(mozconfig_path)
configure_args = mozconfig["configure_args"] configure_args = mozconfig["configure_args"]
if "--enable-application=%s" % app not in configure_args: if "--enable-application=%s" % app not in configure_args:
@ -315,7 +310,7 @@ no shell found in %s -- must build the JS shell with `mach hazards build-shell`
raise Exception("mozconfig must wrap compiles") raise Exception("mozconfig must wrap compiles")
# Communicate mozconfig to build subprocesses. # Communicate mozconfig to build subprocesses.
env["MOZCONFIG"] = os.path.join(command_context.topsrcdir, mozconfig_path) env["MOZCONFIG"] = os.path.join(self.topsrcdir, mozconfig_path)
# hazard mozconfigs need to find binaries in .mozbuild # hazard mozconfigs need to find binaries in .mozbuild
env["MOZBUILD_STATE_PATH"] = self.state_dir() env["MOZBUILD_STATE_PATH"] = self.state_dir()
@ -328,8 +323,8 @@ no shell found in %s -- must build the JS shell with `mach hazards build-shell`
if "haz_objdir" in kwargs: if "haz_objdir" in kwargs:
env["MOZ_OBJDIR"] = kwargs.pop("haz_objdir") env["MOZ_OBJDIR"] = kwargs.pop("haz_objdir")
return command_context._mach_context.commands.dispatch( return self._mach_context.commands.dispatch(
"build", command_context._mach_context, **kwargs "build", self._mach_context, **kwargs
) )
@SubCommand( @SubCommand(
@ -351,9 +346,9 @@ no shell found in %s -- must build the JS shell with `mach hazards build-shell`
def analyze(self, command_context, application, shell_objdir, work_dir): def analyze(self, command_context, application, shell_objdir, work_dir):
"""Analyzed gathered data for rooting hazards""" """Analyzed gathered data for rooting hazards"""
shell = self.ensure_shell(command_context, shell_objdir) shell = self.ensure_shell(shell_objdir)
args = [ args = [
os.path.join(self.script_dir(command_context), "analyze.py"), os.path.join(self.script_dir(), "analyze.py"),
"--js", "--js",
shell, shell,
"gcTypes", "gcTypes",
@ -363,8 +358,8 @@ no shell found in %s -- must build the JS shell with `mach hazards build-shell`
self.setup_env_for_tools(os.environ) self.setup_env_for_tools(os.environ)
os.environ["LD_LIBRARY_PATH"] += ":" + os.path.dirname(shell) os.environ["LD_LIBRARY_PATH"] += ":" + os.path.dirname(shell)
work_dir = self.work_dir(command_context, application, work_dir) work_dir = self.work_dir(application, work_dir)
return command_context.run_process(args=args, cwd=work_dir, pass_thru=True) return self.run_process(args=args, cwd=work_dir, pass_thru=True)
@SubCommand( @SubCommand(
"hazards", "hazards",
@ -378,9 +373,9 @@ no shell found in %s -- must build the JS shell with `mach hazards build-shell`
) )
def self_test(self, command_context, shell_objdir): def self_test(self, command_context, shell_objdir):
"""Analyzed gathered data for rooting hazards""" """Analyzed gathered data for rooting hazards"""
shell = self.ensure_shell(command_context, shell_objdir) shell = self.ensure_shell(shell_objdir)
args = [ args = [
os.path.join(self.script_dir(command_context), "run-test.py"), os.path.join(self.script_dir(), "run-test.py"),
"-v", "-v",
"--js", "--js",
shell, shell,
@ -392,4 +387,4 @@ no shell found in %s -- must build the JS shell with `mach hazards build-shell`
self.setup_env_for_tools(os.environ) self.setup_env_for_tools(os.environ)
os.environ["LD_LIBRARY_PATH"] += ":" + os.path.dirname(shell) os.environ["LD_LIBRARY_PATH"] += ":" + os.path.dirname(shell)
return command_context.run_process(args=args, pass_thru=True) return self.run_process(args=args, pass_thru=True)

22
layout/tools/reftest/mach_commands.py
View file

@ -238,7 +238,7 @@ class MachCommands(MachCommandBase):
) )
def run_reftest(self, command_context, **kwargs): def run_reftest(self, command_context, **kwargs):
kwargs["suite"] = "reftest" kwargs["suite"] = "reftest"
return self._run_reftest(command_context, **kwargs) return self._run_reftest(**kwargs)
@Command( @Command(
"jstestbrowser", "jstestbrowser",
@ -247,15 +247,15 @@ class MachCommands(MachCommandBase):
parser=get_parser, parser=get_parser,
) )
def run_jstestbrowser(self, command_context, **kwargs): def run_jstestbrowser(self, command_context, **kwargs):
if "--enable-js-shell" not in command_context.mozconfig["configure_args"]: if "--enable-js-shell" not in self.mozconfig["configure_args"]:
raise Exception( raise Exception(
"jstestbrowser requires --enable-js-shell be specified in mozconfig." "jstestbrowser requires --enable-js-shell be specified in mozconfig."
) )
command_context._mach_context.commands.dispatch( self._mach_context.commands.dispatch(
"build", command_context._mach_context, what=["stage-jstests"] "build", self._mach_context, what=["stage-jstests"]
) )
kwargs["suite"] = "jstestbrowser" kwargs["suite"] = "jstestbrowser"
return self._run_reftest(command_context, **kwargs) return self._run_reftest(**kwargs)
@Command( @Command(
"crashtest", "crashtest",
@ -265,16 +265,16 @@ class MachCommands(MachCommandBase):
) )
def run_crashtest(self, command_context, **kwargs): def run_crashtest(self, command_context, **kwargs):
kwargs["suite"] = "crashtest" kwargs["suite"] = "crashtest"
return self._run_reftest(command_context, **kwargs) return self._run_reftest(**kwargs)
def _run_reftest(self, command_context, **kwargs): def _run_reftest(self, **kwargs):
kwargs["topsrcdir"] = command_context.topsrcdir kwargs["topsrcdir"] = self.topsrcdir
process_test_objects(kwargs) process_test_objects(kwargs)
reftest = command_context._spawn(ReftestRunner) reftest = self._spawn(ReftestRunner)
# Unstructured logging must be enabled prior to calling # Unstructured logging must be enabled prior to calling
# adb which uses an unstructured logger in its constructor. # adb which uses an unstructured logger in its constructor.
reftest.log_manager.enable_unstructured() reftest.log_manager.enable_unstructured()
if conditions.is_android(command_context): if conditions.is_android(self):
from mozrunner.devices.android_device import ( from mozrunner.devices.android_device import (
verify_android_device, verify_android_device,
InstallIntent, InstallIntent,
@ -292,7 +292,7 @@ class MachCommands(MachCommandBase):
): ):
verbose = True verbose = True
verify_android_device( verify_android_device(
command_context, self,
install=install, install=install,
xre=True, xre=True,
network=True, network=True,

4
layout/tools/reftest/mach_test_package_commands.py
View file

@ -116,6 +116,6 @@ class ReftestCommands(MachCommandBase):
parser=setup_argument_parser, parser=setup_argument_parser,
) )
def reftest(self, command_context, **kwargs): def reftest(self, command_context, **kwargs):
command_context._mach_context.activate_mozharness_venv() self._mach_context.activate_mozharness_venv()
kwargs["suite"] = "reftest" kwargs["suite"] = "reftest"
return run_reftest(command_context._mach_context, **kwargs) return run_reftest(self._mach_context, **kwargs)

86
mobile/android/mach_commands.py
View file

@ -70,7 +70,7 @@ class MachCommands(MachCommandBase):
def android_assemble_app(self, command_context, args): def android_assemble_app(self, command_context, args):
ret = self.gradle( ret = self.gradle(
command_context, command_context,
command_context.substs["GRADLE_ANDROID_APP_TASKS"] + ["-x", "lint"] + args, self.substs["GRADLE_ANDROID_APP_TASKS"] + ["-x", "lint"] + args,
verbose=True, verbose=True,
) )
@ -103,7 +103,7 @@ class MachCommands(MachCommandBase):
ret = self.gradle( ret = self.gradle(
command_context, command_context,
command_context.substs["GRADLE_ANDROID_GENERATE_SDK_BINDINGS_TASKS"] self.substs["GRADLE_ANDROID_GENERATE_SDK_BINDINGS_TASKS"]
+ [bindings_args] + [bindings_args]
+ args, + args,
verbose=True, verbose=True,
@ -120,10 +120,7 @@ class MachCommands(MachCommandBase):
def android_generate_generated_jni_wrappers(self, command_context, args): def android_generate_generated_jni_wrappers(self, command_context, args):
ret = self.gradle( ret = self.gradle(
command_context, command_context,
command_context.substs[ self.substs["GRADLE_ANDROID_GENERATE_GENERATED_JNI_WRAPPERS_TASKS"] + args,
"GRADLE_ANDROID_GENERATE_GENERATED_JNI_WRAPPERS_TASKS"
]
+ args,
verbose=True, verbose=True,
) )
@ -182,9 +179,7 @@ REMOVED/DEPRECATED: Use 'mach lint --linter android-checkstyle'.""",
# can change the outputs for those processes. # can change the outputs for those processes.
self.gradle( self.gradle(
command_context, command_context,
command_context.substs["GRADLE_ANDROID_DEPENDENCIES_TASKS"] self.substs["GRADLE_ANDROID_DEPENDENCIES_TASKS"] + ["--continue"] + args,
+ ["--continue"]
+ args,
verbose=True, verbose=True,
) )
@ -200,7 +195,7 @@ REMOVED/DEPRECATED: Use 'mach lint --linter android-checkstyle'.""",
def android_archive_geckoview(self, command_context, args): def android_archive_geckoview(self, command_context, args):
ret = self.gradle( ret = self.gradle(
command_context, command_context,
command_context.substs["GRADLE_ANDROID_ARCHIVE_GECKOVIEW_TASKS"] + args, self.substs["GRADLE_ANDROID_ARCHIVE_GECKOVIEW_TASKS"] + args,
verbose=True, verbose=True,
) )
@ -211,8 +206,7 @@ REMOVED/DEPRECATED: Use 'mach lint --linter android-checkstyle'.""",
def android_build_geckoview_example(self, command_context, args): def android_build_geckoview_example(self, command_context, args):
self.gradle( self.gradle(
command_context, command_context,
command_context.substs["GRADLE_ANDROID_BUILD_GECKOVIEW_EXAMPLE_TASKS"] self.substs["GRADLE_ANDROID_BUILD_GECKOVIEW_EXAMPLE_TASKS"] + args,
+ args,
verbose=True, verbose=True,
) )
@ -230,8 +224,7 @@ REMOVED/DEPRECATED: Use 'mach lint --linter android-checkstyle'.""",
def android_install_geckoview_example(self, command_context, args): def android_install_geckoview_example(self, command_context, args):
self.gradle( self.gradle(
command_context, command_context,
command_context.substs["GRADLE_ANDROID_INSTALL_GECKOVIEW_EXAMPLE_TASKS"] self.substs["GRADLE_ANDROID_INSTALL_GECKOVIEW_EXAMPLE_TASKS"] + args,
+ args,
verbose=True, verbose=True,
) )
@ -285,9 +278,9 @@ REMOVED/DEPRECATED: Use 'mach lint --linter android-checkstyle'.""",
): ):
tasks = ( tasks = (
command_context.substs["GRADLE_ANDROID_GECKOVIEW_DOCS_ARCHIVE_TASKS"] self.substs["GRADLE_ANDROID_GECKOVIEW_DOCS_ARCHIVE_TASKS"]
if archive or upload if archive or upload
else command_context.substs["GRADLE_ANDROID_GECKOVIEW_DOCS_TASKS"] else self.substs["GRADLE_ANDROID_GECKOVIEW_DOCS_TASKS"]
) )
ret = self.gradle(command_context, tasks, verbose=True) ret = self.gradle(command_context, tasks, verbose=True)
@ -328,7 +321,7 @@ REMOVED/DEPRECATED: Use 'mach lint --linter android-checkstyle'.""",
branch = upload_branch.format(**fmt) branch = upload_branch.format(**fmt)
repo_url = "git@github.com:%s.git" % upload repo_url = "git@github.com:%s.git" % upload
repo_path = mozpath.abspath("gv-docs-repo") repo_path = mozpath.abspath("gv-docs-repo")
command_context.run_process( self.run_process(
[ [
"git", "git",
"clone", "clone",
@ -352,7 +345,7 @@ REMOVED/DEPRECATED: Use 'mach lint --linter android-checkstyle'.""",
# Extract new javadoc to specified directory inside repo. # Extract new javadoc to specified directory inside repo.
src_tar = mozpath.join( src_tar = mozpath.join(
command_context.topobjdir, self.topobjdir,
"gradle", "gradle",
"build", "build",
"mobile", "mobile",
@ -366,11 +359,9 @@ REMOVED/DEPRECATED: Use 'mach lint --linter android-checkstyle'.""",
mozfile.extract_zip(src_tar, dst_path) mozfile.extract_zip(src_tar, dst_path)
# Commit and push. # Commit and push.
command_context.run_process( self.run_process(["git", "add", "--all"], append_env=env, pass_thru=True)
["git", "add", "--all"], append_env=env, pass_thru=True
)
if ( if (
command_context.run_process( self.run_process(
["git", "diff", "--cached", "--quiet"], ["git", "diff", "--cached", "--quiet"],
append_env=env, append_env=env,
pass_thru=True, pass_thru=True,
@ -379,12 +370,12 @@ REMOVED/DEPRECATED: Use 'mach lint --linter android-checkstyle'.""",
!= 0 != 0
): ):
# We have something to commit. # We have something to commit.
command_context.run_process( self.run_process(
["git", "commit", "--message", upload_message.format(**fmt)], ["git", "commit", "--message", upload_message.format(**fmt)],
append_env=env, append_env=env,
pass_thru=True, pass_thru=True,
) )
command_context.run_process( self.run_process(
["git", "push", "origin", branch], append_env=env, pass_thru=True ["git", "push", "origin", branch], append_env=env, pass_thru=True
) )
@ -409,14 +400,14 @@ REMOVED/DEPRECATED: Use 'mach lint --linter android-checkstyle'.""",
def gradle(self, command_context, args, verbose=False): def gradle(self, command_context, args, verbose=False):
if not verbose: if not verbose:
# Avoid logging the command # Avoid logging the command
command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL) self.log_manager.terminal_handler.setLevel(logging.CRITICAL)
# In automation, JAVA_HOME is set via mozconfig, which needs # In automation, JAVA_HOME is set via mozconfig, which needs
# to be specially handled in each mach command. This turns # to be specially handled in each mach command. This turns
# $JAVA_HOME/bin/java into $JAVA_HOME. # $JAVA_HOME/bin/java into $JAVA_HOME.
java_home = os.path.dirname(os.path.dirname(command_context.substs["JAVA"])) java_home = os.path.dirname(os.path.dirname(self.substs["JAVA"]))
gradle_flags = command_context.substs.get("GRADLE_FLAGS", "") or os.environ.get( gradle_flags = self.substs.get("GRADLE_FLAGS", "") or os.environ.get(
"GRADLE_FLAGS", "" "GRADLE_FLAGS", ""
) )
gradle_flags = shell_split(gradle_flags) gradle_flags = shell_split(gradle_flags)
@ -439,7 +430,7 @@ REMOVED/DEPRECATED: Use 'mach lint --linter android-checkstyle'.""",
# https://discuss.gradle.org/t/unmappable-character-for-encoding-ascii-when-building-a-utf-8-project/10692/11 # NOQA: E501 # https://discuss.gradle.org/t/unmappable-character-for-encoding-ascii-when-building-a-utf-8-project/10692/11 # NOQA: E501
# and especially https://stackoverflow.com/a/21755671. # and especially https://stackoverflow.com/a/21755671.
if command_context.substs.get("MOZ_AUTOMATION"): if self.substs.get("MOZ_AUTOMATION"):
gradle_flags += ["--console=plain"] gradle_flags += ["--console=plain"]
env = os.environ.copy() env = os.environ.copy()
@ -452,16 +443,16 @@ REMOVED/DEPRECATED: Use 'mach lint --linter android-checkstyle'.""",
) )
# Set ANDROID_SDK_ROOT if --with-android-sdk was set. # Set ANDROID_SDK_ROOT if --with-android-sdk was set.
# See https://bugzilla.mozilla.org/show_bug.cgi?id=1576471 # See https://bugzilla.mozilla.org/show_bug.cgi?id=1576471
android_sdk_root = command_context.substs.get("ANDROID_SDK_ROOT", "") android_sdk_root = self.substs.get("ANDROID_SDK_ROOT", "")
if android_sdk_root: if android_sdk_root:
env["ANDROID_SDK_ROOT"] = android_sdk_root env["ANDROID_SDK_ROOT"] = android_sdk_root
return command_context.run_process( return self.run_process(
[command_context.substs["GRADLE"]] + gradle_flags + args, [self.substs["GRADLE"]] + gradle_flags + args,
explicit_env=env, explicit_env=env,
pass_thru=True, # Allow user to run gradle interactively. pass_thru=True, # Allow user to run gradle interactively.
ensure_exit_code=False, # Don't throw on non-zero exit code. ensure_exit_code=False, # Don't throw on non-zero exit code.
cwd=mozpath.join(command_context.topsrcdir), cwd=mozpath.join(self.topsrcdir),
) )
@Command("gradle-install", category="devenv", conditions=[REMOVED]) @Command("gradle-install", category="devenv", conditions=[REMOVED])
@ -520,10 +511,7 @@ class AndroidEmulatorCommands(MachCommandBase):
from mozrunner.devices.android_device import AndroidEmulator from mozrunner.devices.android_device import AndroidEmulator
emulator = AndroidEmulator( emulator = AndroidEmulator(
version, version, verbose, substs=self.substs, device_serial="emulator-5554"
verbose,
substs=command_context.substs,
device_serial="emulator-5554",
) )
if emulator.is_running(): if emulator.is_running():
# It is possible to run multiple emulators simultaneously, but: # It is possible to run multiple emulators simultaneously, but:
@ -532,7 +520,7 @@ class AndroidEmulatorCommands(MachCommandBase):
# - additional parameters must be specified when running tests, # - additional parameters must be specified when running tests,
# to select a specific device. # to select a specific device.
# To avoid these complications, allow just one emulator at a time. # To avoid these complications, allow just one emulator at a time.
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"emulator", "emulator",
{}, {},
@ -542,7 +530,7 @@ class AndroidEmulatorCommands(MachCommandBase):
return 1 return 1
if not emulator.is_available(): if not emulator.is_available():
command_context.log( self.log(
logging.WARN, logging.WARN,
"emulator", "emulator",
{}, {},
@ -552,7 +540,7 @@ class AndroidEmulatorCommands(MachCommandBase):
return 2 return 2
if not emulator.check_avd(force_update): if not emulator.check_avd(force_update):
command_context.log( self.log(
logging.INFO, logging.INFO,
"emulator", "emulator",
{}, {},
@ -560,7 +548,7 @@ class AndroidEmulatorCommands(MachCommandBase):
) )
emulator.update_avd(force_update) emulator.update_avd(force_update)
command_context.log( self.log(
logging.INFO, logging.INFO,
"emulator", "emulator",
{}, {},
@ -568,27 +556,25 @@ class AndroidEmulatorCommands(MachCommandBase):
) )
emulator.start(gpu) emulator.start(gpu)
if emulator.wait_for_start(): if emulator.wait_for_start():
command_context.log( self.log(logging.INFO, "emulator", {}, "Android emulator is running.")
logging.INFO, "emulator", {}, "Android emulator is running."
)
else: else:
# This is unusual but the emulator may still function. # This is unusual but the emulator may still function.
command_context.log( self.log(
logging.WARN, logging.WARN,
"emulator", "emulator",
{}, {},
"Unable to verify that emulator is running.", "Unable to verify that emulator is running.",
) )
if conditions.is_android(command_context): if conditions.is_android(self):
command_context.log( self.log(
logging.INFO, logging.INFO,
"emulator", "emulator",
{}, {},
"Use 'mach install' to install or update Firefox on your emulator.", "Use 'mach install' to install or update Firefox on your emulator.",
) )
else: else:
command_context.log( self.log(
logging.WARN, logging.WARN,
"emulator", "emulator",
{}, {},
@ -598,19 +584,19 @@ class AndroidEmulatorCommands(MachCommandBase):
) )
if wait: if wait:
command_context.log( self.log(
logging.INFO, "emulator", {}, "Waiting for Android emulator to close..." logging.INFO, "emulator", {}, "Waiting for Android emulator to close..."
) )
rc = emulator.wait() rc = emulator.wait()
if rc is not None: if rc is not None:
command_context.log( self.log(
logging.INFO, logging.INFO,
"emulator", "emulator",
{}, {},
"Android emulator completed with return code %d." % rc, "Android emulator completed with return code %d." % rc,
) )
else: else:
command_context.log( self.log(
logging.WARN, logging.WARN,
"emulator", "emulator",
{}, {},

76
python/mach/mach/commands/commandinfo.py
View file

@ -13,9 +13,14 @@ from itertools import chain
import attr import attr
from mach.decorators import CommandProvider, Command, CommandArgument, SubCommand from mach.decorators import (
CommandProvider,
Command,
CommandArgument,
SubCommand,
)
from mozbuild.base import MachCommandBase from mozbuild.base import MachCommandBase
from mozbuild.util import memoize from mozbuild.util import memoize, memoized_property
here = os.path.abspath(os.path.dirname(__file__)) here = os.path.abspath(os.path.dirname(__file__))
COMPLETION_TEMPLATES_DIR = os.path.join(here, "completion_templates") COMPLETION_TEMPLATES_DIR = os.path.join(here, "completion_templates")
@ -39,15 +44,15 @@ def render_template(shell, context):
@CommandProvider @CommandProvider
class BuiltinCommands(MachCommandBase): class BuiltinCommands(MachCommandBase):
@memoize @memoized_property
def command_handlers(self, command_context): def command_handlers(self):
"""A dictionary of command handlers keyed by command name.""" """A dictionary of command handlers keyed by command name."""
return command_context._mach_context.commands.command_handlers return self._mach_context.commands.command_handlers
@memoize @memoized_property
def commands(self, command_context): def commands(self):
"""A sorted list of all command names.""" """A sorted list of all command names."""
return sorted(self.command_handlers(command_context)) return sorted(self.command_handlers)
def _get_parser_options(self, parser): def _get_parser_options(self, parser):
options = {} options = {}
@ -63,13 +68,13 @@ class BuiltinCommands(MachCommandBase):
options[tuple(action.option_strings)] = action.help or "" options[tuple(action.option_strings)] = action.help or ""
return options return options
@memoize @memoized_property
def global_options(self, command_context): def global_options(self):
"""Return a dict of global options. """Return a dict of global options.
Of the form `{("-o", "--option"): "description"}`. Of the form `{("-o", "--option"): "description"}`.
""" """
for group in command_context._mach_context.global_parser._action_groups: for group in self._mach_context.global_parser._action_groups:
if group.title == "Global Arguments": if group.title == "Global Arguments":
return self._get_parser_options(group) return self._get_parser_options(group)
@ -112,21 +117,19 @@ class BuiltinCommands(MachCommandBase):
subcommand=handler.subcommand, subcommand=handler.subcommand,
) )
@memoize @memoized_property
def commands_info(self, command_context): def commands_info(self):
"""Return a list of CommandInfo objects for each command.""" """Return a list of CommandInfo objects for each command."""
commands_info = [] commands_info = []
# Loop over self.commands() rather than self.command_handlers().items() for # Loop over self.commands rather than self.command_handlers.items() for
# alphabetical order. # alphabetical order.
for c in self.commands(command_context): for c in self.commands:
commands_info.append( commands_info.append(self._get_handler_info(self.command_handlers[c]))
self._get_handler_info(self.command_handlers(command_context)[c])
)
return commands_info return commands_info
@Command("mach-commands", category="misc", description="List all mach commands.") @Command("mach-commands", category="misc", description="List all mach commands.")
def run_commands(self, command_context): def run_commands(self, command_context):
print("\n".join(self.commands(command_context))) print("\n".join(self.commands))
@Command( @Command(
"mach-debug-commands", "mach-debug-commands",
@ -143,7 +146,7 @@ class BuiltinCommands(MachCommandBase):
def run_debug_commands(self, command_context, match=None): def run_debug_commands(self, command_context, match=None):
import inspect import inspect
for command, handler in self.command_handlers(command_context).items(): for command, handler in self.command_handlers.items():
if match and match not in command: if match and match not in command:
continue continue
@ -168,23 +171,23 @@ class BuiltinCommands(MachCommandBase):
) )
def run_completion(self, command_context, args): def run_completion(self, command_context, args):
if not args: if not args:
print("\n".join(self.commands(command_context))) print("\n".join(self.commands))
return return
is_help = "help" in args is_help = "help" in args
command = None command = None
for i, arg in enumerate(args): for i, arg in enumerate(args):
if arg in self.commands(command_context): if arg in self.commands:
command = arg command = arg
args = args[i + 1 :] args = args[i + 1 :]
break break
# If no command is typed yet, just offer the commands. # If no command is typed yet, just offer the commands.
if not command: if not command:
print("\n".join(self.commands(command_context))) print("\n".join(self.commands))
return return
handler = self.command_handlers(command_context)[command] handler = self.command_handlers[command]
# If a subcommand was typed, update the handler. # If a subcommand was typed, update the handler.
for arg in args: for arg in args:
if arg in handler.subcommand_handlers: if arg in handler.subcommand_handlers:
@ -232,7 +235,7 @@ class BuiltinCommands(MachCommandBase):
commands_subcommands = [] commands_subcommands = []
case_options = [] case_options = []
case_subcommands = [] case_subcommands = []
for i, cmd in enumerate(self.commands_info(command_context)): for i, cmd in enumerate(self.commands_info):
# Build case statement for options. # Build case statement for options.
options = [] options = []
for opt_strs, description in cmd.options.items(): for opt_strs, description in cmd.options.items():
@ -298,13 +301,11 @@ class BuiltinCommands(MachCommandBase):
) )
) )
globalopts = [ globalopts = [opt for opt_strs in self.global_options for opt in opt_strs]
opt for opt_strs in self.global_options(command_context) for opt in opt_strs
]
context = { context = {
"case_options": "\n".join(case_options), "case_options": "\n".join(case_options),
"case_subcommands": "\n".join(case_subcommands), "case_subcommands": "\n".join(case_subcommands),
"commands": " ".join(self.commands(command_context)), "commands": " ".join(self.commands),
"commands_subcommands": " ".join(sorted(commands_subcommands)), "commands_subcommands": " ".join(sorted(commands_subcommands)),
"globalopts": " ".join(sorted(globalopts)), "globalopts": " ".join(sorted(globalopts)),
} }
@ -329,7 +330,7 @@ class BuiltinCommands(MachCommandBase):
commands_subcommands = [] commands_subcommands = []
case_options = [] case_options = []
case_subcommands = [] case_subcommands = []
for i, cmd in enumerate(self.commands_info(command_context)): for i, cmd in enumerate(self.commands_info):
commands_descriptions.append(self._zsh_describe(cmd.name, cmd.description)) commands_descriptions.append(self._zsh_describe(cmd.name, cmd.description))
# Build case statement for options. # Build case statement for options.
@ -392,7 +393,7 @@ class BuiltinCommands(MachCommandBase):
) )
globalopts = [] globalopts = []
for opt_strings, description in self.global_options(command_context).items(): for opt_strings, description in self.global_options.items():
for opt in opt_strings: for opt in opt_strings:
globalopts.append(self._zsh_describe(opt, description)) globalopts.append(self._zsh_describe(opt, description))
@ -429,7 +430,7 @@ class BuiltinCommands(MachCommandBase):
return comp return comp
globalopts = [] globalopts = []
for opt_strs, description in self.global_options(command_context).items(): for opt_strs, description in self.global_options.items():
comp = ( comp = (
"complete -c mach -n '__fish_mach_complete_no_command' " "complete -c mach -n '__fish_mach_complete_no_command' "
"-d '{}'".format(description.replace("'", "\\'")) "-d '{}'".format(description.replace("'", "\\'"))
@ -439,10 +440,13 @@ class BuiltinCommands(MachCommandBase):
cmds = [] cmds = []
cmds_opts = [] cmds_opts = []
for i, cmd in enumerate(self.commands_info(command_context)): for i, cmd in enumerate(self.commands_info):
cmds.append( cmds.append(
"complete -c mach -f -n '__fish_mach_complete_no_command' " "complete -c mach -f -n '__fish_mach_complete_no_command' "
"-a {} -d '{}'".format(cmd.name, cmd.description.replace("'", "\\'")) "-a {} -d '{}'".format(
cmd.name,
cmd.description.replace("'", "\\'"),
)
) )
cmds_opts += ["# {}".format(cmd.name)] cmds_opts += ["# {}".format(cmd.name)]
@ -480,11 +484,11 @@ class BuiltinCommands(MachCommandBase):
) )
cmds_opts.append(comp) cmds_opts.append(comp)
if i < len(self.commands(command_context)) - 1: if i < len(self.commands) - 1:
cmds_opts.append("") cmds_opts.append("")
context = { context = {
"commands": " ".join(self.commands(command_context)), "commands": " ".join(self.commands),
"command_completions": "\n".join(cmds), "command_completions": "\n".join(cmds),
"command_option_completions": "\n".join(cmds_opts), "command_option_completions": "\n".join(cmds_opts),
"global_option_completions": "\n".join(globalopts), "global_option_completions": "\n".join(globalopts),

14
python/mach/mach/commands/settings.py
View file

@ -7,7 +7,11 @@ from __future__ import absolute_import, print_function, unicode_literals
from textwrap import TextWrapper from textwrap import TextWrapper
from mach.config import TYPE_CLASSES from mach.config import TYPE_CLASSES
from mach.decorators import CommandArgument, CommandProvider, Command from mach.decorators import (
CommandArgument,
CommandProvider,
Command,
)
from mozbuild.base import MachCommandBase from mozbuild.base import MachCommandBase
@ -34,14 +38,12 @@ class Settings(MachCommandBase):
"""List available settings.""" """List available settings."""
types = {v: k for k, v in TYPE_CLASSES.items()} types = {v: k for k, v in TYPE_CLASSES.items()}
wrapper = TextWrapper(initial_indent="# ", subsequent_indent="# ") wrapper = TextWrapper(initial_indent="# ", subsequent_indent="# ")
for i, section in enumerate(sorted(command_context._mach_context.settings)): for i, section in enumerate(sorted(self._mach_context.settings)):
if not short: if not short:
print("%s[%s]" % ("" if i == 0 else "\n", section)) print("%s[%s]" % ("" if i == 0 else "\n", section))
for option in sorted( for option in sorted(self._mach_context.settings[section]._settings):
command_context._mach_context.settings[section]._settings meta = self._mach_context.settings[section].get_meta(option)
):
meta = command_context._mach_context.settings[section].get_meta(option)
desc = meta["description"] desc = meta["description"]
if short: if short:

8
python/mach/mach/test/test_decorators.py
View file

@ -57,10 +57,12 @@ def test_register_command_with_metrics_path(registrar):
class CommandFoo(MachCommandBase): class CommandFoo(MachCommandBase):
@Command("cmd_foo", category="testing", metrics_path=metrics_path) @Command("cmd_foo", category="testing", metrics_path=metrics_path)
def run_foo(self, command_context): def run_foo(self, command_context):
assert self.metrics == metrics_mock
assert command_context.metrics == metrics_mock assert command_context.metrics == metrics_mock
@SubCommand("cmd_foo", "sub_foo", metrics_path=metrics_path + "2") @SubCommand("cmd_foo", "sub_foo", metrics_path=metrics_path + "2")
def run_subfoo(self, command_context): def run_subfoo(self, command_context):
assert self.metrics == metrics_mock
assert command_context.metrics == metrics_mock assert command_context.metrics == metrics_mock
registrar.dispatch("cmd_foo", context) registrar.dispatch("cmd_foo", context)
@ -84,6 +86,9 @@ def test_register_command_sets_up_class_at_runtime(registrar):
class CommandFoo(MachCommandBase): class CommandFoo(MachCommandBase):
@Command("cmd_foo", category="testing", virtualenv_name="env_foo") @Command("cmd_foo", category="testing", virtualenv_name="env_foo")
def run_foo(self, command_context): def run_foo(self, command_context):
assert (
os.path.basename(self.virtualenv_manager.virtualenv_root) == "env_foo"
)
assert ( assert (
os.path.basename(command_context.virtualenv_manager.virtualenv_root) os.path.basename(command_context.virtualenv_manager.virtualenv_root)
== "env_foo" == "env_foo"
@ -92,6 +97,9 @@ def test_register_command_sets_up_class_at_runtime(registrar):
@Command("cmd_bar", category="testing", virtualenv_name="env_bar") @Command("cmd_bar", category="testing", virtualenv_name="env_bar")
def run_bar(self, command_context): def run_bar(self, command_context):
assert (
os.path.basename(self.virtualenv_manager.virtualenv_root) == "env_bar"
)
assert ( assert (
os.path.basename(command_context.virtualenv_manager.virtualenv_root) os.path.basename(command_context.virtualenv_manager.virtualenv_root)
== "env_bar" == "env_bar"

66
python/mach_commands.py
View file

@ -63,7 +63,7 @@ class MachCommands(MachCommandBase):
args, args,
): ):
# Avoid logging the command # Avoid logging the command
command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL) self.log_manager.terminal_handler.setLevel(logging.CRITICAL)
# Note: subprocess requires native strings in os.environ on Windows. # Note: subprocess requires native strings in os.environ on Windows.
append_env = {"PYTHONDONTWRITEBYTECODE": str("1")} append_env = {"PYTHONDONTWRITEBYTECODE": str("1")}
@ -75,16 +75,14 @@ class MachCommands(MachCommandBase):
from mach_bootstrap import mach_sys_path from mach_bootstrap import mach_sys_path
python_path = sys.executable python_path = sys.executable
append_env["PYTHONPATH"] = os.pathsep.join( append_env["PYTHONPATH"] = os.pathsep.join(mach_sys_path(self.topsrcdir))
mach_sys_path(command_context.topsrcdir)
)
else: else:
command_context.virtualenv_manager.ensure() self.virtualenv_manager.ensure()
if not no_activate: if not no_activate:
command_context.virtualenv_manager.activate() self.virtualenv_manager.activate()
python_path = command_context.virtualenv_manager.python_path python_path = self.virtualenv_manager.python_path
if requirements: if requirements:
command_context.virtualenv_manager.install_pip_requirements( self.virtualenv_manager.install_pip_requirements(
requirements, require_hashes=False requirements, require_hashes=False
) )
@ -99,14 +97,14 @@ class MachCommands(MachCommandBase):
if not no_virtualenv: if not no_virtualenv:
# Use `_run_pip` directly rather than `install_pip_package` to bypass # Use `_run_pip` directly rather than `install_pip_package` to bypass
# `req.check_if_exists()` which may detect a system installed ipython. # `req.check_if_exists()` which may detect a system installed ipython.
command_context.virtualenv_manager._run_pip(["install", "ipython"]) self.virtualenv_manager._run_pip(["install", "ipython"])
python_path = which("ipython", path=bindir) python_path = which("ipython", path=bindir)
if not python_path: if not python_path:
print("error: could not detect or install ipython") print("error: could not detect or install ipython")
return 1 return 1
return command_context.run_process( return self.run_process(
[python_path] + args, [python_path] + args,
pass_thru=True, # Allow user to run Python interactively. pass_thru=True, # Allow user to run Python interactively.
ensure_exit_code=False, # Don't throw on non-zero exit code. ensure_exit_code=False, # Don't throw on non-zero exit code.
@ -171,7 +169,7 @@ class MachCommands(MachCommandBase):
os.environ[b"PYTHON_TEST_TMP"] = tempdir os.environ[b"PYTHON_TEST_TMP"] = tempdir
else: else:
os.environ["PYTHON_TEST_TMP"] = tempdir os.environ["PYTHON_TEST_TMP"] = tempdir
return self.run_python_tests(command_context, *args, **kwargs) return self.run_python_tests(*args, **kwargs)
finally: finally:
import mozfile import mozfile
@ -179,7 +177,6 @@ class MachCommands(MachCommandBase):
def run_python_tests( def run_python_tests(
self, self,
command_context,
tests=None, tests=None,
test_objects=None, test_objects=None,
subsuite=None, subsuite=None,
@ -190,11 +187,11 @@ class MachCommands(MachCommandBase):
**kwargs **kwargs
): ):
command_context.activate_virtualenv() self.activate_virtualenv()
if test_objects is None: if test_objects is None:
from moztest.resolve import TestResolver from moztest.resolve import TestResolver
resolver = command_context._spawn(TestResolver) resolver = self._spawn(TestResolver)
# If we were given test paths, try to find tests matching them. # If we were given test paths, try to find tests matching them.
test_objects = resolver.resolve_tests(paths=tests, flavor="python") test_objects = resolver.resolve_tests(paths=tests, flavor="python")
else: else:
@ -215,7 +212,7 @@ class MachCommands(MachCommandBase):
tests = mp.active_tests( tests = mp.active_tests(
filters=filters, filters=filters,
disabled=False, disabled=False,
python=command_context.virtualenv_manager.version_info()[0], python=self.virtualenv_manager.version_info()[0],
**mozinfo.info **mozinfo.info
) )
@ -225,7 +222,7 @@ class MachCommands(MachCommandBase):
"TEST-UNEXPECTED-FAIL | No tests collected " "TEST-UNEXPECTED-FAIL | No tests collected "
+ "{}(Not in PYTHON_UNITTEST_MANIFESTS?)".format(submsg) + "{}(Not in PYTHON_UNITTEST_MANIFESTS?)".format(submsg)
) )
command_context.log(logging.WARN, "python-test", {}, message) self.log(logging.WARN, "python-test", {}, message)
return 1 return 1
parallel = [] parallel = []
@ -241,7 +238,7 @@ class MachCommands(MachCommandBase):
test.get("requirements") test.get("requirements")
and test["requirements"] not in installed_requirements and test["requirements"] not in installed_requirements
): ):
command_context.virtualenv_manager.install_pip_requirements( self.virtualenv_manager.install_pip_requirements(
test["requirements"], quiet=True test["requirements"], quiet=True
) )
installed_requirements.add(test["requirements"]) installed_requirements.add(test["requirements"])
@ -256,7 +253,9 @@ class MachCommands(MachCommandBase):
else: else:
parallel.append(test) parallel.append(test)
jobs = jobs or cpu_count() self.jobs = jobs or cpu_count()
self.terminate = False
self.verbose = verbose
return_code = 0 return_code = 0
@ -264,12 +263,10 @@ class MachCommands(MachCommandBase):
output, ret, test_path = result output, ret, test_path = result
for line in output: for line in output:
command_context.log( self.log(logging.INFO, "python-test", {"line": line.rstrip()}, "{line}")
logging.INFO, "python-test", {"line": line.rstrip()}, "{line}"
)
if ret and not return_code: if ret and not return_code:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"python-test", "python-test",
{"test_path": test_path, "ret": ret}, {"test_path": test_path, "ret": ret},
@ -277,12 +274,9 @@ class MachCommands(MachCommandBase):
) )
return return_code or ret return return_code or ret
with ThreadPoolExecutor(max_workers=jobs) as executor: with ThreadPoolExecutor(max_workers=self.jobs) as executor:
futures = [ futures = [
executor.submit( executor.submit(self._run_python_test, test) for test in parallel
self._run_python_test, command_context, test, jobs, verbose
)
for test in parallel
] ]
try: try:
@ -296,13 +290,11 @@ class MachCommands(MachCommandBase):
raise raise
for test in sequential: for test in sequential:
return_code = on_test_finished( return_code = on_test_finished(self._run_python_test(test))
self._run_python_test(command_context, test, jobs, verbose)
)
if return_code and exitfirst: if return_code and exitfirst:
break break
command_context.log( self.log(
logging.INFO, logging.INFO,
"python-test", "python-test",
{"return_code": return_code}, {"return_code": return_code},
@ -310,19 +302,17 @@ class MachCommands(MachCommandBase):
) )
return return_code return return_code
def _run_python_test(self, command_context, test, jobs, verbose): def _run_python_test(self, test):
from mozprocess import ProcessHandler from mozprocess import ProcessHandler
output = [] output = []
def _log(line): def _log(line):
# Buffer messages if more than one worker to avoid interleaving # Buffer messages if more than one worker to avoid interleaving
if jobs > 1: if self.jobs > 1:
output.append(line) output.append(line)
else: else:
command_context.log( self.log(logging.INFO, "python-test", {"line": line.rstrip()}, "{line}")
logging.INFO, "python-test", {"line": line.rstrip()}, "{line}"
)
file_displayed_test = [] # used as boolean file_displayed_test = [] # used as boolean
@ -342,7 +332,7 @@ class MachCommands(MachCommandBase):
_log(line) _log(line)
_log(test["path"]) _log(test["path"])
python = command_context.virtualenv_manager.python_path python = self.virtualenv_manager.python_path
cmd = [python, test["path"]] cmd = [python, test["path"]]
env = os.environ.copy() env = os.environ.copy()
if six.PY2: if six.PY2:
@ -363,7 +353,7 @@ class MachCommands(MachCommandBase):
"call?): {}".format(test["path"]) "call?): {}".format(test["path"])
) )
if verbose: if self.verbose:
if return_code != 0: if return_code != 0:
_log("Test failed: {}".format(test["path"])) _log("Test failed: {}".format(test["path"]))
else: else:

35
python/mozboot/mozboot/mach_commands.py
View file

@ -7,7 +7,11 @@ from __future__ import absolute_import, print_function, unicode_literals
import errno import errno
import sys import sys
from mach.decorators import CommandArgument, CommandProvider, Command from mach.decorators import (
CommandArgument,
CommandProvider,
Command,
)
from mozbuild.base import MachCommandBase from mozbuild.base import MachCommandBase
from mozboot.bootstrap import APPLICATIONS from mozboot.bootstrap import APPLICATIONS
@ -35,17 +39,20 @@ class Bootstrap(MachCommandBase):
help="Only execute actions that leave the system " "configuration alone.", help="Only execute actions that leave the system " "configuration alone.",
) )
def bootstrap( def bootstrap(
self, command_context, application_choice=None, no_system_changes=False self,
command_context,
application_choice=None,
no_system_changes=False,
): ):
from mozboot.bootstrap import Bootstrapper from mozboot.bootstrap import Bootstrapper
bootstrapper = Bootstrapper( bootstrapper = Bootstrapper(
choice=application_choice, choice=application_choice,
no_interactive=not command_context._mach_context.is_interactive, no_interactive=not self._mach_context.is_interactive,
no_system_changes=no_system_changes, no_system_changes=no_system_changes,
mach_context=command_context._mach_context, mach_context=self._mach_context,
) )
bootstrapper.bootstrap(command_context.settings) bootstrapper.bootstrap(self.settings)
@CommandProvider @CommandProvider
@ -80,9 +87,7 @@ class VersionControlCommands(MachCommandBase):
import mozversioncontrol import mozversioncontrol
from mozfile import which from mozfile import which
repo = mozversioncontrol.get_repository_object( repo = mozversioncontrol.get_repository_object(self._mach_context.topdir)
command_context._mach_context.topdir
)
tool = "hg" tool = "hg"
if repo.name == "git": if repo.name == "git":
tool = "git" tool = "git"
@ -100,21 +105,17 @@ class VersionControlCommands(MachCommandBase):
if update_only: if update_only:
if repo.name == "git": if repo.name == "git":
bootstrap.update_git_tools( bootstrap.update_git_tools(
vcs, vcs, self._mach_context.state_dir, self._mach_context.topdir
command_context._mach_context.state_dir,
command_context._mach_context.topdir,
) )
else: else:
bootstrap.update_vct(vcs, command_context._mach_context.state_dir) bootstrap.update_vct(vcs, self._mach_context.state_dir)
else: else:
if repo.name == "git": if repo.name == "git":
bootstrap.configure_git( bootstrap.configure_git(
vcs, vcs,
which("git-cinnabar"), which("git-cinnabar"),
command_context._mach_context.state_dir, self._mach_context.state_dir,
command_context._mach_context.topdir, self._mach_context.topdir,
) )
else: else:
bootstrap.configure_mercurial( bootstrap.configure_mercurial(vcs, self._mach_context.state_dir)
vcs, command_context._mach_context.state_dir
)

112
python/mozbuild/mozbuild/artifact_commands.py
View file

@ -13,9 +13,17 @@ import six
from collections import OrderedDict from collections import OrderedDict
from mach.decorators import CommandArgument, CommandProvider, Command, SubCommand from mach.decorators import (
CommandArgument,
CommandProvider,
Command,
SubCommand,
)
from mozbuild.artifact_builds import JOB_CHOICES from mozbuild.artifact_builds import JOB_CHOICES
from mozbuild.base import MachCommandBase, MachCommandConditions as conditions from mozbuild.base import (
MachCommandBase,
MachCommandConditions as conditions,
)
from mozbuild.util import ensureParentDir from mozbuild.util import ensureParentDir
import mozversioncontrol import mozversioncontrol
@ -80,7 +88,6 @@ class PackageFrontend(MachCommandBase):
def _make_artifacts( def _make_artifacts(
self, self,
command_context,
tree=None, tree=None,
job=None, job=None,
skip_cache=False, skip_cache=False,
@ -90,21 +97,19 @@ class PackageFrontend(MachCommandBase):
download_maven_zip=False, download_maven_zip=False,
no_process=False, no_process=False,
): ):
state_dir = command_context._mach_context.state_dir state_dir = self._mach_context.state_dir
cache_dir = os.path.join(state_dir, "package-frontend") cache_dir = os.path.join(state_dir, "package-frontend")
hg = None hg = None
if conditions.is_hg(command_context): if conditions.is_hg(self):
hg = command_context.substs["HG"] hg = self.substs["HG"]
git = None git = None
if conditions.is_git(command_context): if conditions.is_git(self):
git = command_context.substs["GIT"] git = self.substs["GIT"]
# If we're building Thunderbird, we should be checking for comm-central artifacts. # If we're building Thunderbird, we should be checking for comm-central artifacts.
topsrcdir = command_context.substs.get( topsrcdir = self.substs.get("commtopsrcdir", self.topsrcdir)
"commtopsrcdir", command_context.topsrcdir
)
if download_maven_zip: if download_maven_zip:
if download_tests: if download_tests:
@ -120,10 +125,10 @@ class PackageFrontend(MachCommandBase):
artifacts = Artifacts( artifacts = Artifacts(
tree, tree,
command_context.substs, self.substs,
command_context.defines, self.defines,
job, job,
log=command_context.log, log=self.log,
cache_dir=cache_dir, cache_dir=cache_dir,
skip_cache=skip_cache, skip_cache=skip_cache,
hg=hg, hg=hg,
@ -134,7 +139,7 @@ class PackageFrontend(MachCommandBase):
download_host_bins=download_host_bins, download_host_bins=download_host_bins,
download_maven_zip=download_maven_zip, download_maven_zip=download_maven_zip,
no_process=no_process, no_process=no_process,
mozbuild=command_context, mozbuild=self,
) )
return artifacts return artifacts
@ -184,9 +189,8 @@ class PackageFrontend(MachCommandBase):
no_process=False, no_process=False,
maven_zip=False, maven_zip=False,
): ):
command_context._set_log_level(verbose) self._set_log_level(verbose)
artifacts = self._make_artifacts( artifacts = self._make_artifacts(
command_context,
tree=tree, tree=tree,
job=job, job=job,
skip_cache=skip_cache, skip_cache=skip_cache,
@ -197,7 +201,7 @@ class PackageFrontend(MachCommandBase):
no_process=no_process, no_process=no_process,
) )
return artifacts.install_from(source, distdir or command_context.distdir) return artifacts.install_from(source, distdir or self.distdir)
@ArtifactSubCommand( @ArtifactSubCommand(
"artifact", "artifact",
@ -205,8 +209,8 @@ class PackageFrontend(MachCommandBase):
"Delete local artifacts and reset local artifact cache.", "Delete local artifacts and reset local artifact cache.",
) )
def artifact_clear_cache(self, command_context, tree=None, job=None, verbose=False): def artifact_clear_cache(self, command_context, tree=None, job=None, verbose=False):
command_context._set_log_level(verbose) self._set_log_level(verbose)
artifacts = self._make_artifacts(command_context, tree=tree, job=job) artifacts = self._make_artifacts(tree=tree, job=job)
artifacts.clear_cache() artifacts.clear_cache()
return 0 return 0
@ -269,7 +273,11 @@ class PackageFrontend(MachCommandBase):
): ):
"""Download, cache and install pre-built toolchains.""" """Download, cache and install pre-built toolchains."""
from mozbuild.artifacts import ArtifactCache from mozbuild.artifacts import ArtifactCache
from mozbuild.action.tooltool import FileRecord, open_manifest, unpack_file from mozbuild.action.tooltool import (
FileRecord,
open_manifest,
unpack_file,
)
import redo import redo
import requests import requests
import time import time
@ -277,26 +285,22 @@ class PackageFrontend(MachCommandBase):
from taskgraph.util.taskcluster import get_artifact_url from taskgraph.util.taskcluster import get_artifact_url
start = time.time() start = time.time()
command_context._set_log_level(verbose) self._set_log_level(verbose)
# Normally, we'd use command_context.log_manager.enable_unstructured(), # Normally, we'd use self.log_manager.enable_unstructured(),
# but that enables all logging, while we only really want tooltool's # but that enables all logging, while we only really want tooltool's
# and it also makes structured log output twice. # and it also makes structured log output twice.
# So we manually do what it does, and limit that to the tooltool # So we manually do what it does, and limit that to the tooltool
# logger. # logger.
if command_context.log_manager.terminal_handler: if self.log_manager.terminal_handler:
logging.getLogger("mozbuild.action.tooltool").addHandler( logging.getLogger("mozbuild.action.tooltool").addHandler(
command_context.log_manager.terminal_handler self.log_manager.terminal_handler
) )
logging.getLogger("redo").addHandler( logging.getLogger("redo").addHandler(self.log_manager.terminal_handler)
command_context.log_manager.terminal_handler self.log_manager.terminal_handler.addFilter(
) self.log_manager.structured_filter
command_context.log_manager.terminal_handler.addFilter(
command_context.log_manager.structured_filter
) )
if not cache_dir: if not cache_dir:
cache_dir = os.path.join( cache_dir = os.path.join(self._mach_context.state_dir, "toolchains")
command_context._mach_context.state_dir, "toolchains"
)
tooltool_host = os.environ.get("TOOLTOOL_HOST", "tooltool.mozilla-releng.net") tooltool_host = os.environ.get("TOOLTOOL_HOST", "tooltool.mozilla-releng.net")
taskcluster_proxy_url = os.environ.get("TASKCLUSTER_PROXY_URL") taskcluster_proxy_url = os.environ.get("TASKCLUSTER_PROXY_URL")
@ -305,9 +309,7 @@ class PackageFrontend(MachCommandBase):
else: else:
tooltool_url = "https://{}".format(tooltool_host) tooltool_url = "https://{}".format(tooltool_host)
cache = ArtifactCache( cache = ArtifactCache(cache_dir=cache_dir, log=self.log, skip_cache=skip_cache)
cache_dir=cache_dir, log=command_context.log, skip_cache=skip_cache
)
class DownloadRecord(FileRecord): class DownloadRecord(FileRecord):
def __init__(self, url, *args, **kwargs): def __init__(self, url, *args, **kwargs):
@ -374,7 +376,7 @@ class PackageFrontend(MachCommandBase):
if from_build: if from_build:
if "MOZ_AUTOMATION" in os.environ: if "MOZ_AUTOMATION" in os.environ:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"artifact", "artifact",
{}, {},
@ -395,7 +397,7 @@ class PackageFrontend(MachCommandBase):
task = tasks.get(b) task = tasks.get(b)
if not task: if not task:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"artifact", "artifact",
{"build": user_value}, {"build": user_value},
@ -408,7 +410,7 @@ class PackageFrontend(MachCommandBase):
# are built on trunk projects, so the task will be available to # are built on trunk projects, so the task will be available to
# install here. # install here.
if bootstrap and not task.attributes.get("local-toolchain"): if bootstrap and not task.attributes.get("local-toolchain"):
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"artifact", "artifact",
{"build": user_value}, {"build": user_value},
@ -417,7 +419,7 @@ class PackageFrontend(MachCommandBase):
return 1 return 1
artifact_name = task.attributes.get("toolchain-artifact") artifact_name = task.attributes.get("toolchain-artifact")
command_context.log( self.log(
logging.DEBUG, logging.DEBUG,
"artifact", "artifact",
{ {
@ -431,21 +433,19 @@ class PackageFrontend(MachCommandBase):
task, {}, deadline, task.optimization.get("index-search", []) task, {}, deadline, task.optimization.get("index-search", [])
) )
if task_id in (True, False) or not artifact_name: if task_id in (True, False) or not artifact_name:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"artifact", "artifact",
{"build": user_value}, {"build": user_value},
_COULD_NOT_FIND_ARTIFACTS_TEMPLATE, _COULD_NOT_FIND_ARTIFACTS_TEMPLATE,
) )
# Get and print some helpful info for diagnosis. # Get and print some helpful info for diagnosis.
repo = mozversioncontrol.get_repository_object( repo = mozversioncontrol.get_repository_object(self.topsrcdir)
command_context.topsrcdir
)
changed_files = set(repo.get_outgoing_files()) | set( changed_files = set(repo.get_outgoing_files()) | set(
repo.get_changed_files() repo.get_changed_files()
) )
if changed_files: if changed_files:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"artifact", "artifact",
{}, {},
@ -453,7 +453,7 @@ class PackageFrontend(MachCommandBase):
"to the following files: %s" % sorted(changed_files), "to the following files: %s" % sorted(changed_files),
) )
if "TASKCLUSTER_ROOT_URL" in os.environ: if "TASKCLUSTER_ROOT_URL" in os.environ:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"artifact", "artifact",
{"build": user_value}, {"build": user_value},
@ -466,7 +466,7 @@ class PackageFrontend(MachCommandBase):
) )
return 1 return 1
command_context.log( self.log(
logging.DEBUG, logging.DEBUG,
"artifact", "artifact",
{"name": artifact_name, "task_id": task_id}, {"name": artifact_name, "task_id": task_id},
@ -477,7 +477,7 @@ class PackageFrontend(MachCommandBase):
records[record.filename] = record records[record.filename] = record
for record in six.itervalues(records): for record in six.itervalues(records):
command_context.log( self.log(
logging.INFO, logging.INFO,
"artifact", "artifact",
{"name": record.basename}, {"name": record.basename},
@ -507,11 +507,11 @@ class PackageFrontend(MachCommandBase):
level = logging.WARN level = logging.WARN
else: else:
level = logging.ERROR level = logging.ERROR
command_context.log(level, "artifact", {}, str(e)) self.log(level, "artifact", {}, str(e))
if not should_retry: if not should_retry:
break break
if attempt < retry: if attempt < retry:
command_context.log( self.log(
logging.INFO, "artifact", {}, "Will retry in a moment..." logging.INFO, "artifact", {}, "Will retry in a moment..."
) )
continue continue
@ -522,7 +522,7 @@ class PackageFrontend(MachCommandBase):
if not valid: if not valid:
os.unlink(record.filename) os.unlink(record.filename)
if attempt < retry: if attempt < retry:
command_context.log( self.log(
logging.INFO, logging.INFO,
"artifact", "artifact",
{}, {},
@ -534,7 +534,7 @@ class PackageFrontend(MachCommandBase):
break break
if not valid: if not valid:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"artifact", "artifact",
{"name": record.basename}, {"name": record.basename},
@ -566,13 +566,15 @@ class PackageFrontend(MachCommandBase):
if not data: if not data:
break break
h.update(data) h.update(data)
artifacts[record.url] = {"sha256": h.hexdigest()} artifacts[record.url] = {
"sha256": h.hexdigest(),
}
if record.unpack and not no_unpack: if record.unpack and not no_unpack:
unpack_file(local) unpack_file(local)
os.unlink(local) os.unlink(local)
if not downloaded: if not downloaded:
command_context.log(logging.ERROR, "artifact", {}, "Nothing to download") self.log(logging.ERROR, "artifact", {}, "Nothing to download")
if artifacts: if artifacts:
ensureParentDir(artifact_manifest) ensureParentDir(artifact_manifest)
@ -594,7 +596,7 @@ class PackageFrontend(MachCommandBase):
} }
], ],
} }
command_context.log( self.log(
logging.INFO, logging.INFO,
"perfherder", "perfherder",
{"data": json.dumps(perfherder_data)}, {"data": json.dumps(perfherder_data)},

103
python/mozbuild/mozbuild/backend/mach_commands.py
View file

@ -34,13 +34,13 @@ class MachCommands(MachCommandBase):
backend = "Clangd" backend = "Clangd"
if ide == "eclipse" and not which("eclipse"): if ide == "eclipse" and not which("eclipse"):
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"ide", "ide",
{}, {},
"Eclipse CDT 8.4 or later must be installed in your PATH.", "Eclipse CDT 8.4 or later must be installed in your PATH.",
) )
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"ide", "ide",
{}, {},
@ -50,15 +50,12 @@ class MachCommands(MachCommandBase):
if ide == "vscode": if ide == "vscode":
# Verify if platform has VSCode installed # Verify if platform has VSCode installed
vscode_cmd = self.found_vscode_path(command_context) if not self.found_vscode_path():
if vscode_cmd is None: self.log(logging.ERROR, "ide", {}, "VSCode cannot be found, abording!")
command_context.log(
logging.ERROR, "ide", {}, "VSCode cannot be found, aborting!"
)
return 1 return 1
# Create the Build environment to configure the tree # Create the Build environment to configure the tree
builder = Build(command_context._mach_context, None) builder = Build(self._mach_context, None)
rc = builder.configure(command_context) rc = builder.configure(command_context)
if rc != 0: if rc != 0:
@ -66,9 +63,7 @@ class MachCommands(MachCommandBase):
# First install what we can through install manifests. # First install what we can through install manifests.
rc = builder._run_make( rc = builder._run_make(
directory=command_context.topobjdir, directory=self.topobjdir, target="pre-export", line_handler=None
target="pre-export",
line_handler=None,
) )
if rc != 0: if rc != 0:
return rc return rc
@ -77,9 +72,7 @@ class MachCommands(MachCommandBase):
# export target, because we can't do anything better. # export target, because we can't do anything better.
for target in ("export", "pre-compile"): for target in ("export", "pre-compile"):
rc = builder._run_make( rc = builder._run_make(
directory=command_context.topobjdir, directory=self.topobjdir, target=target, line_handler=None
target=target,
line_handler=None,
) )
if rc != 0: if rc != 0:
return rc return rc
@ -87,53 +80,47 @@ class MachCommands(MachCommandBase):
# Here we refresh the whole build. 'build export' is sufficient here and is # Here we refresh the whole build. 'build export' is sufficient here and is
# probably more correct but it's also nice having a single target to get a fully # probably more correct but it's also nice having a single target to get a fully
# built and indexed project (gives a easy target to use before go out to lunch). # built and indexed project (gives a easy target to use before go out to lunch).
res = command_context._mach_context.commands.dispatch( res = self._mach_context.commands.dispatch("build", self._mach_context)
"build", command_context._mach_context
)
if res != 0: if res != 0:
return 1 return 1
# Generate or refresh the IDE backend. # Generate or refresh the IDE backend.
python = command_context.virtualenv_manager.python_path python = self.virtualenv_manager.python_path
config_status = os.path.join(command_context.topobjdir, "config.status") config_status = os.path.join(self.topobjdir, "config.status")
args = [python, config_status, "--backend=%s" % backend] args = [python, config_status, "--backend=%s" % backend]
res = command_context._run_command_in_objdir( res = self._run_command_in_objdir(
args=args, pass_thru=True, ensure_exit_code=False args=args, pass_thru=True, ensure_exit_code=False
) )
if res != 0: if res != 0:
return 1 return 1
if ide == "eclipse": if ide == "eclipse":
eclipse_workspace_dir = self.get_eclipse_workspace_path(command_context) eclipse_workspace_dir = self.get_eclipse_workspace_path()
subprocess.check_call(["eclipse", "-data", eclipse_workspace_dir]) subprocess.check_call(["eclipse", "-data", eclipse_workspace_dir])
elif ide == "visualstudio": elif ide == "visualstudio":
visual_studio_workspace_dir = self.get_visualstudio_workspace_path( visual_studio_workspace_dir = self.get_visualstudio_workspace_path()
command_context
)
subprocess.check_call(["explorer.exe", visual_studio_workspace_dir]) subprocess.check_call(["explorer.exe", visual_studio_workspace_dir])
elif ide == "vscode": elif ide == "vscode":
return self.setup_vscode(command_context, vscode_cmd) return self.setup_vscode()
def get_eclipse_workspace_path(self, command_context): def get_eclipse_workspace_path(self):
from mozbuild.backend.cpp_eclipse import CppEclipseBackend from mozbuild.backend.cpp_eclipse import CppEclipseBackend
return CppEclipseBackend.get_workspace_path( return CppEclipseBackend.get_workspace_path(self.topsrcdir, self.topobjdir)
command_context.topsrcdir, command_context.topobjdir
)
def get_visualstudio_workspace_path(self, command_context): def get_visualstudio_workspace_path(self):
return os.path.join(command_context.topobjdir, "msvc", "mozilla.sln") return os.path.join(self.topobjdir, "msvc", "mozilla.sln")
def found_vscode_path(self, command_context): def found_vscode_path(self):
if "linux" in command_context.platform[0]: if "linux" in self.platform[0]:
cmd_and_path = [ cmd_and_path = [
{"path": "/usr/local/bin/code", "cmd": ["/usr/local/bin/code"]}, {"path": "/usr/local/bin/code", "cmd": ["/usr/local/bin/code"]},
{"path": "/snap/bin/code", "cmd": ["/snap/bin/code"]}, {"path": "/snap/bin/code", "cmd": ["/snap/bin/code"]},
{"path": "/usr/bin/code", "cmd": ["/usr/bin/code"]}, {"path": "/usr/bin/code", "cmd": ["/usr/bin/code"]},
{"path": "/usr/bin/code-insiders", "cmd": ["/usr/bin/code-insiders"]}, {"path": "/usr/bin/code-insiders", "cmd": ["/usr/bin/code-insiders"]},
] ]
elif "macos" in command_context.platform[0]: elif "macos" in self.platform[0]:
cmd_and_path = [ cmd_and_path = [
{"path": "/usr/local/bin/code", "cmd": ["/usr/local/bin/code"]}, {"path": "/usr/local/bin/code", "cmd": ["/usr/local/bin/code"]},
{ {
@ -149,7 +136,7 @@ class MachCommands(MachCommandBase):
], ],
}, },
] ]
elif "win64" in command_context.platform[0]: elif "win64" in self.platform[0]:
from pathlib import Path from pathlib import Path
vscode_path = mozpath.join( vscode_path = mozpath.join(
@ -176,44 +163,42 @@ class MachCommands(MachCommandBase):
# Did we guess the path? # Did we guess the path?
for element in cmd_and_path: for element in cmd_and_path:
if os.path.exists(element["path"]): if os.path.exists(element["path"]):
return element["cmd"] self.vscode_cmd = element["cmd"]
return True
for _ in range(5): for _ in range(5):
vscode_path = input( vscode_path = input(
"Could not find the VSCode binary. Please provide the full path to it:\n" "Could not find the VSCode binary. Please provide the full path to it:\n"
) )
if os.path.exists(vscode_path): if os.path.exists(vscode_path):
return [vscode_path] self.vscode_cmd = [vscode_path]
return True
# Path cannot be found # Path cannot be found
return None return False
def setup_vscode(self, command_context, vscode_cmd): def setup_vscode(self):
vscode_settings = mozpath.join( vscode_settings = mozpath.join(self.topsrcdir, ".vscode", "settings.json")
command_context.topsrcdir, ".vscode", "settings.json"
)
clangd_cc_path = mozpath.join(command_context.topobjdir, "clangd") clangd_cc_path = mozpath.join(self.topobjdir, "clangd")
# Verify if the required files are present # Verify if the required files are present
clang_tools_path = mozpath.join( clang_tools_path = mozpath.join(self._mach_context.state_dir, "clang-tools")
command_context._mach_context.state_dir, "clang-tools"
)
clang_tidy_bin = mozpath.join(clang_tools_path, "clang-tidy", "bin") clang_tidy_bin = mozpath.join(clang_tools_path, "clang-tidy", "bin")
clangd_path = mozpath.join( clangd_path = mozpath.join(
clang_tidy_bin, clang_tidy_bin,
"clangd" + command_context.config_environment.substs.get("BIN_SUFFIX", ""), "clangd" + self.config_environment.substs.get("BIN_SUFFIX", ""),
) )
if not os.path.exists(clangd_path): if not os.path.exists(clangd_path):
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"ide", "ide",
{}, {},
"Unable to locate clangd in {}.".format(clang_tidy_bin), "Unable to locate clangd in {}.".format(clang_tidy_bin),
) )
rc = self._get_clang_tools(command_context, clang_tools_path) rc = self._get_clang_tools(clang_tools_path)
if rc != 0: if rc != 0:
return rc return rc
@ -222,7 +207,7 @@ class MachCommands(MachCommandBase):
import json import json
from mozbuild.code_analysis.utils import ClangTidyConfig from mozbuild.code_analysis.utils import ClangTidyConfig
clang_tidy_cfg = ClangTidyConfig(command_context.topsrcdir) clang_tidy_cfg = ClangTidyConfig(self.topsrcdir)
clangd_json = json.loads( clangd_json = json.loads(
""" """
@ -300,21 +285,21 @@ class MachCommands(MachCommandBase):
fh.write(json.dumps(settings, indent=4)) fh.write(json.dumps(settings, indent=4))
# Open vscode with new configuration # Open vscode with new configuration
rc = subprocess.call(vscode_cmd + [command_context.topsrcdir]) rc = subprocess.call(self.vscode_cmd + [self.topsrcdir])
if rc != 0: if rc != 0:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"ide", "ide",
{}, {},
"Unable to open VS Code. Please open VS Code manually and load " "Unable to open VS Code. Please open VS Code manually and load "
"directory: {}".format(command_context.topsrcdir), "directory: {}".format(self.topsrcdir),
) )
return rc return rc
return 0 return 0
def _get_clang_tools(self, command_context, clang_tools_path): def _get_clang_tools(self, clang_tools_path):
import shutil import shutil
@ -326,12 +311,12 @@ class MachCommands(MachCommandBase):
from mozbuild.artifact_commands import PackageFrontend from mozbuild.artifact_commands import PackageFrontend
_artifact_manager = PackageFrontend(command_context._mach_context) self._artifact_manager = PackageFrontend(self._mach_context)
job, _ = command_context.platform job, _ = self.platform
if job is None: if job is None:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"ide", "ide",
{}, {},
@ -346,8 +331,8 @@ class MachCommands(MachCommandBase):
# We want to unpack data in the clang-tidy mozbuild folder # We want to unpack data in the clang-tidy mozbuild folder
currentWorkingDir = os.getcwd() currentWorkingDir = os.getcwd()
os.chdir(clang_tools_path) os.chdir(clang_tools_path)
rc = _artifact_manager.artifact_toolchain( rc = self._artifact_manager.artifact_toolchain(
command_context, verbose=False, from_build=[job], no_unpack=False, retry=0 self, verbose=False, from_build=[job], no_unpack=False, retry=0
) )
# Change back the cwd # Change back the cwd
os.chdir(currentWorkingDir) os.chdir(currentWorkingDir)

32
python/mozbuild/mozbuild/build_commands.py
View file

@ -91,9 +91,9 @@ class Build(MachCommandBase):
""" """
from mozbuild.controller.building import BuildDriver from mozbuild.controller.building import BuildDriver
command_context.log_manager.enable_all_structured_loggers() self.log_manager.enable_all_structured_loggers()
loader = MozconfigLoader(command_context.topsrcdir) loader = MozconfigLoader(self.topsrcdir)
mozconfig = loader.read_mozconfig(loader.AUTODETECT) mozconfig = loader.read_mozconfig(loader.AUTODETECT)
configure_args = mozconfig["configure_args"] configure_args = mozconfig["configure_args"]
doing_pgo = configure_args and "MOZ_PGO=1" in configure_args doing_pgo = configure_args and "MOZ_PGO=1" in configure_args
@ -104,19 +104,19 @@ class Build(MachCommandBase):
raise Exception( raise Exception(
"Cannot specify targets (%s) in MOZ_PGO=1 builds" % what "Cannot specify targets (%s) in MOZ_PGO=1 builds" % what
) )
instr = command_context._spawn(BuildDriver) instr = self._spawn(BuildDriver)
orig_topobjdir = instr._topobjdir orig_topobjdir = instr._topobjdir
instr._topobjdir = mozpath.join(instr._topobjdir, "instrumented") instr._topobjdir = mozpath.join(instr._topobjdir, "instrumented")
append_env = {"MOZ_PROFILE_GENERATE": "1"} append_env = {"MOZ_PROFILE_GENERATE": "1"}
status = instr.build( status = instr.build(
command_context.metrics, self.metrics,
what=what, what=what,
jobs=jobs, jobs=jobs,
directory=directory, directory=directory,
verbose=verbose, verbose=verbose,
keep_going=keep_going, keep_going=keep_going,
mach_context=command_context._mach_context, mach_context=self._mach_context,
append_env=append_env, append_env=append_env,
) )
if status != 0: if status != 0:
@ -142,22 +142,22 @@ class Build(MachCommandBase):
pgo_env["JARLOG_FILE"] = mozpath.join(orig_topobjdir, "jarlog/en-US.log") pgo_env["JARLOG_FILE"] = mozpath.join(orig_topobjdir, "jarlog/en-US.log")
pgo_cmd = [ pgo_cmd = [
instr.virtualenv_manager.python_path, instr.virtualenv_manager.python_path,
mozpath.join(command_context.topsrcdir, "build/pgo/profileserver.py"), mozpath.join(self.topsrcdir, "build/pgo/profileserver.py"),
] ]
subprocess.check_call(pgo_cmd, cwd=instr.topobjdir, env=pgo_env) subprocess.check_call(pgo_cmd, cwd=instr.topobjdir, env=pgo_env)
# Set the default build to MOZ_PROFILE_USE # Set the default build to MOZ_PROFILE_USE
append_env = {"MOZ_PROFILE_USE": "1"} append_env = {"MOZ_PROFILE_USE": "1"}
driver = command_context._spawn(BuildDriver) driver = self._spawn(BuildDriver)
return driver.build( return driver.build(
command_context.metrics, self.metrics,
what=what, what=what,
jobs=jobs, jobs=jobs,
directory=directory, directory=directory,
verbose=verbose, verbose=verbose,
keep_going=keep_going, keep_going=keep_going,
mach_context=command_context._mach_context, mach_context=self._mach_context,
append_env=append_env, append_env=append_env,
) )
@ -179,11 +179,11 @@ class Build(MachCommandBase):
): ):
from mozbuild.controller.building import BuildDriver from mozbuild.controller.building import BuildDriver
command_context.log_manager.enable_all_structured_loggers() self.log_manager.enable_all_structured_loggers()
driver = command_context._spawn(BuildDriver) driver = self._spawn(BuildDriver)
return driver.configure( return driver.configure(
command_context.metrics, self.metrics,
options=options, options=options,
buildstatus_messages=buildstatus_messages, buildstatus_messages=buildstatus_messages,
line_handler=line_handler, line_handler=line_handler,
@ -222,7 +222,7 @@ class Build(MachCommandBase):
if url: if url:
server.add_resource_json_url("url", url) server.add_resource_json_url("url", url)
else: else:
last = command_context._get_state_filename("build_resources.json") last = self._get_state_filename("build_resources.json")
if not os.path.exists(last): if not os.path.exists(last):
print( print(
"Build resources not available. If you have performed a " "Build resources not available. If you have performed a "
@ -271,8 +271,8 @@ class Build(MachCommandBase):
def build_backend( def build_backend(
self, command_context, backend, diff=False, verbose=False, dry_run=False self, command_context, backend, diff=False, verbose=False, dry_run=False
): ):
python = command_context.virtualenv_manager.python_path python = self.virtualenv_manager.python_path
config_status = os.path.join(command_context.topobjdir, "config.status") config_status = os.path.join(self.topobjdir, "config.status")
if not os.path.exists(config_status): if not os.path.exists(config_status):
print( print(
@ -292,6 +292,6 @@ class Build(MachCommandBase):
if dry_run: if dry_run:
args.append("--dry-run") args.append("--dry-run")
return command_context._run_command_in_objdir( return self._run_command_in_objdir(
args=args, pass_thru=True, ensure_exit_code=False args=args, pass_thru=True, ensure_exit_code=False
) )

1143
python/mozbuild/mozbuild/code_analysis/mach_commands.py
View file

File diff suppressed because it is too large Load diff

17
python/mozbuild/mozbuild/compilation/codecomplete.py
View file

@ -6,10 +6,17 @@
from __future__ import absolute_import, print_function from __future__ import absolute_import, print_function
from mach.decorators import CommandArgument, CommandProvider, Command from mach.decorators import (
CommandArgument,
CommandProvider,
Command,
)
from mozbuild.base import MachCommandBase from mozbuild.base import MachCommandBase
from mozbuild.shellutil import split as shell_split, quote as shell_quote from mozbuild.shellutil import (
split as shell_split,
quote as shell_quote,
)
@CommandProvider @CommandProvider
@ -28,13 +35,13 @@ class Introspection(MachCommandBase):
from mozbuild.util import resolve_target_to_make from mozbuild.util import resolve_target_to_make
from mozbuild.compilation import util from mozbuild.compilation import util
if not util.check_top_objdir(command_context.topobjdir): if not util.check_top_objdir(self.topobjdir):
return 1 return 1
path_arg = command_context._wrap_path_argument(what) path_arg = self._wrap_path_argument(what)
make_dir, make_target = resolve_target_to_make( make_dir, make_target = resolve_target_to_make(
command_context.topobjdir, path_arg.relpath() self.topobjdir, path_arg.relpath()
) )
if make_dir is None and make_target is None: if make_dir is None and make_target is None:

24
python/mozbuild/mozbuild/frontend/mach_commands.py
View file

@ -44,8 +44,8 @@ class MozbuildFileCommands(MachCommandBase):
def reference(self, command_context, symbol, name_only=False): def reference(self, command_context, symbol, name_only=False):
# mozbuild.sphinx imports some Sphinx modules, so we need to be sure # mozbuild.sphinx imports some Sphinx modules, so we need to be sure
# the optional Sphinx package is installed. # the optional Sphinx package is installed.
command_context.activate_virtualenv() self.activate_virtualenv()
command_context.virtualenv_manager.install_pip_package("Sphinx==1.1.3") self.virtualenv_manager.install_pip_package("Sphinx==1.1.3")
from mozbuild.sphinx import ( from mozbuild.sphinx import (
format_module, format_module,
@ -127,7 +127,7 @@ class MozbuildFileCommands(MachCommandBase):
""" """
components = defaultdict(set) components = defaultdict(set)
try: try:
for p, m in self._get_files_info(command_context, paths, rev=rev).items(): for p, m in self._get_files_info(paths, rev=rev).items():
components[m.get("BUG_COMPONENT")].add(p) components[m.get("BUG_COMPONENT")].add(p)
except InvalidPathException as e: except InvalidPathException as e:
print(e) print(e)
@ -179,7 +179,7 @@ class MozbuildFileCommands(MachCommandBase):
missing = set() missing = set()
try: try:
for p, m in self._get_files_info(command_context, paths, rev=rev).items(): for p, m in self._get_files_info(paths, rev=rev).items():
if "BUG_COMPONENT" not in m: if "BUG_COMPONENT" not in m:
missing.add(p) missing.add(p)
except InvalidPathException as e: except InvalidPathException as e:
@ -218,7 +218,7 @@ class MozbuildFileCommands(MachCommandBase):
# TODO operate in VCS space. This requires teaching the VCS reader # TODO operate in VCS space. This requires teaching the VCS reader
# to understand wildcards and/or for the relative path issue in the # to understand wildcards and/or for the relative path issue in the
# VCS finder to be worked out. # VCS finder to be worked out.
for p, m in sorted(self._get_files_info(command_context, ["**"]).items()): for p, m in sorted(self._get_files_info(["**"]).items()):
if "BUG_COMPONENT" not in m: if "BUG_COMPONENT" not in m:
missing_component.add(p) missing_component.add(p)
print( print(
@ -284,17 +284,17 @@ class MozbuildFileCommands(MachCommandBase):
if missing_component: if missing_component:
return 1 return 1
def _get_files_info(self, command_context, paths, rev=None): def _get_files_info(self, paths, rev=None):
reader = command_context.mozbuild_reader(config_mode="empty", vcs_revision=rev) reader = self.mozbuild_reader(config_mode="empty", vcs_revision=rev)
# Normalize to relative from topsrcdir. # Normalize to relative from topsrcdir.
relpaths = [] relpaths = []
for p in paths: for p in paths:
a = mozpath.abspath(p) a = mozpath.abspath(p)
if not mozpath.basedir(a, [command_context.topsrcdir]): if not mozpath.basedir(a, [self.topsrcdir]):
raise InvalidPathException("path is outside topsrcdir: %s" % p) raise InvalidPathException("path is outside topsrcdir: %s" % p)
relpaths.append(mozpath.relpath(a, command_context.topsrcdir)) relpaths.append(mozpath.relpath(a, self.topsrcdir))
# Expand wildcards. # Expand wildcards.
# One variable is for ordering. The other for membership tests. # One variable is for ordering. The other for membership tests.
@ -304,7 +304,7 @@ class MozbuildFileCommands(MachCommandBase):
for p in relpaths: for p in relpaths:
if "*" not in p: if "*" not in p:
if p not in all_paths_set: if p not in all_paths_set:
if not os.path.exists(mozpath.join(command_context.topsrcdir, p)): if not os.path.exists(mozpath.join(self.topsrcdir, p)):
print("(%s does not exist; ignoring)" % p, file=sys.stderr) print("(%s does not exist; ignoring)" % p, file=sys.stderr)
continue continue
@ -319,9 +319,9 @@ class MozbuildFileCommands(MachCommandBase):
# finder is rooted at / for now. # finder is rooted at / for now.
# TODO bug 1171069 tracks changing to relative. # TODO bug 1171069 tracks changing to relative.
search = mozpath.join(command_context.topsrcdir, p)[1:] search = mozpath.join(self.topsrcdir, p)[1:]
for path, f in reader.finder.find(search): for path, f in reader.finder.find(search):
path = path[len(command_context.topsrcdir) :] path = path[len(self.topsrcdir) :]
if path not in all_paths_set: if path not in all_paths_set:
all_paths_set.add(path) all_paths_set.add(path)
allpaths.append(path) allpaths.append(path)

376
python/mozbuild/mozbuild/mach_commands.py
View file

@ -85,21 +85,21 @@ class Watch(MachCommandBase):
) )
def watch(self, command_context, verbose=False): def watch(self, command_context, verbose=False):
"""Watch and re-build (parts of) the source tree.""" """Watch and re-build (parts of) the source tree."""
if not conditions.is_artifact_build(command_context): if not conditions.is_artifact_build(self):
print( print(
"WARNING: mach watch only rebuilds the `mach build faster` parts of the tree!" "WARNING: mach watch only rebuilds the `mach build faster` parts of the tree!"
) )
if not command_context.substs.get("WATCHMAN", None): if not self.substs.get("WATCHMAN", None):
print( print(
"mach watch requires watchman to be installed and found at configure time. See " "mach watch requires watchman to be installed and found at configure time. See "
"https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Incremental_builds_with_filesystem_watching" # noqa "https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Incremental_builds_with_filesystem_watching" # noqa
) )
return 1 return 1
command_context.activate_virtualenv() self.activate_virtualenv()
try: try:
command_context.virtualenv_manager.install_pip_package("pywatchman==1.4.1") self.virtualenv_manager.install_pip_package("pywatchman==1.4.1")
except Exception: except Exception:
print( print(
"Could not install pywatchman from pip. See " "Could not install pywatchman from pip. See "
@ -109,7 +109,7 @@ class Watch(MachCommandBase):
from mozbuild.faster_daemon import Daemon from mozbuild.faster_daemon import Daemon
daemon = Daemon(command_context.config_environment) daemon = Daemon(self.config_environment)
try: try:
return daemon.watch() return daemon.watch()
@ -183,7 +183,7 @@ class CargoProvider(MachCommandBase):
"force-cargo-host-program-check", "force-cargo-host-program-check",
] ]
ret = command_context._run_make( ret = self._run_make(
srcdir=False, srcdir=False,
directory=root, directory=root,
ensure_exit_code=0, ensure_exit_code=0,
@ -220,14 +220,11 @@ class Doctor(MachCommandBase):
help="Print verbose information found by checks.", help="Print verbose information found by checks.",
) )
def doctor(self, command_context, fix=False, verbose=False): def doctor(self, command_context, fix=False, verbose=False):
command_context.activate_virtualenv() self.activate_virtualenv()
from mozbuild.doctor import run_doctor from mozbuild.doctor import run_doctor
return run_doctor( return run_doctor(
topsrcdir=command_context.topsrcdir, topsrcdir=self.topsrcdir, topobjdir=self.topobjdir, fix=fix, verbose=verbose
topobjdir=command_context.topobjdir,
fix=fix,
verbose=verbose,
) )
@ -285,18 +282,16 @@ class Clobber(MachCommandBase):
from mozbuild.controller.clobber import Clobberer from mozbuild.controller.clobber import Clobberer
try: try:
substs = command_context.substs substs = self.substs
except BuildEnvironmentNotFoundException: except BuildEnvironmentNotFoundException:
substs = {} substs = {}
try: try:
Clobberer( Clobberer(self.topsrcdir, self.topobjdir, substs).remove_objdir(full)
command_context.topsrcdir, command_context.topobjdir, substs
).remove_objdir(full)
except OSError as e: except OSError as e:
if sys.platform.startswith("win"): if sys.platform.startswith("win"):
if isinstance(e, WindowsError) and e.winerror in (5, 32): if isinstance(e, WindowsError) and e.winerror in (5, 32):
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"file_access_error", "file_access_error",
{"error": e}, {"error": e},
@ -307,7 +302,7 @@ class Clobber(MachCommandBase):
raise raise
if "python" in what: if "python" in what:
if conditions.is_hg(command_context): if conditions.is_hg(self):
cmd = [ cmd = [
"hg", "hg",
"--config", "--config",
@ -319,11 +314,11 @@ class Clobber(MachCommandBase):
"-I", "-I",
"glob:**/__pycache__", "glob:**/__pycache__",
] ]
elif conditions.is_git(command_context): elif conditions.is_git(self):
cmd = ["git", "clean", "-d", "-f", "-x", "*.py[cdo]", "*/__pycache__/*"] cmd = ["git", "clean", "-d", "-f", "-x", "*.py[cdo]", "*/__pycache__/*"]
else: else:
cmd = ["find", ".", "-type", "f", "-name", "*.py[cdo]", "-delete"] cmd = ["find", ".", "-type", "f", "-name", "*.py[cdo]", "-delete"]
subprocess.call(cmd, cwd=command_context.topsrcdir) subprocess.call(cmd, cwd=self.topsrcdir)
cmd = [ cmd = [
"find", "find",
".", ".",
@ -334,16 +329,13 @@ class Clobber(MachCommandBase):
"-empty", "-empty",
"-delete", "-delete",
] ]
ret = subprocess.call(cmd, cwd=command_context.topsrcdir) ret = subprocess.call(cmd, cwd=self.topsrcdir)
shutil.rmtree( shutil.rmtree(
mozpath.join(command_context.topobjdir, "_virtualenvs"), mozpath.join(self.topobjdir, "_virtualenvs"), ignore_errors=True
ignore_errors=True,
) )
if "gradle" in what: if "gradle" in what:
shutil.rmtree( shutil.rmtree(mozpath.join(self.topobjdir, "gradle"), ignore_errors=True)
mozpath.join(command_context.topobjdir, "gradle"), ignore_errors=True
)
return ret return ret
@ -364,7 +356,7 @@ class Logs(MachCommandBase):
) )
def show_log(self, command_context, log_file=None): def show_log(self, command_context, log_file=None):
if not log_file: if not log_file:
path = command_context._get_state_filename("last_log.json") path = self._get_state_filename("last_log.json")
log_file = open(path, "rb") log_file = open(path, "rb")
if os.isatty(sys.stdout.fileno()): if os.isatty(sys.stdout.fileno()):
@ -386,23 +378,21 @@ class Logs(MachCommandBase):
created, action, params = json.loads(line) created, action, params = json.loads(line)
if not startTime: if not startTime:
startTime = created startTime = created
command_context.log_manager.terminal_handler.formatter.start_time = ( self.log_manager.terminal_handler.formatter.start_time = created
created
)
if "line" in params: if "line" in params:
record = logging.makeLogRecord( record = logging.makeLogRecord(
{ {
"created": created, "created": created,
"name": command_context._logger.name, "name": self._logger.name,
"levelno": logging.INFO, "levelno": logging.INFO,
"msg": "{line}", "msg": "{line}",
"params": params, "params": params,
"action": action, "action": action,
} }
) )
command_context._logger.handle(record) self._logger.handle(record)
if command_context.log_manager.terminal: if self.log_manager.terminal:
# Close less's input so that it knows that we're done sending data. # Close less's input so that it knows that we're done sending data.
less.stdin.close() less.stdin.close()
# Since the less's input file descriptor is now also the stdout # Since the less's input file descriptor is now also the stdout
@ -418,13 +408,13 @@ class Logs(MachCommandBase):
class Warnings(MachCommandBase): class Warnings(MachCommandBase):
"""Provide commands for inspecting warnings.""" """Provide commands for inspecting warnings."""
def database_path(self, command_context): def database_path(self):
return command_context._get_state_filename("warnings.json") return self._get_state_filename("warnings.json")
def database(self, command_context): def database(self):
from mozbuild.compilation.warnings import WarningsDatabase from mozbuild.compilation.warnings import WarningsDatabase
path = self.database_path(command_context) path = self.database_path()
database = WarningsDatabase() database = WarningsDatabase()
@ -452,10 +442,10 @@ class Warnings(MachCommandBase):
"recent report.", "recent report.",
) )
def summary(self, command_context, directory=None, report=None): def summary(self, command_context, directory=None, report=None):
database = self.database(command_context) database = self.database()
if directory: if directory:
dirpath = self.join_ensure_dir(command_context.topsrcdir, directory) dirpath = self.join_ensure_dir(self.topsrcdir, directory)
if not dirpath: if not dirpath:
return 1 return 1
else: else:
@ -493,11 +483,11 @@ class Warnings(MachCommandBase):
"recent report.", "recent report.",
) )
def list(self, command_context, directory=None, flags=None, report=None): def list(self, command_context, directory=None, flags=None, report=None):
database = self.database(command_context) database = self.database()
by_name = sorted(database.warnings) by_name = sorted(database.warnings)
topsrcdir = mozpath.normpath(command_context.topsrcdir) topsrcdir = mozpath.normpath(self.topsrcdir)
if directory: if directory:
directory = mozpath.normsep(directory) directory = mozpath.normsep(directory)
@ -674,29 +664,29 @@ class GTestCommands(MachCommandBase):
# We lazy build gtest because it's slow to link # We lazy build gtest because it's slow to link
try: try:
command_context.config_environment self.config_environment
except Exception: except Exception:
print("Please run |./mach build| before |./mach gtest|.") print("Please run |./mach build| before |./mach gtest|.")
return 1 return 1
res = command_context._mach_context.commands.dispatch( res = self._mach_context.commands.dispatch(
"build", command_context._mach_context, what=["recurse_gtest"] "build", self._mach_context, what=["recurse_gtest"]
) )
if res: if res:
print("Could not build xul-gtest") print("Could not build xul-gtest")
return res return res
if command_context.substs.get("MOZ_WIDGET_TOOLKIT") == "cocoa": if self.substs.get("MOZ_WIDGET_TOOLKIT") == "cocoa":
command_context._run_make( self._run_make(
directory="browser/app", target="repackage", ensure_exit_code=True directory="browser/app", target="repackage", ensure_exit_code=True
) )
cwd = os.path.join(command_context.topobjdir, "_tests", "gtest") cwd = os.path.join(self.topobjdir, "_tests", "gtest")
if not os.path.isdir(cwd): if not os.path.isdir(cwd):
os.makedirs(cwd) os.makedirs(cwd)
if conditions.is_android(command_context): if conditions.is_android(self):
if jobs != 1: if jobs != 1:
print("--jobs is not supported on Android and will be ignored") print("--jobs is not supported on Android and will be ignored")
if debug or debugger or debugger_args: if debug or debugger or debugger_args:
@ -706,7 +696,6 @@ class GTestCommands(MachCommandBase):
from mozrunner.devices.android_device import InstallIntent from mozrunner.devices.android_device import InstallIntent
return self.android_gtest( return self.android_gtest(
command_context,
cwd, cwd,
shuffle, shuffle,
gtest_filter, gtest_filter,
@ -729,13 +718,10 @@ class GTestCommands(MachCommandBase):
): ):
print("One or more Android-only options will be ignored") print("One or more Android-only options will be ignored")
app_path = command_context.get_binary_path("app") app_path = self.get_binary_path("app")
args = [app_path, "-unittest", "--gtest_death_test_style=threadsafe"] args = [app_path, "-unittest", "--gtest_death_test_style=threadsafe"]
if ( if sys.platform.startswith("win") and "MOZ_LAUNCHER_PROCESS" in self.defines:
sys.platform.startswith("win")
and "MOZ_LAUNCHER_PROCESS" in command_context.defines
):
args.append("--wait-for-browser") args.append("--wait-for-browser")
if list_tests: if list_tests:
@ -754,9 +740,7 @@ class GTestCommands(MachCommandBase):
# Note: we must normalize the path here so that gtest on Windows sees # Note: we must normalize the path here so that gtest on Windows sees
# a MOZ_GMP_PATH which has only Windows dir seperators, because # a MOZ_GMP_PATH which has only Windows dir seperators, because
# nsIFile cannot open the paths with non-Windows dir seperators. # nsIFile cannot open the paths with non-Windows dir seperators.
xre_path = os.path.join( xre_path = os.path.join(os.path.normpath(self.topobjdir), "dist", "bin")
os.path.normpath(command_context.topobjdir), "dist", "bin"
)
gtest_env["MOZ_XRE_DIR"] = xre_path gtest_env["MOZ_XRE_DIR"] = xre_path
gtest_env["MOZ_GMP_PATH"] = os.pathsep.join( gtest_env["MOZ_GMP_PATH"] = os.pathsep.join(
os.path.join(xre_path, p, "1.0") for p in ("gmp-fake", "gmp-fakeopenh264") os.path.join(xre_path, p, "1.0") for p in ("gmp-fake", "gmp-fakeopenh264")
@ -777,7 +761,7 @@ class GTestCommands(MachCommandBase):
gtest_env["MOZ_WEBRENDER"] = "0" gtest_env["MOZ_WEBRENDER"] = "0"
if jobs == 1: if jobs == 1:
return command_context.run_process( return self.run_process(
args=args, args=args,
append_env=gtest_env, append_env=gtest_env,
cwd=cwd, cwd=cwd,
@ -791,7 +775,7 @@ class GTestCommands(MachCommandBase):
def handle_line(job_id, line): def handle_line(job_id, line):
# Prepend the jobId # Prepend the jobId
line = "[%d] %s" % (job_id + 1, line.strip()) line = "[%d] %s" % (job_id + 1, line.strip())
command_context.log(logging.INFO, "GTest", {"line": line}, "{line}") self.log(logging.INFO, "GTest", {"line": line}, "{line}")
gtest_env["GTEST_TOTAL_SHARDS"] = str(jobs) gtest_env["GTEST_TOTAL_SHARDS"] = str(jobs)
processes = {} processes = {}
@ -821,7 +805,6 @@ class GTestCommands(MachCommandBase):
def android_gtest( def android_gtest(
self, self,
command_context,
test_dir, test_dir,
shuffle, shuffle,
gtest_filter, gtest_filter,
@ -836,22 +819,22 @@ class GTestCommands(MachCommandBase):
# setup logging for mozrunner # setup logging for mozrunner
from mozlog.commandline import setup_logging from mozlog.commandline import setup_logging
format_args = {"level": command_context._mach_context.settings["test"]["level"]} format_args = {"level": self._mach_context.settings["test"]["level"]}
default_format = command_context._mach_context.settings["test"]["format"] default_format = self._mach_context.settings["test"]["format"]
setup_logging("mach-gtest", {}, {default_format: sys.stdout}, format_args) setup_logging("mach-gtest", {}, {default_format: sys.stdout}, format_args)
# ensure that a device is available and test app is installed # ensure that a device is available and test app is installed
from mozrunner.devices.android_device import verify_android_device, get_adb_path from mozrunner.devices.android_device import verify_android_device, get_adb_path
verify_android_device( verify_android_device(
command_context, install=install, app=package, device_serial=device_serial self, install=install, app=package, device_serial=device_serial
) )
if not adb_path: if not adb_path:
adb_path = get_adb_path(command_context) adb_path = get_adb_path(self)
if not libxul_path: if not libxul_path:
libxul_path = os.path.join( libxul_path = os.path.join(
command_context.topobjdir, "dist", "bin", "gtest", "libxul.so" self.topobjdir, "dist", "bin", "gtest", "libxul.so"
) )
# run gtest via remotegtests.py # run gtest via remotegtests.py
@ -898,11 +881,11 @@ class Package(MachCommandBase):
help="Verbose output for what commands the packaging process is running.", help="Verbose output for what commands the packaging process is running.",
) )
def package(self, command_context, verbose=False): def package(self, command_context, verbose=False):
ret = command_context._run_make( ret = self._run_make(
directory=".", target="package", silent=not verbose, ensure_exit_code=False directory=".", target="package", silent=not verbose, ensure_exit_code=False
) )
if ret == 0: if ret == 0:
command_context.notify("Packaging complete") self.notify("Packaging complete")
return ret return ret
@ -941,25 +924,20 @@ class Install(MachCommandBase):
description="Install the package on the machine (or device in the case of Android).", description="Install the package on the machine (or device in the case of Android).",
) )
def install(self, command_context, **kwargs): def install(self, command_context, **kwargs):
if conditions.is_android(command_context): if conditions.is_android(self):
from mozrunner.devices.android_device import ( from mozrunner.devices.android_device import (
verify_android_device, verify_android_device,
InstallIntent, InstallIntent,
) )
ret = ( ret = verify_android_device(self, install=InstallIntent.YES, **kwargs) == 0
verify_android_device(
command_context, install=InstallIntent.YES, **kwargs
)
== 0
)
else: else:
ret = command_context._run_make( ret = self._run_make(
directory=".", target="install", ensure_exit_code=False directory=".", target="install", ensure_exit_code=False
) )
if ret == 0: if ret == 0:
command_context.notify("Install complete") self.notify("Install complete")
return ret return ret
@ -1255,15 +1233,14 @@ class RunProgram(MachCommandBase):
description="Run the compiled program, possibly under a debugger or DMD.", description="Run the compiled program, possibly under a debugger or DMD.",
) )
def run(self, command_context, **kwargs): def run(self, command_context, **kwargs):
if conditions.is_android(command_context): if conditions.is_android(self):
return self._run_android(command_context, **kwargs) return self._run_android(**kwargs)
if conditions.is_jsshell(command_context): if conditions.is_jsshell(self):
return self._run_jsshell(command_context, **kwargs) return self._run_jsshell(**kwargs)
return self._run_desktop(command_context, **kwargs) return self._run_desktop(**kwargs)
def _run_android( def _run_android(
self, self,
command_context,
app="org.mozilla.geckoview_example", app="org.mozilla.geckoview_example",
intent=None, intent=None,
env=[], env=[],
@ -1303,7 +1280,7 @@ class RunProgram(MachCommandBase):
# `verify_android_device` respects `DEVICE_SERIAL` if it is set and sets it otherwise. # `verify_android_device` respects `DEVICE_SERIAL` if it is set and sets it otherwise.
verify_android_device( verify_android_device(
command_context, self,
app=app, app=app,
debugger=debug, debugger=debug,
install=InstallIntent.NO if no_install else InstallIntent.YES, install=InstallIntent.NO if no_install else InstallIntent.YES,
@ -1313,13 +1290,13 @@ class RunProgram(MachCommandBase):
print("No ADB devices connected.") print("No ADB devices connected.")
return 1 return 1
device = _get_device(command_context.substs, device_serial=device_serial) device = _get_device(self.substs, device_serial=device_serial)
if debug: if debug:
# This will terminate any existing processes, so we skip it when we # This will terminate any existing processes, so we skip it when we
# want to attach to an existing one. # want to attach to an existing one.
if not use_existing_process: if not use_existing_process:
command_context.log( self.log(
logging.INFO, logging.INFO,
"run", "run",
{"app": app}, {"app": app},
@ -1341,7 +1318,7 @@ class RunProgram(MachCommandBase):
target_profile = "/data/local/tmp/{}-profile".format(app) target_profile = "/data/local/tmp/{}-profile".format(app)
device.rm(target_profile, recursive=True, force=True) device.rm(target_profile, recursive=True, force=True)
device.push(host_profile, target_profile) device.push(host_profile, target_profile)
command_context.log( self.log(
logging.INFO, logging.INFO,
"run", "run",
{ {
@ -1353,7 +1330,7 @@ class RunProgram(MachCommandBase):
) )
else: else:
target_profile = profile target_profile = profile
command_context.log( self.log(
logging.INFO, logging.INFO,
"run", "run",
{"target_profile": target_profile}, {"target_profile": target_profile},
@ -1376,7 +1353,7 @@ class RunProgram(MachCommandBase):
if restart: if restart:
fail_if_running = False fail_if_running = False
command_context.log( self.log(
logging.INFO, logging.INFO,
"run", "run",
{"app": app}, {"app": app},
@ -1386,7 +1363,7 @@ class RunProgram(MachCommandBase):
# We'd prefer to log the actual `am start ...` command, but it's not trivial # We'd prefer to log the actual `am start ...` command, but it's not trivial
# to wire the device's logger to mach's logger. # to wire the device's logger to mach's logger.
command_context.log( self.log(
logging.INFO, logging.INFO,
"run", "run",
{"app": app, "activity_name": activity_name}, {"app": app, "activity_name": activity_name},
@ -1408,9 +1385,9 @@ class RunProgram(MachCommandBase):
from mozrunner.devices.android_device import run_lldb_server from mozrunner.devices.android_device import run_lldb_server
socket_file = run_lldb_server(app, command_context.substs, device_serial) socket_file = run_lldb_server(app, self.substs, device_serial)
if not socket_file: if not socket_file:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"run", "run",
{"msg": "Failed to obtain a socket file!"}, {"msg": "Failed to obtain a socket file!"},
@ -1419,7 +1396,7 @@ class RunProgram(MachCommandBase):
return 1 return 1
# Give lldb-server a chance to start # Give lldb-server a chance to start
command_context.log( self.log(
logging.INFO, logging.INFO,
"run", "run",
{"msg": "Pausing to ensure lldb-server has started..."}, {"msg": "Pausing to ensure lldb-server has started..."},
@ -1452,7 +1429,7 @@ class RunProgram(MachCommandBase):
] ]
if not proc_list: if not proc_list:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"run", "run",
{"app": app}, {"app": app},
@ -1474,16 +1451,14 @@ class RunProgram(MachCommandBase):
response = int(input(prompt).strip()) response = int(input(prompt).strip())
if response in valid_range: if response in valid_range:
break break
command_context.log( self.log(logging.ERROR, "run", {"msg": "Invalid response"}, "{msg}")
logging.ERROR, "run", {"msg": "Invalid response"}, "{msg}"
)
pid = proc_list[response - 1][0] pid = proc_list[response - 1][0]
else: else:
# We're not using an existing process, so there should only be our # We're not using an existing process, so there should only be our
# parent process at this time. # parent process at this time.
pids = device.pidof(app_name=app) pids = device.pidof(app_name=app)
if len(pids) != 1: if len(pids) != 1:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"run", "run",
{"msg": "Not sure which pid to attach to!"}, {"msg": "Not sure which pid to attach to!"},
@ -1492,21 +1467,19 @@ class RunProgram(MachCommandBase):
return 1 return 1
pid = pids[0] pid = pids[0]
command_context.log( self.log(logging.INFO, "run", {"pid": str(pid)}, "Debuggee pid set to {pid}...")
logging.INFO, "run", {"pid": str(pid)}, "Debuggee pid set to {pid}..."
)
lldb_connect_url = "unix-abstract-connect://" + socket_file lldb_connect_url = "unix-abstract-connect://" + socket_file
local_jdb_port = device.forward("tcp:0", "jdwp:%d" % pid) local_jdb_port = device.forward("tcp:0", "jdwp:%d" % pid)
if no_attach: if no_attach:
command_context.log( self.log(
logging.INFO, logging.INFO,
"run", "run",
{"pid": str(pid), "url": lldb_connect_url}, {"pid": str(pid), "url": lldb_connect_url},
"To debug native code, connect lldb to {url} and attach to pid {pid}", "To debug native code, connect lldb to {url} and attach to pid {pid}",
) )
command_context.log( self.log(
logging.INFO, logging.INFO,
"run", "run",
{"port": str(local_jdb_port)}, {"port": str(local_jdb_port)},
@ -1517,9 +1490,7 @@ class RunProgram(MachCommandBase):
# Beyond this point we want to be able to automatically clean up after ourselves, # Beyond this point we want to be able to automatically clean up after ourselves,
# so we enter the following try block. # so we enter the following try block.
try: try:
command_context.log( self.log(logging.INFO, "run", {"msg": "Starting debugger..."}, "{msg}")
logging.INFO, "run", {"msg": "Starting debugger..."}, "{msg}"
)
if not use_existing_process: if not use_existing_process:
# The app is waiting for jdb to attach and will not continue running # The app is waiting for jdb to attach and will not continue running
@ -1556,11 +1527,9 @@ platform connect {connect_url}
process attach {continue_flag}-p {pid!s} process attach {continue_flag}-p {pid!s}
""".lstrip() """.lstrip()
obj_xul = os.path.join( obj_xul = os.path.join(self.topobjdir, "toolkit", "library", "build")
command_context.topobjdir, "toolkit", "library", "build" obj_mozglue = os.path.join(self.topobjdir, "mozglue", "build")
) obj_nss = os.path.join(self.topobjdir, "security")
obj_mozglue = os.path.join(command_context.topobjdir, "mozglue", "build")
obj_nss = os.path.join(command_context.topobjdir, "security")
if use_existing_process: if use_existing_process:
continue_flag = "" continue_flag = ""
@ -1599,7 +1568,7 @@ process attach {continue_flag}-p {pid!s}
if not args: if not args:
return 1 return 1
return command_context.run_process( return self.run_process(
args=args, ensure_exit_code=False, pass_thru=True args=args, ensure_exit_code=False, pass_thru=True
) )
finally: finally:
@ -1610,14 +1579,12 @@ process attach {continue_flag}-p {pid!s}
if not use_existing_process: if not use_existing_process:
device.shell("am clear-debug-app") device.shell("am clear-debug-app")
def _run_jsshell(self, command_context, params, debug, debugger, debugger_args): def _run_jsshell(self, params, debug, debugger, debugger_args):
try: try:
binpath = command_context.get_binary_path("app") binpath = self.get_binary_path("app")
except BinaryNotFoundException as e: except BinaryNotFoundException as e:
command_context.log( self.log(logging.ERROR, "run", {"error": str(e)}, "ERROR: {error}")
logging.ERROR, "run", {"error": str(e)}, "ERROR: {error}" self.log(logging.INFO, "run", {"help": e.help()}, "{help}")
)
command_context.log(logging.INFO, "run", {"help": e.help()}, "{help}")
return 1 return 1
args = [binpath] args = [binpath]
@ -1629,7 +1596,7 @@ process attach {continue_flag}-p {pid!s}
if debug or debugger or debugger_args: if debug or debugger or debugger_args:
if "INSIDE_EMACS" in os.environ: if "INSIDE_EMACS" in os.environ:
command_context.log_manager.terminal_handler.setLevel(logging.WARNING) self.log_manager.terminal_handler.setLevel(logging.WARNING)
import mozdebug import mozdebug
@ -1641,22 +1608,21 @@ process attach {continue_flag}-p {pid!s}
) )
if debugger: if debugger:
debuggerInfo = mozdebug.get_debugger_info(debugger, debugger_args) self.debuggerInfo = mozdebug.get_debugger_info(debugger, debugger_args)
if not debugger or not debuggerInfo: if not debugger or not self.debuggerInfo:
print("Could not find a suitable debugger in your PATH.") print("Could not find a suitable debugger in your PATH.")
return 1 return 1
# Prepend the debugger args. # Prepend the debugger args.
args = [debuggerInfo.path] + debuggerInfo.args + args args = [self.debuggerInfo.path] + self.debuggerInfo.args + args
return command_context.run_process( return self.run_process(
args=args, ensure_exit_code=False, pass_thru=True, append_env=extra_env args=args, ensure_exit_code=False, pass_thru=True, append_env=extra_env
) )
def _run_desktop( def _run_desktop(
self, self,
command_context,
params, params,
packaged, packaged,
app, app,
@ -1681,15 +1647,13 @@ process attach {continue_flag}-p {pid!s}
try: try:
if packaged: if packaged:
binpath = command_context.get_binary_path(where="staged-package") binpath = self.get_binary_path(where="staged-package")
else: else:
binpath = app or command_context.get_binary_path("app") binpath = app or self.get_binary_path("app")
except BinaryNotFoundException as e: except BinaryNotFoundException as e:
command_context.log( self.log(logging.ERROR, "run", {"error": str(e)}, "ERROR: {error}")
logging.ERROR, "run", {"error": str(e)}, "ERROR: {error}"
)
if packaged: if packaged:
command_context.log( self.log(
logging.INFO, logging.INFO,
"run", "run",
{ {
@ -1699,7 +1663,7 @@ process attach {continue_flag}-p {pid!s}
"{help}", "{help}",
) )
else: else:
command_context.log(logging.INFO, "run", {"help": e.help()}, "{help}") self.log(logging.INFO, "run", {"help": e.help()}, "{help}")
return 1 return 1
args = [] args = []
@ -1733,10 +1697,7 @@ process attach {continue_flag}-p {pid!s}
if not background and sys.platform == "darwin": if not background and sys.platform == "darwin":
args.append("-foreground") args.append("-foreground")
if ( if sys.platform.startswith("win") and "MOZ_LAUNCHER_PROCESS" in self.defines:
sys.platform.startswith("win")
and "MOZ_LAUNCHER_PROCESS" in command_context.defines
):
args.append("-wait-for-browser") args.append("-wait-for-browser")
no_profile_option_given = all( no_profile_option_given = all(
@ -1748,12 +1709,12 @@ process attach {continue_flag}-p {pid!s}
"browser.shell.checkDefaultBrowser": False, "browser.shell.checkDefaultBrowser": False,
"general.warnOnAboutConfig": False, "general.warnOnAboutConfig": False,
} }
prefs.update(command_context._mach_context.settings.runprefs) prefs.update(self._mach_context.settings.runprefs)
prefs.update([p.split("=", 1) for p in setpref]) prefs.update([p.split("=", 1) for p in setpref])
for pref in prefs: for pref in prefs:
prefs[pref] = Preferences.cast(prefs[pref]) prefs[pref] = Preferences.cast(prefs[pref])
tmpdir = os.path.join(command_context.topobjdir, "tmp") tmpdir = os.path.join(self.topobjdir, "tmp")
if not os.path.exists(tmpdir): if not os.path.exists(tmpdir):
os.makedirs(tmpdir) os.makedirs(tmpdir)
@ -1793,8 +1754,8 @@ process attach {continue_flag}-p {pid!s}
args.append("-attach-console") args.append("-attach-console")
extra_env = { extra_env = {
"MOZ_DEVELOPER_REPO_DIR": command_context.topsrcdir, "MOZ_DEVELOPER_REPO_DIR": self.topsrcdir,
"MOZ_DEVELOPER_OBJ_DIR": command_context.topobjdir, "MOZ_DEVELOPER_OBJ_DIR": self.topobjdir,
"RUST_BACKTRACE": "full", "RUST_BACKTRACE": "full",
} }
@ -1805,7 +1766,7 @@ process attach {continue_flag}-p {pid!s}
if disable_e10s: if disable_e10s:
version_file = os.path.join( version_file = os.path.join(
command_context.topsrcdir, "browser", "config", "version.txt" self.topsrcdir, "browser", "config", "version.txt"
) )
f = open(version_file, "r") f = open(version_file, "r")
extra_env["MOZ_FORCE_DISABLE_E10S"] = f.read().strip() extra_env["MOZ_FORCE_DISABLE_E10S"] = f.read().strip()
@ -1815,7 +1776,7 @@ process attach {continue_flag}-p {pid!s}
if some_debugging_option: if some_debugging_option:
if "INSIDE_EMACS" in os.environ: if "INSIDE_EMACS" in os.environ:
command_context.log_manager.terminal_handler.setLevel(logging.WARNING) self.log_manager.terminal_handler.setLevel(logging.WARNING)
import mozdebug import mozdebug
@ -1827,9 +1788,9 @@ process attach {continue_flag}-p {pid!s}
) )
if debugger: if debugger:
debuggerInfo = mozdebug.get_debugger_info(debugger, debugger_args) self.debuggerInfo = mozdebug.get_debugger_info(debugger, debugger_args)
if not debugger or not debuggerInfo: if not debugger or not self.debuggerInfo:
print("Could not find a suitable debugger in your PATH.") print("Could not find a suitable debugger in your PATH.")
return 1 return 1
@ -1848,7 +1809,7 @@ process attach {continue_flag}-p {pid!s}
return 1 return 1
# Prepend the debugger args. # Prepend the debugger args.
args = [debuggerInfo.path] + debuggerInfo.args + args args = [self.debuggerInfo.path] + self.debuggerInfo.args + args
if dmd: if dmd:
dmd_params = [] dmd_params = []
@ -1865,7 +1826,7 @@ process attach {continue_flag}-p {pid!s}
else: else:
extra_env["DMD"] = "1" extra_env["DMD"] = "1"
return command_context.run_process( return self.run_process(
args=args, ensure_exit_code=False, pass_thru=True, append_env=extra_env args=args, ensure_exit_code=False, pass_thru=True, append_env=extra_env
) )
@ -1880,7 +1841,7 @@ class Buildsymbols(MachCommandBase):
description="Produce a package of Breakpad-format symbols.", description="Produce a package of Breakpad-format symbols.",
) )
def buildsymbols(self, command_context): def buildsymbols(self, command_context):
return command_context._run_make( return self._run_make(
directory=".", target="buildsymbols", ensure_exit_code=False directory=".", target="buildsymbols", ensure_exit_code=False
) )
@ -1911,43 +1872,43 @@ class MachDebug(MachCommandBase):
from mozbuild.util import FileAvoidWrite from mozbuild.util import FileAvoidWrite
with FileAvoidWrite(output) as out: with FileAvoidWrite(output) as out:
return func(command_context, out, verbose) return func(out, verbose)
return func(command_context, sys.stdout, verbose) return func(sys.stdout, verbose)
def _environment_pretty(self, command_context, out, verbose): def _environment_pretty(self, out, verbose):
state_dir = command_context._mach_context.state_dir state_dir = self._mach_context.state_dir
print("platform:\n\t%s" % platform.platform(), file=out) print("platform:\n\t%s" % platform.platform(), file=out)
print("python version:\n\t%s" % sys.version, file=out) print("python version:\n\t%s" % sys.version, file=out)
print("python prefix:\n\t%s" % sys.prefix, file=out) print("python prefix:\n\t%s" % sys.prefix, file=out)
print("mach cwd:\n\t%s" % command_context._mach_context.cwd, file=out) print("mach cwd:\n\t%s" % self._mach_context.cwd, file=out)
print("os cwd:\n\t%s" % os.getcwd(), file=out) print("os cwd:\n\t%s" % os.getcwd(), file=out)
print("mach directory:\n\t%s" % command_context._mach_context.topdir, file=out) print("mach directory:\n\t%s" % self._mach_context.topdir, file=out)
print("state directory:\n\t%s" % state_dir, file=out) print("state directory:\n\t%s" % state_dir, file=out)
print("object directory:\n\t%s" % command_context.topobjdir, file=out) print("object directory:\n\t%s" % self.topobjdir, file=out)
if command_context.mozconfig["path"]: if self.mozconfig["path"]:
print("mozconfig path:\n\t%s" % command_context.mozconfig["path"], file=out) print("mozconfig path:\n\t%s" % self.mozconfig["path"], file=out)
if command_context.mozconfig["configure_args"]: if self.mozconfig["configure_args"]:
print("mozconfig configure args:", file=out) print("mozconfig configure args:", file=out)
for arg in command_context.mozconfig["configure_args"]: for arg in self.mozconfig["configure_args"]:
print("\t%s" % arg, file=out) print("\t%s" % arg, file=out)
if command_context.mozconfig["make_extra"]: if self.mozconfig["make_extra"]:
print("mozconfig extra make args:", file=out) print("mozconfig extra make args:", file=out)
for arg in command_context.mozconfig["make_extra"]: for arg in self.mozconfig["make_extra"]:
print("\t%s" % arg, file=out) print("\t%s" % arg, file=out)
if command_context.mozconfig["make_flags"]: if self.mozconfig["make_flags"]:
print("mozconfig make flags:", file=out) print("mozconfig make flags:", file=out)
for arg in command_context.mozconfig["make_flags"]: for arg in self.mozconfig["make_flags"]:
print("\t%s" % arg, file=out) print("\t%s" % arg, file=out)
config = None config = None
try: try:
config = command_context.config_environment config = self.config_environment
except Exception: except Exception:
pass pass
@ -1965,7 +1926,7 @@ class MachDebug(MachCommandBase):
for k in sorted(config.defines): for k in sorted(config.defines):
print("\t%s" % k, file=out) print("\t%s" % k, file=out)
def _environment_json(self, command_context, out, verbose): def _environment_json(self, out, verbose):
import json import json
class EnvironmentEncoder(json.JSONEncoder): class EnvironmentEncoder(json.JSONEncoder):
@ -1984,7 +1945,7 @@ class MachDebug(MachCommandBase):
return list(obj) return list(obj)
return json.JSONEncoder.default(self, obj) return json.JSONEncoder.default(self, obj)
json.dump(command_context, cls=EnvironmentEncoder, sort_keys=True, fp=out) json.dump(self, cls=EnvironmentEncoder, sort_keys=True, fp=out)
@CommandProvider @CommandProvider
@ -2014,7 +1975,7 @@ class Repackage(MachCommandBase):
print("Input file does not exist: %s" % input) print("Input file does not exist: %s" % input)
return 1 return 1
if not os.path.exists(os.path.join(command_context.topobjdir, "config.status")): if not os.path.exists(os.path.join(self.topobjdir, "config.status")):
print( print(
"config.status not found. Please run |mach configure| " "config.status not found. Please run |mach configure| "
"prior to |mach repackage|." "prior to |mach repackage|."
@ -2076,7 +2037,7 @@ class Repackage(MachCommandBase):
from mozbuild.repackaging.installer import repackage_installer from mozbuild.repackaging.installer import repackage_installer
repackage_installer( repackage_installer(
topsrcdir=command_context.topsrcdir, topsrcdir=self.topsrcdir,
tag=tag, tag=tag,
setupexe=setupexe, setupexe=setupexe,
package=package, package=package,
@ -2128,7 +2089,7 @@ class Repackage(MachCommandBase):
from mozbuild.repackaging.msi import repackage_msi from mozbuild.repackaging.msi import repackage_msi
repackage_msi( repackage_msi(
topsrcdir=command_context.topsrcdir, topsrcdir=self.topsrcdir,
wsx=wsx, wsx=wsx,
version=version, version=version,
locale=locale, locale=locale,
@ -2151,12 +2112,7 @@ class Repackage(MachCommandBase):
from mozbuild.repackaging.mar import repackage_mar from mozbuild.repackaging.mar import repackage_mar
repackage_mar( repackage_mar(
command_context.topsrcdir, self.topsrcdir, input, mar, output, arch=arch, mar_channel_id=mar_channel_id
input,
mar,
output,
arch=arch,
mar_channel_id=mar_channel_id,
) )
@ -2179,7 +2135,7 @@ class L10NCommands(MachCommandBase):
"--verbose", action="store_true", help="Log informative status messages." "--verbose", action="store_true", help="Log informative status messages."
) )
def package_l10n(self, command_context, verbose=False, locales=[]): def package_l10n(self, command_context, verbose=False, locales=[]):
if "RecursiveMake" not in command_context.substs["BUILD_BACKENDS"]: if "RecursiveMake" not in self.substs["BUILD_BACKENDS"]:
print( print(
"Artifact builds do not support localization. " "Artifact builds do not support localization. "
"If you know what you are doing, you can use:\n" "If you know what you are doing, you can use:\n"
@ -2190,7 +2146,7 @@ class L10NCommands(MachCommandBase):
return 1 return 1
if "en-US" not in locales: if "en-US" not in locales:
command_context.log( self.log(
logging.WARN, logging.WARN,
"package-multi-locale", "package-multi-locale",
{"locales": locales}, {"locales": locales},
@ -2209,7 +2165,7 @@ class L10NCommands(MachCommandBase):
for locale in locales: for locale in locales:
if locale == "en-US": if locale == "en-US":
command_context.log( self.log(
logging.INFO, logging.INFO,
"package-multi-locale", "package-multi-locale",
{"locale": locale}, {"locale": locale},
@ -2217,90 +2173,80 @@ class L10NCommands(MachCommandBase):
) )
continue continue
command_context.log( self.log(
logging.INFO, logging.INFO,
"package-multi-locale", "package-multi-locale",
{"locale": locale}, {"locale": locale},
"Processing chrome Gecko resources for locale {locale}", "Processing chrome Gecko resources for locale {locale}",
) )
command_context.run_process( self.run_process(
[ [
mozpath.join(command_context.topsrcdir, "mach"), mozpath.join(self.topsrcdir, "mach"),
"build", "build",
"chrome-{}".format(locale), "chrome-{}".format(locale),
], ],
append_env=append_env, append_env=append_env,
pass_thru=True, pass_thru=True,
ensure_exit_code=True, ensure_exit_code=True,
cwd=mozpath.join(command_context.topsrcdir), cwd=mozpath.join(self.topsrcdir),
) )
if command_context.substs["MOZ_BUILD_APP"] == "mobile/android": if self.substs["MOZ_BUILD_APP"] == "mobile/android":
command_context.log( self.log(
logging.INFO, logging.INFO,
"package-multi-locale", "package-multi-locale",
{}, {},
"Invoking `mach android assemble-app`", "Invoking `mach android assemble-app`",
) )
command_context.run_process( self.run_process(
[ [mozpath.join(self.topsrcdir, "mach"), "android", "assemble-app"],
mozpath.join(command_context.topsrcdir, "mach"),
"android",
"assemble-app",
],
append_env=append_env, append_env=append_env,
pass_thru=True, pass_thru=True,
ensure_exit_code=True, ensure_exit_code=True,
cwd=mozpath.join(command_context.topsrcdir), cwd=mozpath.join(self.topsrcdir),
) )
if command_context.substs["MOZ_BUILD_APP"] == "browser": if self.substs["MOZ_BUILD_APP"] == "browser":
command_context.log( self.log(logging.INFO, "package-multi-locale", {}, "Repackaging browser")
logging.INFO, "package-multi-locale", {}, "Repackaging browser" self._run_make(
) directory=mozpath.join(self.topobjdir, "browser", "app"),
command_context._run_make(
directory=mozpath.join(command_context.topobjdir, "browser", "app"),
target=["tools"], target=["tools"],
append_env=append_env, append_env=append_env,
pass_thru=True, pass_thru=True,
ensure_exit_code=True, ensure_exit_code=True,
) )
command_context.log( self.log(
logging.INFO, logging.INFO,
"package-multi-locale", "package-multi-locale",
{}, {},
"Invoking multi-locale `mach package`", "Invoking multi-locale `mach package`",
) )
target = ["package"] target = ["package"]
if command_context.substs["MOZ_BUILD_APP"] == "mobile/android": if self.substs["MOZ_BUILD_APP"] == "mobile/android":
target.append("AB_CD=multi") target.append("AB_CD=multi")
command_context._run_make( self._run_make(
directory=command_context.topobjdir, directory=self.topobjdir,
target=target, target=target,
append_env=append_env, append_env=append_env,
pass_thru=True, pass_thru=True,
ensure_exit_code=True, ensure_exit_code=True,
) )
if command_context.substs["MOZ_BUILD_APP"] == "mobile/android": if self.substs["MOZ_BUILD_APP"] == "mobile/android":
command_context.log( self.log(
logging.INFO, logging.INFO,
"package-multi-locale", "package-multi-locale",
{}, {},
"Invoking `mach android archive-geckoview`", "Invoking `mach android archive-geckoview`",
) )
command_context.run_process( self.run_process(
[ [mozpath.join(self.topsrcdir, "mach"), "android", "archive-geckoview"],
mozpath.join(command_context.topsrcdir, "mach"),
"android",
"archive-geckoview",
],
append_env=append_env, append_env=append_env,
pass_thru=True, pass_thru=True,
ensure_exit_code=True, ensure_exit_code=True,
cwd=mozpath.join(command_context.topsrcdir), cwd=mozpath.join(self.topsrcdir),
) )
return 0 return 0
@ -2357,12 +2303,10 @@ class CreateMachEnvironment(MachCommandBase):
return 1 return 1
manager = VirtualenvManager( manager = VirtualenvManager(
command_context.topsrcdir, self.topsrcdir,
virtualenv_path, virtualenv_path,
sys.stdout, sys.stdout,
os.path.join( os.path.join(self.topsrcdir, "build", "mach_virtualenv_packages.txt"),
command_context.topsrcdir, "build", "mach_virtualenv_packages.txt"
),
populate_local_paths=False, populate_local_paths=False,
) )
@ -2375,9 +2319,7 @@ class CreateMachEnvironment(MachCommandBase):
# `mach` can handle it perfectly fine if `psutil` is missing, so # `mach` can handle it perfectly fine if `psutil` is missing, so
# there's no reason to freak out in this case. # there's no reason to freak out in this case.
manager.install_pip_requirements( manager.install_pip_requirements(
os.path.join( os.path.join(self.topsrcdir, "build", "psutil_requirements.txt")
command_context.topsrcdir, "build", "psutil_requirements.txt"
)
) )
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
print( print(
@ -2386,18 +2328,14 @@ class CreateMachEnvironment(MachCommandBase):
) )
manager.install_pip_requirements( manager.install_pip_requirements(
os.path.join( os.path.join(self.topsrcdir, "build", "zstandard_requirements.txt")
command_context.topsrcdir, "build", "zstandard_requirements.txt"
)
) )
# This can fail on some platforms. See # This can fail on some platforms. See
# https://bugzilla.mozilla.org/show_bug.cgi?id=1660120 # https://bugzilla.mozilla.org/show_bug.cgi?id=1660120
try: try:
manager.install_pip_requirements( manager.install_pip_requirements(
os.path.join( os.path.join(self.topsrcdir, "build", "glean_requirements.txt")
command_context.topsrcdir, "build", "glean_requirements.txt"
)
) )
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
print( print(

27
python/mozbuild/mozbuild/test/code_analysis/test_mach_commands.py
View file

@ -39,31 +39,25 @@ class TestStaticAnalysis(unittest.TestCase):
context.cwd = config.topsrcdir context.cwd = config.topsrcdir
cmd = StaticAnalysis(context) cmd = StaticAnalysis(context)
command_context = mock.MagicMock() cmd.topsrcdir = os.path.join("/root", "dir")
command_context.topsrcdir = os.path.join("/root", "dir")
path = os.path.join("/root", "dir", "path1") path = os.path.join("/root", "dir", "path1")
ignored_dirs_re = r"path1|path2/here|path3\there" ignored_dirs_re = r"path1|path2/here|path3\there"
self.assertTrue( self.assertTrue(cmd._is_ignored_path(ignored_dirs_re, path) is not None)
cmd._is_ignored_path(command_context, ignored_dirs_re, path) is not None
)
# simulating a win32 env # simulating a win32 env
win32_path = "\\root\\dir\\path1" win32_path = "\\root\\dir\\path1"
command_context.topsrcdir = "\\root\\dir" cmd.topsrcdir = "\\root\\dir"
old_sep = os.sep old_sep = os.sep
os.sep = "\\" os.sep = "\\"
try: try:
self.assertTrue( self.assertTrue(
cmd._is_ignored_path(command_context, ignored_dirs_re, win32_path) cmd._is_ignored_path(ignored_dirs_re, win32_path) is not None
is not None
) )
finally: finally:
os.sep = old_sep os.sep = old_sep
self.assertTrue( self.assertTrue(cmd._is_ignored_path(ignored_dirs_re, "path2") is None)
cmd._is_ignored_path(command_context, ignored_dirs_re, "path2") is None
)
def test_get_files(self): def test_get_files(self):
from mozbuild.code_analysis.mach_commands import StaticAnalysis from mozbuild.code_analysis.mach_commands import StaticAnalysis
@ -73,17 +67,14 @@ class TestStaticAnalysis(unittest.TestCase):
context.cwd = config.topsrcdir context.cwd = config.topsrcdir
cmd = StaticAnalysis(context) cmd = StaticAnalysis(context)
command_context = mock.MagicMock() cmd.topsrcdir = mozpath.join("/root", "dir")
command_context.topsrcdir = mozpath.join("/root", "dir") source = cmd.get_abspath_files(["file1", mozpath.join("directory", "file2")])
source = cmd.get_abspath_files(
command_context, ["file1", mozpath.join("directory", "file2")]
)
self.assertTrue( self.assertTrue(
source source
== [ == [
mozpath.join(command_context.topsrcdir, "file1"), mozpath.join(cmd.topsrcdir, "file1"),
mozpath.join(command_context.topsrcdir, "directory", "file2"), mozpath.join(cmd.topsrcdir, "directory", "file2"),
] ]
) )

18
python/mozbuild/mozbuild/vendor/mach_commands.py vendored
View file

@ -62,8 +62,8 @@ class Vendor(MachCommandBase):
library = library[0] library = library[0]
assert library not in ["rust", "python"] assert library not in ["rust", "python"]
command_context.populate_logger() self.populate_logger()
command_context.log_manager.enable_unstructured() self.log_manager.enable_unstructured()
if check_for_update: if check_for_update:
logging.disable() logging.disable()
@ -77,26 +77,26 @@ class Vendor(MachCommandBase):
sys.exit(1) sys.exit(1)
if not ignore_modified and not check_for_update: if not ignore_modified and not check_for_update:
self.check_modified_files(command_context) self.check_modified_files()
if not revision: if not revision:
revision = "HEAD" revision = "HEAD"
from mozbuild.vendor.vendor_manifest import VendorManifest from mozbuild.vendor.vendor_manifest import VendorManifest
vendor_command = command_context._spawn(VendorManifest) vendor_command = self._spawn(VendorManifest)
vendor_command.vendor(library, manifest, revision, check_for_update) vendor_command.vendor(library, manifest, revision, check_for_update)
sys.exit(0) sys.exit(0)
def check_modified_files(self, command_context): def check_modified_files(self):
""" """
Ensure that there aren't any uncommitted changes to files Ensure that there aren't any uncommitted changes to files
in the working copy, since we're going to change some state in the working copy, since we're going to change some state
on the user. on the user.
""" """
modified = command_context.repository.get_changed_files("M") modified = self.repository.get_changed_files("M")
if modified: if modified:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"modified_files", "modified_files",
{}, {},
@ -137,7 +137,7 @@ Please commit or stash these changes before vendoring, or re-run with `--ignore-
def vendor_rust(self, command_context, **kwargs): def vendor_rust(self, command_context, **kwargs):
from mozbuild.vendor.vendor_rust import VendorRust from mozbuild.vendor.vendor_rust import VendorRust
vendor_command = command_context._spawn(VendorRust) vendor_command = self._spawn(VendorRust)
vendor_command.vendor(**kwargs) vendor_command.vendor(**kwargs)
# ===================================================================== # =====================================================================
@ -168,5 +168,5 @@ Please commit or stash these changes before vendoring, or re-run with `--ignore-
) )
return 1 return 1
vendor_command = command_context._spawn(VendorPython) vendor_command = self._spawn(VendorPython)
vendor_command.vendor(**kwargs) vendor_command.vendor(**kwargs)

47
python/mozperftest/mozperftest/mach_commands.py
View file

@ -56,7 +56,7 @@ class Perftest(MachCommandBase):
from moztest.resolve import TestResolver from moztest.resolve import TestResolver
from mozperftest.fzf.fzf import select from mozperftest.fzf.fzf import select
resolver = command_context._spawn(TestResolver) resolver = self._spawn(TestResolver)
test_objects = list(resolver.resolve_tests(paths=None, flavor="perftest")) test_objects = list(resolver.resolve_tests(paths=None, flavor="perftest"))
selected = select(test_objects) selected = select(test_objects)
@ -64,7 +64,7 @@ class Perftest(MachCommandBase):
__, script_name, __, location = selection.split(" ") __, script_name, __, location = selection.split(" ")
return str( return str(
Path( Path(
command_context.topsrcdir.rstrip(os.sep), self.topsrcdir.rstrip(os.sep),
location.strip(os.sep), location.strip(os.sep),
script_name, script_name,
) )
@ -102,7 +102,7 @@ class Perftest(MachCommandBase):
push_to_try = kwargs.pop("push_to_try", False) push_to_try = kwargs.pop("push_to_try", False)
if push_to_try: if push_to_try:
sys.path.append(str(Path(command_context.topsrcdir, "tools", "tryselect"))) sys.path.append(str(Path(self.topsrcdir, "tools", "tryselect")))
from tryselect.push import push_to_try from tryselect.push import push_to_try
@ -126,8 +126,8 @@ class Perftest(MachCommandBase):
) )
def relative(path): def relative(path):
if path.startswith(command_context.topsrcdir): if path.startswith(self.topsrcdir):
return path[len(command_context.topsrcdir) :].lstrip(os.sep) return path[len(self.topsrcdir) :].lstrip(os.sep)
return path return path
for name, value in args.items(): for name, value in args.items():
@ -156,14 +156,18 @@ class Perftest(MachCommandBase):
from mozperftest.runner import run_tests from mozperftest.runner import run_tests
run_tests(command_context, kwargs, original_parser.get_user_args(kwargs)) run_tests(self, kwargs, original_parser.get_user_args(kwargs))
print("\nFirefox. Fast For Good.\n") print("\nFirefox. Fast For Good.\n")
@CommandProvider @CommandProvider
class PerftestTests(MachCommandBase): class PerftestTests(MachCommandBase):
@Command("perftest-test", category="testing", description="Run perftest tests") @Command(
"perftest-test",
category="testing",
description="Run perftest tests",
)
@CommandArgument( @CommandArgument(
"tests", default=None, nargs="*", help="Tests to run. By default will run all" "tests", default=None, nargs="*", help="Tests to run. By default will run all"
) )
@ -175,10 +179,14 @@ class PerftestTests(MachCommandBase):
help="Skip flake8 and black", help="Skip flake8 and black",
) )
@CommandArgument( @CommandArgument(
"-v", "--verbose", action="store_true", default=False, help="Verbose mode" "-v",
"--verbose",
action="store_true",
default=False,
help="Verbose mode",
) )
def run_tests(self, command_context, **kwargs): def run_tests(self, command_context, **kwargs):
command_context.activate_virtualenv() MachCommandBase.activate_virtualenv(self)
from pathlib import Path from pathlib import Path
from mozperftest.utils import temporary_env from mozperftest.utils import temporary_env
@ -186,9 +194,9 @@ class PerftestTests(MachCommandBase):
with temporary_env( with temporary_env(
COVERAGE_RCFILE=str(Path(HERE, ".coveragerc")), RUNNING_TESTS="YES" COVERAGE_RCFILE=str(Path(HERE, ".coveragerc")), RUNNING_TESTS="YES"
): ):
self._run_tests(command_context, **kwargs) self._run_tests(**kwargs)
def _run_tests(self, command_context, **kwargs): def _run_tests(self, **kwargs):
from pathlib import Path from pathlib import Path
from mozperftest.runner import _setup_path from mozperftest.runner import _setup_path
from mozperftest.utils import ( from mozperftest.utils import (
@ -198,7 +206,7 @@ class PerftestTests(MachCommandBase):
checkout_python_script, checkout_python_script,
) )
venv = command_context.virtualenv_manager venv = self.virtualenv_manager
skip_linters = kwargs.get("skip_linters", False) skip_linters = kwargs.get("skip_linters", False)
verbose = kwargs.get("verbose", False) verbose = kwargs.get("verbose", False)
@ -207,16 +215,14 @@ class PerftestTests(MachCommandBase):
try: try:
import coverage # noqa import coverage # noqa
except ImportError: except ImportError:
pydeps = Path(command_context.topsrcdir, "third_party", "python") pydeps = Path(self.topsrcdir, "third_party", "python")
vendors = ["coverage"] vendors = ["coverage"]
if not ON_TRY: if not ON_TRY:
vendors.append("attrs") vendors.append("attrs")
# pip-installing dependencies that require compilation or special setup # pip-installing dependencies that require compilation or special setup
for dep in vendors: for dep in vendors:
install_package( install_package(self.virtualenv_manager, str(Path(pydeps, dep)))
command_context.virtualenv_manager, str(Path(pydeps, dep))
)
if not ON_TRY and not skip_linters: if not ON_TRY and not skip_linters:
cmd = "./mach lint " cmd = "./mach lint "
@ -264,7 +270,14 @@ class PerftestTests(MachCommandBase):
assert checkout_python_script( assert checkout_python_script(
venv, "coverage", ["erase"], label="remove old coverage data" venv, "coverage", ["erase"], label="remove old coverage data"
) )
args = ["run", pytest.__file__, options, "--duration", "10", tests] args = [
"run",
pytest.__file__,
options,
"--duration",
"10",
tests,
]
assert checkout_python_script( assert checkout_python_script(
venv, "coverage", args, label="running tests", verbose=verbose venv, "coverage", args, label="running tests", verbose=verbose
) )

21
python/mozrelease/mozrelease/mach_commands.py
View file

@ -10,7 +10,12 @@ from __future__ import absolute_import, print_function, unicode_literals
import sys import sys
import logging import logging
from mach.decorators import CommandArgument, CommandProvider, Command, SubCommand from mach.decorators import (
CommandArgument,
CommandProvider,
Command,
SubCommand,
)
from mozbuild.base import MachCommandBase from mozbuild.base import MachCommandBase
from mozilla_version.gecko import GeckoVersion from mozilla_version.gecko import GeckoVersion
@ -43,7 +48,7 @@ class MachCommands(MachCommandBase):
@CommandArgument("--repo", help="The repo being built.") @CommandArgument("--repo", help="The repo being built.")
@CommandArgument("--revision", required=True, help="The revision being built.") @CommandArgument("--revision", required=True, help="The revision being built.")
def buglist(self, command_context, version, product, revision, repo): def buglist(self, command_context, version, product, revision, repo):
self.setup_logging(command_context) self.setup_logging()
from mozrelease.buglist_creator import create_bugs_url from mozrelease.buglist_creator import create_bugs_url
print( print(
@ -80,7 +85,7 @@ class MachCommands(MachCommandBase):
@CommandArgument("--build-number", required=True, help="The build number") @CommandArgument("--build-number", required=True, help="The build number")
@CommandArgument("--task-group-id", help="The task group of the build.") @CommandArgument("--task-group-id", help="The task group of the build.")
def buglist_email(self, command_context, **options): def buglist_email(self, command_context, **options):
self.setup_logging(command_context) self.setup_logging()
from mozrelease.buglist_creator import email_release_drivers from mozrelease.buglist_creator import email_release_drivers
email_release_drivers(**options) email_release_drivers(**options)
@ -113,7 +118,7 @@ class MachCommands(MachCommandBase):
def push_scriptworker_canary( def push_scriptworker_canary(
self, command_context, scriptworkers, addresses, ssh_key_secret self, command_context, scriptworkers, addresses, ssh_key_secret
): ):
self.setup_logging(command_context) self.setup_logging()
from mozrelease.scriptworker_canary import push_canary from mozrelease.scriptworker_canary import push_canary
push_canary( push_canary(
@ -122,19 +127,19 @@ class MachCommands(MachCommandBase):
ssh_key_secret=ssh_key_secret, ssh_key_secret=ssh_key_secret,
) )
def setup_logging(self, command_context, quiet=False, verbose=True): def setup_logging(self, quiet=False, verbose=True):
""" """
Set up Python logging for all loggers, sending results to stderr (so Set up Python logging for all loggers, sending results to stderr (so
that command output can be redirected easily) and adding the typical that command output can be redirected easily) and adding the typical
mach timestamp. mach timestamp.
""" """
# remove the old terminal handler # remove the old terminal handler
old = command_context.log_manager.replace_terminal_handler(None) old = self.log_manager.replace_terminal_handler(None)
# re-add it, with level and fh set appropriately # re-add it, with level and fh set appropriately
if not quiet: if not quiet:
level = logging.DEBUG if verbose else logging.INFO level = logging.DEBUG if verbose else logging.INFO
command_context.log_manager.add_terminal_logging( self.log_manager.add_terminal_logging(
fh=sys.stderr, fh=sys.stderr,
level=level, level=level,
write_interval=old.formatter.write_interval, write_interval=old.formatter.write_interval,
@ -142,4 +147,4 @@ class MachCommands(MachCommandBase):
) )
# all of the taskgraph logging is unstructured logging # all of the taskgraph logging is unstructured logging
command_context.log_manager.enable_unstructured() self.log_manager.enable_unstructured()

40
remote/mach_commands.py
View file

@ -54,15 +54,15 @@ def setup():
@CommandProvider @CommandProvider
class RemoteCommands(MachCommandBase): class RemoteCommands(MachCommandBase):
def remotedir(self, command_context): def remotedir(self):
return os.path.join(command_context.topsrcdir, "remote") return os.path.join(self.topsrcdir, "remote")
@Command( @Command(
"remote", category="misc", description="Remote protocol related operations." "remote", category="misc", description="Remote protocol related operations."
) )
def remote(self, command_context): def remote(self, command_context):
"""The remote subcommands all relate to the remote protocol.""" """The remote subcommands all relate to the remote protocol."""
command_context._sub_mach(["help", "remote"]) self._sub_mach(["help", "remote"])
return 1 return 1
@SubCommand( @SubCommand(
@ -88,14 +88,11 @@ class RemoteCommands(MachCommandBase):
help="Do not install the just-pulled Puppeteer package,", help="Do not install the just-pulled Puppeteer package,",
) )
def vendor_puppeteer(self, command_context, repository, commitish, install): def vendor_puppeteer(self, command_context, repository, commitish, install):
puppeteer_dir = os.path.join( puppeteer_dir = os.path.join(self.remotedir(), "test", "puppeteer")
self.remotedir(command_context), "test", "puppeteer"
)
# Preserve our custom mocha reporter # Preserve our custom mocha reporter
shutil.move( shutil.move(
os.path.join(puppeteer_dir, "json-mocha-reporter.js"), os.path.join(puppeteer_dir, "json-mocha-reporter.js"), self.remotedir()
self.remotedir(command_context),
) )
shutil.rmtree(puppeteer_dir, ignore_errors=True) shutil.rmtree(puppeteer_dir, ignore_errors=True)
os.makedirs(puppeteer_dir) os.makedirs(puppeteer_dir)
@ -126,8 +123,7 @@ class RemoteCommands(MachCommandBase):
shutil.rmtree(dir_path) shutil.rmtree(dir_path)
shutil.move( shutil.move(
os.path.join(self.remotedir(command_context), "json-mocha-reporter.js"), os.path.join(self.remotedir(), "json-mocha-reporter.js"), puppeteer_dir
puppeteer_dir,
) )
import yaml import yaml
@ -157,11 +153,7 @@ class RemoteCommands(MachCommandBase):
if install: if install:
env = {"PUPPETEER_SKIP_DOWNLOAD": "1"} env = {"PUPPETEER_SKIP_DOWNLOAD": "1"}
npm( npm("install", cwd=os.path.join(self.topsrcdir, puppeteer_dir), env=env)
"install",
cwd=os.path.join(command_context.topsrcdir, puppeteer_dir),
env=env,
)
def git(*args, **kwargs): def git(*args, **kwargs):
@ -610,6 +602,8 @@ class PuppeteerTest(MachCommandBase):
**kwargs **kwargs
): ):
self.ci = ci
logger = mozlog.commandline.setup_logging( logger = mozlog.commandline.setup_logging(
"puppeteer-test", kwargs, {"mach": sys.stdout} "puppeteer-test", kwargs, {"mach": sys.stdout}
) )
@ -654,7 +648,7 @@ class PuppeteerTest(MachCommandBase):
if verbosity > 2: if verbosity > 2:
prefs["remote.log.truncate"] = False prefs["remote.log.truncate"] = False
self.install_puppeteer(command_context, product, ci) self.install_puppeteer(product)
params = { params = {
"binary": binary, "binary": binary,
@ -666,7 +660,7 @@ class PuppeteerTest(MachCommandBase):
"write_results": write_results, "write_results": write_results,
"subset": subset, "subset": subset,
} }
puppeteer = command_context._spawn(PuppeteerRunner) puppeteer = self._spawn(PuppeteerRunner)
try: try:
return puppeteer.run_test(logger, *tests, **params) return puppeteer.run_test(logger, *tests, **params)
except BinaryNotFoundException as e: except BinaryNotFoundException as e:
@ -676,12 +670,12 @@ class PuppeteerTest(MachCommandBase):
except Exception as e: except Exception as e:
exit(EX_SOFTWARE, e) exit(EX_SOFTWARE, e)
def install_puppeteer(self, command_context, product, ci): def install_puppeteer(self, product):
setup() setup()
env = {} env = {}
from mozversioncontrol import get_repository_object from mozversioncontrol import get_repository_object
repo = get_repository_object(command_context.topsrcdir) repo = get_repository_object(self.topsrcdir)
puppeteer_dir = os.path.join("remote", "test", "puppeteer") puppeteer_dir = os.path.join("remote", "test", "puppeteer")
changed_files = False changed_files = False
for f in repo.get_changed_files(): for f in repo.get_changed_files():
@ -691,15 +685,13 @@ class PuppeteerTest(MachCommandBase):
if product != "chrome": if product != "chrome":
env["PUPPETEER_SKIP_DOWNLOAD"] = "1" env["PUPPETEER_SKIP_DOWNLOAD"] = "1"
lib_dir = os.path.join(command_context.topsrcdir, puppeteer_dir, "lib") lib_dir = os.path.join(self.topsrcdir, puppeteer_dir, "lib")
if changed_files and os.path.isdir(lib_dir): if changed_files and os.path.isdir(lib_dir):
# clobber lib to force `tsc compile` step # clobber lib to force `tsc compile` step
shutil.rmtree(lib_dir) shutil.rmtree(lib_dir)
command = "ci" if ci else "install" command = "ci" if self.ci else "install"
npm( npm(command, cwd=os.path.join(self.topsrcdir, puppeteer_dir), env=env)
command, cwd=os.path.join(command_context.topsrcdir, puppeteer_dir), env=env
)
def exit(code, error=None): def exit(code, error=None):

10
security/manager/tools/mach_commands.py
View file

@ -79,12 +79,12 @@ class MachCommands(MachCommandBase):
def generate_test_certs(self, command_context, specifications): def generate_test_certs(self, command_context, specifications):
"""Generate test certificates and keys from specifications.""" """Generate test certificates and keys from specifications."""
command_context.activate_virtualenv() self.activate_virtualenv()
import pycert import pycert
import pykey import pykey
if not specifications: if not specifications:
specifications = self.find_all_specifications(command_context) specifications = self.find_all_specifications()
for specification in specifications: for specification in specifications:
if is_certspec_file(specification): if is_certspec_file(specification):
@ -98,7 +98,7 @@ class MachCommands(MachCommandBase):
run_module_main_on(module, os.path.abspath(specification)) run_module_main_on(module, os.path.abspath(specification))
return 0 return 0
def find_all_specifications(self, command_context): def find_all_specifications(self):
"""Searches the source tree for all specification files """Searches the source tree for all specification files
and returns them as a list.""" and returns them as a list."""
specifications = [] specifications = []
@ -109,11 +109,11 @@ class MachCommands(MachCommandBase):
"testing/xpcshell/moz-http2", "testing/xpcshell/moz-http2",
] ]
exclusions = ["security/manager/ssl/tests/unit/test_signed_apps"] exclusions = ["security/manager/ssl/tests/unit/test_signed_apps"]
finder = FileFinder(command_context.topsrcdir) finder = FileFinder(self.topsrcdir)
for inclusion_path in inclusions: for inclusion_path in inclusions:
for f, _ in finder.find(inclusion_path): for f, _ in finder.find(inclusion_path):
if basedir(f, exclusions): if basedir(f, exclusions):
continue continue
if is_specification_file(f): if is_specification_file(f):
specifications.append(os.path.join(command_context.topsrcdir, f)) specifications.append(os.path.join(self.topsrcdir, f))
return specifications return specifications

44
taskcluster/mach_commands.py
View file

@ -173,35 +173,35 @@ class MachCommands(MachCommandBase):
"taskgraph", "tasks", description="Show all tasks in the taskgraph" "taskgraph", "tasks", description="Show all tasks in the taskgraph"
) )
def taskgraph_tasks(self, command_context, **options): def taskgraph_tasks(self, command_context, **options):
return self.show_taskgraph(command_context, "full_task_set", options) return self.show_taskgraph("full_task_set", options)
@ShowTaskGraphSubCommand("taskgraph", "full", description="Show the full taskgraph") @ShowTaskGraphSubCommand("taskgraph", "full", description="Show the full taskgraph")
def taskgraph_full(self, command_context, **options): def taskgraph_full(self, command_context, **options):
return self.show_taskgraph(command_context, "full_task_graph", options) return self.show_taskgraph("full_task_graph", options)
@ShowTaskGraphSubCommand( @ShowTaskGraphSubCommand(
"taskgraph", "target", description="Show the target task set" "taskgraph", "target", description="Show the target task set"
) )
def taskgraph_target(self, command_context, **options): def taskgraph_target(self, command_context, **options):
return self.show_taskgraph(command_context, "target_task_set", options) return self.show_taskgraph("target_task_set", options)
@ShowTaskGraphSubCommand( @ShowTaskGraphSubCommand(
"taskgraph", "target-graph", description="Show the target taskgraph" "taskgraph", "target-graph", description="Show the target taskgraph"
) )
def taskgraph_target_taskgraph(self, command_context, **options): def taskgraph_target_taskgraph(self, command_context, **options):
return self.show_taskgraph(command_context, "target_task_graph", options) return self.show_taskgraph("target_task_graph", options)
@ShowTaskGraphSubCommand( @ShowTaskGraphSubCommand(
"taskgraph", "optimized", description="Show the optimized taskgraph" "taskgraph", "optimized", description="Show the optimized taskgraph"
) )
def taskgraph_optimized(self, command_context, **options): def taskgraph_optimized(self, command_context, **options):
return self.show_taskgraph(command_context, "optimized_task_graph", options) return self.show_taskgraph("optimized_task_graph", options)
@ShowTaskGraphSubCommand( @ShowTaskGraphSubCommand(
"taskgraph", "morphed", description="Show the morphed taskgraph" "taskgraph", "morphed", description="Show the morphed taskgraph"
) )
def taskgraph_morphed(self, command_context, **options): def taskgraph_morphed(self, command_context, **options):
return self.show_taskgraph(command_context, "morphed_task_graph", options) return self.show_taskgraph("morphed_task_graph", options)
@SubCommand("taskgraph", "actions", description="Write actions.json to stdout") @SubCommand("taskgraph", "actions", description="Write actions.json to stdout")
@CommandArgument( @CommandArgument(
@ -224,7 +224,7 @@ class MachCommands(MachCommandBase):
"`taskcluster/docs/parameters.rst`)`", "`taskcluster/docs/parameters.rst`)`",
) )
def taskgraph_actions(self, command_context, **options): def taskgraph_actions(self, command_context, **options):
return self.show_actions(command_context, options) return self.show_actions(options)
@SubCommand("taskgraph", "decision", description="Run the decision task") @SubCommand("taskgraph", "decision", description="Run the decision task")
@CommandArgument( @CommandArgument(
@ -349,7 +349,7 @@ class MachCommands(MachCommandBase):
import taskgraph.decision import taskgraph.decision
try: try:
self.setup_logging(command_context) self.setup_logging()
start = time.monotonic() start = time.monotonic()
ret = taskgraph.decision.taskgraph_decision(options) ret = taskgraph.decision.taskgraph_decision(options)
end = time.monotonic() end = time.monotonic()
@ -403,7 +403,7 @@ class MachCommands(MachCommandBase):
from taskgraph.actions.util import get_parameters from taskgraph.actions.util import get_parameters
try: try:
self.setup_logging(command_context) self.setup_logging()
# the target task for this action (or null if it's a group action) # the target task for this action (or null if it's a group action)
task_id = json.loads(os.environ.get("ACTION_TASK_ID", "null")) task_id = json.loads(os.environ.get("ACTION_TASK_ID", "null"))
@ -471,7 +471,7 @@ class MachCommands(MachCommandBase):
raise Exception("unknown filename {}".format(filename)) raise Exception("unknown filename {}".format(filename))
try: try:
self.setup_logging(command_context) self.setup_logging()
task_id = options["task_id"] task_id = options["task_id"]
if options["input"]: if options["input"]:
@ -502,19 +502,19 @@ class MachCommands(MachCommandBase):
traceback.print_exc() traceback.print_exc()
sys.exit(1) sys.exit(1)
def setup_logging(self, command_context, quiet=False, verbose=True): def setup_logging(self, quiet=False, verbose=True):
""" """
Set up Python logging for all loggers, sending results to stderr (so Set up Python logging for all loggers, sending results to stderr (so
that command output can be redirected easily) and adding the typical that command output can be redirected easily) and adding the typical
mach timestamp. mach timestamp.
""" """
# remove the old terminal handler # remove the old terminal handler
old = command_context.log_manager.replace_terminal_handler(None) old = self.log_manager.replace_terminal_handler(None)
# re-add it, with level and fh set appropriately # re-add it, with level and fh set appropriately
if not quiet: if not quiet:
level = logging.DEBUG if verbose else logging.INFO level = logging.DEBUG if verbose else logging.INFO
command_context.log_manager.add_terminal_logging( self.log_manager.add_terminal_logging(
fh=sys.stderr, fh=sys.stderr,
level=level, level=level,
write_interval=old.formatter.write_interval, write_interval=old.formatter.write_interval,
@ -522,12 +522,10 @@ class MachCommands(MachCommandBase):
) )
# all of the taskgraph logging is unstructured logging # all of the taskgraph logging is unstructured logging
command_context.log_manager.enable_unstructured() self.log_manager.enable_unstructured()
def show_taskgraph(self, command_context, graph_attr, options): def show_taskgraph(self, graph_attr, options):
self.setup_logging( self.setup_logging(quiet=options["quiet"], verbose=options["verbose"])
command_context, quiet=options["quiet"], verbose=options["verbose"]
)
vcs = None vcs = None
base_out = "" base_out = ""
base_ref = None base_ref = None
@ -536,7 +534,7 @@ class MachCommands(MachCommandBase):
if options["diff"]: if options["diff"]:
from mozversioncontrol import get_repository_object from mozversioncontrol import get_repository_object
vcs = get_repository_object(command_context.topsrcdir) vcs = get_repository_object(self.topsrcdir)
with vcs: with vcs:
if not vcs.working_directory_clean(): if not vcs.working_directory_clean():
print("abort: can't diff taskgraph with dirty working directory") print("abort: can't diff taskgraph with dirty working directory")
@ -573,7 +571,7 @@ class MachCommands(MachCommandBase):
finally: finally:
vcs.update(cur_ref) vcs.update(cur_ref)
diffcmd = command_context._mach_context.settings["taskgraph"]["diffcmd"] diffcmd = self._mach_context.settings["taskgraph"]["diffcmd"]
diffcmd = diffcmd.format(attr=graph_attr, base=base_ref, cur=cur_ref) diffcmd = diffcmd.format(attr=graph_attr, base=base_ref, cur=cur_ref)
with tempfile.NamedTemporaryFile(mode="w") as base: with tempfile.NamedTemporaryFile(mode="w") as base:
@ -679,16 +677,14 @@ class MachCommands(MachCommandBase):
) )
return filtered_taskgraph return filtered_taskgraph
def show_actions(self, command_context, options): def show_actions(self, options):
import taskgraph import taskgraph
import taskgraph.actions import taskgraph.actions
import taskgraph.generator import taskgraph.generator
import taskgraph.parameters import taskgraph.parameters
try: try:
self.setup_logging( self.setup_logging(quiet=options["quiet"], verbose=options["verbose"])
command_context, quiet=options["quiet"], verbose=options["verbose"]
)
parameters = taskgraph.parameters.parameters_loader(options["parameters"]) parameters = taskgraph.parameters.parameters_loader(options["parameters"])
tgg = taskgraph.generator.TaskGraphGenerator( tgg = taskgraph.generator.TaskGraphGenerator(

31
testing/awsy/mach_commands.py
View file

@ -44,7 +44,7 @@ class MachCommands(MachCommandBase):
sys.path.append(AWSY_PATH) sys.path.append(AWSY_PATH)
from awsy import ITERATIONS, PER_TAB_PAUSE, SETTLE_WAIT_TIME, MAX_TABS from awsy import ITERATIONS, PER_TAB_PAUSE, SETTLE_WAIT_TIME, MAX_TABS
def run_awsy(self, command_context, tests, binary=None, **kwargs): def run_awsy(self, tests, binary=None, **kwargs):
import json import json
from mozlog.structured import commandline from mozlog.structured import commandline
@ -52,7 +52,7 @@ class MachCommands(MachCommandBase):
parser = setup_awsy_argument_parser() parser = setup_awsy_argument_parser()
awsy_source_dir = os.path.join(command_context.topsrcdir, "testing", "awsy") awsy_source_dir = os.path.join(self.topsrcdir, "testing", "awsy")
if not tests: if not tests:
tests = [os.path.join(awsy_source_dir, "awsy", "test_memory_usage.py")] tests = [os.path.join(awsy_source_dir, "awsy", "test_memory_usage.py")]
@ -88,7 +88,7 @@ class MachCommands(MachCommandBase):
runtime_testvars[arg] = kwargs[arg] runtime_testvars[arg] = kwargs[arg]
if "webRootDir" not in runtime_testvars: if "webRootDir" not in runtime_testvars:
awsy_tests_dir = os.path.join(command_context.topobjdir, "_tests", "awsy") awsy_tests_dir = os.path.join(self.topobjdir, "_tests", "awsy")
web_root_dir = os.path.join(awsy_tests_dir, "html") web_root_dir = os.path.join(awsy_tests_dir, "html")
runtime_testvars["webRootDir"] = web_root_dir runtime_testvars["webRootDir"] = web_root_dir
else: else:
@ -122,16 +122,15 @@ class MachCommands(MachCommandBase):
tooltool_args = { tooltool_args = {
"args": [ "args": [
sys.executable, sys.executable,
os.path.join(command_context.topsrcdir, "mach"), os.path.join(self.topsrcdir, "mach"),
"artifact", "artifact",
"toolchain", "toolchain",
"-v", "-v",
"--tooltool-manifest=%s" % manifest_file, "--tooltool-manifest=%s" % manifest_file,
"--cache-dir=%s" "--cache-dir=%s" % os.path.join(self.topsrcdir, "tooltool-cache"),
% os.path.join(command_context.topsrcdir, "tooltool-cache"),
] ]
} }
command_context.run_process(cwd=page_load_test_dir, **tooltool_args) self.run_process(cwd=page_load_test_dir, **tooltool_args)
tp5nzip = os.path.join(page_load_test_dir, "tp5n.zip") tp5nzip = os.path.join(page_load_test_dir, "tp5n.zip")
tp5nmanifest = os.path.join(page_load_test_dir, "tp5n", "tp5n.manifest") tp5nmanifest = os.path.join(page_load_test_dir, "tp5n", "tp5n.manifest")
if not os.path.exists(tp5nmanifest): if not os.path.exists(tp5nmanifest):
@ -139,7 +138,7 @@ class MachCommands(MachCommandBase):
"args": ["unzip", "-q", "-o", tp5nzip, "-d", page_load_test_dir] "args": ["unzip", "-q", "-o", tp5nzip, "-d", page_load_test_dir]
} }
try: try:
command_context.run_process(**unzip_args) self.run_process(**unzip_args)
except Exception as exc: except Exception as exc:
troubleshoot = "" troubleshoot = ""
if mozinfo.os == "win": if mozinfo.os == "win":
@ -148,7 +147,7 @@ class MachCommands(MachCommandBase):
"directory closer to the drive root." "directory closer to the drive root."
) )
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"awsy", "awsy",
{"directory": page_load_test_dir, "exception": exc}, {"directory": page_load_test_dir, "exception": exc},
@ -173,7 +172,7 @@ class MachCommands(MachCommandBase):
# Work around a startup crash with DMD on windows # Work around a startup crash with DMD on windows
if mozinfo.os == "win": if mozinfo.os == "win":
kwargs["pref"] = "security.sandbox.content.level:0" kwargs["pref"] = "security.sandbox.content.level:0"
command_context.log( self.log(
logging.WARNING, logging.WARNING,
"awsy", "awsy",
{}, {},
@ -345,13 +344,11 @@ class MachCommands(MachCommandBase):
tests.append(obj["file_relpath"]) tests.append(obj["file_relpath"])
del kwargs["test_objects"] del kwargs["test_objects"]
if not kwargs.get("binary") and conditions.is_firefox(command_context): if not kwargs.get("binary") and conditions.is_firefox(self):
try: try:
kwargs["binary"] = command_context.get_binary_path("app") kwargs["binary"] = self.get_binary_path("app")
except BinaryNotFoundException as e: except BinaryNotFoundException as e:
command_context.log( self.log(logging.ERROR, "awsy", {"error": str(e)}, "ERROR: {error}")
logging.ERROR, "awsy", {"error": str(e)}, "ERROR: {error}" self.log(logging.INFO, "awsy", {"help": e.help()}, "{help}")
)
command_context.log(logging.INFO, "awsy", {"help": e.help()}, "{help}")
return 1 return 1
return self.run_awsy(command_context, tests, **kwargs) return self.run_awsy(tests, **kwargs)

18
testing/condprofile/mach_commands.py
View file

@ -15,9 +15,9 @@ requirements = os.path.join(os.path.dirname(__file__), "requirements", "base.txt
@CommandProvider @CommandProvider
class CondprofileCommandProvider(MachCommandBase): class CondprofileCommandProvider(MachCommandBase):
def _init(self, command_context): def _init(self):
command_context.activate_virtualenv() self.activate_virtualenv()
command_context.virtualenv_manager.install_pip_requirements( self.virtualenv_manager.install_pip_requirements(
requirements, require_hashes=False requirements, require_hashes=False
) )
@ -45,7 +45,7 @@ class CondprofileCommandProvider(MachCommandBase):
download_cache, download_cache,
repo, repo,
): ):
self._init(command_context) self._init()
from condprof.client import get_profile from condprof.client import get_profile
from condprof.util import get_current_platform from condprof.util import get_current_platform
@ -91,21 +91,19 @@ class CondprofileCommandProvider(MachCommandBase):
@CommandArgument("--device-name", help="Name of the device", type=str, default=None) @CommandArgument("--device-name", help="Name of the device", type=str, default=None)
def run(self, command_context, **kw): def run(self, command_context, **kw):
os.environ["MANUAL_MACH_RUN"] = "1" os.environ["MANUAL_MACH_RUN"] = "1"
self._init(command_context) self._init()
if kw["firefox"] is None: if kw["firefox"] is None:
try: try:
kw["firefox"] = command_context.get_binary_path() kw["firefox"] = self.get_binary_path()
except BinaryNotFoundException as e: except BinaryNotFoundException as e:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"run-condprofile", "run-condprofile",
{"error": str(e)}, {"error": str(e)},
"ERROR: {error}", "ERROR: {error}",
) )
command_context.log( self.log(logging.INFO, "run-condprofile", {"help": e.help()}, "{help}")
logging.INFO, "run-condprofile", {"help": e.help()}, "{help}"
)
return 1 return 1
from condprof.runner import run from condprof.runner import run

10
testing/firefox-ui/mach_commands.py
View file

@ -100,21 +100,19 @@ class MachCommands(MachCommandBase):
) )
def run_firefox_ui_functional(self, command_context, **kwargs): def run_firefox_ui_functional(self, command_context, **kwargs):
try: try:
kwargs["binary"] = kwargs["binary"] or command_context.get_binary_path( kwargs["binary"] = kwargs["binary"] or self.get_binary_path("app")
"app"
)
except BinaryNotFoundException as e: except BinaryNotFoundException as e:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"firefox-ui-functional", "firefox-ui-functional",
{"error": str(e)}, {"error": str(e)},
"ERROR: {error}", "ERROR: {error}",
) )
command_context.log( self.log(
logging.INFO, "firefox-ui-functional", {"help": e.help()}, "{help}" logging.INFO, "firefox-ui-functional", {"help": e.help()}, "{help}"
) )
return 1 return 1
return run_firefox_ui_test( return run_firefox_ui_test(
testtype="functional", topsrcdir=command_context.topsrcdir, **kwargs testtype="functional", topsrcdir=self.topsrcdir, **kwargs
) )

28
testing/geckodriver/mach_commands.py
View file

@ -57,12 +57,10 @@ class GeckoDriver(MachCommandBase):
) )
def run(self, command_context, binary, params, debug, debugger, debugger_args): def run(self, command_context, binary, params, debug, debugger, debugger_args):
try: try:
binpath = command_context.get_binary_path("geckodriver") binpath = self.get_binary_path("geckodriver")
except BinaryNotFoundException as e: except BinaryNotFoundException as e:
command_context.log( self.log(logging.ERROR, "geckodriver", {"error": str(e)}, "ERROR: {error}")
logging.ERROR, "geckodriver", {"error": str(e)}, "ERROR: {error}" self.log(
)
command_context.log(
logging.INFO, logging.INFO,
"geckodriver", "geckodriver",
{}, {},
@ -80,21 +78,19 @@ class GeckoDriver(MachCommandBase):
if binary is None: if binary is None:
try: try:
binary = command_context.get_binary_path("app") binary = self.get_binary_path("app")
except BinaryNotFoundException as e: except BinaryNotFoundException as e:
command_context.log( self.log(
logging.ERROR, "geckodriver", {"error": str(e)}, "ERROR: {error}" logging.ERROR, "geckodriver", {"error": str(e)}, "ERROR: {error}"
) )
command_context.log( self.log(logging.INFO, "geckodriver", {"help": e.help()}, "{help}")
logging.INFO, "geckodriver", {"help": e.help()}, "{help}"
)
return 1 return 1
args.extend(["--binary", binary]) args.extend(["--binary", binary])
if debug or debugger or debugger_args: if debug or debugger or debugger_args:
if "INSIDE_EMACS" in os.environ: if "INSIDE_EMACS" in os.environ:
command_context.log_manager.terminal_handler.setLevel(logging.WARNING) self.log_manager.terminal_handler.setLevel(logging.WARNING)
import mozdebug import mozdebug
@ -106,8 +102,8 @@ class GeckoDriver(MachCommandBase):
) )
if debugger: if debugger:
debuggerInfo = mozdebug.get_debugger_info(debugger, debugger_args) self.debuggerInfo = mozdebug.get_debugger_info(debugger, debugger_args)
if not debuggerInfo: if not self.debuggerInfo:
print("Could not find a suitable debugger in your PATH.") print("Could not find a suitable debugger in your PATH.")
return 1 return 1
@ -126,8 +122,6 @@ class GeckoDriver(MachCommandBase):
return 1 return 1
# Prepend the debugger args. # Prepend the debugger args.
args = [debuggerInfo.path] + debuggerInfo.args + args args = [self.debuggerInfo.path] + self.debuggerInfo.args + args
return command_context.run_process( return self.run_process(args=args, ensure_exit_code=False, pass_thru=True)
args=args, ensure_exit_code=False, pass_thru=True
)

4
testing/gtest/mach_test_package_commands.py
View file

@ -128,6 +128,6 @@ class GtestCommands(MachCommandBase):
parser=setup_argument_parser, parser=setup_argument_parser,
) )
def gtest(self, command_context, **kwargs): def gtest(self, command_context, **kwargs):
command_context._mach_context.activate_mozharness_venv() self._mach_context.activate_mozharness_venv()
result = run_gtest(command_context._mach_context, **kwargs) result = run_gtest(self._mach_context, **kwargs)
return 0 if result else 1 return 0 if result else 1

119
testing/mach_commands.py
View file

@ -232,7 +232,7 @@ class AddTest(MachCommandBase):
print("Sorry, `addtest` doesn't currently know how to add {}".format(suite)) print("Sorry, `addtest` doesn't currently know how to add {}".format(suite))
return 1 return 1
creator = creator_cls(command_context.topsrcdir, test, suite, doc, **kwargs) creator = creator_cls(self.topsrcdir, test, suite, doc, **kwargs)
creator.check_args() creator.check_args()
@ -391,7 +391,7 @@ class Test(MachCommandBase):
from mozlog.handlers import StreamHandler from mozlog.handlers import StreamHandler
from moztest.resolve import get_suite_definition, TestResolver, TEST_SUITES from moztest.resolve import get_suite_definition, TestResolver, TEST_SUITES
resolver = command_context._spawn(TestResolver) resolver = self._spawn(TestResolver)
run_suites, run_tests = resolver.resolve_metadata(what) run_suites, run_tests = resolver.resolve_metadata(what)
if not run_suites and not run_tests: if not run_suites and not run_tests:
@ -412,12 +412,12 @@ class Test(MachCommandBase):
extra_args = [extra_args_debugger_notation] extra_args = [extra_args_debugger_notation]
# Create shared logger # Create shared logger
format_args = {"level": command_context._mach_context.settings["test"]["level"]} format_args = {"level": self._mach_context.settings["test"]["level"]}
if not run_suites and len(run_tests) == 1: if not run_suites and len(run_tests) == 1:
format_args["verbose"] = True format_args["verbose"] = True
format_args["compact"] = False format_args["compact"] = False
default_format = command_context._mach_context.settings["test"]["format"] default_format = self._mach_context.settings["test"]["format"]
log = setup_logging( log = setup_logging(
"mach-test", log_args, {default_format: sys.stdout}, format_args "mach-test", log_args, {default_format: sys.stdout}, format_args
) )
@ -433,11 +433,8 @@ class Test(MachCommandBase):
kwargs.setdefault("subsuite", None) kwargs.setdefault("subsuite", None)
if "mach_command" in suite: if "mach_command" in suite:
res = command_context._mach_context.commands.dispatch( res = self._mach_context.commands.dispatch(
suite["mach_command"], suite["mach_command"], self._mach_context, argv=extra_args, **kwargs
command_context._mach_context,
argv=extra_args,
**kwargs
) )
if res: if res:
status = res status = res
@ -459,9 +456,9 @@ class Test(MachCommandBase):
kwargs["log"] = log kwargs["log"] = log
kwargs.setdefault("subsuite", None) kwargs.setdefault("subsuite", None)
res = command_context._mach_context.commands.dispatch( res = self._mach_context.commands.dispatch(
m["mach_command"], m["mach_command"],
command_context._mach_context, self._mach_context,
argv=extra_args, argv=extra_args,
test_objects=tests, test_objects=tests,
**kwargs **kwargs
@ -501,38 +498,34 @@ class MachCommands(MachCommandBase):
log = commandline.setup_logging("cppunittest", {}, {"tbpl": sys.stdout}) log = commandline.setup_logging("cppunittest", {}, {"tbpl": sys.stdout})
# See if we have crash symbols # See if we have crash symbols
symbols_path = os.path.join(command_context.distdir, "crashreporter-symbols") symbols_path = os.path.join(self.distdir, "crashreporter-symbols")
if not os.path.isdir(symbols_path): if not os.path.isdir(symbols_path):
symbols_path = None symbols_path = None
# If no tests specified, run all tests in main manifest # If no tests specified, run all tests in main manifest
tests = params["test_files"] tests = params["test_files"]
if not tests: if not tests:
tests = [os.path.join(command_context.distdir, "cppunittests")] tests = [os.path.join(self.distdir, "cppunittests")]
manifest_path = os.path.join( manifest_path = os.path.join(self.topsrcdir, "testing", "cppunittest.ini")
command_context.topsrcdir, "testing", "cppunittest.ini"
)
else: else:
manifest_path = None manifest_path = None
utility_path = command_context.bindir utility_path = self.bindir
if conditions.is_android(command_context): if conditions.is_android(self):
from mozrunner.devices.android_device import ( from mozrunner.devices.android_device import (
verify_android_device, verify_android_device,
InstallIntent, InstallIntent,
) )
verify_android_device(command_context, install=InstallIntent.NO) verify_android_device(self, install=InstallIntent.NO)
return self.run_android_test(tests, symbols_path, manifest_path, log) return self.run_android_test(tests, symbols_path, manifest_path, log)
return self.run_desktop_test( return self.run_desktop_test(
tests, symbols_path, manifest_path, utility_path, log tests, symbols_path, manifest_path, utility_path, log
) )
def run_desktop_test( def run_desktop_test(self, tests, symbols_path, manifest_path, utility_path, log):
self, command_context, tests, symbols_path, manifest_path, utility_path, log
):
import runcppunittests as cppunittests import runcppunittests as cppunittests
from mozlog import commandline from mozlog import commandline
@ -543,7 +536,7 @@ class MachCommands(MachCommandBase):
options.symbols_path = symbols_path options.symbols_path = symbols_path
options.manifest_path = manifest_path options.manifest_path = manifest_path
options.utility_path = utility_path options.utility_path = utility_path
options.xre_path = command_context.bindir options.xre_path = self.bindir
try: try:
result = cppunittests.run_test_harness(options, tests) result = cppunittests.run_test_harness(options, tests)
@ -554,9 +547,7 @@ class MachCommands(MachCommandBase):
return 0 if result else 1 return 0 if result else 1
def run_android_test( def run_android_test(self, tests, symbols_path, manifest_path, log):
self, command_context, tests, symbols_path, manifest_path, log
):
import remotecppunittests as remotecppunittests import remotecppunittests as remotecppunittests
from mozlog import commandline from mozlog import commandline
@ -567,16 +558,14 @@ class MachCommands(MachCommandBase):
if not options.adb_path: if not options.adb_path:
from mozrunner.devices.android_device import get_adb_path from mozrunner.devices.android_device import get_adb_path
options.adb_path = get_adb_path(command_context) options.adb_path = get_adb_path(self)
options.symbols_path = symbols_path options.symbols_path = symbols_path
options.manifest_path = manifest_path options.manifest_path = manifest_path
options.xre_path = command_context.bindir options.xre_path = self.bindir
options.local_lib = command_context.bindir.replace("bin", "fennec") options.local_lib = self.bindir.replace("bin", "fennec")
for file in os.listdir(os.path.join(command_context.topobjdir, "dist")): for file in os.listdir(os.path.join(self.topobjdir, "dist")):
if file.endswith(".apk") and file.startswith("fennec"): if file.endswith(".apk") and file.startswith("fennec"):
options.local_apk = os.path.join( options.local_apk = os.path.join(self.topobjdir, "dist", file)
command_context.topobjdir, "dist", file
)
log.info("using APK: " + options.local_apk) log.info("using APK: " + options.local_apk)
break break
@ -610,13 +599,13 @@ class SpiderMonkeyTests(MachCommandBase):
def run_jstests(self, command_context, shell, params): def run_jstests(self, command_context, shell, params):
import subprocess import subprocess
command_context.virtualenv_manager.ensure() self.virtualenv_manager.ensure()
python = command_context.virtualenv_manager.python_path python = self.virtualenv_manager.python_path
js = shell or os.path.join(command_context.bindir, executable_name("js")) js = shell or os.path.join(self.bindir, executable_name("js"))
jstest_cmd = [ jstest_cmd = [
python, python,
os.path.join(command_context.topsrcdir, "js", "src", "tests", "jstests.py"), os.path.join(self.topsrcdir, "js", "src", "tests", "jstests.py"),
js, js,
] + params ] + params
@ -643,15 +632,13 @@ class SpiderMonkeyTests(MachCommandBase):
def run_jittests(self, command_context, shell, cgc, params): def run_jittests(self, command_context, shell, cgc, params):
import subprocess import subprocess
command_context.virtualenv_manager.ensure() self.virtualenv_manager.ensure()
python = command_context.virtualenv_manager.python_path python = self.virtualenv_manager.python_path
js = shell or os.path.join(command_context.bindir, executable_name("js")) js = shell or os.path.join(self.bindir, executable_name("js"))
jittest_cmd = [ jittest_cmd = [
python, python,
os.path.join( os.path.join(self.topsrcdir, "js", "src", "jit-test", "jit_test.py"),
command_context.topsrcdir, "js", "src", "jit-test", "jit_test.py"
),
js, js,
] + params ] + params
@ -674,28 +661,24 @@ class SpiderMonkeyTests(MachCommandBase):
def run_jsapitests(self, command_context, test_name=None): def run_jsapitests(self, command_context, test_name=None):
import subprocess import subprocess
jsapi_tests_cmd = [ jsapi_tests_cmd = [os.path.join(self.bindir, executable_name("jsapi-tests"))]
os.path.join(command_context.bindir, executable_name("jsapi-tests"))
]
if test_name: if test_name:
jsapi_tests_cmd.append(test_name) jsapi_tests_cmd.append(test_name)
test_env = os.environ.copy() test_env = os.environ.copy()
test_env["TOPSRCDIR"] = command_context.topsrcdir test_env["TOPSRCDIR"] = self.topsrcdir
return subprocess.call(jsapi_tests_cmd, env=test_env) return subprocess.call(jsapi_tests_cmd, env=test_env)
def run_check_js_msg(self, command_context): def run_check_js_msg(self):
import subprocess import subprocess
command_context.virtualenv_manager.ensure() self.virtualenv_manager.ensure()
python = command_context.virtualenv_manager.python_path python = self.virtualenv_manager.python_path
check_cmd = [ check_cmd = [
python, python,
os.path.join( os.path.join(self.topsrcdir, "config", "check_js_msg_encoding.py"),
command_context.topsrcdir, "config", "check_js_msg_encoding.py"
),
] ]
return subprocess.call(check_cmd) return subprocess.call(check_cmd)
@ -716,7 +699,7 @@ class JsShellTests(MachCommandBase):
description="Run benchmarks in the SpiderMonkey JS shell.", description="Run benchmarks in the SpiderMonkey JS shell.",
) )
def run_jsshelltests(self, command_context, **kwargs): def run_jsshelltests(self, command_context, **kwargs):
command_context.activate_virtualenv() self.activate_virtualenv()
from jsshell import benchmark from jsshell import benchmark
return benchmark.run(**kwargs) return benchmark.run(**kwargs)
@ -745,14 +728,14 @@ class CramTest(MachCommandBase):
def cramtest( def cramtest(
self, command_context, cram_args=None, test_paths=None, test_objects=None self, command_context, cram_args=None, test_paths=None, test_objects=None
): ):
command_context.activate_virtualenv() self.activate_virtualenv()
import mozinfo import mozinfo
from manifestparser import TestManifest from manifestparser import TestManifest
if test_objects is None: if test_objects is None:
from moztest.resolve import TestResolver from moztest.resolve import TestResolver
resolver = command_context._spawn(TestResolver) resolver = self._spawn(TestResolver)
if test_paths: if test_paths:
# If we were given test paths, try to find tests matching them. # If we were given test paths, try to find tests matching them.
test_objects = resolver.resolve_tests(paths=test_paths, flavor="cram") test_objects = resolver.resolve_tests(paths=test_paths, flavor="cram")
@ -762,16 +745,16 @@ class CramTest(MachCommandBase):
if not test_objects: if not test_objects:
message = "No tests were collected, check spelling of the test paths." message = "No tests were collected, check spelling of the test paths."
command_context.log(logging.WARN, "cramtest", {}, message) self.log(logging.WARN, "cramtest", {}, message)
return 1 return 1
mp = TestManifest() mp = TestManifest()
mp.tests.extend(test_objects) mp.tests.extend(test_objects)
tests = mp.active_tests(disabled=False, **mozinfo.info) tests = mp.active_tests(disabled=False, **mozinfo.info)
python = command_context.virtualenv_manager.python_path python = self.virtualenv_manager.python_path
cmd = [python, "-m", "cram"] + cram_args + [t["relpath"] for t in tests] cmd = [python, "-m", "cram"] + cram_args + [t["relpath"] for t in tests]
return subprocess.call(cmd, cwd=command_context.topsrcdir) return subprocess.call(cmd, cwd=self.topsrcdir)
@CommandProvider @CommandProvider
@ -915,10 +898,10 @@ class TestInfoCommand(MachCommandBase):
from mozbuild.build_commands import Build from mozbuild.build_commands import Build
try: try:
command_context.config_environment self.config_environment
except BuildEnvironmentNotFoundException: except BuildEnvironmentNotFoundException:
print("Looks like configure has not run yet, running it now...") print("Looks like configure has not run yet, running it now...")
builder = Build(command_context._mach_context, None) builder = Build(self._mach_context, None)
builder.configure(command_context) builder.configure(command_context)
ti = testinfo.TestInfoReport(verbose) ti = testinfo.TestInfoReport(verbose)
@ -972,9 +955,9 @@ class RustTests(MachCommandBase):
description="Run rust unit tests (via cargo test).", description="Run rust unit tests (via cargo test).",
) )
def run_rusttests(self, command_context, **kwargs): def run_rusttests(self, command_context, **kwargs):
return command_context._mach_context.commands.dispatch( return self._mach_context.commands.dispatch(
"build", "build",
command_context._mach_context, self._mach_context,
what=["pre-export", "export", "recurse_rusttests"], what=["pre-export", "export", "recurse_rusttests"],
) )
@ -990,7 +973,7 @@ class TestFluentMigration(MachCommandBase):
def run_migration_tests(self, command_context, test_paths=None, **kwargs): def run_migration_tests(self, command_context, test_paths=None, **kwargs):
if not test_paths: if not test_paths:
test_paths = [] test_paths = []
command_context.activate_virtualenv() self.activate_virtualenv()
from test_fluent_migrations import fmt from test_fluent_migrations import fmt
rv = 0 rv = 0
@ -999,7 +982,7 @@ class TestFluentMigration(MachCommandBase):
try: try:
context = fmt.inspect_migration(to_test) context = fmt.inspect_migration(to_test)
for issue in context["issues"]: for issue in context["issues"]:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"fluent-migration-test", "fluent-migration-test",
{ {
@ -1017,14 +1000,14 @@ class TestFluentMigration(MachCommandBase):
} }
) )
except Exception as e: except Exception as e:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"fluent-migration-test", "fluent-migration-test",
{"error": str(e), "file": to_test}, {"error": str(e), "file": to_test},
"ERROR in {file}: {error}", "ERROR in {file}: {error}",
) )
rv |= 1 rv |= 1
obj_dir = fmt.prepare_object_dir(command_context) obj_dir = fmt.prepare_object_dir(self)
for context in with_context: for context in with_context:
rv |= fmt.test_migration(command_context, obj_dir, **context) rv |= fmt.test_migration(self, obj_dir, **context)
return rv return rv

20
testing/marionette/mach_commands.py
View file

@ -80,37 +80,33 @@ class MarionetteTest(MachCommandBase):
del kwargs["test_objects"] del kwargs["test_objects"]
if not tests: if not tests:
if conditions.is_thunderbird(command_context): if conditions.is_thunderbird(self):
tests = [ tests = [
os.path.join( os.path.join(
command_context.topsrcdir, self.topsrcdir, "comm/testing/marionette/unit-tests.ini"
"comm/testing/marionette/unit-tests.ini",
) )
] ]
else: else:
tests = [ tests = [
os.path.join( os.path.join(
command_context.topsrcdir, self.topsrcdir,
"testing/marionette/harness/marionette_harness/tests/unit-tests.ini", "testing/marionette/harness/marionette_harness/tests/unit-tests.ini",
) )
] ]
if not kwargs.get("binary") and ( if not kwargs.get("binary") and (
conditions.is_firefox(command_context) conditions.is_firefox(self) or conditions.is_thunderbird(self)
or conditions.is_thunderbird(command_context)
): ):
try: try:
kwargs["binary"] = command_context.get_binary_path("app") kwargs["binary"] = self.get_binary_path("app")
except BinaryNotFoundException as e: except BinaryNotFoundException as e:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"marionette-test", "marionette-test",
{"error": str(e)}, {"error": str(e)},
"ERROR: {error}", "ERROR: {error}",
) )
command_context.log( self.log(logging.INFO, "marionette-test", {"help": e.help()}, "{help}")
logging.INFO, "marionette-test", {"help": e.help()}, "{help}"
)
return 1 return 1
return run_marionette(tests, topsrcdir=command_context.topsrcdir, **kwargs) return run_marionette(tests, topsrcdir=self.topsrcdir, **kwargs)

4
testing/marionette/mach_test_package_commands.py
View file

@ -71,5 +71,5 @@ class MachCommands(MachCommandBase):
parser=setup_marionette_argument_parser, parser=setup_marionette_argument_parser,
) )
def run_marionette_test(self, command_context, **kwargs): def run_marionette_test(self, command_context, **kwargs):
command_context.context.activate_mozharness_venv() self.context.activate_mozharness_venv()
return run_marionette(command_context.context, **kwargs) return run_marionette(self.context, **kwargs)

52
testing/mochitest/mach_commands.py
View file

@ -165,7 +165,7 @@ class MochitestRunner(MozbuildObject):
from mozrunner.devices.android_device import get_adb_path from mozrunner.devices.android_device import get_adb_path
if not kwargs["adbPath"]: if not kwargs["adbPath"]:
kwargs["adbPath"] = get_adb_path(command_context) kwargs["adbPath"] = get_adb_path(self)
options = Namespace(**kwargs) options = Namespace(**kwargs)
@ -321,11 +321,11 @@ class MachCommands(MachCommandBase):
# TODO: This is only strictly necessary while mochitest is using Python # TODO: This is only strictly necessary while mochitest is using Python
# 2 and can be removed once the command is migrated to Python 3. # 2 and can be removed once the command is migrated to Python 3.
command_context.activate_virtualenv() self.activate_virtualenv()
buildapp = None buildapp = None
for app in SUPPORTED_APPS: for app in SUPPORTED_APPS:
if conditions.is_buildapp_in(command_context, apps=[app]): if conditions.is_buildapp_in(self, apps=[app]):
buildapp = app buildapp = app
break break
@ -346,7 +346,7 @@ class MachCommands(MachCommandBase):
from mozbuild.controller.building import BuildDriver from mozbuild.controller.building import BuildDriver
command_context._ensure_state_subdir_exists(".") self._ensure_state_subdir_exists(".")
test_paths = kwargs["test_paths"] test_paths = kwargs["test_paths"]
kwargs["test_paths"] = [] kwargs["test_paths"] = []
@ -357,23 +357,21 @@ class MachCommands(MachCommandBase):
if not mozdebug.get_debugger_info(kwargs.get("debugger")): if not mozdebug.get_debugger_info(kwargs.get("debugger")):
sys.exit(1) sys.exit(1)
mochitest = command_context._spawn(MochitestRunner) mochitest = self._spawn(MochitestRunner)
tests = [] tests = []
if resolve_tests: if resolve_tests:
tests = mochitest.resolve_tests( tests = mochitest.resolve_tests(
test_paths, test_objects, cwd=command_context._mach_context.cwd test_paths, test_objects, cwd=self._mach_context.cwd
) )
if not kwargs.get("log"): if not kwargs.get("log"):
# Create shared logger # Create shared logger
format_args = { format_args = {"level": self._mach_context.settings["test"]["level"]}
"level": command_context._mach_context.settings["test"]["level"]
}
if len(tests) == 1: if len(tests) == 1:
format_args["verbose"] = True format_args["verbose"] = True
format_args["compact"] = False format_args["compact"] = False
default_format = command_context._mach_context.settings["test"]["format"] default_format = self._mach_context.settings["test"]["format"]
kwargs["log"] = setup_logging( kwargs["log"] = setup_logging(
"mach-mochitest", kwargs, {default_format: sys.stdout}, format_args "mach-mochitest", kwargs, {default_format: sys.stdout}, format_args
) )
@ -381,7 +379,7 @@ class MachCommands(MachCommandBase):
if isinstance(handler, StreamHandler): if isinstance(handler, StreamHandler):
handler.formatter.inner.summary_on_shutdown = True handler.formatter.inner.summary_on_shutdown = True
driver = command_context._spawn(BuildDriver) driver = self._spawn(BuildDriver)
driver.install_tests() driver.install_tests()
subsuite = kwargs.get("subsuite") subsuite = kwargs.get("subsuite")
@ -425,14 +423,12 @@ class MachCommands(MachCommandBase):
"websocketprocessbridge", "websocketprocessbridge",
"websocketprocessbridge_requirements_3.txt", "websocketprocessbridge_requirements_3.txt",
) )
command_context.virtualenv_manager.activate() self.virtualenv_manager.activate()
command_context.virtualenv_manager.install_pip_requirements( self.virtualenv_manager.install_pip_requirements(req, require_hashes=False)
req, require_hashes=False
)
# sys.executable is used to start the websocketprocessbridge, though for some # sys.executable is used to start the websocketprocessbridge, though for some
# reason it doesn't get set when calling `activate_this.py` in the virtualenv. # reason it doesn't get set when calling `activate_this.py` in the virtualenv.
sys.executable = command_context.virtualenv_manager.python_path sys.executable = self.virtualenv_manager.python_path
# This is a hack to introduce an option in mach to not send # This is a hack to introduce an option in mach to not send
# filtered tests to the mochitest harness. Mochitest harness will read # filtered tests to the mochitest harness. Mochitest harness will read
@ -484,7 +480,7 @@ class MachCommands(MachCommandBase):
# verify installation # verify installation
verify_android_device( verify_android_device(
command_context, self,
install=install, install=install,
xre=False, xre=False,
network=True, network=True,
@ -508,9 +504,7 @@ class MachCommands(MachCommandBase):
# specific mochitest suite has to be loaded. See Bug 1637463. # specific mochitest suite has to be loaded. See Bug 1637463.
harness_args.update({"suite_name": suite_name}) harness_args.update({"suite_name": suite_name})
result = run_mochitest( result = run_mochitest(self._mach_context, tests=tests, **harness_args)
command_context._mach_context, tests=tests, **harness_args
)
if result: if result:
overall = result overall = result
@ -543,7 +537,7 @@ class GeckoviewJunitCommands(MachCommandBase):
default=False, default=False,
) )
def run_junit(self, command_context, no_install, **kwargs): def run_junit(self, command_context, no_install, **kwargs):
command_context._ensure_state_subdir_exists(".") self._ensure_state_subdir_exists(".")
from mozrunner.devices.android_device import ( from mozrunner.devices.android_device import (
get_adb_path, get_adb_path,
@ -555,7 +549,7 @@ class GeckoviewJunitCommands(MachCommandBase):
app = kwargs.get("app") app = kwargs.get("app")
device_serial = kwargs.get("deviceSerial") device_serial = kwargs.get("deviceSerial")
verify_android_device( verify_android_device(
command_context, self,
install=InstallIntent.NO if no_install else InstallIntent.YES, install=InstallIntent.NO if no_install else InstallIntent.YES,
xre=False, xre=False,
app=app, app=app,
@ -563,20 +557,16 @@ class GeckoviewJunitCommands(MachCommandBase):
) )
if not kwargs.get("adbPath"): if not kwargs.get("adbPath"):
kwargs["adbPath"] = get_adb_path(command_context) kwargs["adbPath"] = get_adb_path(self)
if not kwargs.get("log"): if not kwargs.get("log"):
from mozlog.commandline import setup_logging from mozlog.commandline import setup_logging
format_args = { format_args = {"level": self._mach_context.settings["test"]["level"]}
"level": command_context._mach_context.settings["test"]["level"] default_format = self._mach_context.settings["test"]["format"]
}
default_format = command_context._mach_context.settings["test"]["format"]
kwargs["log"] = setup_logging( kwargs["log"] = setup_logging(
"mach-mochitest", kwargs, {default_format: sys.stdout}, format_args "mach-mochitest", kwargs, {default_format: sys.stdout}, format_args
) )
mochitest = command_context._spawn(MochitestRunner) mochitest = self._spawn(MochitestRunner)
return mochitest.run_geckoview_junit_test( return mochitest.run_geckoview_junit_test(self._mach_context, **kwargs)
command_context._mach_context, **kwargs
)

8
testing/mochitest/mach_test_package_commands.py
View file

@ -202,8 +202,8 @@ class MochitestCommands(MachCommandBase):
parser=setup_mochitest_argument_parser, parser=setup_mochitest_argument_parser,
) )
def mochitest(self, command_context, **kwargs): def mochitest(self, command_context, **kwargs):
command_context._mach_context.activate_mozharness_venv() self._mach_context.activate_mozharness_venv()
return run_test(command_context._mach_context, False, **kwargs) return run_test(self._mach_context, False, **kwargs)
@Command( @Command(
"geckoview-junit", "geckoview-junit",
@ -212,5 +212,5 @@ class MochitestCommands(MachCommandBase):
parser=setup_junit_argument_parser, parser=setup_junit_argument_parser,
) )
def geckoview_junit(self, command_context, **kwargs): def geckoview_junit(self, command_context, **kwargs):
command_context._mach_context.activate_mozharness_venv() self._mach_context.activate_mozharness_venv()
return run_test(command_context._mach_context, True, **kwargs) return run_test(self._mach_context, True, **kwargs)

2
testing/mozharness/mach_commands.py
View file

@ -217,5 +217,5 @@ class MozharnessCommands(MachCommandBase):
parser=get_parser, parser=get_parser,
) )
def mozharness(self, command_context, **kwargs): def mozharness(self, command_context, **kwargs):
runner = command_context._spawn(MozharnessRunner) runner = self._spawn(MozharnessRunner)
return runner.run_suite(kwargs.pop("suite_name")[0], **kwargs) return runner.run_suite(kwargs.pop("suite_name")[0], **kwargs)

10
testing/raptor/mach_commands.py
View file

@ -323,7 +323,7 @@ class MachRaptor(MachCommandBase):
# stop |mach bootstrap| from running # stop |mach bootstrap| from running
from raptor.power import enable_charging, disable_charging from raptor.power import enable_charging, disable_charging
build_obj = command_context build_obj = self
is_android = ( is_android = (
Conditions.is_android(build_obj) or kwargs["app"] in ANDROID_BROWSERS Conditions.is_android(build_obj) or kwargs["app"] in ANDROID_BROWSERS
@ -369,7 +369,7 @@ class MachRaptor(MachCommandBase):
if arg.startswith("raptor"): if arg.startswith("raptor"):
in_mach = False in_mach = False
raptor = command_context._spawn(RaptorRunner) raptor = self._spawn(RaptorRunner)
device = None device = None
try: try:
@ -378,10 +378,8 @@ class MachRaptor(MachCommandBase):
disable_charging(device) disable_charging(device)
return raptor.run_test(argv, kwargs) return raptor.run_test(argv, kwargs)
except BinaryNotFoundException as e: except BinaryNotFoundException as e:
command_context.log( self.log(logging.ERROR, "raptor", {"error": str(e)}, "ERROR: {error}")
logging.ERROR, "raptor", {"error": str(e)}, "ERROR: {error}" self.log(logging.INFO, "raptor", {"help": e.help()}, "{help}")
)
command_context.log(logging.INFO, "raptor", {"help": e.help()}, "{help}")
return 1 return 1
except Exception as e: except Exception as e:
print(repr(e)) print(repr(e))

2
testing/talos/mach_commands.py
View file

@ -132,7 +132,7 @@ class MachCommands(MachCommandBase):
parser=create_parser, parser=create_parser,
) )
def run_talos_test(self, command_context, **kwargs): def run_talos_test(self, command_context, **kwargs):
talos = command_context._spawn(TalosRunner) talos = self._spawn(TalosRunner)
try: try:
return talos.run_test(sys.argv[2:]) return talos.run_test(sys.argv[2:])

5
testing/tps/mach_commands.py
View file

@ -19,11 +19,10 @@ class MachCommands(MachCommandBase):
@CommandArgument("--dest", default=None, help="Where to write add-on.") @CommandArgument("--dest", default=None, help="Where to write add-on.")
def build(self, command_context, dest): def build(self, command_context, dest):
src = os.path.join( src = os.path.join(
command_context.topsrcdir, "services", "sync", "tps", "extensions", "tps" self.topsrcdir, "services", "sync", "tps", "extensions", "tps"
) )
dest = os.path.join( dest = os.path.join(
dest or os.path.join(command_context.topobjdir, "services", "sync"), dest or os.path.join(self.topobjdir, "services", "sync"), "tps.xpi"
"tps.xpi",
) )
if not os.path.exists(os.path.dirname(dest)): if not os.path.exists(os.path.dirname(dest)):

48
testing/web-platform/mach_commands.py
View file

@ -470,9 +470,8 @@ def create_parser_testpaths():
@CommandProvider @CommandProvider
class MachCommands(MachCommandBase): class MachCommands(MachCommandBase):
@staticmethod def setup(self):
def setup(command_context): self.activate_virtualenv()
command_context.activate_virtualenv()
@Command( @Command(
"web-platform-tests", "web-platform-tests",
@ -482,9 +481,9 @@ class MachCommands(MachCommandBase):
parser=create_parser_wpt, parser=create_parser_wpt,
) )
def run_web_platform_tests(self, command_context, **params): def run_web_platform_tests(self, command_context, **params):
self.setup(command_context) self.setup()
if params["product"] is None: if params["product"] is None:
if conditions.is_android(command_context): if conditions.is_android(self):
params["product"] = "firefox_android" params["product"] = "firefox_android"
else: else:
params["product"] = "firefox" params["product"] = "firefox"
@ -504,16 +503,13 @@ class MachCommands(MachCommandBase):
if not mozdebug.get_debugger_info(params.get("debugger")): if not mozdebug.get_debugger_info(params.get("debugger")):
sys.exit(1) sys.exit(1)
wpt_setup = command_context._spawn(WebPlatformTestsRunnerSetup) wpt_setup = self._spawn(WebPlatformTestsRunnerSetup)
wpt_setup._mach_context = command_context._mach_context wpt_setup._mach_context = self._mach_context
wpt_runner = WebPlatformTestsRunner(wpt_setup) wpt_runner = WebPlatformTestsRunner(wpt_setup)
logger = wpt_runner.setup_logging(**params) logger = wpt_runner.setup_logging(**params)
if ( if conditions.is_android(self) and params["product"] != "firefox_android":
conditions.is_android(command_context)
and params["product"] != "firefox_android"
):
logger.warning( logger.warning(
"Must specify --product=firefox_android in Android environment." "Must specify --product=firefox_android in Android environment."
) )
@ -537,12 +533,12 @@ class MachCommands(MachCommandBase):
parser=create_parser_update, parser=create_parser_update,
) )
def update_web_platform_tests(self, command_context, **params): def update_web_platform_tests(self, command_context, **params):
self.setup(command_context) self.setup()
command_context.virtualenv_manager.install_pip_package("html5lib==1.0.1") self.virtualenv_manager.install_pip_package("html5lib==1.0.1")
command_context.virtualenv_manager.install_pip_package("ujson") self.virtualenv_manager.install_pip_package("ujson")
command_context.virtualenv_manager.install_pip_package("requests") self.virtualenv_manager.install_pip_package("requests")
wpt_updater = command_context._spawn(WebPlatformTestsUpdater) wpt_updater = self._spawn(WebPlatformTestsUpdater)
logger = wpt_updater.setup_logging(**params) logger = wpt_updater.setup_logging(**params)
return wpt_updater.run_update(logger, **params) return wpt_updater.run_update(logger, **params)
@ -562,8 +558,8 @@ class MachCommands(MachCommandBase):
parser=create_parser_manifest_update, parser=create_parser_manifest_update,
) )
def wpt_manifest_update(self, command_context, **params): def wpt_manifest_update(self, command_context, **params):
self.setup(command_context) self.setup()
wpt_setup = command_context._spawn(WebPlatformTestsRunnerSetup) wpt_setup = self._spawn(WebPlatformTestsRunnerSetup)
wpt_runner = WebPlatformTestsRunner(wpt_setup) wpt_runner = WebPlatformTestsRunner(wpt_setup)
logger = wpt_runner.setup_logging(**params) logger = wpt_runner.setup_logging(**params)
logger.warning( logger.warning(
@ -579,12 +575,12 @@ class MachCommands(MachCommandBase):
parser=create_parser_serve, parser=create_parser_serve,
) )
def wpt_serve(self, command_context, **params): def wpt_serve(self, command_context, **params):
self.setup(command_context) self.setup()
import logging import logging
logger = logging.getLogger("web-platform-tests") logger = logging.getLogger("web-platform-tests")
logger.addHandler(logging.StreamHandler(sys.stdout)) logger.addHandler(logging.StreamHandler(sys.stdout))
wpt_serve = command_context._spawn(WebPlatformTestsServeRunner) wpt_serve = self._spawn(WebPlatformTestsServeRunner)
return wpt_serve.run(**params) return wpt_serve.run(**params)
@Command( @Command(
@ -596,7 +592,7 @@ class MachCommands(MachCommandBase):
def wpt_summary(self, command_context, **params): def wpt_summary(self, command_context, **params):
import metasummary import metasummary
wpt_setup = command_context._spawn(WebPlatformTestsRunnerSetup) wpt_setup = self._spawn(WebPlatformTestsRunnerSetup)
return metasummary.run(wpt_setup.topsrcdir, wpt_setup.topobjdir, **params) return metasummary.run(wpt_setup.topsrcdir, wpt_setup.topobjdir, **params)
@Command( @Command(
@ -616,9 +612,9 @@ class MachCommands(MachCommandBase):
parser=create_parser_unittest, parser=create_parser_unittest,
) )
def wpt_unittest(self, command_context, **params): def wpt_unittest(self, command_context, **params):
self.setup(command_context) self.setup()
command_context.virtualenv_manager.install_pip_package("tox") self.virtualenv_manager.install_pip_package("tox")
runner = command_context._spawn(WebPlatformTestsUnittestRunner) runner = self._spawn(WebPlatformTestsUnittestRunner)
return 0 if runner.run(**params) else 1 return 0 if runner.run(**params) else 1
@Command( @Command(
@ -628,7 +624,7 @@ class MachCommands(MachCommandBase):
parser=create_parser_testpaths, parser=create_parser_testpaths,
) )
def wpt_test_paths(self, command_context, **params): def wpt_test_paths(self, command_context, **params):
runner = command_context._spawn(WebPlatformTestsTestPathsRunner) runner = self._spawn(WebPlatformTestsTestPathsRunner)
runner.run(**params) runner.run(**params)
return 0 return 0
@ -639,6 +635,6 @@ class MachCommands(MachCommandBase):
parser=create_parser_fission_regressions, parser=create_parser_fission_regressions,
) )
def wpt_fission_regressions(self, command_context, **params): def wpt_fission_regressions(self, command_context, **params):
runner = command_context._spawn(WebPlatformTestsFissionRegressionsRunner) runner = self._spawn(WebPlatformTestsFissionRegressionsRunner)
runner.run(**params) runner.run(**params)
return 0 return 0

6
testing/web-platform/mach_test_package_commands.py
View file

@ -76,11 +76,11 @@ class WebPlatformTestsRunnerSetup(object):
class MachCommands(MachCommandBase): class MachCommands(MachCommandBase):
@Command("web-platform-tests", category="testing", parser=create_parser_wpt) @Command("web-platform-tests", category="testing", parser=create_parser_wpt)
def run_web_platform_tests(self, command_context, **kwargs): def run_web_platform_tests(self, command_context, **kwargs):
command_context._mach_context.activate_mozharness_venv() self._mach_context.activate_mozharness_venv()
return WebPlatformTestsRunner( return WebPlatformTestsRunner(
WebPlatformTestsRunnerSetup(command_context._mach_context) WebPlatformTestsRunnerSetup(self._mach_context)
).run(**kwargs) ).run(**kwargs)
@Command("wpt", category="testing", parser=create_parser_wpt) @Command("wpt", category="testing", parser=create_parser_wpt)
def run_wpt(self, command_context, **params): def run_wpt(self, command_context, **params):
return command_context.run_web_platform_tests(**params) return self.run_web_platform_tests(**params)

28
testing/xpcshell/mach_commands.py
View file

@ -238,20 +238,18 @@ class MachCommands(MachCommandBase):
m.tests.extend(test_objects) m.tests.extend(test_objects)
params["manifest"] = m params["manifest"] = m
driver = command_context._spawn(BuildDriver) driver = self._spawn(BuildDriver)
driver.install_tests() driver.install_tests()
# We should probably have a utility function to ensure the tree is # We should probably have a utility function to ensure the tree is
# ready to run tests. Until then, we just create the state dir (in # ready to run tests. Until then, we just create the state dir (in
# case the tree wasn't built with mach). # case the tree wasn't built with mach).
command_context._ensure_state_subdir_exists(".") self._ensure_state_subdir_exists(".")
if not params.get("log"): if not params.get("log"):
log_defaults = { log_defaults = {self._mach_context.settings["test"]["format"]: sys.stdout}
command_context._mach_context.settings["test"]["format"]: sys.stdout
}
fmt_defaults = { fmt_defaults = {
"level": command_context._mach_context.settings["test"]["level"], "level": self._mach_context.settings["test"]["level"],
"verbose": True, "verbose": True,
} }
params["log"] = structured.commandline.setup_logging( params["log"] = structured.commandline.setup_logging(
@ -262,10 +260,7 @@ class MachCommands(MachCommandBase):
# pylint --py3k W1619 # pylint --py3k W1619
params["threadCount"] = int((cpu_count() * 3) / 2) params["threadCount"] = int((cpu_count() * 3) / 2)
if ( if conditions.is_android(self) or self.substs.get("MOZ_BUILD_APP") == "b2g":
conditions.is_android(command_context)
or command_context.substs.get("MOZ_BUILD_APP") == "b2g"
):
from mozrunner.devices.android_device import ( from mozrunner.devices.android_device import (
verify_android_device, verify_android_device,
get_adb_path, get_adb_path,
@ -275,17 +270,14 @@ class MachCommands(MachCommandBase):
install = InstallIntent.YES if params["setup"] else InstallIntent.NO install = InstallIntent.YES if params["setup"] else InstallIntent.NO
device_serial = params.get("deviceSerial") device_serial = params.get("deviceSerial")
verify_android_device( verify_android_device(
command_context, self, network=True, install=install, device_serial=device_serial
network=True,
install=install,
device_serial=device_serial,
) )
if not params["adbPath"]: if not params["adbPath"]:
params["adbPath"] = get_adb_path(command_context) params["adbPath"] = get_adb_path(self)
xpcshell = command_context._spawn(AndroidXPCShellRunner) xpcshell = self._spawn(AndroidXPCShellRunner)
else: else:
xpcshell = command_context._spawn(XPCShellRunner) xpcshell = self._spawn(XPCShellRunner)
xpcshell.cwd = command_context._mach_context.cwd xpcshell.cwd = self._mach_context.cwd
try: try:
return xpcshell.run_test(**params) return xpcshell.run_test(**params)

4
testing/xpcshell/mach_test_package_commands.py
View file

@ -61,5 +61,5 @@ class MochitestCommands(MachCommandBase):
parser=parser_desktop, parser=parser_desktop,
) )
def xpcshell(self, command_context, **kwargs): def xpcshell(self, command_context, **kwargs):
command_context._mach_context.activate_mozharness_venv() self._mach_context.activate_mozharness_venv()
return run_xpcshell(command_context._mach_context, **kwargs) return run_xpcshell(self._mach_context, **kwargs)

10
toolkit/components/telemetry/tests/marionette/mach_commands.py
View file

@ -80,17 +80,17 @@ class TelemetryTest(MachCommandBase):
for obj in kwargs["test_objects"]: for obj in kwargs["test_objects"]:
tests.append(obj["file_relpath"]) tests.append(obj["file_relpath"])
del kwargs["test_objects"] del kwargs["test_objects"]
if not kwargs.get("binary") and conditions.is_firefox(command_context): if not kwargs.get("binary") and conditions.is_firefox(self):
try: try:
kwargs["binary"] = command_context.get_binary_path("app") kwargs["binary"] = self.get_binary_path("app")
except BinaryNotFoundException as e: except BinaryNotFoundException as e:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"telemetry-tests-client", "telemetry-tests-client",
{"error": str(e)}, {"error": str(e)},
"ERROR: {error}", "ERROR: {error}",
) )
command_context.log( self.log(
logging.INFO, "telemetry-tests-client", {"help": e.help()}, "{help}" logging.INFO, "telemetry-tests-client", {"help": e.help()}, "{help}"
) )
return 1 return 1
@ -98,4 +98,4 @@ class TelemetryTest(MachCommandBase):
kwargs[ kwargs[
"server_root" "server_root"
] = "toolkit/components/telemetry/tests/marionette/harness/www" ] = "toolkit/components/telemetry/tests/marionette/harness/www"
return run_telemetry(tests, topsrcdir=command_context.topsrcdir, **kwargs) return run_telemetry(tests, topsrcdir=self.topsrcdir, **kwargs)

135
tools/browsertime/mach_commands.py
View file

@ -155,19 +155,17 @@ host_fetches = {
@CommandProvider @CommandProvider
class MachBrowsertime(MachCommandBase): class MachBrowsertime(MachCommandBase):
def artifact_cache_path(self, command_context): def artifact_cache_path(self):
r"""Downloaded artifacts will be kept here.""" r"""Downloaded artifacts will be kept here."""
# The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE. # The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE.
return mozpath.join( return mozpath.join(self._mach_context.state_dir, "cache", "browsertime")
command_context._mach_context.state_dir, "cache", "browsertime"
)
def state_path(self, command_context): def state_path(self):
r"""Unpacked artifacts will be kept here.""" r"""Unpacked artifacts will be kept here."""
# The convention is $MOZBUILD_STATE_PATH/$FEATURE. # The convention is $MOZBUILD_STATE_PATH/$FEATURE.
return mozpath.join(command_context._mach_context.state_dir, "browsertime") return mozpath.join(self._mach_context.state_dir, "browsertime")
def setup_prerequisites(self, command_context): def setup_prerequisites(self):
r"""Install browsertime and visualmetrics.py prerequisites.""" r"""Install browsertime and visualmetrics.py prerequisites."""
from mozbuild.action.tooltool import unpack_file from mozbuild.action.tooltool import unpack_file
@ -193,9 +191,7 @@ class MachBrowsertime(MachCommandBase):
# Download the visualmetrics.py requirements. # Download the visualmetrics.py requirements.
artifact_cache = ArtifactCache( artifact_cache = ArtifactCache(
self.artifact_cache_path(command_context), self.artifact_cache_path(), log=self.log, skip_cache=False
log=command_context.log,
skip_cache=False,
) )
fetches = host_fetches[host_platform()] fetches = host_fetches[host_platform()]
@ -206,9 +202,9 @@ class MachBrowsertime(MachCommandBase):
if fetch.get("unpack", True): if fetch.get("unpack", True):
cwd = os.getcwd() cwd = os.getcwd()
try: try:
mkdir(self.state_path(command_context)) mkdir(self.state_path())
os.chdir(self.state_path(command_context)) os.chdir(self.state_path())
command_context.log( self.log(
logging.INFO, logging.INFO,
"browsertime", "browsertime",
{"path": archive}, {"path": archive},
@ -219,20 +215,14 @@ class MachBrowsertime(MachCommandBase):
# Windows archive does not contain a subfolder # Windows archive does not contain a subfolder
# so we make one for it here # so we make one for it here
mkdir(fetch.get("path")) mkdir(fetch.get("path"))
os.chdir( os.chdir(os.path.join(self.state_path(), fetch.get("path")))
os.path.join(
self.state_path(command_context), fetch.get("path")
)
)
unpack_file(archive) unpack_file(archive)
os.chdir(self.state_path(command_context)) os.chdir(self.state_path())
else: else:
unpack_file(archive) unpack_file(archive)
# Make sure the expected path exists after extraction # Make sure the expected path exists after extraction
path = os.path.join( path = os.path.join(self.state_path(), fetch.get("path"))
self.state_path(command_context), fetch.get("path")
)
if not os.path.exists(path): if not os.path.exists(path):
raise Exception("Cannot find an extracted directory: %s" % path) raise Exception("Cannot find an extracted directory: %s" % path)
@ -256,21 +246,19 @@ class MachBrowsertime(MachCommandBase):
finally: finally:
os.chdir(cwd) os.chdir(cwd)
def setup(self, command_context, should_clobber=False, new_upstream_url=""): def setup(self, should_clobber=False, new_upstream_url=""):
r"""Install browsertime and visualmetrics.py prerequisites and the Node.js package.""" r"""Install browsertime and visualmetrics.py prerequisites and the Node.js package."""
sys.path.append( sys.path.append(mozpath.join(self.topsrcdir, "tools", "lint", "eslint"))
mozpath.join(command_context.topsrcdir, "tools", "lint", "eslint")
)
import setup_helper import setup_helper
if not new_upstream_url: if not new_upstream_url:
self.setup_prerequisites(command_context) self.setup_prerequisites()
if new_upstream_url: if new_upstream_url:
package_json_path = os.path.join(BROWSERTIME_ROOT, "package.json") package_json_path = os.path.join(BROWSERTIME_ROOT, "package.json")
command_context.log( self.log(
logging.INFO, logging.INFO,
"browsertime", "browsertime",
{ {
@ -311,7 +299,7 @@ class MachBrowsertime(MachCommandBase):
os.environ["CHROMEDRIVER_SKIP_DOWNLOAD"] = "true" os.environ["CHROMEDRIVER_SKIP_DOWNLOAD"] = "true"
os.environ["GECKODRIVER_SKIP_DOWNLOAD"] = "true" os.environ["GECKODRIVER_SKIP_DOWNLOAD"] = "true"
command_context.log( self.log(
logging.INFO, logging.INFO,
"browsertime", "browsertime",
{"package_json": mozpath.join(BROWSERTIME_ROOT, "package.json")}, {"package_json": mozpath.join(BROWSERTIME_ROOT, "package.json")},
@ -331,19 +319,19 @@ class MachBrowsertime(MachCommandBase):
if new_upstream_url or AUTOMATION: if new_upstream_url or AUTOMATION:
return 0 return 0
return self.check(command_context) return self.check()
def node(self, command_context, args): def node(self, args):
r"""Invoke node (interactively) with the given arguments.""" r"""Invoke node (interactively) with the given arguments."""
return command_context.run_process( return self.run_process(
[node_path()] + args, [node_path()] + args,
append_env=self.append_env(command_context), append_env=self.append_env(),
pass_thru=True, # Allow user to run Node interactively. pass_thru=True, # Allow user to run Node interactively.
ensure_exit_code=False, # Don't throw on non-zero exit code. ensure_exit_code=False, # Don't throw on non-zero exit code.
cwd=mozpath.join(command_context.topsrcdir), cwd=mozpath.join(self.topsrcdir),
) )
def append_env(self, command_context, append_path=True): def append_env(self, append_path=True):
fetches = host_fetches[host_platform()] fetches = host_fetches[host_platform()]
# Ensure that bare `ffmpeg` and ImageMagick commands # Ensure that bare `ffmpeg` and ImageMagick commands
@ -351,14 +339,12 @@ class MachBrowsertime(MachCommandBase):
# script doesn't take these as configuration, so we do this (for now). # script doesn't take these as configuration, so we do this (for now).
# We should update the script itself to accept this configuration. # We should update the script itself to accept this configuration.
path = os.environ.get("PATH", "").split(os.pathsep) if append_path else [] path = os.environ.get("PATH", "").split(os.pathsep) if append_path else []
path_to_ffmpeg = mozpath.join( path_to_ffmpeg = mozpath.join(self.state_path(), fetches["ffmpeg"]["path"])
self.state_path(command_context), fetches["ffmpeg"]["path"]
)
path_to_imagemagick = None path_to_imagemagick = None
if "ImageMagick" in fetches: if "ImageMagick" in fetches:
path_to_imagemagick = mozpath.join( path_to_imagemagick = mozpath.join(
self.state_path(command_context), fetches["ImageMagick"]["path"] self.state_path(), fetches["ImageMagick"]["path"]
) )
if path_to_imagemagick: if path_to_imagemagick:
@ -366,7 +352,7 @@ class MachBrowsertime(MachCommandBase):
# want to ensure that our ffmpeg goes first, just in case. # want to ensure that our ffmpeg goes first, just in case.
path.insert( path.insert(
0, 0,
self.state_path(command_context) self.state_path()
if host_platform().startswith("win") if host_platform().startswith("win")
else mozpath.join(path_to_imagemagick, "bin"), else mozpath.join(path_to_imagemagick, "bin"),
) # noqa ) # noqa
@ -417,7 +403,7 @@ class MachBrowsertime(MachCommandBase):
# #
# Our fork of browsertime supports a `PYTHON` environment variable # Our fork of browsertime supports a `PYTHON` environment variable
# that points to the exact python executable to use. # that points to the exact python executable to use.
"PYTHON": command_context.virtualenv_manager.python_path, "PYTHON": self.virtualenv_manager.python_path,
} }
if path_to_imagemagick: if path_to_imagemagick:
@ -433,7 +419,7 @@ class MachBrowsertime(MachCommandBase):
return append_env return append_env
def _need_install(self, command_context, package): def _need_install(self, package):
from pip._internal.req.constructors import install_req_from_line from pip._internal.req.constructors import install_req_from_line
req = install_req_from_line(package) req = install_req_from_line(package)
@ -441,41 +427,38 @@ class MachBrowsertime(MachCommandBase):
if req.satisfied_by is None: if req.satisfied_by is None:
return True return True
venv_site_lib = os.path.abspath( venv_site_lib = os.path.abspath(
os.path.join(command_context.virtualenv_manager.bin_path, "..", "lib") os.path.join(self.virtualenv_manager.bin_path, "..", "lib")
) )
site_packages = os.path.abspath(req.satisfied_by.location) site_packages = os.path.abspath(req.satisfied_by.location)
return not site_packages.startswith(venv_site_lib) return not site_packages.startswith(venv_site_lib)
def activate_browsertime_virtualenv(self, command_context, *args, **kwargs): def activate_virtualenv(self, *args, **kwargs):
r"""Activates virtualenv. r"""Activates virtualenv.
This function will also install Pillow and pyssim if needed. This function will also install Pillow and pyssim if needed.
It will raise an error in case the install failed. It will raise an error in case the install failed.
""" """
MachCommandBase.activate_virtualenv(command_context, *args, **kwargs) MachCommandBase.activate_virtualenv(self, *args, **kwargs)
# installing Python deps on the fly # installing Python deps on the fly
for dep in ("Pillow==%s" % PILLOW_VERSION, "pyssim==%s" % PYSSIM_VERSION): for dep in ("Pillow==%s" % PILLOW_VERSION, "pyssim==%s" % PYSSIM_VERSION):
if self._need_install(command_context, dep): if self._need_install(dep):
command_context.virtualenv_manager._run_pip(["install", dep]) self.virtualenv_manager._run_pip(["install", dep])
def check(self, command_context): def check(self):
r"""Run `visualmetrics.py --check`.""" r"""Run `visualmetrics.py --check`."""
command_context.activate_virtualenv() self.activate_virtualenv()
args = ["--check"] args = ["--check"]
status = command_context.run_process( status = self.run_process(
[command_context.virtualenv_manager.python_path, visualmetrics_path()] [self.virtualenv_manager.python_path, visualmetrics_path()] + args,
+ args,
# For --check, don't allow user's path to interfere with # For --check, don't allow user's path to interfere with
# path testing except on Linux, where ImageMagick needs to # path testing except on Linux, where ImageMagick needs to
# be installed manually. # be installed manually.
append_env=self.append_env( append_env=self.append_env(append_path=host_platform().startswith("linux")),
command_context, append_path=host_platform().startswith("linux")
),
pass_thru=True, pass_thru=True,
ensure_exit_code=False, # Don't throw on non-zero exit code. ensure_exit_code=False, # Don't throw on non-zero exit code.
cwd=mozpath.join(command_context.topsrcdir), cwd=mozpath.join(self.topsrcdir),
) )
sys.stdout.flush() sys.stdout.flush()
@ -485,15 +468,15 @@ class MachBrowsertime(MachCommandBase):
return status return status
# Avoid logging the command (and, on Windows, the environment). # Avoid logging the command (and, on Windows, the environment).
command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL) self.log_manager.terminal_handler.setLevel(logging.CRITICAL)
print("browsertime version:", end=" ") print("browsertime version:", end=" ")
sys.stdout.flush() sys.stdout.flush()
sys.stderr.flush() sys.stderr.flush()
return self.node(command_context, [browsertime_path()] + ["--version"]) return self.node([browsertime_path()] + ["--version"])
def extra_default_args(self, command_context, args=[]): def extra_default_args(self, args=[]):
# Add Mozilla-specific default arguments. This is tricky because browsertime is quite # Add Mozilla-specific default arguments. This is tricky because browsertime is quite
# loose about arguments; repeat arguments are generally accepted but then produce # loose about arguments; repeat arguments are generally accepted but then produce
# difficult to interpret type errors. # difficult to interpret type errors.
@ -544,17 +527,15 @@ class MachBrowsertime(MachCommandBase):
if not specifies_binaryPath: if not specifies_binaryPath:
try: try:
extra_args.extend( extra_args.extend(("--firefox.binaryPath", self.get_binary_path()))
("--firefox.binaryPath", command_context.get_binary_path())
)
except BinaryNotFoundException as e: except BinaryNotFoundException as e:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"browsertime", "browsertime",
{"error": str(e)}, {"error": str(e)},
"ERROR: {error}", "ERROR: {error}",
) )
command_context.log( self.log(
logging.INFO, logging.INFO,
"browsertime", "browsertime",
{}, {},
@ -564,7 +545,7 @@ class MachBrowsertime(MachCommandBase):
return 1 return 1
if extra_args: if extra_args:
command_context.log( self.log(
logging.DEBUG, logging.DEBUG,
"browsertime", "browsertime",
{"extra_args": extra_args}, {"extra_args": extra_args},
@ -573,11 +554,9 @@ class MachBrowsertime(MachCommandBase):
return extra_args return extra_args
def _verify_node_install(self, command_context): def _verify_node_install(self):
# check if Node is installed # check if Node is installed
sys.path.append( sys.path.append(mozpath.join(self.topsrcdir, "tools", "lint", "eslint"))
mozpath.join(command_context.topsrcdir, "tools", "lint", "eslint")
)
import setup_helper import setup_helper
with silence(): with silence():
@ -635,13 +614,13 @@ class MachBrowsertime(MachCommandBase):
check=False, check=False,
browsertime_help=False, browsertime_help=False,
): ):
command_context._set_log_level(verbose) self._set_log_level(verbose)
# Output a message before going further to make sure the # Output a message before going further to make sure the
# user knows that this tool is unsupported by the perftest # user knows that this tool is unsupported by the perftest
# team and point them to our supported tools. Pause a bit to # team and point them to our supported tools. Pause a bit to
# make sure the user sees this message. # make sure the user sees this message.
command_context.log( self.log(
logging.INFO, logging.INFO,
"browsertime", "browsertime",
{}, {},
@ -658,21 +637,21 @@ class MachBrowsertime(MachCommandBase):
time.sleep(5) time.sleep(5)
if update_upstream_url: if update_upstream_url:
return self.setup(command_context, new_upstream_url=update_upstream_url) return self.setup(new_upstream_url=update_upstream_url)
elif setup: elif setup:
return self.setup(command_context, should_clobber=clobber) return self.setup(should_clobber=clobber)
else: else:
if not self._verify_node_install(command_context): if not self._verify_node_install():
return 1 return 1
if check: if check:
return self.check(command_context) return self.check()
if browsertime_help: if browsertime_help:
args.append("--help") args.append("--help")
self.activate_browsertime_virtualenv(command_context) self.activate_virtualenv()
default_args = self.extra_default_args(command_context, args) default_args = self.extra_default_args(args)
if default_args == 1: if default_args == 1:
return 1 return 1
return self.node(command_context, [browsertime_path()] + default_args + args) return self.node([browsertime_path()] + default_args + args)

30
tools/lint/mach_commands.py
View file

@ -71,27 +71,25 @@ class MachCommands(MachCommandBase):
) )
def lint(self, command_context, *runargs, **lintargs): def lint(self, command_context, *runargs, **lintargs):
"""Run linters.""" """Run linters."""
command_context.activate_virtualenv() self.activate_virtualenv()
from mozlint import cli, parser from mozlint import cli, parser
try: try:
buildargs = {} buildargs = {}
buildargs["substs"] = copy.deepcopy(dict(command_context.substs)) buildargs["substs"] = copy.deepcopy(dict(self.substs))
buildargs["defines"] = copy.deepcopy(dict(command_context.defines)) buildargs["defines"] = copy.deepcopy(dict(self.defines))
buildargs["topobjdir"] = command_context.topobjdir buildargs["topobjdir"] = self.topobjdir
lintargs.update(buildargs) lintargs.update(buildargs)
except BuildEnvironmentNotFoundException: except BuildEnvironmentNotFoundException:
pass pass
lintargs.setdefault("root", command_context.topsrcdir) lintargs.setdefault("root", self.topsrcdir)
lintargs["exclude"] = get_global_excludes(lintargs["root"]) lintargs["exclude"] = get_global_excludes(lintargs["root"])
lintargs["config_paths"].insert(0, here) lintargs["config_paths"].insert(0, here)
lintargs["virtualenv_bin_path"] = command_context.virtualenv_manager.bin_path lintargs["virtualenv_bin_path"] = self.virtualenv_manager.bin_path
lintargs["virtualenv_manager"] = command_context.virtualenv_manager lintargs["virtualenv_manager"] = self.virtualenv_manager
for path in EXCLUSION_FILES: for path in EXCLUSION_FILES:
parser.GLOBAL_SUPPORT_FILES.append( parser.GLOBAL_SUPPORT_FILES.append(os.path.join(self.topsrcdir, path))
os.path.join(command_context.topsrcdir, path)
)
return cli.run(*runargs, **lintargs) return cli.run(*runargs, **lintargs)
@Command( @Command(
@ -127,9 +125,9 @@ class MachCommands(MachCommandBase):
help="Extra args that will be forwarded to eslint.", help="Extra args that will be forwarded to eslint.",
) )
def eslint(self, command_context, paths, extra_args=[], **kwargs): def eslint(self, command_context, paths, extra_args=[], **kwargs):
command_context._mach_context.commands.dispatch( self._mach_context.commands.dispatch(
"lint", "lint",
command_context._mach_context, self._mach_context,
linters=["eslint"], linters=["eslint"],
paths=paths, paths=paths,
argv=extra_args, argv=extra_args,
@ -160,10 +158,6 @@ class MachCommands(MachCommandBase):
kwargs["linters"] = list(linters) kwargs["linters"] = list(linters)
kwargs["fix"] = True kwargs["fix"] = True
command_context._mach_context.commands.dispatch( self._mach_context.commands.dispatch(
"lint", "lint", self._mach_context, paths=paths, argv=extra_args, **kwargs
command_context._mach_context,
paths=paths,
argv=extra_args,
**kwargs
) )

26
tools/mach_commands.py
View file

@ -77,7 +77,7 @@ class BustedProvider(MachCommandBase):
if ( if (
against != "general" against != "general"
and against not in command_context._mach_context.commands.command_handlers and against not in self._mach_context.commands.command_handlers
): ):
print( print(
"%s is not a valid value for `against`. `against` must be " "%s is not a valid value for `against`. `against` must be "
@ -95,12 +95,10 @@ class BustedProvider(MachCommandBase):
# Look up the file implementing that command, then cross-refernce # Look up the file implementing that command, then cross-refernce
# moz.build files to get the product/component. # moz.build files to get the product/component.
handler = command_context._mach_context.commands.command_handlers[against] handler = self._mach_context.commands.command_handlers[against]
method = getattr(handler.cls, handler.method) method = getattr(handler.cls, handler.method)
sourcefile = mozpath.relpath( sourcefile = mozpath.relpath(inspect.getsourcefile(method), self.topsrcdir)
inspect.getsourcefile(method), command_context.topsrcdir reader = self.mozbuild_reader(config_mode="empty")
)
reader = command_context.mozbuild_reader(config_mode="empty")
try: try:
res = reader.files_info([sourcefile])[sourcefile]["BUG_COMPONENT"] res = reader.files_info([sourcefile])[sourcefile]["BUG_COMPONENT"]
product, component = res.product, res.component product, component = res.product, res.component
@ -441,7 +439,7 @@ class MozregressionCommand(MachCommandBase):
parser=mozregression_create_parser, parser=mozregression_create_parser,
) )
def run(self, command_context, **options): def run(self, command_context, **options):
command_context.activate_virtualenv() self.activate_virtualenv()
mozregression = PypiBasedTool("mozregression") mozregression = PypiBasedTool("mozregression")
mozregression.run(**options) mozregression.run(**options)
@ -458,11 +456,11 @@ class NodeCommands(MachCommandBase):
from mozbuild.nodeutil import find_node_executable from mozbuild.nodeutil import find_node_executable
# Avoid logging the command # Avoid logging the command
command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL) self.log_manager.terminal_handler.setLevel(logging.CRITICAL)
node_path, _ = find_node_executable() node_path, _ = find_node_executable()
return command_context.run_process( return self.run_process(
[node_path] + args, [node_path] + args,
pass_thru=True, # Allow user to run Node interactively. pass_thru=True, # Allow user to run Node interactively.
ensure_exit_code=False, # Don't throw on non-zero exit code. ensure_exit_code=False, # Don't throw on non-zero exit code.
@ -478,7 +476,7 @@ class NodeCommands(MachCommandBase):
from mozbuild.nodeutil import find_npm_executable from mozbuild.nodeutil import find_npm_executable
# Avoid logging the command # Avoid logging the command
command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL) self.log_manager.terminal_handler.setLevel(logging.CRITICAL)
import os import os
@ -494,7 +492,7 @@ class NodeCommands(MachCommandBase):
path = os.path.abspath(os.path.dirname(npm_path)) path = os.path.abspath(os.path.dirname(npm_path))
os.environ["PATH"] = "{}:{}".format(path, os.environ["PATH"]) os.environ["PATH"] = "{}:{}".format(path, os.environ["PATH"])
return command_context.run_process( return self.run_process(
[npm_path, "--scripts-prepend-node-path=auto"] + args, [npm_path, "--scripts-prepend-node-path=auto"] + args,
pass_thru=True, # Avoid eating npm output/error messages pass_thru=True, # Avoid eating npm output/error messages
ensure_exit_code=False, # Don't throw on non-zero exit code. ensure_exit_code=False, # Don't throw on non-zero exit code.
@ -524,18 +522,18 @@ class LogspamCommand(MachCommandBase):
@SubCommand("logspam", "report", parser=partial(logspam_create_parser, "report")) @SubCommand("logspam", "report", parser=partial(logspam_create_parser, "report"))
def report(self, command_context, **options): def report(self, command_context, **options):
command_context.activate_virtualenv() self.activate_virtualenv()
logspam = PypiBasedTool("logspam") logspam = PypiBasedTool("logspam")
logspam.run(command="report", **options) logspam.run(command="report", **options)
@SubCommand("logspam", "bisect", parser=partial(logspam_create_parser, "bisect")) @SubCommand("logspam", "bisect", parser=partial(logspam_create_parser, "bisect"))
def bisect(self, command_context, **options): def bisect(self, command_context, **options):
command_context.activate_virtualenv() self.activate_virtualenv()
logspam = PypiBasedTool("logspam") logspam = PypiBasedTool("logspam")
logspam.run(command="bisect", **options) logspam.run(command="bisect", **options)
@SubCommand("logspam", "file", parser=partial(logspam_create_parser, "file")) @SubCommand("logspam", "file", parser=partial(logspam_create_parser, "file"))
def create(self, command_context, **options): def create(self, command_context, **options):
command_context.activate_virtualenv() self.activate_virtualenv()
logspam = PypiBasedTool("logspam") logspam = PypiBasedTool("logspam")
logspam.run(command="file", **options) logspam.run(command="file", **options)

23
tools/moztreedocs/mach_commands.py
View file

@ -116,12 +116,10 @@ class Documentation(MachCommandBase):
): ):
# TODO: Bug 1704891 - move the ESLint setup tools to a shared place. # TODO: Bug 1704891 - move the ESLint setup tools to a shared place.
sys.path.append( sys.path.append(mozpath.join(self.topsrcdir, "tools", "lint", "eslint"))
mozpath.join(command_context.topsrcdir, "tools", "lint", "eslint")
)
import setup_helper import setup_helper
setup_helper.set_project_root(command_context.topsrcdir) setup_helper.set_project_root(self.topsrcdir)
if not setup_helper.check_node_executables_valid(): if not setup_helper.check_node_executables_valid():
return 1 return 1
@ -131,15 +129,15 @@ class Documentation(MachCommandBase):
# Set the path so that Sphinx can find jsdoc, unfortunately there isn't # Set the path so that Sphinx can find jsdoc, unfortunately there isn't
# a way to pass this to Sphinx itself at the moment. # a way to pass this to Sphinx itself at the moment.
os.environ["PATH"] = ( os.environ["PATH"] = (
mozpath.join(command_context.topsrcdir, "node_modules", ".bin") mozpath.join(self.topsrcdir, "node_modules", ".bin")
+ os.pathsep + os.pathsep
+ self._node_path() + self._node_path()
+ os.pathsep + os.pathsep
+ os.environ["PATH"] + os.environ["PATH"]
) )
command_context.activate_virtualenv() self.activate_virtualenv()
command_context.virtualenv_manager.install_pip_requirements( self.virtualenv_manager.install_pip_requirements(
os.path.join(here, "requirements.txt") os.path.join(here, "requirements.txt")
) )
@ -149,10 +147,10 @@ class Documentation(MachCommandBase):
unique_id = "%s/%s" % (self.project(), str(uuid.uuid1())) unique_id = "%s/%s" % (self.project(), str(uuid.uuid1()))
outdir = outdir or os.path.join(command_context.topobjdir, "docs") outdir = outdir or os.path.join(self.topobjdir, "docs")
savedir = os.path.join(outdir, fmt) savedir = os.path.join(outdir, fmt)
path = path or command_context.topsrcdir path = path or self.topsrcdir
path = os.path.normpath(os.path.abspath(path)) path = os.path.normpath(os.path.abspath(path))
docdir = self._find_doc_dir(path) docdir = self._find_doc_dir(path)
@ -395,12 +393,7 @@ class Documentation(MachCommandBase):
for handler in Registrar.command_handlers.values() for handler in Registrar.command_handlers.values()
if handler.metrics_path is not None if handler.metrics_path is not None
] ]
args.extend( args.extend([os.path.join(self.topsrcdir, path) for path in set(metrics_paths)])
[
os.path.join(command_context.topsrcdir, path)
for path in set(metrics_paths)
]
)
subprocess.check_call(args) subprocess.check_call(args)

10
tools/phabricator/mach_commands.py
View file

@ -31,7 +31,7 @@ class PhabricatorCommandProvider(MachCommandBase):
existing = mozfile.which("moz-phab") existing = mozfile.which("moz-phab")
if existing and not force: if existing and not force:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"already_installed", "already_installed",
{}, {},
@ -44,7 +44,7 @@ class PhabricatorCommandProvider(MachCommandBase):
# if pip3 is missing. # if pip3 is missing.
pip3 = mozfile.which("pip3") pip3 = mozfile.which("pip3")
if not pip3: if not pip3:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"pip3_not_installed", "pip3_not_installed",
{}, {},
@ -74,7 +74,7 @@ class PhabricatorCommandProvider(MachCommandBase):
else: else:
# Unsupported, default to --user. # Unsupported, default to --user.
command_context.log( self.log(
logging.WARNING, logging.WARNING,
"unsupported_platform", "unsupported_platform",
{}, {},
@ -88,7 +88,7 @@ class PhabricatorCommandProvider(MachCommandBase):
# installation if we're not within one. # installation if we're not within one.
command.append("--user") command.append("--user")
command_context.log(logging.INFO, "run", {}, "Installing moz-phab") self.log(logging.INFO, "run", {}, "Installing moz-phab")
subprocess.run(command) subprocess.run(command)
# There isn't an elegant way of determining the CLI location of a pip-installed package. # There isn't an elegant way of determining the CLI location of a pip-installed package.
@ -110,7 +110,7 @@ class PhabricatorCommandProvider(MachCommandBase):
).findall(info) ).findall(info)
if len(potential_cli_paths) != 1: if len(potential_cli_paths) != 1:
command_context.log( self.log(
logging.WARNING, logging.WARNING,
"no_mozphab_console_script", "no_mozphab_console_script",
{}, {},

2
tools/power/mach_commands.py
View file

@ -47,7 +47,7 @@ class MachCommands(MachCommandBase):
import re import re
import subprocess import subprocess
rapl = os.path.join(command_context.topobjdir, "dist", "bin", "rapl") rapl = os.path.join(self.topobjdir, "dist", "bin", "rapl")
interval = str(interval) interval = str(interval)

110
tools/tryselect/mach_commands.py
View file

@ -70,13 +70,13 @@ class TryConfig:
@CommandProvider @CommandProvider
class TrySelect(MachCommandBase): class TrySelect(MachCommandBase):
def init(self, command_context): def init(self):
from tryselect import push from tryselect import push
push.MAX_HISTORY = command_context._mach_context.settings["try"]["maxhistory"] push.MAX_HISTORY = self._mach_context.settings["try"]["maxhistory"]
@memoize @memoize
def presets(self, command_context): def presets(self):
from tryselect.preset import MergedHandler from tryselect.preset import MergedHandler
# Create our handler using both local and in-tree presets. The first # Create our handler using both local and in-tree presets. The first
@ -88,16 +88,12 @@ class TrySelect(MachCommandBase):
else: else:
preset_paths = [ preset_paths = [
os.path.join(get_state_dir(), "try_presets.yml"), os.path.join(get_state_dir(), "try_presets.yml"),
os.path.join( os.path.join(self.topsrcdir, "tools", "tryselect", "try_presets.yml"),
command_context.topsrcdir, "tools", "tryselect", "try_presets.yml"
),
] ]
return MergedHandler(*preset_paths) return MergedHandler(*preset_paths)
def handle_presets( def handle_presets(self, preset_action=None, save=None, preset=None, **kwargs):
self, command_context, preset_action=None, save=None, preset=None, **kwargs
):
"""Handle preset related arguments. """Handle preset related arguments.
This logic lives here so that the underlying selectors don't need This logic lives here so that the underlying selectors don't need
@ -106,25 +102,23 @@ class TrySelect(MachCommandBase):
""" """
from tryselect.util.dicttools import merge from tryselect.util.dicttools import merge
user_presets = self.presets(command_context).handlers[0] user_presets = self.presets().handlers[0]
if preset_action == "list": if preset_action == "list":
self.presets(command_context).list() self.presets().list()
sys.exit() sys.exit()
if preset_action == "edit": if preset_action == "edit":
user_presets.edit() user_presets.edit()
sys.exit() sys.exit()
parser = command_context._mach_context.handler.parser parser = self._mach_context.handler.parser
subcommand = command_context._mach_context.handler.subcommand subcommand = self._mach_context.handler.subcommand
if "preset" not in parser.common_groups: if "preset" not in parser.common_groups:
return kwargs return kwargs
default = parser.get_default default = parser.get_default
if save: if save:
selector = ( selector = subcommand or self._mach_context.settings["try"]["default"]
subcommand or command_context._mach_context.settings["try"]["default"]
)
# Only save non-default values for simplicity. # Only save non-default values for simplicity.
kwargs = {k: v for k, v in kwargs.items() if v != default(k)} kwargs = {k: v for k, v in kwargs.items() if v != default(k)}
@ -133,13 +127,13 @@ class TrySelect(MachCommandBase):
sys.exit() sys.exit()
if preset: if preset:
if preset not in self.presets(command_context): if preset not in self.presets():
command_context._mach_context.parser.error( self._mach_context.parser.error(
"preset '{}' does not exist".format(preset) "preset '{}' does not exist".format(preset)
) )
name = preset name = preset
preset = self.presets(command_context)[name] preset = self.presets()[name]
selector = preset.pop("selector") selector = preset.pop("selector")
preset.pop("description", None) # description isn't used by any selectors preset.pop("description", None) # description isn't used by any selectors
@ -168,12 +162,12 @@ class TrySelect(MachCommandBase):
return kwargs return kwargs
def handle_try_config(self, command_context, **kwargs): def handle_try_config(self, **kwargs):
from tryselect.util.dicttools import merge from tryselect.util.dicttools import merge
to_validate = [] to_validate = []
kwargs.setdefault("try_config", {}) kwargs.setdefault("try_config", {})
for cls in command_context._mach_context.handler.parser.task_configs.values(): for cls in self._mach_context.handler.parser.task_configs.values():
try_config = cls.try_config(**kwargs) try_config = cls.try_config(**kwargs)
if try_config is not None: if try_config is not None:
to_validate.append(cls) to_validate.append(cls)
@ -188,16 +182,14 @@ class TrySelect(MachCommandBase):
cls.validate(**kwargs) cls.validate(**kwargs)
return kwargs return kwargs
def run(self, command_context, **kwargs): def run(self, **kwargs):
kwargs = self.handle_presets(command_context, **kwargs) kwargs = self.handle_presets(**kwargs)
if command_context._mach_context.handler.parser.task_configs: if self._mach_context.handler.parser.task_configs:
kwargs = self.handle_try_config(command_context, **kwargs) kwargs = self.handle_try_config(**kwargs)
mod = importlib.import_module( mod = importlib.import_module(
"tryselect.selectors.{}".format( "tryselect.selectors.{}".format(self._mach_context.handler.subcommand)
command_context._mach_context.handler.subcommand
)
) )
return mod.run(**kwargs) return mod.run(**kwargs)
@ -219,22 +211,22 @@ class TrySelect(MachCommandBase):
default. Run |mach try auto --help| for more information on default. Run |mach try auto --help| for more information on
scheduling with the `auto` selector. scheduling with the `auto` selector.
""" """
self.init(command_context) self.init()
subcommand = command_context._mach_context.handler.subcommand subcommand = self._mach_context.handler.subcommand
# We do special handling of presets here so that `./mach try --preset foo` # We do special handling of presets here so that `./mach try --preset foo`
# works no matter what subcommand 'foo' was saved with. # works no matter what subcommand 'foo' was saved with.
preset = kwargs["preset"] preset = kwargs["preset"]
if preset: if preset:
if preset not in self.presets(command_context): if preset not in self.presets():
command_context._mach_context.handler.parser.error( self._mach_context.handler.parser.error(
"preset '{}' does not exist".format(preset) "preset '{}' does not exist".format(preset)
) )
subcommand = self.presets(command_context)[preset]["selector"] subcommand = self.presets()[preset]["selector"]
sub = subcommand or command_context._mach_context.settings["try"]["default"] sub = subcommand or self._mach_context.settings["try"]["default"]
return command_context._mach_context.commands.dispatch( return self._mach_context.commands.dispatch(
"try", command_context._mach_context, subcommand=sub, argv=argv, **kwargs "try", self._mach_context, subcommand=sub, argv=argv, **kwargs
) )
@SubCommand( @SubCommand(
@ -318,7 +310,7 @@ class TrySelect(MachCommandBase):
For more detailed documentation, please see: For more detailed documentation, please see:
https://firefox-source-docs.mozilla.org/tools/try/selectors/fuzzy.html https://firefox-source-docs.mozilla.org/tools/try/selectors/fuzzy.html
""" """
self.init(command_context) self.init()
if kwargs.pop("interactive"): if kwargs.pop("interactive"):
kwargs["query"].append("INTERACTIVE") kwargs["query"].append("INTERACTIVE")
@ -332,14 +324,14 @@ class TrySelect(MachCommandBase):
kwargs_copy = kwargs.copy() kwargs_copy = kwargs.copy()
kwargs_copy["push"] = False kwargs_copy["push"] = False
kwargs_copy["save"] = None kwargs_copy["save"] = None
kwargs["query"] = self.run(command_context, save_query=True, **kwargs_copy) kwargs["query"] = self.run(save_query=True, **kwargs_copy)
if not kwargs["query"]: if not kwargs["query"]:
return return
if kwargs.get("paths"): if kwargs.get("paths"):
kwargs["test_paths"] = kwargs["paths"] kwargs["test_paths"] = kwargs["paths"]
return self.run(command_context, **kwargs) return self.run(**kwargs)
@SubCommand( @SubCommand(
"try", "try",
@ -355,14 +347,14 @@ class TrySelect(MachCommandBase):
has been made, pressing the 'Push' button will automatically push the has been made, pressing the 'Push' button will automatically push the
selection to try. selection to try.
""" """
self.init(command_context) self.init()
command_context.activate_virtualenv() self.activate_virtualenv()
path = os.path.join( path = os.path.join(
"tools", "tryselect", "selectors", "chooser", "requirements.txt" "tools", "tryselect", "selectors", "chooser", "requirements.txt"
) )
command_context.virtualenv_manager.install_pip_requirements(path, quiet=True) self.virtualenv_manager.install_pip_requirements(path, quiet=True)
return self.run(command_context, **kwargs) return self.run(**kwargs)
@SubCommand( @SubCommand(
"try", "try",
@ -373,8 +365,8 @@ class TrySelect(MachCommandBase):
parser=get_parser("auto"), parser=get_parser("auto"),
) )
def try_auto(self, command_context, **kwargs): def try_auto(self, command_context, **kwargs):
self.init(command_context) self.init()
return self.run(command_context, **kwargs) return self.run(**kwargs)
@SubCommand( @SubCommand(
"try", "try",
@ -383,8 +375,8 @@ class TrySelect(MachCommandBase):
parser=get_parser("again"), parser=get_parser("again"),
) )
def try_again(self, command_context, **kwargs): def try_again(self, command_context, **kwargs):
self.init(command_context) self.init()
return self.run(command_context, **kwargs) return self.run(**kwargs)
@SubCommand( @SubCommand(
"try", "try",
@ -401,8 +393,8 @@ class TrySelect(MachCommandBase):
via Treeherder's Add New Jobs feature, located in the per-push via Treeherder's Add New Jobs feature, located in the per-push
menu. menu.
""" """
self.init(command_context) self.init()
return self.run(command_context, **kwargs) return self.run(**kwargs)
@SubCommand( @SubCommand(
"try", "try",
@ -448,9 +440,9 @@ class TrySelect(MachCommandBase):
(installable from mach vcs-setup). (installable from mach vcs-setup).
""" """
self.init(command_context) self.init()
try: try:
if command_context.substs.get("MOZ_ARTIFACT_BUILDS"): if self.substs.get("MOZ_ARTIFACT_BUILDS"):
kwargs["local_artifact_build"] = True kwargs["local_artifact_build"] = True
except BuildEnvironmentNotFoundException: except BuildEnvironmentNotFoundException:
# If we don't have a build locally, we can't tell whether # If we don't have a build locally, we can't tell whether
@ -458,12 +450,12 @@ class TrySelect(MachCommandBase):
# command to succeed, if possible. # command to succeed, if possible.
pass pass
config_status = os.path.join(command_context.topobjdir, "config.status") config_status = os.path.join(self.topobjdir, "config.status")
if (kwargs["paths"] or kwargs["tags"]) and not config_status: if (kwargs["paths"] or kwargs["tags"]) and not config_status:
print(CONFIG_ENVIRONMENT_NOT_FOUND) print(CONFIG_ENVIRONMENT_NOT_FOUND)
sys.exit(1) sys.exit(1)
return self.run(command_context, **kwargs) return self.run(**kwargs)
@SubCommand( @SubCommand(
"try", "try",
@ -473,8 +465,8 @@ class TrySelect(MachCommandBase):
) )
def try_coverage(self, command_context, **kwargs): def try_coverage(self, command_context, **kwargs):
"""Select which tasks to use using coverage data.""" """Select which tasks to use using coverage data."""
self.init(command_context) self.init()
return self.run(command_context, **kwargs) return self.run(**kwargs)
@SubCommand( @SubCommand(
"try", "try",
@ -484,8 +476,8 @@ class TrySelect(MachCommandBase):
) )
def try_release(self, command_context, **kwargs): def try_release(self, command_context, **kwargs):
"""Push the current tree to try, configured for a staging release.""" """Push the current tree to try, configured for a staging release."""
self.init(command_context) self.init()
return self.run(command_context, **kwargs) return self.run(**kwargs)
@SubCommand( @SubCommand(
"try", "try",
@ -498,5 +490,5 @@ class TrySelect(MachCommandBase):
Requires VPN and shipit access. Requires VPN and shipit access.
""" """
self.init(command_context) self.init()
return self.run(command_context, **kwargs) return self.run(**kwargs)

36
tools/vcs/mach_commands.py
View file

@ -91,7 +91,7 @@ class PullRequestImporter(MachCommandBase):
break break
if repository is None: if repository is None:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"unrecognized_repo", "unrecognized_repo",
{}, {},
@ -100,7 +100,7 @@ class PullRequestImporter(MachCommandBase):
) )
sys.exit(1) sys.exit(1)
command_context.log( self.log(
logging.INFO, logging.INFO,
"import_pr", "import_pr",
{"pr_url": pull_request}, {"pr_url": pull_request},
@ -108,24 +108,22 @@ class PullRequestImporter(MachCommandBase):
) )
dirty = [ dirty = [
f f
for f in command_context.repository.get_changed_files(mode="all") for f in self.repository.get_changed_files(mode="all")
if f.startswith(repository["path"]) if f.startswith(repository["path"])
] ]
if dirty: if dirty:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"dirty_tree", "dirty_tree",
repository, repository,
"Local {path} tree is dirty; aborting!", "Local {path} tree is dirty; aborting!",
) )
sys.exit(1) sys.exit(1)
target_dir = mozpath.join( target_dir = mozpath.join(self.topsrcdir, os.path.normpath(repository["path"]))
command_context.topsrcdir, os.path.normpath(repository["path"])
)
if bug_number is None: if bug_number is None:
if bugzilla_token is None: if bugzilla_token is None:
command_context.log( self.log(
logging.WARNING, logging.WARNING,
"no_token", "no_token",
{}, {},
@ -133,11 +131,9 @@ class PullRequestImporter(MachCommandBase):
"be added to commit messages.", "be added to commit messages.",
) )
else: else:
bug_number = self._file_bug( bug_number = self._file_bug(bugzilla_token, repository, pr_number)
command_context, bugzilla_token, repository, pr_number
)
elif bugzilla_token is not None: elif bugzilla_token is not None:
command_context.log( self.log(
logging.WARNING, logging.WARNING,
"too_much_bug", "too_much_bug",
{}, {},
@ -150,7 +146,7 @@ class PullRequestImporter(MachCommandBase):
for patch in self._split_patches( for patch in self._split_patches(
pr_patch.content, bug_number, pull_request, reviewer pr_patch.content, bug_number, pull_request, reviewer
): ):
command_context.log( self.log(
logging.INFO, logging.INFO,
"commit_msg", "commit_msg",
patch, patch,
@ -163,21 +159,19 @@ class PullRequestImporter(MachCommandBase):
patch_cmd.stdin.close() patch_cmd.stdin.close()
patch_cmd.wait() patch_cmd.wait()
if patch_cmd.returncode != 0: if patch_cmd.returncode != 0:
command_context.log( self.log(
logging.ERROR, logging.ERROR,
"commit_fail", "commit_fail",
{}, {},
'Error applying diff from commit via "patch -p1 -s". Aborting...', 'Error applying diff from commit via "patch -p1 -s". Aborting...',
) )
sys.exit(patch_cmd.returncode) sys.exit(patch_cmd.returncode)
command_context.repository.commit( self.repository.commit(
patch["commit_msg"], patch["author"], patch["date"], [target_dir] patch["commit_msg"], patch["author"], patch["date"], [target_dir]
) )
command_context.log( self.log(logging.INFO, "commit_pass", {}, "Committed successfully.")
logging.INFO, "commit_pass", {}, "Committed successfully."
)
def _file_bug(self, command_context, token, repo, pr_number): def _file_bug(self, token, repo, pr_number):
import requests import requests
bug = requests.post( bug = requests.post(
@ -191,9 +185,9 @@ class PullRequestImporter(MachCommandBase):
}, },
) )
bug.raise_for_status() bug.raise_for_status()
command_context.log(logging.DEBUG, "new_bug", {}, bug.content) self.log(logging.DEBUG, "new_bug", {}, bug.content)
bugnumber = json.loads(bug.content)["id"] bugnumber = json.loads(bug.content)["id"]
command_context.log( self.log(
logging.INFO, "new_bug", {"bugnumber": bugnumber}, "Filed bug {bugnumber}" logging.INFO, "new_bug", {"bugnumber": bugnumber}, "Filed bug {bugnumber}"
) )
return bugnumber return bugnumber