Backed out 3 changesets (bug 1857279) for causing try failures with exit status 1 CLOSED TREE

Backed out changeset 4c1965a65011 (bug 1857279)
Backed out changeset b9885d1b6980 (bug 1857279)
Backed out changeset 76706ea52991 (bug 1857279)

build/mach_initialize.py

@@ -174,6 +174,30 @@ def initialize(topsrcdir, args=()):
     )
     from mach.main import get_argument_parser
 
+    parser = get_argument_parser(
+        action=DetermineCommandVenvAction,
+        topsrcdir=topsrcdir,
+    )
+    namespace = parser.parse_args()
+
+    command_name = getattr(namespace, "command_name", None)
+    site_name = getattr(namespace, "site_name", "common")
+    command_site_manager = None
+
+    # the 'clobber' command needs to run in the 'mach' venv, so we
+    # don't want to activate any other virtualenv for it.
+    if command_name != "clobber":
+        from mach.site import CommandSiteManager
+
+        command_site_manager = CommandSiteManager.from_environment(
+            topsrcdir,
+            lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir)),
+            site_name,
+            get_virtualenv_base_dir(topsrcdir),
+        )
+
+        command_site_manager.activate()
+
     # Set a reasonable limit to the number of open files.
     #
     # Some linux systems set `ulimit -n` to a very high number, which works
@@ -297,7 +321,7 @@ def initialize(topsrcdir, args=()):
     if "MACH_MAIN_PID" not in os.environ:
         setenv("MACH_MAIN_PID", str(os.getpid()))
 
-    driver = mach.main.Mach(os.getcwd())
+    driver = mach.main.Mach(os.getcwd(), command_site_manager)
     driver.populate_context_handler = populate_context
 
     if not driver.settings_paths:
@@ -305,42 +329,6 @@ def initialize(topsrcdir, args=()):
         driver.settings_paths.append(state_dir)
     # always load local repository configuration
     driver.settings_paths.append(topsrcdir)
-    driver.load_settings()
-
-    aliases = driver.settings.alias
-
-    command = args[0]
-
-    if command in aliases:
-        import shlex
-
-        alias = aliases[command]
-        defaults = shlex.split(alias)
-        args = defaults + args[1:]
-
-    parser = get_argument_parser(
-        action=DetermineCommandVenvAction,
-        topsrcdir=topsrcdir,
-    )
-    namespace = parser.parse_args(args)
-
-    command_name = getattr(namespace, "command_name", None)
-    site_name = getattr(namespace, "site_name", "common")
-    command_site_manager = None
-
-    # the 'clobber' command needs to run in the 'mach' venv, so we
-    # don't want to activate any other virtualenv for it.
-    if command_name != "clobber":
-        from mach.site import CommandSiteManager
-
-        command_site_manager = CommandSiteManager.from_environment(
-            topsrcdir,
-            lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir)),
-            site_name,
-            get_virtualenv_base_dir(topsrcdir),
-        )
-
-        command_site_manager.activate()
 
     for category, meta in CATEGORIES.items():
         driver.define_category(category, meta["short"], meta["long"], meta["priority"])
@@ -394,7 +382,6 @@ def initialize(topsrcdir, args=()):
             for command_name in command_names_to_load
         }
 
-    driver.command_site_manager = command_site_manager
     load_commands_from_spec(command_modules_to_load, topsrcdir, missing_ok=missing_ok)
 
     return driver

python/mach/mach/dispatcher.py

@@ -9,6 +9,22 @@ import sys
 from operator import itemgetter
 
 from .base import NoCommandError, UnknownCommandError, UnrecognizedArgumentError
+from .decorators import SettingsProvider
+
+
+@SettingsProvider
+class DispatchSettings:
+    config_settings = [
+        (
+            "alias.*",
+            "string",
+            """
+Create a command alias of the form `<alias>=<command> <args>`.
+Aliases can also be used to set default arguments:
+<command>=<command> <args>
+""".strip(),
+        ),
+    ]
 
 
 class CommandFormatter(argparse.HelpFormatter):

python/mach/mach/main.py

@@ -12,7 +12,9 @@ import os
 import sys
 import traceback
 from pathlib import Path
-from typing import List
+from typing import List, Optional
+
+from mach.site import CommandSiteManager
 
 from .base import (
     CommandContext,
@@ -198,7 +200,9 @@ To see more help for a specific command, run:
   %(prog)s help <command>
 """
 
-    def __init__(self, cwd: str):
+    def __init__(
+        self, cwd: str, command_site_manager: Optional[CommandSiteManager] = None
+    ):
         assert Path(cwd).is_dir()
 
         self.cwd = cwd
@@ -206,8 +210,7 @@ To see more help for a specific command, run:
         self.logger = logging.getLogger(__name__)
         self.settings = ConfigSettings()
         self.settings_paths = []
-        self.settings_loaded = False
+        self.command_site_manager = command_site_manager
-        self.command_site_manager = None
 
         if "MACHRC" in os.environ:
             self.settings_paths.append(os.environ["MACHRC"])
@@ -256,7 +259,13 @@ To see more help for a specific command, run:
         orig_env = dict(os.environ)
 
         try:
-            self.load_settings()
+            # Load settings as early as possible so things in dispatcher.py
+            # can use them.
+            for provider in Registrar.settings_providers:
+                self.settings.register_provider(provider)
+
+            setting_paths_to_pass = [Path(path) for path in self.settings_paths]
+            self.load_settings(setting_paths_to_pass)
 
             if sys.version_info < (3, 0):
                 if stdin.encoding is None:
@@ -394,7 +403,7 @@ To see more help for a specific command, run:
         if args.settings_file:
             # Argument parsing has already happened, so settings that apply
             # to command line handling (e.g alias, defaults) will be ignored.
-            self.load_settings_by_file([Path(args.settings_file)])
+            self.load_settings([Path(args.settings_file)])
 
         try:
             return Registrar._run_command_handler(
@@ -499,18 +508,7 @@ To see more help for a specific command, run:
 
         fh.write("\nSentry event ID: {}\n".format(sentry_event_id))
 
-    def load_settings(self):
+    def load_settings(self, paths: List[Path]):
-        if not self.settings_loaded:
-            import mach.settings  # noqa need @SettingsProvider hook to execute
-
-            for provider in Registrar.settings_providers:
-                self.settings.register_provider(provider)
-
-            setting_paths_to_pass = [Path(path) for path in self.settings_paths]
-            self.load_settings_by_file(setting_paths_to_pass)
-            self.settings_loaded = True
-
-    def load_settings_by_file(self, paths: List[Path]):
         """Load the specified settings files.
 
         If a directory is specified, the following basenames will be

python/mach/mach/settings.py

@@ -1,136 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from mach.decorators import SettingsProvider
-
-
-@SettingsProvider
-class MachSettings:
-    @classmethod
-    def config_settings(cls):
-        def telemetry_config_settings():
-            return [
-                (
-                    "build.telemetry",
-                    "boolean",
-                    "Enable submission of build system telemetry "
-                    '(Deprecated, replaced by "telemetry.is_enabled")',
-                ),
-                (
-                    "mach_telemetry.is_enabled",
-                    "boolean",
-                    "Build system telemetry is allowed",
-                    False,
-                ),
-                (
-                    "mach_telemetry.is_set_up",
-                    "boolean",
-                    "The telemetry setup workflow has been completed "
-                    "(e.g.: user has been prompted to opt-in)",
-                    False,
-                ),
-            ]
-
-        def dispatch_config_settings():
-            return [
-                (
-                    "alias.*",
-                    "string",
-                    """
-        Create a command alias of the form `<alias>=<command> <args>`.
-        Aliases can also be used to set default arguments:
-        <command>=<command> <args>
-        """.strip(),
-                ),
-            ]
-
-        def run_config_settings():
-            return [
-                (
-                    "runprefs.*",
-                    "string",
-                    """
-        Pass a pref into Firefox when using `mach run`, of the form `foo.bar=value`.
-        Prefs will automatically be cast into the appropriate type. Integers can be
-        single quoted to force them to be strings.
-        """.strip(),
-                ),
-            ]
-
-        def try_config_settings():
-            desc = "The default selector to use when running `mach try` without a subcommand."
-            choices = [
-                "fuzzy",
-                "chooser",
-                "auto",
-                "again",
-                "empty",
-                "syntax",
-                "coverage",
-                "release",
-                "scriptworker",
-                "compare",
-                "perf",
-            ]
-
-            return [
-                ("try.default", "string", desc, "auto", {"choices": choices}),
-                (
-                    "try.maxhistory",
-                    "int",
-                    "Maximum number of pushes to save in history.",
-                    10,
-                ),
-            ]
-
-        def taskgraph_config_settings():
-            return [
-                (
-                    "taskgraph.diffcmd",
-                    "string",
-                    "The command to run with `./mach taskgraph --diff`",
-                    "diff --report-identical-files "
-                    "--label={attr}@{base} --label={attr}@{cur} -U20",
-                    {},
-                )
-            ]
-
-        def test_config_settings():
-            from mozlog.commandline import log_formatters
-            from mozlog.structuredlog import log_levels
-
-            format_desc = (
-                "The default format to use when running tests with `mach test`."
-            )
-            format_choices = list(log_formatters)
-            level_desc = (
-                "The default log level to use when running tests with `mach test`."
-            )
-            level_choices = [l.lower() for l in log_levels]
-            return [
-                (
-                    "test.format",
-                    "string",
-                    format_desc,
-                    "mach",
-                    {"choices": format_choices},
-                ),
-                (
-                    "test.level",
-                    "string",
-                    level_desc,
-                    "info",
-                    {"choices": level_choices},
-                ),
-            ]
-
-        settings = []
-        settings.extend(telemetry_config_settings())
-        settings.extend(dispatch_config_settings())
-        settings.extend(run_config_settings())
-        settings.extend(try_config_settings())
-        settings.extend(taskgraph_config_settings())
-        settings.extend(test_config_settings())
-
-        return settings

python/mach/mach/telemetry.py

@@ -12,12 +12,12 @@ from textwrap import dedent
 import requests
 import six.moves.urllib.parse as urllib_parse
 from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
+from mozbuild.settings import TelemetrySettings
 from mozbuild.telemetry import filter_args
 from mozversioncontrol import InvalidRepoPath, get_repository_object
 from six.moves import configparser, input
 
 from mach.config import ConfigSettings
-from mach.settings import MachSettings
 from mach.site import MozSiteMetadata
 from mach.telemetry_interface import GleanTelemetry, NoopTelemetry
 from mach.util import get_state_dir
@@ -213,7 +213,7 @@ def record_telemetry_settings(
     # settings, update it, then write to it.
     settings_path = state_dir / "machrc"
     file_settings = ConfigSettings()
-    file_settings.register_provider(MachSettings)
+    file_settings.register_provider(TelemetrySettings)
     try:
         file_settings.load_file(settings_path)
     except configparser.Error as error:

python/mozbuild/mozbuild/mach_commands.py

@@ -25,10 +25,12 @@ from mach.decorators import (
     Command,
     CommandArgument,
     CommandArgumentGroup,
+    SettingsProvider,
     SubCommand,
 )
 from mozfile import load_source
 
+import mozbuild.settings  # noqa need @SettingsProvider hook to execute
 from mozbuild.base import (
     BinaryNotFoundException,
     BuildEnvironmentNotFoundException,
@@ -1216,6 +1218,21 @@ def install(command_context, **kwargs):
     return ret
 
 
+@SettingsProvider
+class RunSettings:
+    config_settings = [
+        (
+            "runprefs.*",
+            "string",
+            """
+Pass a pref into Firefox when using `mach run`, of the form `foo.bar=value`.
+Prefs will automatically be cast into the appropriate type. Integers can be
+single quoted to force them to be strings.
+""".strip(),
+        )
+    ]
+
+
 def _get_android_run_parser():
     parser = argparse.ArgumentParser()
     group = parser.add_argument_group("The compiled program")

python/mozbuild/mozbuild/settings.py

@@ -0,0 +1,30 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mach.decorators import SettingsProvider
+
+
+@SettingsProvider
+class TelemetrySettings:
+    config_settings = [
+        (
+            "build.telemetry",
+            "boolean",
+            "Enable submission of build system telemetry "
+            '(Deprecated, replaced by "telemetry.is_enabled")',
+        ),
+        (
+            "mach_telemetry.is_enabled",
+            "boolean",
+            "Build system telemetry is allowed",
+            False,
+        ),
+        (
+            "mach_telemetry.is_set_up",
+            "boolean",
+            "The telemetry setup workflow has been completed "
+            "(e.g.: user has been prompted to opt-in)",
+            False,
+        ),
+    ]

python/mozbuild/mozbuild/test/test_telemetry_settings.py

@@ -11,13 +11,14 @@ import pytest
 import requests
 from mach.config import ConfigSettings
 from mach.decorators import SettingsProvider
-from mach.settings import MachSettings
 from mach.telemetry import (
     initialize_telemetry_setting,
     record_telemetry_settings,
     resolve_is_employee,
 )
 
+from mozbuild.settings import TelemetrySettings
+
 
 @SettingsProvider
 class OtherSettings:
@@ -31,7 +32,7 @@ def record_enabled_telemetry(mozbuild_path, settings):
 @pytest.fixture
 def settings():
     s = ConfigSettings()
-    s.register_provider(MachSettings)
+    s.register_provider(TelemetrySettings)
     s.register_provider(OtherSettings)
     return s
 

taskcluster/mach_commands.py

@@ -16,11 +16,27 @@ from functools import partial
 
 import gecko_taskgraph.main
 from gecko_taskgraph.main import commands as taskgraph_commands
-from mach.decorators import Command, CommandArgument, SubCommand
+from mach.decorators import Command, CommandArgument, SettingsProvider, SubCommand
 
 logger = logging.getLogger("taskcluster")
 
 
+@SettingsProvider
+class TaskgraphConfig(object):
+    @classmethod
+    def config_settings(cls):
+        return [
+            (
+                "taskgraph.diffcmd",
+                "string",
+                "The command to run with `./mach taskgraph --diff`",
+                "diff --report-identical-files "
+                "--label={attr}@{base} --label={attr}@{cur} -U20",
+                {},
+            )
+        ]
+
+
 def strtobool(value):
     """Convert string to boolean.
 

testing/mach_commands.py

@@ -9,7 +9,7 @@ import subprocess
 import sys
 
 import requests
-from mach.decorators import Command, CommandArgument, SubCommand
+from mach.decorators import Command, CommandArgument, SettingsProvider, SubCommand
 from mozbuild.base import BuildEnvironmentNotFoundException
 from mozbuild.base import MachCommandConditions as conditions
 
@@ -38,6 +38,23 @@ The following test suites and aliases are supported: {}
 """.strip()
 
 
+@SettingsProvider
+class TestConfig(object):
+    @classmethod
+    def config_settings(cls):
+        from mozlog.commandline import log_formatters
+        from mozlog.structuredlog import log_levels
+
+        format_desc = "The default format to use when running tests with `mach test`."
+        format_choices = list(log_formatters)
+        level_desc = "The default log level to use when running tests with `mach test`."
+        level_choices = [l.lower() for l in log_levels]
+        return [
+            ("test.format", "string", format_desc, "mach", {"choices": format_choices}),
+            ("test.level", "string", level_desc, "info", {"choices": level_choices}),
+        ]
+
+
 def get_test_parser():
     from mozlog.commandline import add_logging_group
     from moztest.resolve import TEST_SUITES

tools/tryselect/mach_commands.py

@@ -8,7 +8,7 @@ import importlib
 import os
 import sys
 
-from mach.decorators import Command, SubCommand
+from mach.decorators import Command, SettingsProvider, SubCommand
 from mach.util import get_state_dir
 from mozbuild.base import BuildEnvironmentNotFoundException
 from mozbuild.util import memoize
@@ -40,6 +40,28 @@ def generic_parser():
     return parser
 
 
+@SettingsProvider
+class TryConfig:
+    @classmethod
+    def config_settings(cls):
+        from mach.registrar import Registrar
+
+        desc = (
+            "The default selector to use when running `mach try` without a subcommand."
+        )
+        choices = Registrar.command_handlers["try"].subcommand_handlers.keys()
+
+        return [
+            ("try.default", "string", desc, "auto", {"choices": choices}),
+            (
+                "try.maxhistory",
+                "int",
+                "Maximum number of pushes to save in history.",
+                10,
+            ),
+        ]
+
+
 def init(command_context):
     from tryselect import push