Bug 1729383 - Simplify the parallel build setup. r=firefox-build-system-reviewers,mhentges

Historically, client.mk was not invoked with -jn because it would create
race conditions, but that was actually mostly solved by the addition of
`.NOTPARALLEL` in bug 422986, although the mechanism of adding -jn via
`MOZ_MAKE_FLAGS` or `MOZ_PARALLEL_BUILD` has continued well past that.

Nowadays, client.mk is only invoked by mach (it will even bail out if
that's not the case) and only has one target (`build`) and no
dependencies.

This means we don't need to rely on `MOZ_PARALLEL_BUILD` to pass `-jn` in
some cases, and can just always invoke `make -f client.mk` with `-jn`, even
when we just want no parallelism, in which case we can use `-j1`.

This, in turn, allows to remove the extra allow_parallel argument to
`_run_make`, and only rely on `num_jobs`, and to remove some of the
multiple ways the `n` in `-jn` could be set.

Differential Revision: https://phabricator.services.mozilla.com/D124729

client.mk

@@ -23,17 +23,6 @@ endif
 ### Load mozconfig options
 include $(OBJDIR)/.mozconfig-client-mk
 
-ifdef MOZ_PARALLEL_BUILD
-  MOZ_MAKE_FLAGS := $(filter-out -j%,$(MOZ_MAKE_FLAGS))
-  MOZ_MAKE_FLAGS += -j$(MOZ_PARALLEL_BUILD)
-endif
-
-# Automatically add -jN to make flags if not defined. N defaults to number of cores.
-ifeq (,$(findstring -j,$(MOZ_MAKE_FLAGS)))
-  cores=$(shell python3 -c 'import multiprocessing; print(multiprocessing.cpu_count())')
-  MOZ_MAKE_FLAGS += -j$(cores)
-endif
-
 ### Set up make flags
 ifdef MOZ_AUTOMATION
 ifeq (4.0,$(firstword $(sort 4.0 $(MAKE_VERSION))))
@@ -75,10 +64,5 @@ build::
 	-$(MOZBUILD_MANAGE_SCCACHE_DAEMON) --stop-server
 endif
 
-# This makefile doesn't support parallel execution. It does pass
-# MOZ_MAKE_FLAGS to sub-make processes, so they will correctly execute
-# in parallel.
-.NOTPARALLEL:
-
 .PHONY: \
     build

config/makefiles/rust.mk

@@ -49,9 +49,7 @@ endif
 
 # Without -j > 1, make will not pass jobserver info down to cargo. Force
 # one job when requested as a special case.
-ifeq (1,$(MOZ_PARALLEL_BUILD))
-cargo_build_flags += -j1
-endif
+cargo_build_flags += $(filter -j1,$(MAKEFLAGS))
 
 # We also need to rebuild the rust stdlib so that it's instrumented. Because
 # build-std is still pretty experimental, we need to explicitly request

python/mozbuild/mozbuild/base.py

@@ -722,7 +722,6 @@ class MozbuildObject(ProcessExecutionMixin):
         target=None,
         log=True,
         srcdir=False,
-        allow_parallel=True,
         line_handler=None,
         append_env=None,
         explicit_env=None,
@@ -776,20 +775,17 @@ class MozbuildObject(ProcessExecutionMixin):
                 else:
                     args.append(flag)
 
-        if allow_parallel:
-            if num_jobs > 0:
-                args.append("-j%d" % num_jobs)
-            else:
-                args.append("-j%d" % multiprocessing.cpu_count())
-        elif num_jobs > 0:
-            args.append("MOZ_PARALLEL_BUILD=%d" % num_jobs)
+        if num_jobs > 0:
+            args.append("-j%d" % num_jobs)
         elif os.environ.get("MOZ_LOW_PARALLELISM_BUILD"):
             cpus = multiprocessing.cpu_count()
             jobs = max(1, int(0.75 * cpus))
             print(
                 "  Low parallelism requested: using %d jobs for %d cores" % (jobs, cpus)
             )
-            args.append("MOZ_PARALLEL_BUILD=%d" % jobs)
+            args.append("-j%d" % jobs)
+        else:
+            args.append("-j%d" % multiprocessing.cpu_count())
 
         if ignore_errors:
             args.append("-k")

python/mozbuild/mozbuild/compilation/util.py

@@ -36,7 +36,7 @@ def get_build_vars(directory, cmd):
             target="showbuild",
             log=False,
             print_directory=False,
-            allow_parallel=False,
+            num_jobs=1,
             silent=True,
             line_handler=on_line,
         )

python/mozbuild/mozbuild/controller/building.py

@@ -1190,18 +1190,13 @@ class BuildDriver(MozbuildObject):
                 ]
                 self.run_process(args, cwd=self.topobjdir, pass_thru=True)
 
-            if "Make" not in active_backend:
-                # client.mk has its own handling of MOZ_PARALLEL_BUILD so the
-                # make backend can determine when to run in single-threaded mode
-                # or parallel mode. For other backends, we can pass in the value
-                # of MOZ_PARALLEL_BUILD if -jX was not specified on the
-                # commandline.
-                if jobs == 0 and "make_extra" in self.mozconfig:
-                    for param in self.mozconfig["make_extra"]:
-                        key, value = param.split("=", 1)
-                        if key == "MOZ_PARALLEL_BUILD":
-                            jobs = int(value)
+            if jobs == 0 and "make_extra" in self.mozconfig:
+                for param in self.mozconfig["make_extra"]:
+                    key, value = param.split("=", 1)
+                    if key == "MOZ_PARALLEL_BUILD":
+                        jobs = int(value)
 
+            if "Make" not in active_backend:
                 backend_cls = get_backend_class(active_backend)(config)
                 status = backend_cls.build(self, output, jobs, verbose, what)
 
@@ -1700,7 +1695,6 @@ class BuildDriver(MozbuildObject):
         return self._run_make(
             srcdir=True,
             filename="client.mk",
-            allow_parallel=False,
             ensure_exit_code=False,
             print_directory=False,
             target=target,