forked from mirrors/gecko-dev
Bug 1316140 - Allow use of multiprocessing from config.status on windows. r=mshal
This generalizes the monkey-patching of the main module added in bug 914563 to allow multiprocessing to be used from config.status (which triggers the same bug as when it's used with `mach` as main). MozReview-Commit-ID: AdOdpKzmbsp
This commit is contained in:
Chris Manchester
parent
eb8660f21f
commit
2d2292267f
4 changed files with 88 additions and 64 deletions
|
|
@ -204,6 +204,9 @@ def bootstrap(topsrcdir, mozilla_dir=None):
|
|||
import mach.main
|
||||
from mozboot.util import get_state_dir
|
||||
|
||||
from mozbuild.util import patch_main
|
||||
patch_main()
|
||||
|
||||
def telemetry_handler(context, data):
|
||||
# We have not opted-in to telemetry
|
||||
if 'BUILD_SYSTEM_TELEMETRY' not in os.environ:
|
||||
|
|
|
|||
|
|
@ -83,6 +83,8 @@ def config_status(config):
|
|||
if config.get('MOZ_BUILD_APP') != 'js' or config.get('JS_STANDALONE'):
|
||||
fh.write(textwrap.dedent('''
|
||||
if __name__ == '__main__':
|
||||
from mozbuild.util import patch_main
|
||||
patch_main()
|
||||
from mozbuild.config_status import config_status
|
||||
args = dict([(name, globals()[name]) for name in __all__])
|
||||
config_status(**args)
|
||||
|
|
|
|||
64
mach
64
mach
|
|
@ -83,68 +83,4 @@ def main(args):
|
|||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if sys.platform == 'win32':
|
||||
# This is a complete hack to work around the fact that Windows
|
||||
# multiprocessing needs to import the original module (ie: this
|
||||
# file), but only works if it has a .py extension.
|
||||
#
|
||||
# We do this by a sort of two-level function interposing. The first
|
||||
# level interposes forking.get_command_line() with our version defined
|
||||
# in my_get_command_line(). Our version of get_command_line will
|
||||
# replace the command string with the contents of the fork_interpose()
|
||||
# function to be used in the subprocess.
|
||||
#
|
||||
# The subprocess then gets an interposed imp.find_module(), which we
|
||||
# hack up to find 'mach' without the .py extension, since we already
|
||||
# know where it is (it's us!). If we're not looking for 'mach', then
|
||||
# the original find_module will suffice.
|
||||
#
|
||||
# See also: http://bugs.python.org/issue19946
|
||||
# And: https://bugzilla.mozilla.org/show_bug.cgi?id=914563
|
||||
import inspect
|
||||
from multiprocessing import forking
|
||||
global orig_command_line
|
||||
|
||||
def fork_interpose():
|
||||
import imp
|
||||
import os
|
||||
import sys
|
||||
orig_find_module = imp.find_module
|
||||
def my_find_module(name, dirs):
|
||||
if name == 'mach':
|
||||
path = os.path.join(dirs[0], 'mach')
|
||||
f = open(path)
|
||||
return (f, path, ('', 'r', imp.PY_SOURCE))
|
||||
return orig_find_module(name, dirs)
|
||||
|
||||
# Don't allow writing bytecode file for mach module.
|
||||
orig_load_module = imp.load_module
|
||||
def my_load_module(name, file, path, description):
|
||||
# multiprocess.forking invokes imp.load_module manually and
|
||||
# hard-codes the name __parents_main__ as the module name.
|
||||
if name == '__parents_main__':
|
||||
old_bytecode = sys.dont_write_bytecode
|
||||
sys.dont_write_bytecode = True
|
||||
try:
|
||||
return orig_load_module(name, file, path, description)
|
||||
finally:
|
||||
sys.dont_write_bytecode = old_bytecode
|
||||
|
||||
return orig_load_module(name, file, path, description)
|
||||
|
||||
imp.find_module = my_find_module
|
||||
imp.load_module = my_load_module
|
||||
from multiprocessing.forking import main; main()
|
||||
|
||||
def my_get_command_line():
|
||||
fork_code, lineno = inspect.getsourcelines(fork_interpose)
|
||||
# Remove the first line (for 'def fork_interpose():') and the three
|
||||
# levels of indentation (12 spaces).
|
||||
fork_string = ''.join(x[12:] for x in fork_code[1:])
|
||||
cmdline = orig_command_line()
|
||||
cmdline[2] = fork_string
|
||||
return cmdline
|
||||
orig_command_line = forking.get_command_line
|
||||
forking.get_command_line = my_get_command_line
|
||||
|
||||
main(sys.argv[1:])
|
||||
|
|
|
|||
|
|
@ -1262,3 +1262,86 @@ def encode(obj, encoding='utf-8'):
|
|||
if isinstance(obj, Iterable):
|
||||
return [encode(i, encoding) for i in obj]
|
||||
return obj
|
||||
|
||||
|
||||
def patch_main():
|
||||
'''This is a hack to work around the fact that Windows multiprocessing needs
|
||||
to import the original main module, and assumes that it corresponds to a file
|
||||
ending in .py.
|
||||
|
||||
We do this by a sort of two-level function interposing. The first
|
||||
level interposes forking.get_command_line() with our version defined
|
||||
in my_get_command_line(). Our version of get_command_line will
|
||||
replace the command string with the contents of the fork_interpose()
|
||||
function to be used in the subprocess.
|
||||
|
||||
The subprocess then gets an interposed imp.find_module(), which we
|
||||
hack up to find the main module name multiprocessing will assume, since we
|
||||
know what this will be based on the main module in the parent. If we're not
|
||||
looking for our main module, then the original find_module will suffice.
|
||||
|
||||
See also: http://bugs.python.org/issue19946
|
||||
And: https://bugzilla.mozilla.org/show_bug.cgi?id=914563
|
||||
'''
|
||||
if sys.platform == 'win32':
|
||||
import inspect
|
||||
import os
|
||||
from multiprocessing import forking
|
||||
global orig_command_line
|
||||
|
||||
# Figure out what multiprocessing will assume our main module
|
||||
# is called (see python/Lib/multiprocessing/forking.py).
|
||||
main_path = getattr(sys.modules['__main__'], '__file__', None)
|
||||
if main_path is None:
|
||||
# If someone deleted or modified __main__, there's nothing left for
|
||||
# us to do.
|
||||
return
|
||||
main_file_name = os.path.basename(main_path)
|
||||
main_module_name, ext = os.path.splitext(main_file_name)
|
||||
if ext == '.py':
|
||||
# If main is a .py file, everything ought to work as expected.
|
||||
return
|
||||
|
||||
def fork_interpose():
|
||||
import imp
|
||||
import os
|
||||
import sys
|
||||
orig_find_module = imp.find_module
|
||||
def my_find_module(name, dirs):
|
||||
if name == main_module_name:
|
||||
path = os.path.join(dirs[0], main_file_name)
|
||||
f = open(path)
|
||||
return (f, path, ('', 'r', imp.PY_SOURCE))
|
||||
return orig_find_module(name, dirs)
|
||||
|
||||
# Don't allow writing bytecode file for the main module.
|
||||
orig_load_module = imp.load_module
|
||||
def my_load_module(name, file, path, description):
|
||||
# multiprocess.forking invokes imp.load_module manually and
|
||||
# hard-codes the name __parents_main__ as the module name.
|
||||
if name == '__parents_main__':
|
||||
old_bytecode = sys.dont_write_bytecode
|
||||
sys.dont_write_bytecode = True
|
||||
try:
|
||||
return orig_load_module(name, file, path, description)
|
||||
finally:
|
||||
sys.dont_write_bytecode = old_bytecode
|
||||
|
||||
return orig_load_module(name, file, path, description)
|
||||
|
||||
imp.find_module = my_find_module
|
||||
imp.load_module = my_load_module
|
||||
from multiprocessing.forking import main; main()
|
||||
|
||||
def my_get_command_line():
|
||||
fork_code, lineno = inspect.getsourcelines(fork_interpose)
|
||||
# Remove the first line (for 'def fork_interpose():') and the three
|
||||
# levels of indentation (12 spaces), add our relevant globals.
|
||||
fork_string = ("main_file_name = '%s'\n" % main_file_name +
|
||||
"main_module_name = '%s'\n" % main_module_name +
|
||||
''.join(x[12:] for x in fork_code[1:]))
|
||||
cmdline = orig_command_line()
|
||||
cmdline[2] = fork_string
|
||||
return cmdline
|
||||
orig_command_line = forking.get_command_line
|
||||
forking.get_command_line = my_get_command_line
|
||||
|
|
|
|||
Loading…
Reference in a new issue