Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1###############################################################################
2# Prepares and processes the data to setup the new process environment
3#
4# author: Thomas Moreau and Olivier Grisel
5#
6# adapted from multiprocessing/spawn.py (17/02/2017)
7# * Improve logging data
8#
9import os
10import sys
11import runpy
12import textwrap
13import types
14from multiprocessing import process, util
17if sys.platform != "win32":
18 WINEXE = False
19 WINSERVICE = False
20else:
21 import msvcrt
22 from multiprocessing.reduction import duplicate
24 WINEXE = sys.platform == "win32" and getattr(sys, "frozen", False)
25 WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
27if WINSERVICE:
28 _python_exe = os.path.join(sys.exec_prefix, "python.exe")
29else:
30 _python_exe = sys.executable
33def get_executable():
34 return _python_exe
37def _check_not_importing_main():
38 if getattr(process.current_process(), "_inheriting", False):
39 raise RuntimeError(
40 textwrap.dedent(
41 """\
42 An attempt has been made to start a new process before the
43 current process has finished its bootstrapping phase.
45 This probably means that you are not using fork to start your
46 child processes and you have forgotten to use the proper idiom
47 in the main module:
49 if __name__ == '__main__':
50 freeze_support()
51 ...
53 The "freeze_support()" line can be omitted if the program
54 is not going to be frozen to produce an executable."""
55 )
56 )
59def get_preparation_data(name, init_main_module=True):
60 """Return info about parent needed by child to unpickle process object."""
61 _check_not_importing_main()
62 d = dict(
63 log_to_stderr=util._log_to_stderr,
64 authkey=bytes(process.current_process().authkey),
65 name=name,
66 sys_argv=sys.argv,
67 orig_dir=process.ORIGINAL_DIR,
68 dir=os.getcwd(),
69 )
71 # Send sys_path and make sure the current directory will not be changed
72 d["sys_path"] = [p if p != "" else process.ORIGINAL_DIR for p in sys.path]
74 # Make sure to pass the information if the multiprocessing logger is active
75 if util._logger is not None:
76 d["log_level"] = util._logger.getEffectiveLevel()
77 if util._logger.handlers:
78 h = util._logger.handlers[0]
79 d["log_fmt"] = h.formatter._fmt
81 # Tell the child how to communicate with the resource_tracker
82 from .resource_tracker import _resource_tracker
84 _resource_tracker.ensure_running()
85 if sys.platform == "win32":
86 d["tracker_fd"] = msvcrt.get_osfhandle(_resource_tracker._fd)
87 else:
88 d["tracker_fd"] = _resource_tracker._fd
90 if os.name == "posix":
91 # joblib/loky#242: allow loky processes to retrieve the resource
92 # tracker of their parent in case the child processes depickles
93 # shared_memory objects, that are still tracked by multiprocessing's
94 # resource_tracker by default.
95 # XXX: this is a workaround that may be error prone: in the future, it
96 # would be better to have loky subclass multiprocessing's shared_memory
97 # to force registration of shared_memory segments via loky's
98 # resource_tracker.
99 from multiprocessing.resource_tracker import (
100 _resource_tracker as mp_resource_tracker,
101 )
103 # multiprocessing's resource_tracker must be running before loky
104 # process is created (othewise the child won't be able to use it if it
105 # is created later on)
106 mp_resource_tracker.ensure_running()
107 d["mp_tracker_fd"] = mp_resource_tracker._fd
109 # Figure out whether to initialise main in the subprocess as a module
110 # or through direct execution (or to leave it alone entirely)
111 if init_main_module:
112 main_module = sys.modules["__main__"]
113 try:
114 main_mod_name = getattr(main_module.__spec__, "name", None)
115 except BaseException:
116 main_mod_name = None
117 if main_mod_name is not None:
118 d["init_main_from_name"] = main_mod_name
119 elif sys.platform != "win32" or (not WINEXE and not WINSERVICE):
120 main_path = getattr(main_module, "__file__", None)
121 if main_path is not None:
122 if (
123 not os.path.isabs(main_path)
124 and process.ORIGINAL_DIR is not None
125 ):
126 main_path = os.path.join(process.ORIGINAL_DIR, main_path)
127 d["init_main_from_path"] = os.path.normpath(main_path)
129 return d
132#
133# Prepare current process
134#
135old_main_modules = []
138def prepare(data, parent_sentinel=None):
139 """Try to get current process ready to unpickle process object."""
140 if "name" in data:
141 process.current_process().name = data["name"]
143 if "authkey" in data:
144 process.current_process().authkey = data["authkey"]
146 if "log_to_stderr" in data and data["log_to_stderr"]:
147 util.log_to_stderr()
149 if "log_level" in data:
150 util.get_logger().setLevel(data["log_level"])
152 if "log_fmt" in data:
153 import logging
155 util.get_logger().handlers[0].setFormatter(
156 logging.Formatter(data["log_fmt"])
157 )
159 if "sys_path" in data:
160 sys.path = data["sys_path"]
162 if "sys_argv" in data:
163 sys.argv = data["sys_argv"]
165 if "dir" in data:
166 os.chdir(data["dir"])
168 if "orig_dir" in data:
169 process.ORIGINAL_DIR = data["orig_dir"]
171 if "mp_tracker_fd" in data:
172 from multiprocessing.resource_tracker import (
173 _resource_tracker as mp_resource_tracker,
174 )
176 mp_resource_tracker._fd = data["mp_tracker_fd"]
177 if "tracker_fd" in data:
178 from .resource_tracker import _resource_tracker
180 if sys.platform == "win32":
181 handle = data["tracker_fd"]
182 handle = duplicate(handle, source_process=parent_sentinel)
183 _resource_tracker._fd = msvcrt.open_osfhandle(handle, os.O_RDONLY)
184 else:
185 _resource_tracker._fd = data["tracker_fd"]
187 if "init_main_from_name" in data:
188 _fixup_main_from_name(data["init_main_from_name"])
189 elif "init_main_from_path" in data:
190 _fixup_main_from_path(data["init_main_from_path"])
193# Multiprocessing module helpers to fix up the main module in
194# spawned subprocesses
195def _fixup_main_from_name(mod_name):
196 # __main__.py files for packages, directories, zip archives, etc, run
197 # their "main only" code unconditionally, so we don't even try to
198 # populate anything in __main__, nor do we make any changes to
199 # __main__ attributes
200 current_main = sys.modules["__main__"]
201 if mod_name == "__main__" or mod_name.endswith(".__main__"):
202 return
204 # If this process was forked, __main__ may already be populated
205 if getattr(current_main.__spec__, "name", None) == mod_name:
206 return
208 # Otherwise, __main__ may contain some non-main code where we need to
209 # support unpickling it properly. We rerun it as __mp_main__ and make
210 # the normal __main__ an alias to that
211 old_main_modules.append(current_main)
212 main_module = types.ModuleType("__mp_main__")
213 main_content = runpy.run_module(
214 mod_name, run_name="__mp_main__", alter_sys=True
215 )
216 main_module.__dict__.update(main_content)
217 sys.modules["__main__"] = sys.modules["__mp_main__"] = main_module
220def _fixup_main_from_path(main_path):
221 # If this process was forked, __main__ may already be populated
222 current_main = sys.modules["__main__"]
224 # Unfortunately, the main ipython launch script historically had no
225 # "if __name__ == '__main__'" guard, so we work around that
226 # by treating it like a __main__.py file
227 # See https://github.com/ipython/ipython/issues/4698
228 main_name = os.path.splitext(os.path.basename(main_path))[0]
229 if main_name == "ipython":
230 return
232 # Otherwise, if __file__ already has the setting we expect,
233 # there's nothing more to do
234 if getattr(current_main, "__file__", None) == main_path:
235 return
237 # If the parent process has sent a path through rather than a module
238 # name we assume it is an executable script that may contain
239 # non-main code that needs to be executed
240 old_main_modules.append(current_main)
241 main_module = types.ModuleType("__mp_main__")
242 main_content = runpy.run_path(main_path, run_name="__mp_main__")
243 main_module.__dict__.update(main_content)
244 sys.modules["__main__"] = sys.modules["__mp_main__"] = main_module