Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/joblib/externals/loky/backend/spawn.py: 16%

1############################################################################### 

2# Prepares and processes the data to setup the new process environment 

3# 

4# author: Thomas Moreau and Olivier Grisel 

5# 

6# adapted from multiprocessing/spawn.py (17/02/2017) 

7# * Improve logging data 

8# 

9import os 

10import sys 

11import runpy 

12import textwrap 

13import types 

14from multiprocessing import process, util 

15 

16 

17if sys.platform != "win32": 

18 WINEXE = False 

19 WINSERVICE = False 

20else: 

21 import msvcrt 

22 from multiprocessing.reduction import duplicate 

23 

24 WINEXE = sys.platform == "win32" and getattr(sys, "frozen", False) 

25 WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") 

26 

27if WINSERVICE: 

28 _python_exe = os.path.join(sys.exec_prefix, "python.exe") 

29else: 

30 _python_exe = sys.executable 

31 

32 

33def get_executable(): 

34 return _python_exe 

35 

36 

37def _check_not_importing_main(): 

38 if getattr(process.current_process(), "_inheriting", False): 

39 raise RuntimeError( 

40 textwrap.dedent( 

41 """\ 

42 An attempt has been made to start a new process before the 

43 current process has finished its bootstrapping phase. 

44 

45 This probably means that you are not using fork to start your 

46 child processes and you have forgotten to use the proper idiom 

47 in the main module: 

48 

49 if __name__ == '__main__': 

50 freeze_support() 

51 ... 

52 

53 The "freeze_support()" line can be omitted if the program 

54 is not going to be frozen to produce an executable.""" 

55 ) 

56 ) 

57 

58 

59def get_preparation_data(name, init_main_module=True): 

60 """Return info about parent needed by child to unpickle process object.""" 

61 _check_not_importing_main() 

62 d = dict( 

63 log_to_stderr=util._log_to_stderr, 

64 authkey=bytes(process.current_process().authkey), 

65 name=name, 

66 sys_argv=sys.argv, 

67 orig_dir=process.ORIGINAL_DIR, 

68 dir=os.getcwd(), 

69 ) 

70 

71 # Send sys_path and make sure the current directory will not be changed 

72 d["sys_path"] = [p if p != "" else process.ORIGINAL_DIR for p in sys.path] 

73 

74 # Make sure to pass the information if the multiprocessing logger is active 

75 if util._logger is not None: 

76 d["log_level"] = util._logger.getEffectiveLevel() 

77 if util._logger.handlers: 

78 h = util._logger.handlers[0] 

79 d["log_fmt"] = h.formatter._fmt 

80 

81 # Tell the child how to communicate with the resource_tracker 

82 from .resource_tracker import _resource_tracker 

83 

84 _resource_tracker.ensure_running() 

85 d["tracker_args"] = {"pid": _resource_tracker._pid} 

86 if sys.platform == "win32": 

87 d["tracker_args"]["fh"] = msvcrt.get_osfhandle(_resource_tracker._fd) 

88 else: 

89 d["tracker_args"]["fd"] = _resource_tracker._fd 

90 

91 if sys.version_info >= (3, 8) and os.name == "posix": 

92 # joblib/loky#242: allow loky processes to retrieve the resource 

93 # tracker of their parent in case the child processes depickles 

94 # shared_memory objects, that are still tracked by multiprocessing's 

95 # resource_tracker by default. 

96 # XXX: this is a workaround that may be error prone: in the future, it 

97 # would be better to have loky subclass multiprocessing's shared_memory 

98 # to force registration of shared_memory segments via loky's 

99 # resource_tracker. 

100 from multiprocessing.resource_tracker import ( 

101 _resource_tracker as mp_resource_tracker, 

102 ) 

103 

104 # multiprocessing's resource_tracker must be running before loky 

105 # process is created (othewise the child won't be able to use it if it 

106 # is created later on) 

107 mp_resource_tracker.ensure_running() 

108 d["mp_tracker_args"] = { 

109 "fd": mp_resource_tracker._fd, 

110 "pid": mp_resource_tracker._pid, 

111 } 

112 

113 # Figure out whether to initialise main in the subprocess as a module 

114 # or through direct execution (or to leave it alone entirely) 

115 if init_main_module: 

116 main_module = sys.modules["__main__"] 

117 try: 

118 main_mod_name = getattr(main_module.__spec__, "name", None) 

119 except BaseException: 

120 main_mod_name = None 

121 if main_mod_name is not None: 

122 d["init_main_from_name"] = main_mod_name 

123 elif sys.platform != "win32" or (not WINEXE and not WINSERVICE): 

124 main_path = getattr(main_module, "__file__", None) 

125 if main_path is not None: 

126 if ( 

127 not os.path.isabs(main_path) 

128 and process.ORIGINAL_DIR is not None 

129 ): 

130 main_path = os.path.join(process.ORIGINAL_DIR, main_path) 

131 d["init_main_from_path"] = os.path.normpath(main_path) 

132 

133 return d 

134 

135 

136# 

137# Prepare current process 

138# 

139old_main_modules = [] 

140 

141 

142def prepare(data, parent_sentinel=None): 

143 """Try to get current process ready to unpickle process object.""" 

144 if "name" in data: 

145 process.current_process().name = data["name"] 

146 

147 if "authkey" in data: 

148 process.current_process().authkey = data["authkey"] 

149 

150 if "log_to_stderr" in data and data["log_to_stderr"]: 

151 util.log_to_stderr() 

152 

153 if "log_level" in data: 

154 util.get_logger().setLevel(data["log_level"]) 

155 

156 if "log_fmt" in data: 

157 import logging 

158 

159 util.get_logger().handlers[0].setFormatter( 

160 logging.Formatter(data["log_fmt"]) 

161 ) 

162 

163 if "sys_path" in data: 

164 sys.path = data["sys_path"] 

165 

166 if "sys_argv" in data: 

167 sys.argv = data["sys_argv"] 

168 

169 if "dir" in data: 

170 os.chdir(data["dir"]) 

171 

172 if "orig_dir" in data: 

173 process.ORIGINAL_DIR = data["orig_dir"] 

174 

175 if "mp_tracker_args" in data: 

176 from multiprocessing.resource_tracker import ( 

177 _resource_tracker as mp_resource_tracker, 

178 ) 

179 

180 mp_resource_tracker._fd = data["mp_tracker_args"]["fd"] 

181 mp_resource_tracker._pid = data["mp_tracker_args"]["pid"] 

182 if "tracker_args" in data: 

183 from .resource_tracker import _resource_tracker 

184 

185 _resource_tracker._pid = data["tracker_args"]["pid"] 

186 if sys.platform == "win32": 

187 handle = data["tracker_args"]["fh"] 

188 handle = duplicate(handle, source_process=parent_sentinel) 

189 _resource_tracker._fd = msvcrt.open_osfhandle(handle, os.O_RDONLY) 

190 else: 

191 _resource_tracker._fd = data["tracker_args"]["fd"] 

192 

193 if "init_main_from_name" in data: 

194 _fixup_main_from_name(data["init_main_from_name"]) 

195 elif "init_main_from_path" in data: 

196 _fixup_main_from_path(data["init_main_from_path"]) 

197 

198 

199# Multiprocessing module helpers to fix up the main module in 

200# spawned subprocesses 

201def _fixup_main_from_name(mod_name): 

202 # __main__.py files for packages, directories, zip archives, etc, run 

203 # their "main only" code unconditionally, so we don't even try to 

204 # populate anything in __main__, nor do we make any changes to 

205 # __main__ attributes 

206 current_main = sys.modules["__main__"] 

207 if mod_name == "__main__" or mod_name.endswith(".__main__"): 

208 return 

209 

210 # If this process was forked, __main__ may already be populated 

211 if getattr(current_main.__spec__, "name", None) == mod_name: 

212 return 

213 

214 # Otherwise, __main__ may contain some non-main code where we need to 

215 # support unpickling it properly. We rerun it as __mp_main__ and make 

216 # the normal __main__ an alias to that 

217 old_main_modules.append(current_main) 

218 main_module = types.ModuleType("__mp_main__") 

219 main_content = runpy.run_module( 

220 mod_name, run_name="__mp_main__", alter_sys=True 

221 ) 

222 main_module.__dict__.update(main_content) 

223 sys.modules["__main__"] = sys.modules["__mp_main__"] = main_module 

224 

225 

226def _fixup_main_from_path(main_path): 

227 # If this process was forked, __main__ may already be populated 

228 current_main = sys.modules["__main__"] 

229 

230 # Unfortunately, the main ipython launch script historically had no 

231 # "if __name__ == '__main__'" guard, so we work around that 

232 # by treating it like a __main__.py file 

233 # See https://github.com/ipython/ipython/issues/4698 

234 main_name = os.path.splitext(os.path.basename(main_path))[0] 

235 if main_name == "ipython": 

236 return 

237 

238 # Otherwise, if __file__ already has the setting we expect, 

239 # there's nothing more to do 

240 if getattr(current_main, "__file__", None) == main_path: 

241 return 

242 

243 # If the parent process has sent a path through rather than a module 

244 # name we assume it is an executable script that may contain 

245 # non-main code that needs to be executed 

246 old_main_modules.append(current_main) 

247 main_module = types.ModuleType("__mp_main__") 

248 main_content = runpy.run_path(main_path, run_name="__mp_main__") 

249 main_module.__dict__.update(main_content) 

250 sys.modules["__main__"] = sys.modules["__mp_main__"] = main_module