Coverage for /pythoncovmergedfiles/medio/medio/usr/lib/python3.9/subprocess.py: 13%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# subprocess - Subprocesses with accessible I/O streams
2#
3# For more information about this module, see PEP 324.
4#
5# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
6#
7# Licensed to PSF under a Contributor Agreement.
8# See http://www.python.org/2.4/license for licensing details.
10r"""Subprocesses with accessible I/O streams
12This module allows you to spawn processes, connect to their
13input/output/error pipes, and obtain their return codes.
15For a complete description of this module see the Python documentation.
17Main API
18========
19run(...): Runs a command, waits for it to complete, then returns a
20 CompletedProcess instance.
21Popen(...): A class for flexibly executing a command in a new process
23Constants
24---------
25DEVNULL: Special value that indicates that os.devnull should be used
26PIPE: Special value that indicates a pipe should be created
27STDOUT: Special value that indicates that stderr should go to stdout
30Older API
31=========
32call(...): Runs a command, waits for it to complete, then returns
33 the return code.
34check_call(...): Same as call() but raises CalledProcessError()
35 if return code is not 0
36check_output(...): Same as check_call() but returns the contents of
37 stdout instead of a return code
38getoutput(...): Runs a command in the shell, waits for it to complete,
39 then returns the output
40getstatusoutput(...): Runs a command in the shell, waits for it to complete,
41 then returns a (exitcode, output) tuple
42"""
44import builtins
45import errno
46import io
47import os
48import time
49import signal
50import sys
51import threading
52import warnings
53import contextlib
54from time import monotonic as _time
55import types
57try:
58 import pwd
59except ImportError:
60 pwd = None
61try:
62 import grp
63except ImportError:
64 grp = None
66__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput",
67 "getoutput", "check_output", "run", "CalledProcessError", "DEVNULL",
68 "SubprocessError", "TimeoutExpired", "CompletedProcess"]
69 # NOTE: We intentionally exclude list2cmdline as it is
70 # considered an internal implementation detail. issue10838.
72try:
73 import msvcrt
74 import _winapi
75 _mswindows = True
76except ModuleNotFoundError:
77 _mswindows = False
78 import _posixsubprocess
79 import select
80 import selectors
81else:
82 from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP,
83 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
84 STD_ERROR_HANDLE, SW_HIDE,
85 STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW,
86 ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS,
87 HIGH_PRIORITY_CLASS, IDLE_PRIORITY_CLASS,
88 NORMAL_PRIORITY_CLASS, REALTIME_PRIORITY_CLASS,
89 CREATE_NO_WINDOW, DETACHED_PROCESS,
90 CREATE_DEFAULT_ERROR_MODE, CREATE_BREAKAWAY_FROM_JOB)
92 __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP",
93 "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE",
94 "STD_ERROR_HANDLE", "SW_HIDE",
95 "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW",
96 "STARTUPINFO",
97 "ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS",
98 "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS",
99 "NORMAL_PRIORITY_CLASS", "REALTIME_PRIORITY_CLASS",
100 "CREATE_NO_WINDOW", "DETACHED_PROCESS",
101 "CREATE_DEFAULT_ERROR_MODE", "CREATE_BREAKAWAY_FROM_JOB"])
104# Exception classes used by this module.
105class SubprocessError(Exception): pass
108class CalledProcessError(SubprocessError):
109 """Raised when run() is called with check=True and the process
110 returns a non-zero exit status.
112 Attributes:
113 cmd, returncode, stdout, stderr, output
114 """
115 def __init__(self, returncode, cmd, output=None, stderr=None):
116 self.returncode = returncode
117 self.cmd = cmd
118 self.output = output
119 self.stderr = stderr
121 def __str__(self):
122 if self.returncode and self.returncode < 0:
123 try:
124 return "Command '%s' died with %r." % (
125 self.cmd, signal.Signals(-self.returncode))
126 except ValueError:
127 return "Command '%s' died with unknown signal %d." % (
128 self.cmd, -self.returncode)
129 else:
130 return "Command '%s' returned non-zero exit status %d." % (
131 self.cmd, self.returncode)
133 @property
134 def stdout(self):
135 """Alias for output attribute, to match stderr"""
136 return self.output
138 @stdout.setter
139 def stdout(self, value):
140 # There's no obvious reason to set this, but allow it anyway so
141 # .stdout is a transparent alias for .output
142 self.output = value
145class TimeoutExpired(SubprocessError):
146 """This exception is raised when the timeout expires while waiting for a
147 child process.
149 Attributes:
150 cmd, output, stdout, stderr, timeout
151 """
152 def __init__(self, cmd, timeout, output=None, stderr=None):
153 self.cmd = cmd
154 self.timeout = timeout
155 self.output = output
156 self.stderr = stderr
158 def __str__(self):
159 return ("Command '%s' timed out after %s seconds" %
160 (self.cmd, self.timeout))
162 @property
163 def stdout(self):
164 return self.output
166 @stdout.setter
167 def stdout(self, value):
168 # There's no obvious reason to set this, but allow it anyway so
169 # .stdout is a transparent alias for .output
170 self.output = value
173if _mswindows:
174 class STARTUPINFO:
175 def __init__(self, *, dwFlags=0, hStdInput=None, hStdOutput=None,
176 hStdError=None, wShowWindow=0, lpAttributeList=None):
177 self.dwFlags = dwFlags
178 self.hStdInput = hStdInput
179 self.hStdOutput = hStdOutput
180 self.hStdError = hStdError
181 self.wShowWindow = wShowWindow
182 self.lpAttributeList = lpAttributeList or {"handle_list": []}
184 def copy(self):
185 attr_list = self.lpAttributeList.copy()
186 if 'handle_list' in attr_list:
187 attr_list['handle_list'] = list(attr_list['handle_list'])
189 return STARTUPINFO(dwFlags=self.dwFlags,
190 hStdInput=self.hStdInput,
191 hStdOutput=self.hStdOutput,
192 hStdError=self.hStdError,
193 wShowWindow=self.wShowWindow,
194 lpAttributeList=attr_list)
197 class Handle(int):
198 closed = False
200 def Close(self, CloseHandle=_winapi.CloseHandle):
201 if not self.closed:
202 self.closed = True
203 CloseHandle(self)
205 def Detach(self):
206 if not self.closed:
207 self.closed = True
208 return int(self)
209 raise ValueError("already closed")
211 def __repr__(self):
212 return "%s(%d)" % (self.__class__.__name__, int(self))
214 __del__ = Close
215else:
216 # When select or poll has indicated that the file is writable,
217 # we can write up to _PIPE_BUF bytes without risk of blocking.
218 # POSIX defines PIPE_BUF as >= 512.
219 _PIPE_BUF = getattr(select, 'PIPE_BUF', 512)
221 # poll/select have the advantage of not requiring any extra file
222 # descriptor, contrarily to epoll/kqueue (also, they require a single
223 # syscall).
224 if hasattr(selectors, 'PollSelector'):
225 _PopenSelector = selectors.PollSelector
226 else:
227 _PopenSelector = selectors.SelectSelector
230if _mswindows:
231 # On Windows we just need to close `Popen._handle` when we no longer need
232 # it, so that the kernel can free it. `Popen._handle` gets closed
233 # implicitly when the `Popen` instance is finalized (see `Handle.__del__`,
234 # which is calling `CloseHandle` as requested in [1]), so there is nothing
235 # for `_cleanup` to do.
236 #
237 # [1] https://docs.microsoft.com/en-us/windows/desktop/ProcThread/
238 # creating-processes
239 _active = None
241 def _cleanup():
242 pass
243else:
244 # This lists holds Popen instances for which the underlying process had not
245 # exited at the time its __del__ method got called: those processes are
246 # wait()ed for synchronously from _cleanup() when a new Popen object is
247 # created, to avoid zombie processes.
248 _active = []
250 def _cleanup():
251 if _active is None:
252 return
253 for inst in _active[:]:
254 res = inst._internal_poll(_deadstate=sys.maxsize)
255 if res is not None:
256 try:
257 _active.remove(inst)
258 except ValueError:
259 # This can happen if two threads create a new Popen instance.
260 # It's harmless that it was already removed, so ignore.
261 pass
263PIPE = -1
264STDOUT = -2
265DEVNULL = -3
268# XXX This function is only used by multiprocessing and the test suite,
269# but it's here so that it can be imported when Python is compiled without
270# threads.
272def _optim_args_from_interpreter_flags():
273 """Return a list of command-line arguments reproducing the current
274 optimization settings in sys.flags."""
275 args = []
276 value = sys.flags.optimize
277 if value > 0:
278 args.append('-' + 'O' * value)
279 return args
282def _args_from_interpreter_flags():
283 """Return a list of command-line arguments reproducing the current
284 settings in sys.flags, sys.warnoptions and sys._xoptions."""
285 flag_opt_map = {
286 'debug': 'd',
287 # 'inspect': 'i',
288 # 'interactive': 'i',
289 'dont_write_bytecode': 'B',
290 'no_site': 'S',
291 'verbose': 'v',
292 'bytes_warning': 'b',
293 'quiet': 'q',
294 # -O is handled in _optim_args_from_interpreter_flags()
295 }
296 args = _optim_args_from_interpreter_flags()
297 for flag, opt in flag_opt_map.items():
298 v = getattr(sys.flags, flag)
299 if v > 0:
300 args.append('-' + opt * v)
302 if sys.flags.isolated:
303 args.append('-I')
304 else:
305 if sys.flags.ignore_environment:
306 args.append('-E')
307 if sys.flags.no_user_site:
308 args.append('-s')
310 # -W options
311 warnopts = sys.warnoptions[:]
312 bytes_warning = sys.flags.bytes_warning
313 xoptions = getattr(sys, '_xoptions', {})
314 dev_mode = ('dev' in xoptions)
316 if bytes_warning > 1:
317 warnopts.remove("error::BytesWarning")
318 elif bytes_warning:
319 warnopts.remove("default::BytesWarning")
320 if dev_mode:
321 warnopts.remove('default')
322 for opt in warnopts:
323 args.append('-W' + opt)
325 # -X options
326 if dev_mode:
327 args.extend(('-X', 'dev'))
328 for opt in ('faulthandler', 'tracemalloc', 'importtime',
329 'showrefcount', 'utf8', 'oldparser'):
330 if opt in xoptions:
331 value = xoptions[opt]
332 if value is True:
333 arg = opt
334 else:
335 arg = '%s=%s' % (opt, value)
336 args.extend(('-X', arg))
338 return args
341def call(*popenargs, timeout=None, **kwargs):
342 """Run command with arguments. Wait for command to complete or
343 timeout, then return the returncode attribute.
345 The arguments are the same as for the Popen constructor. Example:
347 retcode = call(["ls", "-l"])
348 """
349 with Popen(*popenargs, **kwargs) as p:
350 try:
351 return p.wait(timeout=timeout)
352 except: # Including KeyboardInterrupt, wait handled that.
353 p.kill()
354 # We don't call p.wait() again as p.__exit__ does that for us.
355 raise
358def check_call(*popenargs, **kwargs):
359 """Run command with arguments. Wait for command to complete. If
360 the exit code was zero then return, otherwise raise
361 CalledProcessError. The CalledProcessError object will have the
362 return code in the returncode attribute.
364 The arguments are the same as for the call function. Example:
366 check_call(["ls", "-l"])
367 """
368 retcode = call(*popenargs, **kwargs)
369 if retcode:
370 cmd = kwargs.get("args")
371 if cmd is None:
372 cmd = popenargs[0]
373 raise CalledProcessError(retcode, cmd)
374 return 0
377def check_output(*popenargs, timeout=None, **kwargs):
378 r"""Run command with arguments and return its output.
380 If the exit code was non-zero it raises a CalledProcessError. The
381 CalledProcessError object will have the return code in the returncode
382 attribute and output in the output attribute.
384 The arguments are the same as for the Popen constructor. Example:
386 >>> check_output(["ls", "-l", "/dev/null"])
387 b'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n'
389 The stdout argument is not allowed as it is used internally.
390 To capture standard error in the result, use stderr=STDOUT.
392 >>> check_output(["/bin/sh", "-c",
393 ... "ls -l non_existent_file ; exit 0"],
394 ... stderr=STDOUT)
395 b'ls: non_existent_file: No such file or directory\n'
397 There is an additional optional argument, "input", allowing you to
398 pass a string to the subprocess's stdin. If you use this argument
399 you may not also use the Popen constructor's "stdin" argument, as
400 it too will be used internally. Example:
402 >>> check_output(["sed", "-e", "s/foo/bar/"],
403 ... input=b"when in the course of fooman events\n")
404 b'when in the course of barman events\n'
406 By default, all communication is in bytes, and therefore any "input"
407 should be bytes, and the return value will be bytes. If in text mode,
408 any "input" should be a string, and the return value will be a string
409 decoded according to locale encoding, or by "encoding" if set. Text mode
410 is triggered by setting any of text, encoding, errors or universal_newlines.
411 """
412 if 'stdout' in kwargs:
413 raise ValueError('stdout argument not allowed, it will be overridden.')
415 if 'input' in kwargs and kwargs['input'] is None:
416 # Explicitly passing input=None was previously equivalent to passing an
417 # empty string. That is maintained here for backwards compatibility.
418 if kwargs.get('universal_newlines') or kwargs.get('text'):
419 empty = ''
420 else:
421 empty = b''
422 kwargs['input'] = empty
424 return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
425 **kwargs).stdout
428class CompletedProcess(object):
429 """A process that has finished running.
431 This is returned by run().
433 Attributes:
434 args: The list or str args passed to run().
435 returncode: The exit code of the process, negative for signals.
436 stdout: The standard output (None if not captured).
437 stderr: The standard error (None if not captured).
438 """
439 def __init__(self, args, returncode, stdout=None, stderr=None):
440 self.args = args
441 self.returncode = returncode
442 self.stdout = stdout
443 self.stderr = stderr
445 def __repr__(self):
446 args = ['args={!r}'.format(self.args),
447 'returncode={!r}'.format(self.returncode)]
448 if self.stdout is not None:
449 args.append('stdout={!r}'.format(self.stdout))
450 if self.stderr is not None:
451 args.append('stderr={!r}'.format(self.stderr))
452 return "{}({})".format(type(self).__name__, ', '.join(args))
454 __class_getitem__ = classmethod(types.GenericAlias)
457 def check_returncode(self):
458 """Raise CalledProcessError if the exit code is non-zero."""
459 if self.returncode:
460 raise CalledProcessError(self.returncode, self.args, self.stdout,
461 self.stderr)
464def run(*popenargs,
465 input=None, capture_output=False, timeout=None, check=False, **kwargs):
466 """Run command with arguments and return a CompletedProcess instance.
468 The returned instance will have attributes args, returncode, stdout and
469 stderr. By default, stdout and stderr are not captured, and those attributes
470 will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them.
472 If check is True and the exit code was non-zero, it raises a
473 CalledProcessError. The CalledProcessError object will have the return code
474 in the returncode attribute, and output & stderr attributes if those streams
475 were captured.
477 If timeout is given, and the process takes too long, a TimeoutExpired
478 exception will be raised.
480 There is an optional argument "input", allowing you to
481 pass bytes or a string to the subprocess's stdin. If you use this argument
482 you may not also use the Popen constructor's "stdin" argument, as
483 it will be used internally.
485 By default, all communication is in bytes, and therefore any "input" should
486 be bytes, and the stdout and stderr will be bytes. If in text mode, any
487 "input" should be a string, and stdout and stderr will be strings decoded
488 according to locale encoding, or by "encoding" if set. Text mode is
489 triggered by setting any of text, encoding, errors or universal_newlines.
491 The other arguments are the same as for the Popen constructor.
492 """
493 if input is not None:
494 if kwargs.get('stdin') is not None:
495 raise ValueError('stdin and input arguments may not both be used.')
496 kwargs['stdin'] = PIPE
498 if capture_output:
499 if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None:
500 raise ValueError('stdout and stderr arguments may not be used '
501 'with capture_output.')
502 kwargs['stdout'] = PIPE
503 kwargs['stderr'] = PIPE
505 with Popen(*popenargs, **kwargs) as process:
506 try:
507 stdout, stderr = process.communicate(input, timeout=timeout)
508 except TimeoutExpired as exc:
509 process.kill()
510 if _mswindows:
511 # Windows accumulates the output in a single blocking
512 # read() call run on child threads, with the timeout
513 # being done in a join() on those threads. communicate()
514 # _after_ kill() is required to collect that and add it
515 # to the exception.
516 exc.stdout, exc.stderr = process.communicate()
517 else:
518 # POSIX _communicate already populated the output so
519 # far into the TimeoutExpired exception.
520 process.wait()
521 raise
522 except: # Including KeyboardInterrupt, communicate handled that.
523 process.kill()
524 # We don't call process.wait() as .__exit__ does that for us.
525 raise
526 retcode = process.poll()
527 if check and retcode:
528 raise CalledProcessError(retcode, process.args,
529 output=stdout, stderr=stderr)
530 return CompletedProcess(process.args, retcode, stdout, stderr)
533def list2cmdline(seq):
534 """
535 Translate a sequence of arguments into a command line
536 string, using the same rules as the MS C runtime:
538 1) Arguments are delimited by white space, which is either a
539 space or a tab.
541 2) A string surrounded by double quotation marks is
542 interpreted as a single argument, regardless of white space
543 contained within. A quoted string can be embedded in an
544 argument.
546 3) A double quotation mark preceded by a backslash is
547 interpreted as a literal double quotation mark.
549 4) Backslashes are interpreted literally, unless they
550 immediately precede a double quotation mark.
552 5) If backslashes immediately precede a double quotation mark,
553 every pair of backslashes is interpreted as a literal
554 backslash. If the number of backslashes is odd, the last
555 backslash escapes the next double quotation mark as
556 described in rule 3.
557 """
559 # See
560 # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx
561 # or search http://msdn.microsoft.com for
562 # "Parsing C++ Command-Line Arguments"
563 result = []
564 needquote = False
565 for arg in map(os.fsdecode, seq):
566 bs_buf = []
568 # Add a space to separate this argument from the others
569 if result:
570 result.append(' ')
572 needquote = (" " in arg) or ("\t" in arg) or not arg
573 if needquote:
574 result.append('"')
576 for c in arg:
577 if c == '\\':
578 # Don't know if we need to double yet.
579 bs_buf.append(c)
580 elif c == '"':
581 # Double backslashes.
582 result.append('\\' * len(bs_buf)*2)
583 bs_buf = []
584 result.append('\\"')
585 else:
586 # Normal char
587 if bs_buf:
588 result.extend(bs_buf)
589 bs_buf = []
590 result.append(c)
592 # Add remaining backslashes, if any.
593 if bs_buf:
594 result.extend(bs_buf)
596 if needquote:
597 result.extend(bs_buf)
598 result.append('"')
600 return ''.join(result)
603# Various tools for executing commands and looking at their output and status.
604#
606def getstatusoutput(cmd):
607 """Return (exitcode, output) of executing cmd in a shell.
609 Execute the string 'cmd' in a shell with 'check_output' and
610 return a 2-tuple (status, output). The locale encoding is used
611 to decode the output and process newlines.
613 A trailing newline is stripped from the output.
614 The exit status for the command can be interpreted
615 according to the rules for the function 'wait'. Example:
617 >>> import subprocess
618 >>> subprocess.getstatusoutput('ls /bin/ls')
619 (0, '/bin/ls')
620 >>> subprocess.getstatusoutput('cat /bin/junk')
621 (1, 'cat: /bin/junk: No such file or directory')
622 >>> subprocess.getstatusoutput('/bin/junk')
623 (127, 'sh: /bin/junk: not found')
624 >>> subprocess.getstatusoutput('/bin/kill $$')
625 (-15, '')
626 """
627 try:
628 data = check_output(cmd, shell=True, text=True, stderr=STDOUT)
629 exitcode = 0
630 except CalledProcessError as ex:
631 data = ex.output
632 exitcode = ex.returncode
633 if data[-1:] == '\n':
634 data = data[:-1]
635 return exitcode, data
637def getoutput(cmd):
638 """Return output (stdout or stderr) of executing cmd in a shell.
640 Like getstatusoutput(), except the exit status is ignored and the return
641 value is a string containing the command's output. Example:
643 >>> import subprocess
644 >>> subprocess.getoutput('ls /bin/ls')
645 '/bin/ls'
646 """
647 return getstatusoutput(cmd)[1]
650def _use_posix_spawn():
651 """Check if posix_spawn() can be used for subprocess.
653 subprocess requires a posix_spawn() implementation that properly reports
654 errors to the parent process, & sets errno on the following failures:
656 * Process attribute actions failed.
657 * File actions failed.
658 * exec() failed.
660 Prefer an implementation which can use vfork() in some cases for best
661 performance.
662 """
663 if _mswindows or not hasattr(os, 'posix_spawn'):
664 # os.posix_spawn() is not available
665 return False
667 if sys.platform == 'darwin':
668 # posix_spawn() is a syscall on macOS and properly reports errors
669 return True
671 # Check libc name and runtime libc version
672 try:
673 ver = os.confstr('CS_GNU_LIBC_VERSION')
674 # parse 'glibc 2.28' as ('glibc', (2, 28))
675 parts = ver.split(maxsplit=1)
676 if len(parts) != 2:
677 # reject unknown format
678 raise ValueError
679 libc = parts[0]
680 version = tuple(map(int, parts[1].split('.')))
682 if sys.platform == 'linux' and libc == 'glibc' and version >= (2, 24):
683 # glibc 2.24 has a new Linux posix_spawn implementation using vfork
684 # which properly reports errors to the parent process.
685 return True
686 # Note: Don't use the implementation in earlier glibc because it doesn't
687 # use vfork (even if glibc 2.26 added a pipe to properly report errors
688 # to the parent process).
689 except (AttributeError, ValueError, OSError):
690 # os.confstr() or CS_GNU_LIBC_VERSION value not available
691 pass
693 # By default, assume that posix_spawn() does not properly report errors.
694 return False
697_USE_POSIX_SPAWN = _use_posix_spawn()
700class Popen(object):
701 """ Execute a child program in a new process.
703 For a complete description of the arguments see the Python documentation.
705 Arguments:
706 args: A string, or a sequence of program arguments.
708 bufsize: supplied as the buffering argument to the open() function when
709 creating the stdin/stdout/stderr pipe file objects
711 executable: A replacement program to execute.
713 stdin, stdout and stderr: These specify the executed programs' standard
714 input, standard output and standard error file handles, respectively.
716 preexec_fn: (POSIX only) An object to be called in the child process
717 just before the child is executed.
719 close_fds: Controls closing or inheriting of file descriptors.
721 shell: If true, the command will be executed through the shell.
723 cwd: Sets the current directory before the child is executed.
725 env: Defines the environment variables for the new process.
727 text: If true, decode stdin, stdout and stderr using the given encoding
728 (if set) or the system default otherwise.
730 universal_newlines: Alias of text, provided for backwards compatibility.
732 startupinfo and creationflags (Windows only)
734 restore_signals (POSIX only)
736 start_new_session (POSIX only)
738 group (POSIX only)
740 extra_groups (POSIX only)
742 user (POSIX only)
744 umask (POSIX only)
746 pass_fds (POSIX only)
748 encoding and errors: Text mode encoding and error handling to use for
749 file objects stdin, stdout and stderr.
751 Attributes:
752 stdin, stdout, stderr, pid, returncode
753 """
754 _child_created = False # Set here since __del__ checks it
756 def __init__(self, args, bufsize=-1, executable=None,
757 stdin=None, stdout=None, stderr=None,
758 preexec_fn=None, close_fds=True,
759 shell=False, cwd=None, env=None, universal_newlines=None,
760 startupinfo=None, creationflags=0,
761 restore_signals=True, start_new_session=False,
762 pass_fds=(), *, user=None, group=None, extra_groups=None,
763 encoding=None, errors=None, text=None, umask=-1):
764 """Create new Popen instance."""
765 _cleanup()
766 # Held while anything is calling waitpid before returncode has been
767 # updated to prevent clobbering returncode if wait() or poll() are
768 # called from multiple threads at once. After acquiring the lock,
769 # code must re-check self.returncode to see if another thread just
770 # finished a waitpid() call.
771 self._waitpid_lock = threading.Lock()
773 self._input = None
774 self._communication_started = False
775 if bufsize is None:
776 bufsize = -1 # Restore default
777 if not isinstance(bufsize, int):
778 raise TypeError("bufsize must be an integer")
780 if _mswindows:
781 if preexec_fn is not None:
782 raise ValueError("preexec_fn is not supported on Windows "
783 "platforms")
784 else:
785 # POSIX
786 if pass_fds and not close_fds:
787 warnings.warn("pass_fds overriding close_fds.", RuntimeWarning)
788 close_fds = True
789 if startupinfo is not None:
790 raise ValueError("startupinfo is only supported on Windows "
791 "platforms")
792 if creationflags != 0:
793 raise ValueError("creationflags is only supported on Windows "
794 "platforms")
796 self.args = args
797 self.stdin = None
798 self.stdout = None
799 self.stderr = None
800 self.pid = None
801 self.returncode = None
802 self.encoding = encoding
803 self.errors = errors
805 # Validate the combinations of text and universal_newlines
806 if (text is not None and universal_newlines is not None
807 and bool(universal_newlines) != bool(text)):
808 raise SubprocessError('Cannot disambiguate when both text '
809 'and universal_newlines are supplied but '
810 'different. Pass one or the other.')
812 # Input and output objects. The general principle is like
813 # this:
814 #
815 # Parent Child
816 # ------ -----
817 # p2cwrite ---stdin---> p2cread
818 # c2pread <--stdout--- c2pwrite
819 # errread <--stderr--- errwrite
820 #
821 # On POSIX, the child objects are file descriptors. On
822 # Windows, these are Windows file handles. The parent objects
823 # are file descriptors on both platforms. The parent objects
824 # are -1 when not using PIPEs. The child objects are -1
825 # when not redirecting.
827 (p2cread, p2cwrite,
828 c2pread, c2pwrite,
829 errread, errwrite) = self._get_handles(stdin, stdout, stderr)
831 # We wrap OS handles *before* launching the child, otherwise a
832 # quickly terminating child could make our fds unwrappable
833 # (see #8458).
835 if _mswindows:
836 if p2cwrite != -1:
837 p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
838 if c2pread != -1:
839 c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
840 if errread != -1:
841 errread = msvcrt.open_osfhandle(errread.Detach(), 0)
843 self.text_mode = encoding or errors or text or universal_newlines
845 # How long to resume waiting on a child after the first ^C.
846 # There is no right value for this. The purpose is to be polite
847 # yet remain good for interactive users trying to exit a tool.
848 self._sigint_wait_secs = 0.25 # 1/xkcd221.getRandomNumber()
850 self._closed_child_pipe_fds = False
852 if self.text_mode:
853 if bufsize == 1:
854 line_buffering = True
855 # Use the default buffer size for the underlying binary streams
856 # since they don't support line buffering.
857 bufsize = -1
858 else:
859 line_buffering = False
861 gid = None
862 if group is not None:
863 if not hasattr(os, 'setregid'):
864 raise ValueError("The 'group' parameter is not supported on the "
865 "current platform")
867 elif isinstance(group, str):
868 if grp is None:
869 raise ValueError("The group parameter cannot be a string "
870 "on systems without the grp module")
872 gid = grp.getgrnam(group).gr_gid
873 elif isinstance(group, int):
874 gid = group
875 else:
876 raise TypeError("Group must be a string or an integer, not {}"
877 .format(type(group)))
879 if gid < 0:
880 raise ValueError(f"Group ID cannot be negative, got {gid}")
882 gids = None
883 if extra_groups is not None:
884 if not hasattr(os, 'setgroups'):
885 raise ValueError("The 'extra_groups' parameter is not "
886 "supported on the current platform")
888 elif isinstance(extra_groups, str):
889 raise ValueError("Groups must be a list, not a string")
891 gids = []
892 for extra_group in extra_groups:
893 if isinstance(extra_group, str):
894 if grp is None:
895 raise ValueError("Items in extra_groups cannot be "
896 "strings on systems without the "
897 "grp module")
899 gids.append(grp.getgrnam(extra_group).gr_gid)
900 elif isinstance(extra_group, int):
901 gids.append(extra_group)
902 else:
903 raise TypeError("Items in extra_groups must be a string "
904 "or integer, not {}"
905 .format(type(extra_group)))
907 # make sure that the gids are all positive here so we can do less
908 # checking in the C code
909 for gid_check in gids:
910 if gid_check < 0:
911 raise ValueError(f"Group ID cannot be negative, got {gid_check}")
913 uid = None
914 if user is not None:
915 if not hasattr(os, 'setreuid'):
916 raise ValueError("The 'user' parameter is not supported on "
917 "the current platform")
919 elif isinstance(user, str):
920 if pwd is None:
921 raise ValueError("The user parameter cannot be a string "
922 "on systems without the pwd module")
924 uid = pwd.getpwnam(user).pw_uid
925 elif isinstance(user, int):
926 uid = user
927 else:
928 raise TypeError("User must be a string or an integer")
930 if uid < 0:
931 raise ValueError(f"User ID cannot be negative, got {uid}")
933 try:
934 if p2cwrite != -1:
935 self.stdin = io.open(p2cwrite, 'wb', bufsize)
936 if self.text_mode:
937 self.stdin = io.TextIOWrapper(self.stdin, write_through=True,
938 line_buffering=line_buffering,
939 encoding=encoding, errors=errors)
940 if c2pread != -1:
941 self.stdout = io.open(c2pread, 'rb', bufsize)
942 if self.text_mode:
943 self.stdout = io.TextIOWrapper(self.stdout,
944 encoding=encoding, errors=errors)
945 if errread != -1:
946 self.stderr = io.open(errread, 'rb', bufsize)
947 if self.text_mode:
948 self.stderr = io.TextIOWrapper(self.stderr,
949 encoding=encoding, errors=errors)
951 self._execute_child(args, executable, preexec_fn, close_fds,
952 pass_fds, cwd, env,
953 startupinfo, creationflags, shell,
954 p2cread, p2cwrite,
955 c2pread, c2pwrite,
956 errread, errwrite,
957 restore_signals,
958 gid, gids, uid, umask,
959 start_new_session)
960 except:
961 # Cleanup if the child failed starting.
962 for f in filter(None, (self.stdin, self.stdout, self.stderr)):
963 try:
964 f.close()
965 except OSError:
966 pass # Ignore EBADF or other errors.
968 if not self._closed_child_pipe_fds:
969 to_close = []
970 if stdin == PIPE:
971 to_close.append(p2cread)
972 if stdout == PIPE:
973 to_close.append(c2pwrite)
974 if stderr == PIPE:
975 to_close.append(errwrite)
976 if hasattr(self, '_devnull'):
977 to_close.append(self._devnull)
978 for fd in to_close:
979 try:
980 if _mswindows and isinstance(fd, Handle):
981 fd.Close()
982 else:
983 os.close(fd)
984 except OSError:
985 pass
987 raise
989 def __repr__(self):
990 obj_repr = (
991 f"<{self.__class__.__name__}: "
992 f"returncode: {self.returncode} args: {list(self.args)!r}>"
993 )
994 if len(obj_repr) > 80:
995 obj_repr = obj_repr[:76] + "...>"
996 return obj_repr
998 __class_getitem__ = classmethod(types.GenericAlias)
1000 @property
1001 def universal_newlines(self):
1002 # universal_newlines as retained as an alias of text_mode for API
1003 # compatibility. bpo-31756
1004 return self.text_mode
1006 @universal_newlines.setter
1007 def universal_newlines(self, universal_newlines):
1008 self.text_mode = bool(universal_newlines)
1010 def _translate_newlines(self, data, encoding, errors):
1011 data = data.decode(encoding, errors)
1012 return data.replace("\r\n", "\n").replace("\r", "\n")
1014 def __enter__(self):
1015 return self
1017 def __exit__(self, exc_type, value, traceback):
1018 if self.stdout:
1019 self.stdout.close()
1020 if self.stderr:
1021 self.stderr.close()
1022 try: # Flushing a BufferedWriter may raise an error
1023 if self.stdin:
1024 self.stdin.close()
1025 finally:
1026 if exc_type == KeyboardInterrupt:
1027 # https://bugs.python.org/issue25942
1028 # In the case of a KeyboardInterrupt we assume the SIGINT
1029 # was also already sent to our child processes. We can't
1030 # block indefinitely as that is not user friendly.
1031 # If we have not already waited a brief amount of time in
1032 # an interrupted .wait() or .communicate() call, do so here
1033 # for consistency.
1034 if self._sigint_wait_secs > 0:
1035 try:
1036 self._wait(timeout=self._sigint_wait_secs)
1037 except TimeoutExpired:
1038 pass
1039 self._sigint_wait_secs = 0 # Note that this has been done.
1040 return # resume the KeyboardInterrupt
1042 # Wait for the process to terminate, to avoid zombies.
1043 self.wait()
1045 def __del__(self, _maxsize=sys.maxsize, _warn=warnings.warn):
1046 if not self._child_created:
1047 # We didn't get to successfully create a child process.
1048 return
1049 if self.returncode is None:
1050 # Not reading subprocess exit status creates a zombie process which
1051 # is only destroyed at the parent python process exit
1052 _warn("subprocess %s is still running" % self.pid,
1053 ResourceWarning, source=self)
1054 # In case the child hasn't been waited on, check if it's done.
1055 self._internal_poll(_deadstate=_maxsize)
1056 if self.returncode is None and _active is not None:
1057 # Child is still running, keep us alive until we can wait on it.
1058 _active.append(self)
1060 def _get_devnull(self):
1061 if not hasattr(self, '_devnull'):
1062 self._devnull = os.open(os.devnull, os.O_RDWR)
1063 return self._devnull
1065 def _stdin_write(self, input):
1066 if input:
1067 try:
1068 self.stdin.write(input)
1069 except BrokenPipeError:
1070 pass # communicate() must ignore broken pipe errors.
1071 except OSError as exc:
1072 if exc.errno == errno.EINVAL:
1073 # bpo-19612, bpo-30418: On Windows, stdin.write() fails
1074 # with EINVAL if the child process exited or if the child
1075 # process is still running but closed the pipe.
1076 pass
1077 else:
1078 raise
1080 try:
1081 self.stdin.close()
1082 except BrokenPipeError:
1083 pass # communicate() must ignore broken pipe errors.
1084 except OSError as exc:
1085 if exc.errno == errno.EINVAL:
1086 pass
1087 else:
1088 raise
1090 def communicate(self, input=None, timeout=None):
1091 """Interact with process: Send data to stdin and close it.
1092 Read data from stdout and stderr, until end-of-file is
1093 reached. Wait for process to terminate.
1095 The optional "input" argument should be data to be sent to the
1096 child process, or None, if no data should be sent to the child.
1097 communicate() returns a tuple (stdout, stderr).
1099 By default, all communication is in bytes, and therefore any
1100 "input" should be bytes, and the (stdout, stderr) will be bytes.
1101 If in text mode (indicated by self.text_mode), any "input" should
1102 be a string, and (stdout, stderr) will be strings decoded
1103 according to locale encoding, or by "encoding" if set. Text mode
1104 is triggered by setting any of text, encoding, errors or
1105 universal_newlines.
1106 """
1108 if self._communication_started and input:
1109 raise ValueError("Cannot send input after starting communication")
1111 # Optimization: If we are not worried about timeouts, we haven't
1112 # started communicating, and we have one or zero pipes, using select()
1113 # or threads is unnecessary.
1114 if (timeout is None and not self._communication_started and
1115 [self.stdin, self.stdout, self.stderr].count(None) >= 2):
1116 stdout = None
1117 stderr = None
1118 if self.stdin:
1119 self._stdin_write(input)
1120 elif self.stdout:
1121 stdout = self.stdout.read()
1122 self.stdout.close()
1123 elif self.stderr:
1124 stderr = self.stderr.read()
1125 self.stderr.close()
1126 self.wait()
1127 else:
1128 if timeout is not None:
1129 endtime = _time() + timeout
1130 else:
1131 endtime = None
1133 try:
1134 stdout, stderr = self._communicate(input, endtime, timeout)
1135 except KeyboardInterrupt:
1136 # https://bugs.python.org/issue25942
1137 # See the detailed comment in .wait().
1138 if timeout is not None:
1139 sigint_timeout = min(self._sigint_wait_secs,
1140 self._remaining_time(endtime))
1141 else:
1142 sigint_timeout = self._sigint_wait_secs
1143 self._sigint_wait_secs = 0 # nothing else should wait.
1144 try:
1145 self._wait(timeout=sigint_timeout)
1146 except TimeoutExpired:
1147 pass
1148 raise # resume the KeyboardInterrupt
1150 finally:
1151 self._communication_started = True
1153 sts = self.wait(timeout=self._remaining_time(endtime))
1155 return (stdout, stderr)
1158 def poll(self):
1159 """Check if child process has terminated. Set and return returncode
1160 attribute."""
1161 return self._internal_poll()
1164 def _remaining_time(self, endtime):
1165 """Convenience for _communicate when computing timeouts."""
1166 if endtime is None:
1167 return None
1168 else:
1169 return endtime - _time()
1172 def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq,
1173 skip_check_and_raise=False):
1174 """Convenience for checking if a timeout has expired."""
1175 if endtime is None:
1176 return
1177 if skip_check_and_raise or _time() > endtime:
1178 raise TimeoutExpired(
1179 self.args, orig_timeout,
1180 output=b''.join(stdout_seq) if stdout_seq else None,
1181 stderr=b''.join(stderr_seq) if stderr_seq else None)
1184 def wait(self, timeout=None):
1185 """Wait for child process to terminate; returns self.returncode."""
1186 if timeout is not None:
1187 endtime = _time() + timeout
1188 try:
1189 return self._wait(timeout=timeout)
1190 except KeyboardInterrupt:
1191 # https://bugs.python.org/issue25942
1192 # The first keyboard interrupt waits briefly for the child to
1193 # exit under the common assumption that it also received the ^C
1194 # generated SIGINT and will exit rapidly.
1195 if timeout is not None:
1196 sigint_timeout = min(self._sigint_wait_secs,
1197 self._remaining_time(endtime))
1198 else:
1199 sigint_timeout = self._sigint_wait_secs
1200 self._sigint_wait_secs = 0 # nothing else should wait.
1201 try:
1202 self._wait(timeout=sigint_timeout)
1203 except TimeoutExpired:
1204 pass
1205 raise # resume the KeyboardInterrupt
1207 def _close_pipe_fds(self,
1208 p2cread, p2cwrite,
1209 c2pread, c2pwrite,
1210 errread, errwrite):
1211 # self._devnull is not always defined.
1212 devnull_fd = getattr(self, '_devnull', None)
1214 with contextlib.ExitStack() as stack:
1215 if _mswindows:
1216 if p2cread != -1:
1217 stack.callback(p2cread.Close)
1218 if c2pwrite != -1:
1219 stack.callback(c2pwrite.Close)
1220 if errwrite != -1:
1221 stack.callback(errwrite.Close)
1222 else:
1223 if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd:
1224 stack.callback(os.close, p2cread)
1225 if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd:
1226 stack.callback(os.close, c2pwrite)
1227 if errwrite != -1 and errread != -1 and errwrite != devnull_fd:
1228 stack.callback(os.close, errwrite)
1230 if devnull_fd is not None:
1231 stack.callback(os.close, devnull_fd)
1233 # Prevent a double close of these handles/fds from __init__ on error.
1234 self._closed_child_pipe_fds = True
1236 if _mswindows:
1237 #
1238 # Windows methods
1239 #
1240 def _get_handles(self, stdin, stdout, stderr):
1241 """Construct and return tuple with IO objects:
1242 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
1243 """
1244 if stdin is None and stdout is None and stderr is None:
1245 return (-1, -1, -1, -1, -1, -1)
1247 p2cread, p2cwrite = -1, -1
1248 c2pread, c2pwrite = -1, -1
1249 errread, errwrite = -1, -1
1251 if stdin is None:
1252 p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)
1253 if p2cread is None:
1254 p2cread, _ = _winapi.CreatePipe(None, 0)
1255 p2cread = Handle(p2cread)
1256 _winapi.CloseHandle(_)
1257 elif stdin == PIPE:
1258 p2cread, p2cwrite = _winapi.CreatePipe(None, 0)
1259 p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
1260 elif stdin == DEVNULL:
1261 p2cread = msvcrt.get_osfhandle(self._get_devnull())
1262 elif isinstance(stdin, int):
1263 p2cread = msvcrt.get_osfhandle(stdin)
1264 else:
1265 # Assuming file-like object
1266 p2cread = msvcrt.get_osfhandle(stdin.fileno())
1267 p2cread = self._make_inheritable(p2cread)
1269 if stdout is None:
1270 c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
1271 if c2pwrite is None:
1272 _, c2pwrite = _winapi.CreatePipe(None, 0)
1273 c2pwrite = Handle(c2pwrite)
1274 _winapi.CloseHandle(_)
1275 elif stdout == PIPE:
1276 c2pread, c2pwrite = _winapi.CreatePipe(None, 0)
1277 c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
1278 elif stdout == DEVNULL:
1279 c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
1280 elif isinstance(stdout, int):
1281 c2pwrite = msvcrt.get_osfhandle(stdout)
1282 else:
1283 # Assuming file-like object
1284 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
1285 c2pwrite = self._make_inheritable(c2pwrite)
1287 if stderr is None:
1288 errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
1289 if errwrite is None:
1290 _, errwrite = _winapi.CreatePipe(None, 0)
1291 errwrite = Handle(errwrite)
1292 _winapi.CloseHandle(_)
1293 elif stderr == PIPE:
1294 errread, errwrite = _winapi.CreatePipe(None, 0)
1295 errread, errwrite = Handle(errread), Handle(errwrite)
1296 elif stderr == STDOUT:
1297 errwrite = c2pwrite
1298 elif stderr == DEVNULL:
1299 errwrite = msvcrt.get_osfhandle(self._get_devnull())
1300 elif isinstance(stderr, int):
1301 errwrite = msvcrt.get_osfhandle(stderr)
1302 else:
1303 # Assuming file-like object
1304 errwrite = msvcrt.get_osfhandle(stderr.fileno())
1305 errwrite = self._make_inheritable(errwrite)
1307 return (p2cread, p2cwrite,
1308 c2pread, c2pwrite,
1309 errread, errwrite)
1312 def _make_inheritable(self, handle):
1313 """Return a duplicate of handle, which is inheritable"""
1314 h = _winapi.DuplicateHandle(
1315 _winapi.GetCurrentProcess(), handle,
1316 _winapi.GetCurrentProcess(), 0, 1,
1317 _winapi.DUPLICATE_SAME_ACCESS)
1318 return Handle(h)
1321 def _filter_handle_list(self, handle_list):
1322 """Filter out console handles that can't be used
1323 in lpAttributeList["handle_list"] and make sure the list
1324 isn't empty. This also removes duplicate handles."""
1325 # An handle with it's lowest two bits set might be a special console
1326 # handle that if passed in lpAttributeList["handle_list"], will
1327 # cause it to fail.
1328 return list({handle for handle in handle_list
1329 if handle & 0x3 != 0x3
1330 or _winapi.GetFileType(handle) !=
1331 _winapi.FILE_TYPE_CHAR})
1334 def _execute_child(self, args, executable, preexec_fn, close_fds,
1335 pass_fds, cwd, env,
1336 startupinfo, creationflags, shell,
1337 p2cread, p2cwrite,
1338 c2pread, c2pwrite,
1339 errread, errwrite,
1340 unused_restore_signals,
1341 unused_gid, unused_gids, unused_uid,
1342 unused_umask,
1343 unused_start_new_session):
1344 """Execute program (MS Windows version)"""
1346 assert not pass_fds, "pass_fds not supported on Windows."
1348 if isinstance(args, str):
1349 pass
1350 elif isinstance(args, bytes):
1351 if shell:
1352 raise TypeError('bytes args is not allowed on Windows')
1353 args = list2cmdline([args])
1354 elif isinstance(args, os.PathLike):
1355 if shell:
1356 raise TypeError('path-like args is not allowed when '
1357 'shell is true')
1358 args = list2cmdline([args])
1359 else:
1360 args = list2cmdline(args)
1362 if executable is not None:
1363 executable = os.fsdecode(executable)
1365 # Process startup details
1366 if startupinfo is None:
1367 startupinfo = STARTUPINFO()
1368 else:
1369 # bpo-34044: Copy STARTUPINFO since it is modified above,
1370 # so the caller can reuse it multiple times.
1371 startupinfo = startupinfo.copy()
1373 use_std_handles = -1 not in (p2cread, c2pwrite, errwrite)
1374 if use_std_handles:
1375 startupinfo.dwFlags |= _winapi.STARTF_USESTDHANDLES
1376 startupinfo.hStdInput = p2cread
1377 startupinfo.hStdOutput = c2pwrite
1378 startupinfo.hStdError = errwrite
1380 attribute_list = startupinfo.lpAttributeList
1381 have_handle_list = bool(attribute_list and
1382 "handle_list" in attribute_list and
1383 attribute_list["handle_list"])
1385 # If we were given an handle_list or need to create one
1386 if have_handle_list or (use_std_handles and close_fds):
1387 if attribute_list is None:
1388 attribute_list = startupinfo.lpAttributeList = {}
1389 handle_list = attribute_list["handle_list"] = \
1390 list(attribute_list.get("handle_list", []))
1392 if use_std_handles:
1393 handle_list += [int(p2cread), int(c2pwrite), int(errwrite)]
1395 handle_list[:] = self._filter_handle_list(handle_list)
1397 if handle_list:
1398 if not close_fds:
1399 warnings.warn("startupinfo.lpAttributeList['handle_list'] "
1400 "overriding close_fds", RuntimeWarning)
1402 # When using the handle_list we always request to inherit
1403 # handles but the only handles that will be inherited are
1404 # the ones in the handle_list
1405 close_fds = False
1407 if shell:
1408 startupinfo.dwFlags |= _winapi.STARTF_USESHOWWINDOW
1409 startupinfo.wShowWindow = _winapi.SW_HIDE
1410 comspec = os.environ.get("COMSPEC", "cmd.exe")
1411 args = '{} /c "{}"'.format (comspec, args)
1413 if cwd is not None:
1414 cwd = os.fsdecode(cwd)
1416 sys.audit("subprocess.Popen", executable, args, cwd, env)
1418 # Start the process
1419 try:
1420 hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
1421 # no special security
1422 None, None,
1423 int(not close_fds),
1424 creationflags,
1425 env,
1426 cwd,
1427 startupinfo)
1428 finally:
1429 # Child is launched. Close the parent's copy of those pipe
1430 # handles that only the child should have open. You need
1431 # to make sure that no handles to the write end of the
1432 # output pipe are maintained in this process or else the
1433 # pipe will not close when the child process exits and the
1434 # ReadFile will hang.
1435 self._close_pipe_fds(p2cread, p2cwrite,
1436 c2pread, c2pwrite,
1437 errread, errwrite)
1439 # Retain the process handle, but close the thread handle
1440 self._child_created = True
1441 self._handle = Handle(hp)
1442 self.pid = pid
1443 _winapi.CloseHandle(ht)
1445 def _internal_poll(self, _deadstate=None,
1446 _WaitForSingleObject=_winapi.WaitForSingleObject,
1447 _WAIT_OBJECT_0=_winapi.WAIT_OBJECT_0,
1448 _GetExitCodeProcess=_winapi.GetExitCodeProcess):
1449 """Check if child process has terminated. Returns returncode
1450 attribute.
1452 This method is called by __del__, so it can only refer to objects
1453 in its local scope.
1455 """
1456 if self.returncode is None:
1457 if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0:
1458 self.returncode = _GetExitCodeProcess(self._handle)
1459 return self.returncode
1462 def _wait(self, timeout):
1463 """Internal implementation of wait() on Windows."""
1464 if timeout is None:
1465 timeout_millis = _winapi.INFINITE
1466 else:
1467 timeout_millis = int(timeout * 1000)
1468 if self.returncode is None:
1469 # API note: Returns immediately if timeout_millis == 0.
1470 result = _winapi.WaitForSingleObject(self._handle,
1471 timeout_millis)
1472 if result == _winapi.WAIT_TIMEOUT:
1473 raise TimeoutExpired(self.args, timeout)
1474 self.returncode = _winapi.GetExitCodeProcess(self._handle)
1475 return self.returncode
1478 def _readerthread(self, fh, buffer):
1479 buffer.append(fh.read())
1480 fh.close()
1483 def _communicate(self, input, endtime, orig_timeout):
1484 # Start reader threads feeding into a list hanging off of this
1485 # object, unless they've already been started.
1486 if self.stdout and not hasattr(self, "_stdout_buff"):
1487 self._stdout_buff = []
1488 self.stdout_thread = \
1489 threading.Thread(target=self._readerthread,
1490 args=(self.stdout, self._stdout_buff))
1491 self.stdout_thread.daemon = True
1492 self.stdout_thread.start()
1493 if self.stderr and not hasattr(self, "_stderr_buff"):
1494 self._stderr_buff = []
1495 self.stderr_thread = \
1496 threading.Thread(target=self._readerthread,
1497 args=(self.stderr, self._stderr_buff))
1498 self.stderr_thread.daemon = True
1499 self.stderr_thread.start()
1501 if self.stdin:
1502 self._stdin_write(input)
1504 # Wait for the reader threads, or time out. If we time out, the
1505 # threads remain reading and the fds left open in case the user
1506 # calls communicate again.
1507 if self.stdout is not None:
1508 self.stdout_thread.join(self._remaining_time(endtime))
1509 if self.stdout_thread.is_alive():
1510 raise TimeoutExpired(self.args, orig_timeout)
1511 if self.stderr is not None:
1512 self.stderr_thread.join(self._remaining_time(endtime))
1513 if self.stderr_thread.is_alive():
1514 raise TimeoutExpired(self.args, orig_timeout)
1516 # Collect the output from and close both pipes, now that we know
1517 # both have been read successfully.
1518 stdout = None
1519 stderr = None
1520 if self.stdout:
1521 stdout = self._stdout_buff
1522 self.stdout.close()
1523 if self.stderr:
1524 stderr = self._stderr_buff
1525 self.stderr.close()
1527 # All data exchanged. Translate lists into strings.
1528 stdout = stdout[0] if stdout else None
1529 stderr = stderr[0] if stderr else None
1531 return (stdout, stderr)
1533 def send_signal(self, sig):
1534 """Send a signal to the process."""
1535 # Don't signal a process that we know has already died.
1536 if self.returncode is not None:
1537 return
1538 if sig == signal.SIGTERM:
1539 self.terminate()
1540 elif sig == signal.CTRL_C_EVENT:
1541 os.kill(self.pid, signal.CTRL_C_EVENT)
1542 elif sig == signal.CTRL_BREAK_EVENT:
1543 os.kill(self.pid, signal.CTRL_BREAK_EVENT)
1544 else:
1545 raise ValueError("Unsupported signal: {}".format(sig))
1547 def terminate(self):
1548 """Terminates the process."""
1549 # Don't terminate a process that we know has already died.
1550 if self.returncode is not None:
1551 return
1552 try:
1553 _winapi.TerminateProcess(self._handle, 1)
1554 except PermissionError:
1555 # ERROR_ACCESS_DENIED (winerror 5) is received when the
1556 # process already died.
1557 rc = _winapi.GetExitCodeProcess(self._handle)
1558 if rc == _winapi.STILL_ACTIVE:
1559 raise
1560 self.returncode = rc
1562 kill = terminate
1564 else:
1565 #
1566 # POSIX methods
1567 #
1568 def _get_handles(self, stdin, stdout, stderr):
1569 """Construct and return tuple with IO objects:
1570 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
1571 """
1572 p2cread, p2cwrite = -1, -1
1573 c2pread, c2pwrite = -1, -1
1574 errread, errwrite = -1, -1
1576 if stdin is None:
1577 pass
1578 elif stdin == PIPE:
1579 p2cread, p2cwrite = os.pipe()
1580 elif stdin == DEVNULL:
1581 p2cread = self._get_devnull()
1582 elif isinstance(stdin, int):
1583 p2cread = stdin
1584 else:
1585 # Assuming file-like object
1586 p2cread = stdin.fileno()
1588 if stdout is None:
1589 pass
1590 elif stdout == PIPE:
1591 c2pread, c2pwrite = os.pipe()
1592 elif stdout == DEVNULL:
1593 c2pwrite = self._get_devnull()
1594 elif isinstance(stdout, int):
1595 c2pwrite = stdout
1596 else:
1597 # Assuming file-like object
1598 c2pwrite = stdout.fileno()
1600 if stderr is None:
1601 pass
1602 elif stderr == PIPE:
1603 errread, errwrite = os.pipe()
1604 elif stderr == STDOUT:
1605 if c2pwrite != -1:
1606 errwrite = c2pwrite
1607 else: # child's stdout is not set, use parent's stdout
1608 errwrite = sys.__stdout__.fileno()
1609 elif stderr == DEVNULL:
1610 errwrite = self._get_devnull()
1611 elif isinstance(stderr, int):
1612 errwrite = stderr
1613 else:
1614 # Assuming file-like object
1615 errwrite = stderr.fileno()
1617 return (p2cread, p2cwrite,
1618 c2pread, c2pwrite,
1619 errread, errwrite)
1622 def _posix_spawn(self, args, executable, env, restore_signals,
1623 p2cread, p2cwrite,
1624 c2pread, c2pwrite,
1625 errread, errwrite):
1626 """Execute program using os.posix_spawn()."""
1627 if env is None:
1628 env = os.environ
1630 kwargs = {}
1631 if restore_signals:
1632 # See _Py_RestoreSignals() in Python/pylifecycle.c
1633 sigset = []
1634 for signame in ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ'):
1635 signum = getattr(signal, signame, None)
1636 if signum is not None:
1637 sigset.append(signum)
1638 kwargs['setsigdef'] = sigset
1640 file_actions = []
1641 for fd in (p2cwrite, c2pread, errread):
1642 if fd != -1:
1643 file_actions.append((os.POSIX_SPAWN_CLOSE, fd))
1644 for fd, fd2 in (
1645 (p2cread, 0),
1646 (c2pwrite, 1),
1647 (errwrite, 2),
1648 ):
1649 if fd != -1:
1650 file_actions.append((os.POSIX_SPAWN_DUP2, fd, fd2))
1651 if file_actions:
1652 kwargs['file_actions'] = file_actions
1654 self.pid = os.posix_spawn(executable, args, env, **kwargs)
1655 self._child_created = True
1657 self._close_pipe_fds(p2cread, p2cwrite,
1658 c2pread, c2pwrite,
1659 errread, errwrite)
1661 def _execute_child(self, args, executable, preexec_fn, close_fds,
1662 pass_fds, cwd, env,
1663 startupinfo, creationflags, shell,
1664 p2cread, p2cwrite,
1665 c2pread, c2pwrite,
1666 errread, errwrite,
1667 restore_signals,
1668 gid, gids, uid, umask,
1669 start_new_session):
1670 """Execute program (POSIX version)"""
1672 if isinstance(args, (str, bytes)):
1673 args = [args]
1674 elif isinstance(args, os.PathLike):
1675 if shell:
1676 raise TypeError('path-like args is not allowed when '
1677 'shell is true')
1678 args = [args]
1679 else:
1680 args = list(args)
1682 if shell:
1683 # On Android the default shell is at '/system/bin/sh'.
1684 unix_shell = ('/system/bin/sh' if
1685 hasattr(sys, 'getandroidapilevel') else '/bin/sh')
1686 args = [unix_shell, "-c"] + args
1687 if executable:
1688 args[0] = executable
1690 if executable is None:
1691 executable = args[0]
1693 sys.audit("subprocess.Popen", executable, args, cwd, env)
1695 if (_USE_POSIX_SPAWN
1696 and os.path.dirname(executable)
1697 and preexec_fn is None
1698 and not close_fds
1699 and not pass_fds
1700 and cwd is None
1701 and (p2cread == -1 or p2cread > 2)
1702 and (c2pwrite == -1 or c2pwrite > 2)
1703 and (errwrite == -1 or errwrite > 2)
1704 and not start_new_session
1705 and gid is None
1706 and gids is None
1707 and uid is None
1708 and umask < 0):
1709 self._posix_spawn(args, executable, env, restore_signals,
1710 p2cread, p2cwrite,
1711 c2pread, c2pwrite,
1712 errread, errwrite)
1713 return
1715 orig_executable = executable
1717 # For transferring possible exec failure from child to parent.
1718 # Data format: "exception name:hex errno:description"
1719 # Pickle is not used; it is complex and involves memory allocation.
1720 errpipe_read, errpipe_write = os.pipe()
1721 # errpipe_write must not be in the standard io 0, 1, or 2 fd range.
1722 low_fds_to_close = []
1723 while errpipe_write < 3:
1724 low_fds_to_close.append(errpipe_write)
1725 errpipe_write = os.dup(errpipe_write)
1726 for low_fd in low_fds_to_close:
1727 os.close(low_fd)
1728 try:
1729 try:
1730 # We must avoid complex work that could involve
1731 # malloc or free in the child process to avoid
1732 # potential deadlocks, thus we do all this here.
1733 # and pass it to fork_exec()
1735 if env is not None:
1736 env_list = []
1737 for k, v in env.items():
1738 k = os.fsencode(k)
1739 if b'=' in k:
1740 raise ValueError("illegal environment variable name")
1741 env_list.append(k + b'=' + os.fsencode(v))
1742 else:
1743 env_list = None # Use execv instead of execve.
1744 executable = os.fsencode(executable)
1745 if os.path.dirname(executable):
1746 executable_list = (executable,)
1747 else:
1748 # This matches the behavior of os._execvpe().
1749 executable_list = tuple(
1750 os.path.join(os.fsencode(dir), executable)
1751 for dir in os.get_exec_path(env))
1752 fds_to_keep = set(pass_fds)
1753 fds_to_keep.add(errpipe_write)
1754 self.pid = _posixsubprocess.fork_exec(
1755 args, executable_list,
1756 close_fds, tuple(sorted(map(int, fds_to_keep))),
1757 cwd, env_list,
1758 p2cread, p2cwrite, c2pread, c2pwrite,
1759 errread, errwrite,
1760 errpipe_read, errpipe_write,
1761 restore_signals, start_new_session,
1762 gid, gids, uid, umask,
1763 preexec_fn)
1764 self._child_created = True
1765 finally:
1766 # be sure the FD is closed no matter what
1767 os.close(errpipe_write)
1769 self._close_pipe_fds(p2cread, p2cwrite,
1770 c2pread, c2pwrite,
1771 errread, errwrite)
1773 # Wait for exec to fail or succeed; possibly raising an
1774 # exception (limited in size)
1775 errpipe_data = bytearray()
1776 while True:
1777 part = os.read(errpipe_read, 50000)
1778 errpipe_data += part
1779 if not part or len(errpipe_data) > 50000:
1780 break
1781 finally:
1782 # be sure the FD is closed no matter what
1783 os.close(errpipe_read)
1785 if errpipe_data:
1786 try:
1787 pid, sts = os.waitpid(self.pid, 0)
1788 if pid == self.pid:
1789 self._handle_exitstatus(sts)
1790 else:
1791 self.returncode = sys.maxsize
1792 except ChildProcessError:
1793 pass
1795 try:
1796 exception_name, hex_errno, err_msg = (
1797 errpipe_data.split(b':', 2))
1798 # The encoding here should match the encoding
1799 # written in by the subprocess implementations
1800 # like _posixsubprocess
1801 err_msg = err_msg.decode()
1802 except ValueError:
1803 exception_name = b'SubprocessError'
1804 hex_errno = b'0'
1805 err_msg = 'Bad exception data from child: {!r}'.format(
1806 bytes(errpipe_data))
1807 child_exception_type = getattr(
1808 builtins, exception_name.decode('ascii'),
1809 SubprocessError)
1810 if issubclass(child_exception_type, OSError) and hex_errno:
1811 errno_num = int(hex_errno, 16)
1812 child_exec_never_called = (err_msg == "noexec")
1813 if child_exec_never_called:
1814 err_msg = ""
1815 # The error must be from chdir(cwd).
1816 err_filename = cwd
1817 else:
1818 err_filename = orig_executable
1819 if errno_num != 0:
1820 err_msg = os.strerror(errno_num)
1821 raise child_exception_type(errno_num, err_msg, err_filename)
1822 raise child_exception_type(err_msg)
1825 def _handle_exitstatus(self, sts,
1826 waitstatus_to_exitcode=os.waitstatus_to_exitcode,
1827 _WIFSTOPPED=os.WIFSTOPPED,
1828 _WSTOPSIG=os.WSTOPSIG):
1829 """All callers to this function MUST hold self._waitpid_lock."""
1830 # This method is called (indirectly) by __del__, so it cannot
1831 # refer to anything outside of its local scope.
1832 if _WIFSTOPPED(sts):
1833 self.returncode = -_WSTOPSIG(sts)
1834 else:
1835 self.returncode = waitstatus_to_exitcode(sts)
1837 def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid,
1838 _WNOHANG=os.WNOHANG, _ECHILD=errno.ECHILD):
1839 """Check if child process has terminated. Returns returncode
1840 attribute.
1842 This method is called by __del__, so it cannot reference anything
1843 outside of the local scope (nor can any methods it calls).
1845 """
1846 if self.returncode is None:
1847 if not self._waitpid_lock.acquire(False):
1848 # Something else is busy calling waitpid. Don't allow two
1849 # at once. We know nothing yet.
1850 return None
1851 try:
1852 if self.returncode is not None:
1853 return self.returncode # Another thread waited.
1854 pid, sts = _waitpid(self.pid, _WNOHANG)
1855 if pid == self.pid:
1856 self._handle_exitstatus(sts)
1857 except OSError as e:
1858 if _deadstate is not None:
1859 self.returncode = _deadstate
1860 elif e.errno == _ECHILD:
1861 # This happens if SIGCLD is set to be ignored or
1862 # waiting for child processes has otherwise been
1863 # disabled for our process. This child is dead, we
1864 # can't get the status.
1865 # http://bugs.python.org/issue15756
1866 self.returncode = 0
1867 finally:
1868 self._waitpid_lock.release()
1869 return self.returncode
1872 def _try_wait(self, wait_flags):
1873 """All callers to this function MUST hold self._waitpid_lock."""
1874 try:
1875 (pid, sts) = os.waitpid(self.pid, wait_flags)
1876 except ChildProcessError:
1877 # This happens if SIGCLD is set to be ignored or waiting
1878 # for child processes has otherwise been disabled for our
1879 # process. This child is dead, we can't get the status.
1880 pid = self.pid
1881 sts = 0
1882 return (pid, sts)
1885 def _wait(self, timeout):
1886 """Internal implementation of wait() on POSIX."""
1887 if self.returncode is not None:
1888 return self.returncode
1890 if timeout is not None:
1891 endtime = _time() + timeout
1892 # Enter a busy loop if we have a timeout. This busy loop was
1893 # cribbed from Lib/threading.py in Thread.wait() at r71065.
1894 delay = 0.0005 # 500 us -> initial delay of 1 ms
1895 while True:
1896 if self._waitpid_lock.acquire(False):
1897 try:
1898 if self.returncode is not None:
1899 break # Another thread waited.
1900 (pid, sts) = self._try_wait(os.WNOHANG)
1901 assert pid == self.pid or pid == 0
1902 if pid == self.pid:
1903 self._handle_exitstatus(sts)
1904 break
1905 finally:
1906 self._waitpid_lock.release()
1907 remaining = self._remaining_time(endtime)
1908 if remaining <= 0:
1909 raise TimeoutExpired(self.args, timeout)
1910 delay = min(delay * 2, remaining, .05)
1911 time.sleep(delay)
1912 else:
1913 while self.returncode is None:
1914 with self._waitpid_lock:
1915 if self.returncode is not None:
1916 break # Another thread waited.
1917 (pid, sts) = self._try_wait(0)
1918 # Check the pid and loop as waitpid has been known to
1919 # return 0 even without WNOHANG in odd situations.
1920 # http://bugs.python.org/issue14396.
1921 if pid == self.pid:
1922 self._handle_exitstatus(sts)
1923 return self.returncode
1926 def _communicate(self, input, endtime, orig_timeout):
1927 if self.stdin and not self._communication_started:
1928 # Flush stdio buffer. This might block, if the user has
1929 # been writing to .stdin in an uncontrolled fashion.
1930 try:
1931 self.stdin.flush()
1932 except BrokenPipeError:
1933 pass # communicate() must ignore BrokenPipeError.
1934 if not input:
1935 try:
1936 self.stdin.close()
1937 except BrokenPipeError:
1938 pass # communicate() must ignore BrokenPipeError.
1940 stdout = None
1941 stderr = None
1943 # Only create this mapping if we haven't already.
1944 if not self._communication_started:
1945 self._fileobj2output = {}
1946 if self.stdout:
1947 self._fileobj2output[self.stdout] = []
1948 if self.stderr:
1949 self._fileobj2output[self.stderr] = []
1951 if self.stdout:
1952 stdout = self._fileobj2output[self.stdout]
1953 if self.stderr:
1954 stderr = self._fileobj2output[self.stderr]
1956 self._save_input(input)
1958 if self._input:
1959 input_view = memoryview(self._input)
1961 with _PopenSelector() as selector:
1962 if self.stdin and input:
1963 selector.register(self.stdin, selectors.EVENT_WRITE)
1964 if self.stdout and not self.stdout.closed:
1965 selector.register(self.stdout, selectors.EVENT_READ)
1966 if self.stderr and not self.stderr.closed:
1967 selector.register(self.stderr, selectors.EVENT_READ)
1969 while selector.get_map():
1970 timeout = self._remaining_time(endtime)
1971 if timeout is not None and timeout < 0:
1972 self._check_timeout(endtime, orig_timeout,
1973 stdout, stderr,
1974 skip_check_and_raise=True)
1975 raise RuntimeError( # Impossible :)
1976 '_check_timeout(..., skip_check_and_raise=True) '
1977 'failed to raise TimeoutExpired.')
1979 ready = selector.select(timeout)
1980 self._check_timeout(endtime, orig_timeout, stdout, stderr)
1982 # XXX Rewrite these to use non-blocking I/O on the file
1983 # objects; they are no longer using C stdio!
1985 for key, events in ready:
1986 if key.fileobj is self.stdin:
1987 chunk = input_view[self._input_offset :
1988 self._input_offset + _PIPE_BUF]
1989 try:
1990 self._input_offset += os.write(key.fd, chunk)
1991 except BrokenPipeError:
1992 selector.unregister(key.fileobj)
1993 key.fileobj.close()
1994 else:
1995 if self._input_offset >= len(self._input):
1996 selector.unregister(key.fileobj)
1997 key.fileobj.close()
1998 elif key.fileobj in (self.stdout, self.stderr):
1999 data = os.read(key.fd, 32768)
2000 if not data:
2001 selector.unregister(key.fileobj)
2002 key.fileobj.close()
2003 self._fileobj2output[key.fileobj].append(data)
2005 self.wait(timeout=self._remaining_time(endtime))
2007 # All data exchanged. Translate lists into strings.
2008 if stdout is not None:
2009 stdout = b''.join(stdout)
2010 if stderr is not None:
2011 stderr = b''.join(stderr)
2013 # Translate newlines, if requested.
2014 # This also turns bytes into strings.
2015 if self.text_mode:
2016 if stdout is not None:
2017 stdout = self._translate_newlines(stdout,
2018 self.stdout.encoding,
2019 self.stdout.errors)
2020 if stderr is not None:
2021 stderr = self._translate_newlines(stderr,
2022 self.stderr.encoding,
2023 self.stderr.errors)
2025 return (stdout, stderr)
2028 def _save_input(self, input):
2029 # This method is called from the _communicate_with_*() methods
2030 # so that if we time out while communicating, we can continue
2031 # sending input if we retry.
2032 if self.stdin and self._input is None:
2033 self._input_offset = 0
2034 self._input = input
2035 if input is not None and self.text_mode:
2036 self._input = self._input.encode(self.stdin.encoding,
2037 self.stdin.errors)
2040 def send_signal(self, sig):
2041 """Send a signal to the process."""
2042 # bpo-38630: Polling reduces the risk of sending a signal to the
2043 # wrong process if the process completed, the Popen.returncode
2044 # attribute is still None, and the pid has been reassigned
2045 # (recycled) to a new different process. This race condition can
2046 # happens in two cases.
2047 #
2048 # Case 1. Thread A calls Popen.poll(), thread B calls
2049 # Popen.send_signal(). In thread A, waitpid() succeed and returns
2050 # the exit status. Thread B calls kill() because poll() in thread A
2051 # did not set returncode yet. Calling poll() in thread B prevents
2052 # the race condition thanks to Popen._waitpid_lock.
2053 #
2054 # Case 2. waitpid(pid, 0) has been called directly, without
2055 # using Popen methods: returncode is still None is this case.
2056 # Calling Popen.poll() will set returncode to a default value,
2057 # since waitpid() fails with ProcessLookupError.
2058 self.poll()
2059 if self.returncode is not None:
2060 # Skip signalling a process that we know has already died.
2061 return
2063 # The race condition can still happen if the race condition
2064 # described above happens between the returncode test
2065 # and the kill() call.
2066 try:
2067 os.kill(self.pid, sig)
2068 except ProcessLookupError:
2069 # Supress the race condition error; bpo-40550.
2070 pass
2072 def terminate(self):
2073 """Terminate the process with SIGTERM
2074 """
2075 self.send_signal(signal.SIGTERM)
2077 def kill(self):
2078 """Kill the process with SIGKILL
2079 """
2080 self.send_signal(signal.SIGKILL)