IMP  2.2.1
The Integrative Modeling Platform
compat_subprocess.py
1 # subprocess - Subprocesses with accessible I/O streams
2 #
3 # For more information about this module, see PEP 324.
4 #
5 # This module should remain compatible with Python 2.2, see PEP 291.
6 #
7 # Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
8 #
9 # Licensed to PSF under a Contributor Agreement.
10 # See http://www.python.org/2.4/license for licensing details.
11 
12 r"""subprocess - Subprocesses with accessible I/O streams
13 
14 This module allows you to spawn processes, connect to their
15 input/output/error pipes, and obtain their return codes. This module
16 intends to replace several other, older modules and functions, like:
17 
18 os.system
19 os.spawn*
20 os.popen*
21 popen2.*
22 commands.*
23 
24 Information about how the subprocess module can be used to replace these
25 modules and functions can be found below.
26 
27 
28 
29 Using the subprocess module
30 ===========================
31 This module defines one class called Popen:
32 
33 class Popen(args, bufsize=0, executable=None,
34  stdin=None, stdout=None, stderr=None,
35  preexec_fn=None, close_fds=False, shell=False,
36  cwd=None, env=None, universal_newlines=False,
37  startupinfo=None, creationflags=0):
38 
39 
40 Arguments are:
41 
42 args should be a string, or a sequence of program arguments. The
43 program to execute is normally the first item in the args sequence or
44 string, but can be explicitly set by using the executable argument.
45 
46 On UNIX, with shell=False (default): In this case, the Popen class
47 uses os.execvp() to execute the child program. args should normally
48 be a sequence. A string will be treated as a sequence with the string
49 as the only item (the program to execute).
50 
51 On UNIX, with shell=True: If args is a string, it specifies the
52 command string to execute through the shell. If args is a sequence,
53 the first item specifies the command string, and any additional items
54 will be treated as additional shell arguments.
55 
56 On Windows: the Popen class uses CreateProcess() to execute the child
57 program, which operates on strings. If args is a sequence, it will be
58 converted to a string using the list2cmdline method. Please note that
59 not all MS Windows applications interpret the command line the same
60 way: The list2cmdline is designed for applications using the same
61 rules as the MS C runtime.
62 
63 bufsize, if given, has the same meaning as the corresponding argument
64 to the built-in open() function: 0 means unbuffered, 1 means line
65 buffered, any other positive value means use a buffer of
66 (approximately) that size. A negative bufsize means to use the system
67 default, which usually means fully buffered. The default value for
68 bufsize is 0 (unbuffered).
69 
70 stdin, stdout and stderr specify the executed programs' standard
71 input, standard output and standard error file handles, respectively.
72 Valid values are PIPE, an existing file descriptor (a positive
73 integer), an existing file object, and None. PIPE indicates that a
74 new pipe to the child should be created. With None, no redirection
75 will occur; the child's file handles will be inherited from the
76 parent. Additionally, stderr can be STDOUT, which indicates that the
77 stderr data from the applications should be captured into the same
78 file handle as for stdout.
79 
80 If preexec_fn is set to a callable object, this object will be called
81 in the child process just before the child is executed.
82 
83 If close_fds is true, all file descriptors except 0, 1 and 2 will be
84 closed before the child process is executed.
85 
86 if shell is true, the specified command will be executed through the
87 shell.
88 
89 If cwd is not None, the current directory will be changed to cwd
90 before the child is executed.
91 
92 If env is not None, it defines the environment variables for the new
93 process.
94 
95 If universal_newlines is true, the file objects stdout and stderr are
96 opened as a text files, but lines may be terminated by any of '\n',
97 the Unix end-of-line convention, '\r', the Macintosh convention or
98 '\r\n', the Windows convention. All of these external representations
99 are seen as '\n' by the Python program. Note: This feature is only
100 available if Python is built with universal newline support (the
101 default). Also, the newlines attribute of the file objects stdout,
102 stdin and stderr are not updated by the communicate() method.
103 
104 The startupinfo and creationflags, if given, will be passed to the
105 underlying CreateProcess() function. They can specify things such as
106 appearance of the main window and priority for the new process.
107 (Windows only)
108 
109 
110 This module also defines two shortcut functions:
111 
112 call(*args, **kwargs):
113  Run command with arguments. Wait for command to complete, then
114  return the returncode attribute.
115 
116  The arguments are the same as for the Popen constructor. Example:
117 
118  retcode = call(["ls", "-l"])
119 
120 
121 Exceptions
122 ----------
123 Exceptions raised in the child process, before the new program has
124 started to execute, will be re-raised in the parent. Additionally,
125 the exception object will have one extra attribute called
126 'child_traceback', which is a string containing traceback information
127 from the childs point of view.
128 
129 The most common exception raised is OSError. This occurs, for
130 example, when trying to execute a non-existent file. Applications
131 should prepare for OSErrors.
132 
133 A ValueError will be raised if Popen is called with invalid arguments.
134 
135 
136 Security
137 --------
138 Unlike some other popen functions, this implementation will never call
139 /bin/sh implicitly. This means that all characters, including shell
140 metacharacters, can safely be passed to child processes.
141 
142 
143 Popen objects
144 =============
145 Instances of the Popen class have the following methods:
146 
147 poll()
148  Check if child process has terminated. Returns returncode
149  attribute.
150 
151 wait()
152  Wait for child process to terminate. Returns returncode attribute.
153 
154 communicate(input=None)
155  Interact with process: Send data to stdin. Read data from stdout
156  and stderr, until end-of-file is reached. Wait for process to
157  terminate. The optional stdin argument should be a string to be
158  sent to the child process, or None, if no data should be sent to
159  the child.
160 
161  communicate() returns a tuple (stdout, stderr).
162 
163  Note: The data read is buffered in memory, so do not use this
164  method if the data size is large or unlimited.
165 
166 The following attributes are also available:
167 
168 stdin
169  If the stdin argument is PIPE, this attribute is a file object
170  that provides input to the child process. Otherwise, it is None.
171 
172 stdout
173  If the stdout argument is PIPE, this attribute is a file object
174  that provides output from the child process. Otherwise, it is
175  None.
176 
177 stderr
178  If the stderr argument is PIPE, this attribute is file object that
179  provides error output from the child process. Otherwise, it is
180  None.
181 
182 pid
183  The process ID of the child process.
184 
185 returncode
186  The child return code. A None value indicates that the process
187  hasn't terminated yet. A negative value -N indicates that the
188  child was terminated by signal N (UNIX only).
189 
190 
191 Replacing older functions with the subprocess module
192 ====================================================
193 In this section, "a ==> b" means that b can be used as a replacement
194 for a.
195 
196 Note: All functions in this section fail (more or less) silently if
197 the executed program cannot be found; this module raises an OSError
198 exception.
199 
200 In the following examples, we assume that the subprocess module is
201 imported with "from subprocess import *".
202 
203 
204 Replacing /bin/sh shell backquote
205 ---------------------------------
206 output=`mycmd myarg`
207 ==>
208 output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0]
209 
210 
211 Replacing shell pipe line
212 -------------------------
213 output=`dmesg | grep hda`
214 ==>
215 p1 = Popen(["dmesg"], stdout=PIPE)
216 p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
217 output = p2.communicate()[0]
218 
219 
220 Replacing os.system()
221 ---------------------
222 sts = os.system("mycmd" + " myarg")
223 ==>
224 p = Popen("mycmd" + " myarg", shell=True)
225 sts = os.waitpid(p.pid, 0)
226 
227 Note:
228 
229 * Calling the program through the shell is usually not required.
230 
231 * It's easier to look at the returncode attribute than the
232  exitstatus.
233 
234 A more real-world example would look like this:
235 
236 try:
237  retcode = call("mycmd" + " myarg", shell=True)
238  if retcode < 0:
239  print >>sys.stderr, "Child was terminated by signal", -retcode
240  else:
241  print >>sys.stderr, "Child returned", retcode
242 except OSError, e:
243  print >>sys.stderr, "Execution failed:", e
244 
245 
246 Replacing os.spawn*
247 -------------------
248 P_NOWAIT example:
249 
250 pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg")
251 ==>
252 pid = Popen(["/bin/mycmd", "myarg"]).pid
253 
254 
255 P_WAIT example:
256 
257 retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg")
258 ==>
259 retcode = call(["/bin/mycmd", "myarg"])
260 
261 
262 Vector example:
263 
264 os.spawnvp(os.P_NOWAIT, path, args)
265 ==>
266 Popen([path] + args[1:])
267 
268 
269 Environment example:
270 
271 os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env)
272 ==>
273 Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"})
274 
275 
276 Replacing os.popen*
277 -------------------
278 pipe = os.popen(cmd, mode='r', bufsize)
279 ==>
280 pipe = Popen(cmd, shell=True, bufsize=bufsize, stdout=PIPE).stdout
281 
282 pipe = os.popen(cmd, mode='w', bufsize)
283 ==>
284 pipe = Popen(cmd, shell=True, bufsize=bufsize, stdin=PIPE).stdin
285 
286 
287 (child_stdin, child_stdout) = os.popen2(cmd, mode, bufsize)
288 ==>
289 p = Popen(cmd, shell=True, bufsize=bufsize,
290  stdin=PIPE, stdout=PIPE, close_fds=True)
291 (child_stdin, child_stdout) = (p.stdin, p.stdout)
292 
293 
294 (child_stdin,
295  child_stdout,
296  child_stderr) = os.popen3(cmd, mode, bufsize)
297 ==>
298 p = Popen(cmd, shell=True, bufsize=bufsize,
299  stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
300 (child_stdin,
301  child_stdout,
302  child_stderr) = (p.stdin, p.stdout, p.stderr)
303 
304 
305 (child_stdin, child_stdout_and_stderr) = os.popen4(cmd, mode, bufsize)
306 ==>
307 p = Popen(cmd, shell=True, bufsize=bufsize,
308  stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
309 (child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout)
310 
311 
312 Replacing popen2.*
313 ------------------
314 Note: If the cmd argument to popen2 functions is a string, the command
315 is executed through /bin/sh. If it is a list, the command is directly
316 executed.
317 
318 (child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode)
319 ==>
320 p = Popen(["somestring"], shell=True, bufsize=bufsize
321  stdin=PIPE, stdout=PIPE, close_fds=True)
322 (child_stdout, child_stdin) = (p.stdout, p.stdin)
323 
324 
325 (child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, mode)
326 ==>
327 p = Popen(["mycmd", "myarg"], bufsize=bufsize,
328  stdin=PIPE, stdout=PIPE, close_fds=True)
329 (child_stdout, child_stdin) = (p.stdout, p.stdin)
330 
331 The popen2.Popen3 and popen3.Popen4 basically works as subprocess.Popen,
332 except that:
333 
334 * subprocess.Popen raises an exception if the execution fails
335 * the capturestderr argument is replaced with the stderr argument.
336 * stdin=PIPE and stdout=PIPE must be specified.
337 * popen2 closes all filedescriptors by default, but you have to specify
338  close_fds=True with subprocess.Popen.
339 
340 
341 """
342 
343 import sys
344 mswindows = (sys.platform == "win32")
345 
346 import os
347 import types
348 import traceback
349 
350 if mswindows:
351  import threading
352  import msvcrt
353  if 0: # <-- change this to use pywin32 instead of the _subprocess driver
354  import pywintypes
355  from win32api import GetStdHandle, STD_INPUT_HANDLE, \
356  STD_OUTPUT_HANDLE, STD_ERROR_HANDLE
357  from win32api import GetCurrentProcess, DuplicateHandle, \
358  GetModuleFileName, GetVersion
359  from win32con import DUPLICATE_SAME_ACCESS, SW_HIDE
360  from win32pipe import CreatePipe
361  from win32process import CreateProcess, STARTUPINFO, \
362  GetExitCodeProcess, STARTF_USESTDHANDLES, \
363  STARTF_USESHOWWINDOW, CREATE_NEW_CONSOLE
364  from win32event import WaitForSingleObject, INFINITE, WAIT_OBJECT_0
365  else:
366  from _subprocess import *
367 
368  class STARTUPINFO:
369  dwFlags = 0
370  hStdInput = None
371  hStdOutput = None
372  hStdError = None
373  wShowWindow = 0
374 
375  class pywintypes:
376  error = IOError
377 else:
378  import select
379  import errno
380  import fcntl
381  import pickle
382 
383 __all__ = ["Popen", "PIPE", "STDOUT", "call"]
384 
385 try:
386  MAXFD = os.sysconf("SC_OPEN_MAX")
387 except:
388  MAXFD = 256
389 
390 # True/False does not exist on 2.2.0
391 try:
392  False
393 except NameError:
394  False = 0
395  True = 1
396 
397 _active = []
398 
399 
400 def _cleanup():
401  for inst in _active[:]:
402  inst.poll()
403 
404 PIPE = -1
405 STDOUT = -2
406 
407 
408 def call(*args, **kwargs):
409  """Run command with arguments. Wait for command to complete, then
410  return the returncode attribute.
411 
412  The arguments are the same as for the Popen constructor. Example:
413 
414  retcode = call(["ls", "-l"])
415  """
416  return Popen(*args, **kwargs).wait()
417 
418 
419 def list2cmdline(seq):
420  """
421  Translate a sequence of arguments into a command line
422  string, using the same rules as the MS C runtime:
423 
424  1) Arguments are delimited by white space, which is either a
425  space or a tab.
426 
427  2) A string surrounded by double quotation marks is
428  interpreted as a single argument, regardless of white space
429  contained within. A quoted string can be embedded in an
430  argument.
431 
432  3) A double quotation mark preceded by a backslash is
433  interpreted as a literal double quotation mark.
434 
435  4) Backslashes are interpreted literally, unless they
436  immediately precede a double quotation mark.
437 
438  5) If backslashes immediately precede a double quotation mark,
439  every pair of backslashes is interpreted as a literal
440  backslash. If the number of backslashes is odd, the last
441  backslash escapes the next double quotation mark as
442  described in rule 3.
443  """
444 
445  # See
446  # http://msdn.microsoft.com/library/en-us/vccelng/htm/progs_12.asp
447  result = []
448  needquote = False
449  for arg in seq:
450  bs_buf = []
451 
452  # Add a space to separate this argument from the others
453  if result:
454  result.append(' ')
455 
456  needquote = (" " in arg) or ("\t" in arg)
457  if needquote:
458  result.append('"')
459 
460  for c in arg:
461  if c == '\\':
462  # Don't know if we need to double yet.
463  bs_buf.append(c)
464  elif c == '"':
465  # Double backspaces.
466  result.append('\\' * len(bs_buf) * 2)
467  bs_buf = []
468  result.append('\\"')
469  else:
470  # Normal char
471  if bs_buf:
472  result.extend(bs_buf)
473  bs_buf = []
474  result.append(c)
475 
476  # Add remaining backspaces, if any.
477  if bs_buf:
478  result.extend(bs_buf)
479 
480  if needquote:
481  result.extend(bs_buf)
482  result.append('"')
483 
484  return ''.join(result)
485 
486 
487 class Popen(object):
488 
489  def __init__(self, args, bufsize=0, executable=None,
490  stdin=None, stdout=None, stderr=None,
491  preexec_fn=None, close_fds=False, shell=False,
492  cwd=None, env=None, universal_newlines=False,
493  startupinfo=None, creationflags=0):
494  """Create new Popen instance."""
495  _cleanup()
496 
497  if not isinstance(bufsize, (int, long)):
498  raise TypeError("bufsize must be an integer")
499 
500  if mswindows:
501  if preexec_fn is not None:
502  raise ValueError("preexec_fn is not supported on Windows "
503  "platforms")
504  if close_fds:
505  raise ValueError("close_fds is not supported on Windows "
506  "platforms")
507  else:
508  # POSIX
509  if startupinfo is not None:
510  raise ValueError("startupinfo is only supported on Windows "
511  "platforms")
512  if creationflags != 0:
513  raise ValueError("creationflags is only supported on Windows "
514  "platforms")
515 
516  self.stdin = None
517  self.stdout = None
518  self.stderr = None
519  self.pid = None
520  self.returncode = None
521  self.universal_newlines = universal_newlines
522 
523  # Input and output objects. The general principle is like
524  # this:
525  #
526  # Parent Child
527  # ------ -----
528  # p2cwrite ---stdin---> p2cread
529  # c2pread <--stdout--- c2pwrite
530  # errread <--stderr--- errwrite
531  #
532  # On POSIX, the child objects are file descriptors. On
533  # Windows, these are Windows file handles. The parent objects
534  # are file descriptors on both platforms. The parent objects
535  # are None when not using PIPEs. The child objects are None
536  # when not redirecting.
537 
538  (p2cread, p2cwrite,
539  c2pread, c2pwrite,
540  errread, errwrite) = self._get_handles(stdin, stdout, stderr)
541 
542  self._execute_child(args, executable, preexec_fn, close_fds,
543  cwd, env, universal_newlines,
544  startupinfo, creationflags, shell,
545  p2cread, p2cwrite,
546  c2pread, c2pwrite,
547  errread, errwrite)
548 
549  if p2cwrite:
550  self.stdin = os.fdopen(p2cwrite, 'wb', bufsize)
551  if c2pread:
552  if universal_newlines:
553  self.stdout = os.fdopen(c2pread, 'rU', bufsize)
554  else:
555  self.stdout = os.fdopen(c2pread, 'rb', bufsize)
556  if errread:
557  if universal_newlines:
558  self.stderr = os.fdopen(errread, 'rU', bufsize)
559  else:
560  self.stderr = os.fdopen(errread, 'rb', bufsize)
561 
562  _active.append(self)
563 
564  def _translate_newlines(self, data):
565  data = data.replace("\r\n", "\n")
566  data = data.replace("\r", "\n")
567  return data
568 
569  if mswindows:
570  #
571  # Windows methods
572  #
573  def _get_handles(self, stdin, stdout, stderr):
574  """Construct and return tupel with IO objects:
575  p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
576  """
577  if stdin is None and stdout is None and stderr is None:
578  return (None, None, None, None, None, None)
579 
580  p2cread, p2cwrite = None, None
581  c2pread, c2pwrite = None, None
582  errread, errwrite = None, None
583 
584  if stdin is None:
585  p2cread = GetStdHandle(STD_INPUT_HANDLE)
586  elif stdin == PIPE:
587  p2cread, p2cwrite = CreatePipe(None, 0)
588  # Detach and turn into fd
589  p2cwrite = p2cwrite.Detach()
590  p2cwrite = msvcrt.open_osfhandle(p2cwrite, 0)
591  elif isinstance(stdin, types.IntType):
592  p2cread = msvcrt.get_osfhandle(stdin)
593  else:
594  # Assuming file-like object
595  p2cread = msvcrt.get_osfhandle(stdin.fileno())
596  p2cread = self._make_inheritable(p2cread)
597 
598  if stdout is None:
599  c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE)
600  elif stdout == PIPE:
601  c2pread, c2pwrite = CreatePipe(None, 0)
602  # Detach and turn into fd
603  c2pread = c2pread.Detach()
604  c2pread = msvcrt.open_osfhandle(c2pread, 0)
605  elif isinstance(stdout, types.IntType):
606  c2pwrite = msvcrt.get_osfhandle(stdout)
607  else:
608  # Assuming file-like object
609  c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
610  c2pwrite = self._make_inheritable(c2pwrite)
611 
612  if stderr is None:
613  errwrite = GetStdHandle(STD_ERROR_HANDLE)
614  elif stderr == PIPE:
615  errread, errwrite = CreatePipe(None, 0)
616  # Detach and turn into fd
617  errread = errread.Detach()
618  errread = msvcrt.open_osfhandle(errread, 0)
619  elif stderr == STDOUT:
620  errwrite = c2pwrite
621  elif isinstance(stderr, types.IntType):
622  errwrite = msvcrt.get_osfhandle(stderr)
623  else:
624  # Assuming file-like object
625  errwrite = msvcrt.get_osfhandle(stderr.fileno())
626  errwrite = self._make_inheritable(errwrite)
627 
628  return (p2cread, p2cwrite,
629  c2pread, c2pwrite,
630  errread, errwrite)
631 
632  def _make_inheritable(self, handle):
633  """Return a duplicate of handle, which is inheritable"""
634  return DuplicateHandle(GetCurrentProcess(), handle,
635  GetCurrentProcess(), 0, 1,
636  DUPLICATE_SAME_ACCESS)
637 
638  def _find_w9xpopen(self):
639  """Find and return absolut path to w9xpopen.exe"""
640  w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)),
641  "w9xpopen.exe")
642  if not os.path.exists(w9xpopen):
643  # Eeek - file-not-found - possibly an embedding
644  # situation - see if we can locate it in sys.exec_prefix
645  w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix),
646  "w9xpopen.exe")
647  if not os.path.exists(w9xpopen):
648  raise RuntimeError("Cannot locate w9xpopen.exe, which is "
649  "needed for Popen to work with your "
650  "shell or platform.")
651  return w9xpopen
652 
653  def _execute_child(self, args, executable, preexec_fn, close_fds,
654  cwd, env, universal_newlines,
655  startupinfo, creationflags, shell,
656  p2cread, p2cwrite,
657  c2pread, c2pwrite,
658  errread, errwrite):
659  """Execute program (MS Windows version)"""
660 
661  if not isinstance(args, types.StringTypes):
662  args = list2cmdline(args)
663 
664  # Process startup details
665  if startupinfo is None:
666  startupinfo = STARTUPINFO()
667  if None not in (p2cread, c2pwrite, errwrite):
668  startupinfo.dwFlags |= STARTF_USESTDHANDLES
669  startupinfo.hStdInput = p2cread
670  startupinfo.hStdOutput = c2pwrite
671  startupinfo.hStdError = errwrite
672 
673  if shell:
674  startupinfo.dwFlags |= STARTF_USESHOWWINDOW
675  startupinfo.wShowWindow = SW_HIDE
676  comspec = os.environ.get("COMSPEC", "cmd.exe")
677  args = comspec + " /c " + args
678  if (GetVersion() >= 0x80000000 or
679  os.path.basename(comspec).lower() == "command.com"):
680  # Win9x, or using command.com on NT. We need to
681  # use the w9xpopen intermediate program. For more
682  # information, see KB Q150956
683  # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp)
684  w9xpopen = self._find_w9xpopen()
685  args = '"%s" %s' % (w9xpopen, args)
686  # Not passing CREATE_NEW_CONSOLE has been known to
687  # cause random failures on win9x. Specifically a
688  # dialog: "Your program accessed mem currently in
689  # use at xxx" and a hopeful warning about the
690  # stability of your system. Cost is Ctrl+C wont
691  # kill children.
692  creationflags |= CREATE_NEW_CONSOLE
693 
694  # Start the process
695  try:
696  hp, ht, pid, tid = CreateProcess(executable, args,
697  # no special security
698  None, None,
699  # must inherit handles to pass std
700  # handles
701  1,
702  creationflags,
703  env,
704  cwd,
705  startupinfo)
706  except pywintypes.error, e:
707  # Translate pywintypes.error to WindowsError, which is
708  # a subclass of OSError. FIXME: We should really
709  # translate errno using _sys_errlist (or simliar), but
710  # how can this be done from Python?
711  raise WindowsError(*e.args)
712 
713  # Retain the process handle, but close the thread handle
714  self._handle = hp
715  self.pid = pid
716  ht.Close()
717 
718  # Child is launched. Close the parent's copy of those pipe
719  # handles that only the child should have open. You need
720  # to make sure that no handles to the write end of the
721  # output pipe are maintained in this process or else the
722  # pipe will not close when the child process exits and the
723  # ReadFile will hang.
724  if p2cread is not None:
725  p2cread.Close()
726  if c2pwrite is not None:
727  c2pwrite.Close()
728  if errwrite is not None:
729  errwrite.Close()
730 
731  def poll(self):
732  """Check if child process has terminated. Returns returncode
733  attribute."""
734  if self.returncode is None:
735  if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0:
736  self.returncode = GetExitCodeProcess(self._handle)
737  _active.remove(self)
738  return self.returncode
739 
740  def wait(self):
741  """Wait for child process to terminate. Returns returncode
742  attribute."""
743  if self.returncode is None:
744  obj = WaitForSingleObject(self._handle, INFINITE)
745  self.returncode = GetExitCodeProcess(self._handle)
746  _active.remove(self)
747  return self.returncode
748 
749  def _readerthread(self, fh, buffer):
750  buffer.append(fh.read())
751 
752  def communicate(self, input=None):
753  """Interact with process: Send data to stdin. Read data from
754  stdout and stderr, until end-of-file is reached. Wait for
755  process to terminate. The optional input argument should be a
756  string to be sent to the child process, or None, if no data
757  should be sent to the child.
758 
759  communicate() returns a tuple (stdout, stderr)."""
760  stdout = None # Return
761  stderr = None # Return
762 
763  if self.stdout:
764  stdout = []
765  stdout_thread = threading.Thread(target=self._readerthread,
766  args=(self.stdout, stdout))
767  stdout_thread.setDaemon(True)
768  stdout_thread.start()
769  if self.stderr:
770  stderr = []
771  stderr_thread = threading.Thread(target=self._readerthread,
772  args=(self.stderr, stderr))
773  stderr_thread.setDaemon(True)
774  stderr_thread.start()
775 
776  if self.stdin:
777  if input is not None:
778  self.stdin.write(input)
779  self.stdin.close()
780 
781  if self.stdout:
782  stdout_thread.join()
783  if self.stderr:
784  stderr_thread.join()
785 
786  # All data exchanged. Translate lists into strings.
787  if stdout is not None:
788  stdout = stdout[0]
789  if stderr is not None:
790  stderr = stderr[0]
791 
792  # Translate newlines, if requested. We cannot let the file
793  # object do the translation: It is based on stdio, which is
794  # impossible to combine with select (unless forcing no
795  # buffering).
796  if self.universal_newlines and hasattr(open, 'newlines'):
797  if stdout:
798  stdout = self._translate_newlines(stdout)
799  if stderr:
800  stderr = self._translate_newlines(stderr)
801 
802  self.wait()
803  return (stdout, stderr)
804 
805  else:
806  #
807  # POSIX methods
808  #
809  def _get_handles(self, stdin, stdout, stderr):
810  """Construct and return tupel with IO objects:
811  p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
812  """
813  p2cread, p2cwrite = None, None
814  c2pread, c2pwrite = None, None
815  errread, errwrite = None, None
816 
817  if stdin is None:
818  pass
819  elif stdin == PIPE:
820  p2cread, p2cwrite = os.pipe()
821  elif isinstance(stdin, types.IntType):
822  p2cread = stdin
823  else:
824  # Assuming file-like object
825  p2cread = stdin.fileno()
826 
827  if stdout is None:
828  pass
829  elif stdout == PIPE:
830  c2pread, c2pwrite = os.pipe()
831  elif isinstance(stdout, types.IntType):
832  c2pwrite = stdout
833  else:
834  # Assuming file-like object
835  c2pwrite = stdout.fileno()
836 
837  if stderr is None:
838  pass
839  elif stderr == PIPE:
840  errread, errwrite = os.pipe()
841  elif stderr == STDOUT:
842  errwrite = c2pwrite
843  elif isinstance(stderr, types.IntType):
844  errwrite = stderr
845  else:
846  # Assuming file-like object
847  errwrite = stderr.fileno()
848 
849  return (p2cread, p2cwrite,
850  c2pread, c2pwrite,
851  errread, errwrite)
852 
853  def _set_cloexec_flag(self, fd):
854  try:
855  cloexec_flag = fcntl.FD_CLOEXEC
856  except AttributeError:
857  cloexec_flag = 1
858 
859  old = fcntl.fcntl(fd, fcntl.F_GETFD)
860  fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag)
861 
862  def _close_fds(self, but):
863  for i in xrange(3, MAXFD):
864  if i == but:
865  continue
866  try:
867  os.close(i)
868  except:
869  pass
870 
871  def _execute_child(self, args, executable, preexec_fn, close_fds,
872  cwd, env, universal_newlines,
873  startupinfo, creationflags, shell,
874  p2cread, p2cwrite,
875  c2pread, c2pwrite,
876  errread, errwrite):
877  """Execute program (POSIX version)"""
878 
879  if isinstance(args, types.StringTypes):
880  args = [args]
881 
882  if shell:
883  args = ["/bin/sh", "-c"] + args
884 
885  if executable is None:
886  executable = args[0]
887 
888  # For transferring possible exec failure from child to parent
889  # The first char specifies the exception type: 0 means
890  # OSError, 1 means some other error.
891  errpipe_read, errpipe_write = os.pipe()
892  self._set_cloexec_flag(errpipe_write)
893 
894  self.pid = os.fork()
895  if self.pid == 0:
896  # Child
897  try:
898  # Close parent's pipe ends
899  if p2cwrite:
900  os.close(p2cwrite)
901  if c2pread:
902  os.close(c2pread)
903  if errread:
904  os.close(errread)
905  os.close(errpipe_read)
906 
907  # Dup fds for child
908  if p2cread:
909  os.dup2(p2cread, 0)
910  if c2pwrite:
911  os.dup2(c2pwrite, 1)
912  if errwrite:
913  os.dup2(errwrite, 2)
914 
915  # Close pipe fds. Make sure we doesn't close the same
916  # fd more than once.
917  if p2cread:
918  os.close(p2cread)
919  if c2pwrite and c2pwrite not in (p2cread,):
920  os.close(c2pwrite)
921  if errwrite and errwrite not in (p2cread, c2pwrite):
922  os.close(errwrite)
923 
924  # Close all other fds, if asked for
925  if close_fds:
926  self._close_fds(but=errpipe_write)
927 
928  if cwd is not None:
929  os.chdir(cwd)
930 
931  if preexec_fn:
932  apply(preexec_fn)
933 
934  if env is None:
935  os.execvp(executable, args)
936  else:
937  os.execvpe(executable, args, env)
938 
939  except:
940  exc_type, exc_value, tb = sys.exc_info()
941  # Save the traceback and attach it to the exception object
942  exc_lines = traceback.format_exception(exc_type,
943  exc_value,
944  tb)
945  exc_value.child_traceback = ''.join(exc_lines)
946  os.write(errpipe_write, pickle.dumps(exc_value))
947 
948  # This exitcode won't be reported to applications, so it
949  # really doesn't matter what we return.
950  os._exit(255)
951 
952  # Parent
953  os.close(errpipe_write)
954  if p2cread and p2cwrite:
955  os.close(p2cread)
956  if c2pwrite and c2pread:
957  os.close(c2pwrite)
958  if errwrite and errread:
959  os.close(errwrite)
960 
961  # Wait for exec to fail or succeed; possibly raising exception
962  data = os.read(errpipe_read, 1048576) # Exceptions limited to 1 MB
963  os.close(errpipe_read)
964  if data != "":
965  os.waitpid(self.pid, 0)
966  child_exception = pickle.loads(data)
967  raise child_exception
968 
969  def _handle_exitstatus(self, sts):
970  if os.WIFSIGNALED(sts):
971  self.returncode = -os.WTERMSIG(sts)
972  elif os.WIFEXITED(sts):
973  self.returncode = os.WEXITSTATUS(sts)
974  else:
975  # Should never happen
976  raise RuntimeError("Unknown child exit status!")
977 
978  _active.remove(self)
979 
980  def poll(self):
981  """Check if child process has terminated. Returns returncode
982  attribute."""
983  if self.returncode is None:
984  try:
985  pid, sts = os.waitpid(self.pid, os.WNOHANG)
986  if pid == self.pid:
987  self._handle_exitstatus(sts)
988  except os.error:
989  pass
990  return self.returncode
991 
992  def wait(self):
993  """Wait for child process to terminate. Returns returncode
994  attribute."""
995  if self.returncode is None:
996  pid, sts = os.waitpid(self.pid, 0)
997  self._handle_exitstatus(sts)
998  return self.returncode
999 
1000  def communicate(self, input=None):
1001  """Interact with process: Send data to stdin. Read data from
1002  stdout and stderr, until end-of-file is reached. Wait for
1003  process to terminate. The optional input argument should be a
1004  string to be sent to the child process, or None, if no data
1005  should be sent to the child.
1006 
1007  communicate() returns a tuple (stdout, stderr)."""
1008  read_set = []
1009  write_set = []
1010  stdout = None # Return
1011  stderr = None # Return
1012 
1013  if self.stdin:
1014  # Flush stdio buffer. This might block, if the user has
1015  # been writing to .stdin in an uncontrolled fashion.
1016  self.stdin.flush()
1017  if input:
1018  write_set.append(self.stdin)
1019  else:
1020  self.stdin.close()
1021  if self.stdout:
1022  read_set.append(self.stdout)
1023  stdout = []
1024  if self.stderr:
1025  read_set.append(self.stderr)
1026  stderr = []
1027 
1028  while read_set or write_set:
1029  rlist, wlist, xlist = select.select(read_set, write_set, [])
1030 
1031  if self.stdin in wlist:
1032  # When select has indicated that the file is writable,
1033  # we can write up to PIPE_BUF bytes without risk
1034  # blocking. POSIX defines PIPE_BUF >= 512
1035  bytes_written = os.write(self.stdin.fileno(), input[:512])
1036  input = input[bytes_written:]
1037  if not input:
1038  self.stdin.close()
1039  write_set.remove(self.stdin)
1040 
1041  if self.stdout in rlist:
1042  data = os.read(self.stdout.fileno(), 1024)
1043  if data == "":
1044  self.stdout.close()
1045  read_set.remove(self.stdout)
1046  stdout.append(data)
1047 
1048  if self.stderr in rlist:
1049  data = os.read(self.stderr.fileno(), 1024)
1050  if data == "":
1051  self.stderr.close()
1052  read_set.remove(self.stderr)
1053  stderr.append(data)
1054 
1055  # All data exchanged. Translate lists into strings.
1056  if stdout is not None:
1057  stdout = ''.join(stdout)
1058  if stderr is not None:
1059  stderr = ''.join(stderr)
1060 
1061  # Translate newlines, if requested. We cannot let the file
1062  # object do the translation: It is based on stdio, which is
1063  # impossible to combine with select (unless forcing no
1064  # buffering).
1065  if self.universal_newlines and hasattr(open, 'newlines'):
1066  if stdout:
1067  stdout = self._translate_newlines(stdout)
1068  if stderr:
1069  stderr = self._translate_newlines(stderr)
1070 
1071  self.wait()
1072  return (stdout, stderr)
1073 
1074 
1075 def _demo_posix():
1076  #
1077  # Example 1: Simple redirection: Get process list
1078  #
1079  plist = Popen(["ps"], stdout=PIPE).communicate()[0]
1080  print "Process list:"
1081  print plist
1082 
1083  #
1084  # Example 2: Change uid before executing child
1085  #
1086  if os.getuid() == 0:
1087  p = Popen(["id"], preexec_fn=lambda: os.setuid(100))
1088  p.wait()
1089 
1090  #
1091  # Example 3: Connecting several subprocesses
1092  #
1093  print "Looking for 'hda'..."
1094  p1 = Popen(["dmesg"], stdout=PIPE)
1095  p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
1096  print repr(p2.communicate()[0])
1097 
1098  #
1099  # Example 4: Catch execution error
1100  #
1101  print
1102  print "Trying a weird file..."
1103  try:
1104  print Popen(["/this/path/does/not/exist"]).communicate()
1105  except OSError, e:
1106  if e.errno == errno.ENOENT:
1107  print "The file didn't exist. I thought so..."
1108  print "Child traceback:"
1109  print e.child_traceback
1110  else:
1111  print "Error", e.errno
1112  else:
1113  print >>sys.stderr, "Gosh. No error."
1114 
1115 
1116 def _demo_windows():
1117  #
1118  # Example 1: Connecting several subprocesses
1119  #
1120  print "Looking for 'PROMPT' in set output..."
1121  p1 = Popen("set", stdout=PIPE, shell=True)
1122  p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE)
1123  print repr(p2.communicate()[0])
1124 
1125  #
1126  # Example 2: Simple execution of program
1127  #
1128  print "Executing calc..."
1129  p = Popen("calc")
1130  p.wait()
1131 
1132 
1133 if __name__ == "__main__":
1134  if mswindows:
1135  _demo_windows()
1136  else:
1137  _demo_posix()