IMP  2.0.0
The Integrative Modeling Platform
compat_subprocess.py
1 # subprocess - Subprocesses with accessible I/O streams
2 #
3 # For more information about this module, see PEP 324.
4 #
5 # This module should remain compatible with Python 2.2, see PEP 291.
6 #
7 # Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
8 #
9 # Licensed to PSF under a Contributor Agreement.
10 # See http://www.python.org/2.4/license for licensing details.
11 
12 r"""subprocess - Subprocesses with accessible I/O streams
13 
14 This module allows you to spawn processes, connect to their
15 input/output/error pipes, and obtain their return codes. This module
16 intends to replace several other, older modules and functions, like:
17 
18 os.system
19 os.spawn*
20 os.popen*
21 popen2.*
22 commands.*
23 
24 Information about how the subprocess module can be used to replace these
25 modules and functions can be found below.
26 
27 
28 
29 Using the subprocess module
30 ===========================
31 This module defines one class called Popen:
32 
33 class Popen(args, bufsize=0, executable=None,
34  stdin=None, stdout=None, stderr=None,
35  preexec_fn=None, close_fds=False, shell=False,
36  cwd=None, env=None, universal_newlines=False,
37  startupinfo=None, creationflags=0):
38 
39 
40 Arguments are:
41 
42 args should be a string, or a sequence of program arguments. The
43 program to execute is normally the first item in the args sequence or
44 string, but can be explicitly set by using the executable argument.
45 
46 On UNIX, with shell=False (default): In this case, the Popen class
47 uses os.execvp() to execute the child program. args should normally
48 be a sequence. A string will be treated as a sequence with the string
49 as the only item (the program to execute).
50 
51 On UNIX, with shell=True: If args is a string, it specifies the
52 command string to execute through the shell. If args is a sequence,
53 the first item specifies the command string, and any additional items
54 will be treated as additional shell arguments.
55 
56 On Windows: the Popen class uses CreateProcess() to execute the child
57 program, which operates on strings. If args is a sequence, it will be
58 converted to a string using the list2cmdline method. Please note that
59 not all MS Windows applications interpret the command line the same
60 way: The list2cmdline is designed for applications using the same
61 rules as the MS C runtime.
62 
63 bufsize, if given, has the same meaning as the corresponding argument
64 to the built-in open() function: 0 means unbuffered, 1 means line
65 buffered, any other positive value means use a buffer of
66 (approximately) that size. A negative bufsize means to use the system
67 default, which usually means fully buffered. The default value for
68 bufsize is 0 (unbuffered).
69 
70 stdin, stdout and stderr specify the executed programs' standard
71 input, standard output and standard error file handles, respectively.
72 Valid values are PIPE, an existing file descriptor (a positive
73 integer), an existing file object, and None. PIPE indicates that a
74 new pipe to the child should be created. With None, no redirection
75 will occur; the child's file handles will be inherited from the
76 parent. Additionally, stderr can be STDOUT, which indicates that the
77 stderr data from the applications should be captured into the same
78 file handle as for stdout.
79 
80 If preexec_fn is set to a callable object, this object will be called
81 in the child process just before the child is executed.
82 
83 If close_fds is true, all file descriptors except 0, 1 and 2 will be
84 closed before the child process is executed.
85 
86 if shell is true, the specified command will be executed through the
87 shell.
88 
89 If cwd is not None, the current directory will be changed to cwd
90 before the child is executed.
91 
92 If env is not None, it defines the environment variables for the new
93 process.
94 
95 If universal_newlines is true, the file objects stdout and stderr are
96 opened as a text files, but lines may be terminated by any of '\n',
97 the Unix end-of-line convention, '\r', the Macintosh convention or
98 '\r\n', the Windows convention. All of these external representations
99 are seen as '\n' by the Python program. Note: This feature is only
100 available if Python is built with universal newline support (the
101 default). Also, the newlines attribute of the file objects stdout,
102 stdin and stderr are not updated by the communicate() method.
103 
104 The startupinfo and creationflags, if given, will be passed to the
105 underlying CreateProcess() function. They can specify things such as
106 appearance of the main window and priority for the new process.
107 (Windows only)
108 
109 
110 This module also defines two shortcut functions:
111 
112 call(*args, **kwargs):
113  Run command with arguments. Wait for command to complete, then
114  return the returncode attribute.
115 
116  The arguments are the same as for the Popen constructor. Example:
117 
118  retcode = call(["ls", "-l"])
119 
120 
121 Exceptions
122 ----------
123 Exceptions raised in the child process, before the new program has
124 started to execute, will be re-raised in the parent. Additionally,
125 the exception object will have one extra attribute called
126 'child_traceback', which is a string containing traceback information
127 from the childs point of view.
128 
129 The most common exception raised is OSError. This occurs, for
130 example, when trying to execute a non-existent file. Applications
131 should prepare for OSErrors.
132 
133 A ValueError will be raised if Popen is called with invalid arguments.
134 
135 
136 Security
137 --------
138 Unlike some other popen functions, this implementation will never call
139 /bin/sh implicitly. This means that all characters, including shell
140 metacharacters, can safely be passed to child processes.
141 
142 
143 Popen objects
144 =============
145 Instances of the Popen class have the following methods:
146 
147 poll()
148  Check if child process has terminated. Returns returncode
149  attribute.
150 
151 wait()
152  Wait for child process to terminate. Returns returncode attribute.
153 
154 communicate(input=None)
155  Interact with process: Send data to stdin. Read data from stdout
156  and stderr, until end-of-file is reached. Wait for process to
157  terminate. The optional stdin argument should be a string to be
158  sent to the child process, or None, if no data should be sent to
159  the child.
160 
161  communicate() returns a tuple (stdout, stderr).
162 
163  Note: The data read is buffered in memory, so do not use this
164  method if the data size is large or unlimited.
165 
166 The following attributes are also available:
167 
168 stdin
169  If the stdin argument is PIPE, this attribute is a file object
170  that provides input to the child process. Otherwise, it is None.
171 
172 stdout
173  If the stdout argument is PIPE, this attribute is a file object
174  that provides output from the child process. Otherwise, it is
175  None.
176 
177 stderr
178  If the stderr argument is PIPE, this attribute is file object that
179  provides error output from the child process. Otherwise, it is
180  None.
181 
182 pid
183  The process ID of the child process.
184 
185 returncode
186  The child return code. A None value indicates that the process
187  hasn't terminated yet. A negative value -N indicates that the
188  child was terminated by signal N (UNIX only).
189 
190 
191 Replacing older functions with the subprocess module
192 ====================================================
193 In this section, "a ==> b" means that b can be used as a replacement
194 for a.
195 
196 Note: All functions in this section fail (more or less) silently if
197 the executed program cannot be found; this module raises an OSError
198 exception.
199 
200 In the following examples, we assume that the subprocess module is
201 imported with "from subprocess import *".
202 
203 
204 Replacing /bin/sh shell backquote
205 ---------------------------------
206 output=`mycmd myarg`
207 ==>
208 output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0]
209 
210 
211 Replacing shell pipe line
212 -------------------------
213 output=`dmesg | grep hda`
214 ==>
215 p1 = Popen(["dmesg"], stdout=PIPE)
216 p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
217 output = p2.communicate()[0]
218 
219 
220 Replacing os.system()
221 ---------------------
222 sts = os.system("mycmd" + " myarg")
223 ==>
224 p = Popen("mycmd" + " myarg", shell=True)
225 sts = os.waitpid(p.pid, 0)
226 
227 Note:
228 
229 * Calling the program through the shell is usually not required.
230 
231 * It's easier to look at the returncode attribute than the
232  exitstatus.
233 
234 A more real-world example would look like this:
235 
236 try:
237  retcode = call("mycmd" + " myarg", shell=True)
238  if retcode < 0:
239  print >>sys.stderr, "Child was terminated by signal", -retcode
240  else:
241  print >>sys.stderr, "Child returned", retcode
242 except OSError, e:
243  print >>sys.stderr, "Execution failed:", e
244 
245 
246 Replacing os.spawn*
247 -------------------
248 P_NOWAIT example:
249 
250 pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg")
251 ==>
252 pid = Popen(["/bin/mycmd", "myarg"]).pid
253 
254 
255 P_WAIT example:
256 
257 retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg")
258 ==>
259 retcode = call(["/bin/mycmd", "myarg"])
260 
261 
262 Vector example:
263 
264 os.spawnvp(os.P_NOWAIT, path, args)
265 ==>
266 Popen([path] + args[1:])
267 
268 
269 Environment example:
270 
271 os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env)
272 ==>
273 Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"})
274 
275 
276 Replacing os.popen*
277 -------------------
278 pipe = os.popen(cmd, mode='r', bufsize)
279 ==>
280 pipe = Popen(cmd, shell=True, bufsize=bufsize, stdout=PIPE).stdout
281 
282 pipe = os.popen(cmd, mode='w', bufsize)
283 ==>
284 pipe = Popen(cmd, shell=True, bufsize=bufsize, stdin=PIPE).stdin
285 
286 
287 (child_stdin, child_stdout) = os.popen2(cmd, mode, bufsize)
288 ==>
289 p = Popen(cmd, shell=True, bufsize=bufsize,
290  stdin=PIPE, stdout=PIPE, close_fds=True)
291 (child_stdin, child_stdout) = (p.stdin, p.stdout)
292 
293 
294 (child_stdin,
295  child_stdout,
296  child_stderr) = os.popen3(cmd, mode, bufsize)
297 ==>
298 p = Popen(cmd, shell=True, bufsize=bufsize,
299  stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
300 (child_stdin,
301  child_stdout,
302  child_stderr) = (p.stdin, p.stdout, p.stderr)
303 
304 
305 (child_stdin, child_stdout_and_stderr) = os.popen4(cmd, mode, bufsize)
306 ==>
307 p = Popen(cmd, shell=True, bufsize=bufsize,
308  stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
309 (child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout)
310 
311 
312 Replacing popen2.*
313 ------------------
314 Note: If the cmd argument to popen2 functions is a string, the command
315 is executed through /bin/sh. If it is a list, the command is directly
316 executed.
317 
318 (child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode)
319 ==>
320 p = Popen(["somestring"], shell=True, bufsize=bufsize
321  stdin=PIPE, stdout=PIPE, close_fds=True)
322 (child_stdout, child_stdin) = (p.stdout, p.stdin)
323 
324 
325 (child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, mode)
326 ==>
327 p = Popen(["mycmd", "myarg"], bufsize=bufsize,
328  stdin=PIPE, stdout=PIPE, close_fds=True)
329 (child_stdout, child_stdin) = (p.stdout, p.stdin)
330 
331 The popen2.Popen3 and popen3.Popen4 basically works as subprocess.Popen,
332 except that:
333 
334 * subprocess.Popen raises an exception if the execution fails
335 * the capturestderr argument is replaced with the stderr argument.
336 * stdin=PIPE and stdout=PIPE must be specified.
337 * popen2 closes all filedescriptors by default, but you have to specify
338  close_fds=True with subprocess.Popen.
339 
340 
341 """
342 
343 import sys
344 mswindows = (sys.platform == "win32")
345 
346 import os
347 import types
348 import traceback
349 
350 if mswindows:
351  import threading
352  import msvcrt
353  if 0: # <-- change this to use pywin32 instead of the _subprocess driver
354  import pywintypes
355  from win32api import GetStdHandle, STD_INPUT_HANDLE, \
356  STD_OUTPUT_HANDLE, STD_ERROR_HANDLE
357  from win32api import GetCurrentProcess, DuplicateHandle, \
358  GetModuleFileName, GetVersion
359  from win32con import DUPLICATE_SAME_ACCESS, SW_HIDE
360  from win32pipe import CreatePipe
361  from win32process import CreateProcess, STARTUPINFO, \
362  GetExitCodeProcess, STARTF_USESTDHANDLES, \
363  STARTF_USESHOWWINDOW, CREATE_NEW_CONSOLE
364  from win32event import WaitForSingleObject, INFINITE, WAIT_OBJECT_0
365  else:
366  from _subprocess import *
367  class STARTUPINFO:
368  dwFlags = 0
369  hStdInput = None
370  hStdOutput = None
371  hStdError = None
372  wShowWindow = 0
373  class pywintypes:
374  error = IOError
375 else:
376  import select
377  import errno
378  import fcntl
379  import pickle
380 
381 __all__ = ["Popen", "PIPE", "STDOUT", "call"]
382 
383 try:
384  MAXFD = os.sysconf("SC_OPEN_MAX")
385 except:
386  MAXFD = 256
387 
388 # True/False does not exist on 2.2.0
389 try:
390  False
391 except NameError:
392  False = 0
393  True = 1
394 
395 _active = []
396 
397 def _cleanup():
398  for inst in _active[:]:
399  inst.poll()
400 
401 PIPE = -1
402 STDOUT = -2
403 
404 
405 def call(*args, **kwargs):
406  """Run command with arguments. Wait for command to complete, then
407  return the returncode attribute.
408 
409  The arguments are the same as for the Popen constructor. Example:
410 
411  retcode = call(["ls", "-l"])
412  """
413  return Popen(*args, **kwargs).wait()
414 
415 
416 def list2cmdline(seq):
417  """
418  Translate a sequence of arguments into a command line
419  string, using the same rules as the MS C runtime:
420 
421  1) Arguments are delimited by white space, which is either a
422  space or a tab.
423 
424  2) A string surrounded by double quotation marks is
425  interpreted as a single argument, regardless of white space
426  contained within. A quoted string can be embedded in an
427  argument.
428 
429  3) A double quotation mark preceded by a backslash is
430  interpreted as a literal double quotation mark.
431 
432  4) Backslashes are interpreted literally, unless they
433  immediately precede a double quotation mark.
434 
435  5) If backslashes immediately precede a double quotation mark,
436  every pair of backslashes is interpreted as a literal
437  backslash. If the number of backslashes is odd, the last
438  backslash escapes the next double quotation mark as
439  described in rule 3.
440  """
441 
442  # See
443  # http://msdn.microsoft.com/library/en-us/vccelng/htm/progs_12.asp
444  result = []
445  needquote = False
446  for arg in seq:
447  bs_buf = []
448 
449  # Add a space to separate this argument from the others
450  if result:
451  result.append(' ')
452 
453  needquote = (" " in arg) or ("\t" in arg)
454  if needquote:
455  result.append('"')
456 
457  for c in arg:
458  if c == '\\':
459  # Don't know if we need to double yet.
460  bs_buf.append(c)
461  elif c == '"':
462  # Double backspaces.
463  result.append('\\' * len(bs_buf)*2)
464  bs_buf = []
465  result.append('\\"')
466  else:
467  # Normal char
468  if bs_buf:
469  result.extend(bs_buf)
470  bs_buf = []
471  result.append(c)
472 
473  # Add remaining backspaces, if any.
474  if bs_buf:
475  result.extend(bs_buf)
476 
477  if needquote:
478  result.extend(bs_buf)
479  result.append('"')
480 
481  return ''.join(result)
482 
483 
484 class Popen(object):
485  def __init__(self, args, bufsize=0, executable=None,
486  stdin=None, stdout=None, stderr=None,
487  preexec_fn=None, close_fds=False, shell=False,
488  cwd=None, env=None, universal_newlines=False,
489  startupinfo=None, creationflags=0):
490  """Create new Popen instance."""
491  _cleanup()
492 
493  if not isinstance(bufsize, (int, long)):
494  raise TypeError("bufsize must be an integer")
495 
496  if mswindows:
497  if preexec_fn is not None:
498  raise ValueError("preexec_fn is not supported on Windows "
499  "platforms")
500  if close_fds:
501  raise ValueError("close_fds is not supported on Windows "
502  "platforms")
503  else:
504  # POSIX
505  if startupinfo is not None:
506  raise ValueError("startupinfo is only supported on Windows "
507  "platforms")
508  if creationflags != 0:
509  raise ValueError("creationflags is only supported on Windows "
510  "platforms")
511 
512  self.stdin = None
513  self.stdout = None
514  self.stderr = None
515  self.pid = None
516  self.returncode = None
517  self.universal_newlines = universal_newlines
518 
519  # Input and output objects. The general principle is like
520  # this:
521  #
522  # Parent Child
523  # ------ -----
524  # p2cwrite ---stdin---> p2cread
525  # c2pread <--stdout--- c2pwrite
526  # errread <--stderr--- errwrite
527  #
528  # On POSIX, the child objects are file descriptors. On
529  # Windows, these are Windows file handles. The parent objects
530  # are file descriptors on both platforms. The parent objects
531  # are None when not using PIPEs. The child objects are None
532  # when not redirecting.
533 
534  (p2cread, p2cwrite,
535  c2pread, c2pwrite,
536  errread, errwrite) = self._get_handles(stdin, stdout, stderr)
537 
538  self._execute_child(args, executable, preexec_fn, close_fds,
539  cwd, env, universal_newlines,
540  startupinfo, creationflags, shell,
541  p2cread, p2cwrite,
542  c2pread, c2pwrite,
543  errread, errwrite)
544 
545  if p2cwrite:
546  self.stdin = os.fdopen(p2cwrite, 'wb', bufsize)
547  if c2pread:
548  if universal_newlines:
549  self.stdout = os.fdopen(c2pread, 'rU', bufsize)
550  else:
551  self.stdout = os.fdopen(c2pread, 'rb', bufsize)
552  if errread:
553  if universal_newlines:
554  self.stderr = os.fdopen(errread, 'rU', bufsize)
555  else:
556  self.stderr = os.fdopen(errread, 'rb', bufsize)
557 
558  _active.append(self)
559 
560 
561  def _translate_newlines(self, data):
562  data = data.replace("\r\n", "\n")
563  data = data.replace("\r", "\n")
564  return data
565 
566 
567  if mswindows:
568  #
569  # Windows methods
570  #
571  def _get_handles(self, stdin, stdout, stderr):
572  """Construct and return tupel with IO objects:
573  p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
574  """
575  if stdin == None and stdout == None and stderr == None:
576  return (None, None, None, None, None, None)
577 
578  p2cread, p2cwrite = None, None
579  c2pread, c2pwrite = None, None
580  errread, errwrite = None, None
581 
582  if stdin == None:
583  p2cread = GetStdHandle(STD_INPUT_HANDLE)
584  elif stdin == PIPE:
585  p2cread, p2cwrite = CreatePipe(None, 0)
586  # Detach and turn into fd
587  p2cwrite = p2cwrite.Detach()
588  p2cwrite = msvcrt.open_osfhandle(p2cwrite, 0)
589  elif type(stdin) == types.IntType:
590  p2cread = msvcrt.get_osfhandle(stdin)
591  else:
592  # Assuming file-like object
593  p2cread = msvcrt.get_osfhandle(stdin.fileno())
594  p2cread = self._make_inheritable(p2cread)
595 
596  if stdout == None:
597  c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE)
598  elif stdout == PIPE:
599  c2pread, c2pwrite = CreatePipe(None, 0)
600  # Detach and turn into fd
601  c2pread = c2pread.Detach()
602  c2pread = msvcrt.open_osfhandle(c2pread, 0)
603  elif type(stdout) == types.IntType:
604  c2pwrite = msvcrt.get_osfhandle(stdout)
605  else:
606  # Assuming file-like object
607  c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
608  c2pwrite = self._make_inheritable(c2pwrite)
609 
610  if stderr == None:
611  errwrite = GetStdHandle(STD_ERROR_HANDLE)
612  elif stderr == PIPE:
613  errread, errwrite = CreatePipe(None, 0)
614  # Detach and turn into fd
615  errread = errread.Detach()
616  errread = msvcrt.open_osfhandle(errread, 0)
617  elif stderr == STDOUT:
618  errwrite = c2pwrite
619  elif type(stderr) == types.IntType:
620  errwrite = msvcrt.get_osfhandle(stderr)
621  else:
622  # Assuming file-like object
623  errwrite = msvcrt.get_osfhandle(stderr.fileno())
624  errwrite = self._make_inheritable(errwrite)
625 
626  return (p2cread, p2cwrite,
627  c2pread, c2pwrite,
628  errread, errwrite)
629 
630 
631  def _make_inheritable(self, handle):
632  """Return a duplicate of handle, which is inheritable"""
633  return DuplicateHandle(GetCurrentProcess(), handle,
634  GetCurrentProcess(), 0, 1,
635  DUPLICATE_SAME_ACCESS)
636 
637 
638  def _find_w9xpopen(self):
639  """Find and return absolut path to w9xpopen.exe"""
640  w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)),
641  "w9xpopen.exe")
642  if not os.path.exists(w9xpopen):
643  # Eeek - file-not-found - possibly an embedding
644  # situation - see if we can locate it in sys.exec_prefix
645  w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix),
646  "w9xpopen.exe")
647  if not os.path.exists(w9xpopen):
648  raise RuntimeError("Cannot locate w9xpopen.exe, which is "
649  "needed for Popen to work with your "
650  "shell or platform.")
651  return w9xpopen
652 
653 
654  def _execute_child(self, args, executable, preexec_fn, close_fds,
655  cwd, env, universal_newlines,
656  startupinfo, creationflags, shell,
657  p2cread, p2cwrite,
658  c2pread, c2pwrite,
659  errread, errwrite):
660  """Execute program (MS Windows version)"""
661 
662  if not isinstance(args, types.StringTypes):
663  args = list2cmdline(args)
664 
665  # Process startup details
666  if startupinfo == None:
667  startupinfo = STARTUPINFO()
668  if None not in (p2cread, c2pwrite, errwrite):
669  startupinfo.dwFlags |= STARTF_USESTDHANDLES
670  startupinfo.hStdInput = p2cread
671  startupinfo.hStdOutput = c2pwrite
672  startupinfo.hStdError = errwrite
673 
674  if shell:
675  startupinfo.dwFlags |= STARTF_USESHOWWINDOW
676  startupinfo.wShowWindow = SW_HIDE
677  comspec = os.environ.get("COMSPEC", "cmd.exe")
678  args = comspec + " /c " + args
679  if (GetVersion() >= 0x80000000L or
680  os.path.basename(comspec).lower() == "command.com"):
681  # Win9x, or using command.com on NT. We need to
682  # use the w9xpopen intermediate program. For more
683  # information, see KB Q150956
684  # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp)
685  w9xpopen = self._find_w9xpopen()
686  args = '"%s" %s' % (w9xpopen, args)
687  # Not passing CREATE_NEW_CONSOLE has been known to
688  # cause random failures on win9x. Specifically a
689  # dialog: "Your program accessed mem currently in
690  # use at xxx" and a hopeful warning about the
691  # stability of your system. Cost is Ctrl+C wont
692  # kill children.
693  creationflags |= CREATE_NEW_CONSOLE
694 
695  # Start the process
696  try:
697  hp, ht, pid, tid = CreateProcess(executable, args,
698  # no special security
699  None, None,
700  # must inherit handles to pass std
701  # handles
702  1,
703  creationflags,
704  env,
705  cwd,
706  startupinfo)
707  except pywintypes.error, e:
708  # Translate pywintypes.error to WindowsError, which is
709  # a subclass of OSError. FIXME: We should really
710  # translate errno using _sys_errlist (or simliar), but
711  # how can this be done from Python?
712  raise WindowsError(*e.args)
713 
714  # Retain the process handle, but close the thread handle
715  self._handle = hp
716  self.pid = pid
717  ht.Close()
718 
719  # Child is launched. Close the parent's copy of those pipe
720  # handles that only the child should have open. You need
721  # to make sure that no handles to the write end of the
722  # output pipe are maintained in this process or else the
723  # pipe will not close when the child process exits and the
724  # ReadFile will hang.
725  if p2cread != None:
726  p2cread.Close()
727  if c2pwrite != None:
728  c2pwrite.Close()
729  if errwrite != None:
730  errwrite.Close()
731 
732 
733  def poll(self):
734  """Check if child process has terminated. Returns returncode
735  attribute."""
736  if self.returncode == None:
737  if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0:
738  self.returncode = GetExitCodeProcess(self._handle)
739  _active.remove(self)
740  return self.returncode
741 
742 
743  def wait(self):
744  """Wait for child process to terminate. Returns returncode
745  attribute."""
746  if self.returncode == None:
747  obj = WaitForSingleObject(self._handle, INFINITE)
748  self.returncode = GetExitCodeProcess(self._handle)
749  _active.remove(self)
750  return self.returncode
751 
752 
753  def _readerthread(self, fh, buffer):
754  buffer.append(fh.read())
755 
756 
757  def communicate(self, input=None):
758  """Interact with process: Send data to stdin. Read data from
759  stdout and stderr, until end-of-file is reached. Wait for
760  process to terminate. The optional input argument should be a
761  string to be sent to the child process, or None, if no data
762  should be sent to the child.
763 
764  communicate() returns a tuple (stdout, stderr)."""
765  stdout = None # Return
766  stderr = None # Return
767 
768  if self.stdout:
769  stdout = []
770  stdout_thread = threading.Thread(target=self._readerthread,
771  args=(self.stdout, stdout))
772  stdout_thread.setDaemon(True)
773  stdout_thread.start()
774  if self.stderr:
775  stderr = []
776  stderr_thread = threading.Thread(target=self._readerthread,
777  args=(self.stderr, stderr))
778  stderr_thread.setDaemon(True)
779  stderr_thread.start()
780 
781  if self.stdin:
782  if input != None:
783  self.stdin.write(input)
784  self.stdin.close()
785 
786  if self.stdout:
787  stdout_thread.join()
788  if self.stderr:
789  stderr_thread.join()
790 
791  # All data exchanged. Translate lists into strings.
792  if stdout != None:
793  stdout = stdout[0]
794  if stderr != None:
795  stderr = stderr[0]
796 
797  # Translate newlines, if requested. We cannot let the file
798  # object do the translation: It is based on stdio, which is
799  # impossible to combine with select (unless forcing no
800  # buffering).
801  if self.universal_newlines and hasattr(open, 'newlines'):
802  if stdout:
803  stdout = self._translate_newlines(stdout)
804  if stderr:
805  stderr = self._translate_newlines(stderr)
806 
807  self.wait()
808  return (stdout, stderr)
809 
810  else:
811  #
812  # POSIX methods
813  #
814  def _get_handles(self, stdin, stdout, stderr):
815  """Construct and return tupel with IO objects:
816  p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
817  """
818  p2cread, p2cwrite = None, None
819  c2pread, c2pwrite = None, None
820  errread, errwrite = None, None
821 
822  if stdin == None:
823  pass
824  elif stdin == PIPE:
825  p2cread, p2cwrite = os.pipe()
826  elif type(stdin) == types.IntType:
827  p2cread = stdin
828  else:
829  # Assuming file-like object
830  p2cread = stdin.fileno()
831 
832  if stdout == None:
833  pass
834  elif stdout == PIPE:
835  c2pread, c2pwrite = os.pipe()
836  elif type(stdout) == types.IntType:
837  c2pwrite = stdout
838  else:
839  # Assuming file-like object
840  c2pwrite = stdout.fileno()
841 
842  if stderr == None:
843  pass
844  elif stderr == PIPE:
845  errread, errwrite = os.pipe()
846  elif stderr == STDOUT:
847  errwrite = c2pwrite
848  elif type(stderr) == types.IntType:
849  errwrite = stderr
850  else:
851  # Assuming file-like object
852  errwrite = stderr.fileno()
853 
854  return (p2cread, p2cwrite,
855  c2pread, c2pwrite,
856  errread, errwrite)
857 
858 
859  def _set_cloexec_flag(self, fd):
860  try:
861  cloexec_flag = fcntl.FD_CLOEXEC
862  except AttributeError:
863  cloexec_flag = 1
864 
865  old = fcntl.fcntl(fd, fcntl.F_GETFD)
866  fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag)
867 
868 
869  def _close_fds(self, but):
870  for i in xrange(3, MAXFD):
871  if i == but:
872  continue
873  try:
874  os.close(i)
875  except:
876  pass
877 
878 
879  def _execute_child(self, args, executable, preexec_fn, close_fds,
880  cwd, env, universal_newlines,
881  startupinfo, creationflags, shell,
882  p2cread, p2cwrite,
883  c2pread, c2pwrite,
884  errread, errwrite):
885  """Execute program (POSIX version)"""
886 
887  if isinstance(args, types.StringTypes):
888  args = [args]
889 
890  if shell:
891  args = ["/bin/sh", "-c"] + args
892 
893  if executable == None:
894  executable = args[0]
895 
896  # For transferring possible exec failure from child to parent
897  # The first char specifies the exception type: 0 means
898  # OSError, 1 means some other error.
899  errpipe_read, errpipe_write = os.pipe()
900  self._set_cloexec_flag(errpipe_write)
901 
902  self.pid = os.fork()
903  if self.pid == 0:
904  # Child
905  try:
906  # Close parent's pipe ends
907  if p2cwrite:
908  os.close(p2cwrite)
909  if c2pread:
910  os.close(c2pread)
911  if errread:
912  os.close(errread)
913  os.close(errpipe_read)
914 
915  # Dup fds for child
916  if p2cread:
917  os.dup2(p2cread, 0)
918  if c2pwrite:
919  os.dup2(c2pwrite, 1)
920  if errwrite:
921  os.dup2(errwrite, 2)
922 
923  # Close pipe fds. Make sure we doesn't close the same
924  # fd more than once.
925  if p2cread:
926  os.close(p2cread)
927  if c2pwrite and c2pwrite not in (p2cread,):
928  os.close(c2pwrite)
929  if errwrite and errwrite not in (p2cread, c2pwrite):
930  os.close(errwrite)
931 
932  # Close all other fds, if asked for
933  if close_fds:
934  self._close_fds(but=errpipe_write)
935 
936  if cwd != None:
937  os.chdir(cwd)
938 
939  if preexec_fn:
940  apply(preexec_fn)
941 
942  if env == None:
943  os.execvp(executable, args)
944  else:
945  os.execvpe(executable, args, env)
946 
947  except:
948  exc_type, exc_value, tb = sys.exc_info()
949  # Save the traceback and attach it to the exception object
950  exc_lines = traceback.format_exception(exc_type,
951  exc_value,
952  tb)
953  exc_value.child_traceback = ''.join(exc_lines)
954  os.write(errpipe_write, pickle.dumps(exc_value))
955 
956  # This exitcode won't be reported to applications, so it
957  # really doesn't matter what we return.
958  os._exit(255)
959 
960  # Parent
961  os.close(errpipe_write)
962  if p2cread and p2cwrite:
963  os.close(p2cread)
964  if c2pwrite and c2pread:
965  os.close(c2pwrite)
966  if errwrite and errread:
967  os.close(errwrite)
968 
969  # Wait for exec to fail or succeed; possibly raising exception
970  data = os.read(errpipe_read, 1048576) # Exceptions limited to 1 MB
971  os.close(errpipe_read)
972  if data != "":
973  os.waitpid(self.pid, 0)
974  child_exception = pickle.loads(data)
975  raise child_exception
976 
977 
978  def _handle_exitstatus(self, sts):
979  if os.WIFSIGNALED(sts):
980  self.returncode = -os.WTERMSIG(sts)
981  elif os.WIFEXITED(sts):
982  self.returncode = os.WEXITSTATUS(sts)
983  else:
984  # Should never happen
985  raise RuntimeError("Unknown child exit status!")
986 
987  _active.remove(self)
988 
989 
990  def poll(self):
991  """Check if child process has terminated. Returns returncode
992  attribute."""
993  if self.returncode == None:
994  try:
995  pid, sts = os.waitpid(self.pid, os.WNOHANG)
996  if pid == self.pid:
997  self._handle_exitstatus(sts)
998  except os.error:
999  pass
1000  return self.returncode
1001 
1002 
1003  def wait(self):
1004  """Wait for child process to terminate. Returns returncode
1005  attribute."""
1006  if self.returncode == None:
1007  pid, sts = os.waitpid(self.pid, 0)
1008  self._handle_exitstatus(sts)
1009  return self.returncode
1010 
1011 
1012  def communicate(self, input=None):
1013  """Interact with process: Send data to stdin. Read data from
1014  stdout and stderr, until end-of-file is reached. Wait for
1015  process to terminate. The optional input argument should be a
1016  string to be sent to the child process, or None, if no data
1017  should be sent to the child.
1018 
1019  communicate() returns a tuple (stdout, stderr)."""
1020  read_set = []
1021  write_set = []
1022  stdout = None # Return
1023  stderr = None # Return
1024 
1025  if self.stdin:
1026  # Flush stdio buffer. This might block, if the user has
1027  # been writing to .stdin in an uncontrolled fashion.
1028  self.stdin.flush()
1029  if input:
1030  write_set.append(self.stdin)
1031  else:
1032  self.stdin.close()
1033  if self.stdout:
1034  read_set.append(self.stdout)
1035  stdout = []
1036  if self.stderr:
1037  read_set.append(self.stderr)
1038  stderr = []
1039 
1040  while read_set or write_set:
1041  rlist, wlist, xlist = select.select(read_set, write_set, [])
1042 
1043  if self.stdin in wlist:
1044  # When select has indicated that the file is writable,
1045  # we can write up to PIPE_BUF bytes without risk
1046  # blocking. POSIX defines PIPE_BUF >= 512
1047  bytes_written = os.write(self.stdin.fileno(), input[:512])
1048  input = input[bytes_written:]
1049  if not input:
1050  self.stdin.close()
1051  write_set.remove(self.stdin)
1052 
1053  if self.stdout in rlist:
1054  data = os.read(self.stdout.fileno(), 1024)
1055  if data == "":
1056  self.stdout.close()
1057  read_set.remove(self.stdout)
1058  stdout.append(data)
1059 
1060  if self.stderr in rlist:
1061  data = os.read(self.stderr.fileno(), 1024)
1062  if data == "":
1063  self.stderr.close()
1064  read_set.remove(self.stderr)
1065  stderr.append(data)
1066 
1067  # All data exchanged. Translate lists into strings.
1068  if stdout != None:
1069  stdout = ''.join(stdout)
1070  if stderr != None:
1071  stderr = ''.join(stderr)
1072 
1073  # Translate newlines, if requested. We cannot let the file
1074  # object do the translation: It is based on stdio, which is
1075  # impossible to combine with select (unless forcing no
1076  # buffering).
1077  if self.universal_newlines and hasattr(open, 'newlines'):
1078  if stdout:
1079  stdout = self._translate_newlines(stdout)
1080  if stderr:
1081  stderr = self._translate_newlines(stderr)
1082 
1083  self.wait()
1084  return (stdout, stderr)
1085 
1086 
1087 def _demo_posix():
1088  #
1089  # Example 1: Simple redirection: Get process list
1090  #
1091  plist = Popen(["ps"], stdout=PIPE).communicate()[0]
1092  print "Process list:"
1093  print plist
1094 
1095  #
1096  # Example 2: Change uid before executing child
1097  #
1098  if os.getuid() == 0:
1099  p = Popen(["id"], preexec_fn=lambda: os.setuid(100))
1100  p.wait()
1101 
1102  #
1103  # Example 3: Connecting several subprocesses
1104  #
1105  print "Looking for 'hda'..."
1106  p1 = Popen(["dmesg"], stdout=PIPE)
1107  p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
1108  print repr(p2.communicate()[0])
1109 
1110  #
1111  # Example 4: Catch execution error
1112  #
1113  print
1114  print "Trying a weird file..."
1115  try:
1116  print Popen(["/this/path/does/not/exist"]).communicate()
1117  except OSError, e:
1118  if e.errno == errno.ENOENT:
1119  print "The file didn't exist. I thought so..."
1120  print "Child traceback:"
1121  print e.child_traceback
1122  else:
1123  print "Error", e.errno
1124  else:
1125  print >>sys.stderr, "Gosh. No error."
1126 
1127 
1128 def _demo_windows():
1129  #
1130  # Example 1: Connecting several subprocesses
1131  #
1132  print "Looking for 'PROMPT' in set output..."
1133  p1 = Popen("set", stdout=PIPE, shell=True)
1134  p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE)
1135  print repr(p2.communicate()[0])
1136 
1137  #
1138  # Example 2: Simple execution of program
1139  #
1140  print "Executing calc..."
1141  p = Popen("calc")
1142  p.wait()
1143 
1144 
1145 if __name__ == "__main__":
1146  if mswindows:
1147  _demo_windows()
1148  else:
1149  _demo_posix()