exec_command.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330
  1. """
  2. exec_command
  3. Implements exec_command function that is (almost) equivalent to
  4. commands.getstatusoutput function but on NT, DOS systems the
  5. returned status is actually correct (though, the returned status
  6. values may be different by a factor). In addition, exec_command
  7. takes keyword arguments for (re-)defining environment variables.
  8. Provides functions:
  9. exec_command --- execute command in a specified directory and
  10. in the modified environment.
  11. find_executable --- locate a command using info from environment
  12. variable PATH. Equivalent to posix `which`
  13. command.
  14. Author: Pearu Peterson <pearu@cens.ioc.ee>
  15. Created: 11 January 2003
  16. Requires: Python 2.x
  17. Successfully tested on:
  18. ======== ============ =================================================
  19. os.name sys.platform comments
  20. ======== ============ =================================================
  21. posix linux2 Debian (sid) Linux, Python 2.1.3+, 2.2.3+, 2.3.3
  22. PyCrust 0.9.3, Idle 1.0.2
  23. posix linux2 Red Hat 9 Linux, Python 2.1.3, 2.2.2, 2.3.2
  24. posix sunos5 SunOS 5.9, Python 2.2, 2.3.2
  25. posix darwin Darwin 7.2.0, Python 2.3
  26. nt win32 Windows Me
  27. Python 2.3(EE), Idle 1.0, PyCrust 0.7.2
  28. Python 2.1.1 Idle 0.8
  29. nt win32 Windows 98, Python 2.1.1. Idle 0.8
  30. nt win32 Cygwin 98-4.10, Python 2.1.1(MSC) - echo tests
  31. fail i.e. redefining environment variables may
  32. not work. FIXED: don't use cygwin echo!
  33. Comment: also `cmd /c echo` will not work
  34. but redefining environment variables do work.
  35. posix cygwin Cygwin 98-4.10, Python 2.3.3(cygming special)
  36. nt win32 Windows XP, Python 2.3.3
  37. ======== ============ =================================================
  38. Known bugs:
  39. * Tests, that send messages to stderr, fail when executed from MSYS prompt
  40. because the messages are lost at some point.
  41. """
  42. from __future__ import division, absolute_import, print_function
  43. __all__ = ['exec_command', 'find_executable']
  44. import os
  45. import sys
  46. import subprocess
  47. import locale
  48. import warnings
  49. from numpy.distutils.misc_util import is_sequence, make_temp_file
  50. from numpy.distutils import log
  51. def filepath_from_subprocess_output(output):
  52. """
  53. Convert `bytes` in the encoding used by a subprocess into a filesystem-appropriate `str`.
  54. Inherited from `exec_command`, and possibly incorrect.
  55. """
  56. mylocale = locale.getpreferredencoding(False)
  57. if mylocale is None:
  58. mylocale = 'ascii'
  59. output = output.decode(mylocale, errors='replace')
  60. output = output.replace('\r\n', '\n')
  61. # Another historical oddity
  62. if output[-1:] == '\n':
  63. output = output[:-1]
  64. # stdio uses bytes in python 2, so to avoid issues, we simply
  65. # remove all non-ascii characters
  66. if sys.version_info < (3, 0):
  67. output = output.encode('ascii', errors='replace')
  68. return output
  69. def forward_bytes_to_stdout(val):
  70. """
  71. Forward bytes from a subprocess call to the console, without attempting to
  72. decode them.
  73. The assumption is that the subprocess call already returned bytes in
  74. a suitable encoding.
  75. """
  76. if sys.version_info.major < 3:
  77. # python 2 has binary output anyway
  78. sys.stdout.write(val)
  79. elif hasattr(sys.stdout, 'buffer'):
  80. # use the underlying binary output if there is one
  81. sys.stdout.buffer.write(val)
  82. elif hasattr(sys.stdout, 'encoding'):
  83. # round-trip the encoding if necessary
  84. sys.stdout.write(val.decode(sys.stdout.encoding))
  85. else:
  86. # make a best-guess at the encoding
  87. sys.stdout.write(val.decode('utf8', errors='replace'))
  88. def temp_file_name():
  89. # 2019-01-30, 1.17
  90. warnings.warn('temp_file_name is deprecated since NumPy v1.17, use '
  91. 'tempfile.mkstemp instead', DeprecationWarning, stacklevel=1)
  92. fo, name = make_temp_file()
  93. fo.close()
  94. return name
  95. def get_pythonexe():
  96. pythonexe = sys.executable
  97. if os.name in ['nt', 'dos']:
  98. fdir, fn = os.path.split(pythonexe)
  99. fn = fn.upper().replace('PYTHONW', 'PYTHON')
  100. pythonexe = os.path.join(fdir, fn)
  101. assert os.path.isfile(pythonexe), '%r is not a file' % (pythonexe,)
  102. return pythonexe
  103. def find_executable(exe, path=None, _cache={}):
  104. """Return full path of a executable or None.
  105. Symbolic links are not followed.
  106. """
  107. key = exe, path
  108. try:
  109. return _cache[key]
  110. except KeyError:
  111. pass
  112. log.debug('find_executable(%r)' % exe)
  113. orig_exe = exe
  114. if path is None:
  115. path = os.environ.get('PATH', os.defpath)
  116. if os.name=='posix':
  117. realpath = os.path.realpath
  118. else:
  119. realpath = lambda a:a
  120. if exe.startswith('"'):
  121. exe = exe[1:-1]
  122. suffixes = ['']
  123. if os.name in ['nt', 'dos', 'os2']:
  124. fn, ext = os.path.splitext(exe)
  125. extra_suffixes = ['.exe', '.com', '.bat']
  126. if ext.lower() not in extra_suffixes:
  127. suffixes = extra_suffixes
  128. if os.path.isabs(exe):
  129. paths = ['']
  130. else:
  131. paths = [ os.path.abspath(p) for p in path.split(os.pathsep) ]
  132. for path in paths:
  133. fn = os.path.join(path, exe)
  134. for s in suffixes:
  135. f_ext = fn+s
  136. if not os.path.islink(f_ext):
  137. f_ext = realpath(f_ext)
  138. if os.path.isfile(f_ext) and os.access(f_ext, os.X_OK):
  139. log.info('Found executable %s' % f_ext)
  140. _cache[key] = f_ext
  141. return f_ext
  142. log.warn('Could not locate executable %s' % orig_exe)
  143. return None
  144. ############################################################
  145. def _preserve_environment( names ):
  146. log.debug('_preserve_environment(%r)' % (names))
  147. env = {name: os.environ.get(name) for name in names}
  148. return env
  149. def _update_environment( **env ):
  150. log.debug('_update_environment(...)')
  151. for name, value in env.items():
  152. os.environ[name] = value or ''
  153. def exec_command(command, execute_in='', use_shell=None, use_tee=None,
  154. _with_python = 1, **env ):
  155. """
  156. Return (status,output) of executed command.
  157. .. deprecated:: 1.17
  158. Use subprocess.Popen instead
  159. Parameters
  160. ----------
  161. command : str
  162. A concatenated string of executable and arguments.
  163. execute_in : str
  164. Before running command ``cd execute_in`` and after ``cd -``.
  165. use_shell : {bool, None}, optional
  166. If True, execute ``sh -c command``. Default None (True)
  167. use_tee : {bool, None}, optional
  168. If True use tee. Default None (True)
  169. Returns
  170. -------
  171. res : str
  172. Both stdout and stderr messages.
  173. Notes
  174. -----
  175. On NT, DOS systems the returned status is correct for external commands.
  176. Wild cards will not work for non-posix systems or when use_shell=0.
  177. """
  178. # 2019-01-30, 1.17
  179. warnings.warn('exec_command is deprecated since NumPy v1.17, use '
  180. 'subprocess.Popen instead', DeprecationWarning, stacklevel=1)
  181. log.debug('exec_command(%r,%s)' % (command,
  182. ','.join(['%s=%r'%kv for kv in env.items()])))
  183. if use_tee is None:
  184. use_tee = os.name=='posix'
  185. if use_shell is None:
  186. use_shell = os.name=='posix'
  187. execute_in = os.path.abspath(execute_in)
  188. oldcwd = os.path.abspath(os.getcwd())
  189. if __name__[-12:] == 'exec_command':
  190. exec_dir = os.path.dirname(os.path.abspath(__file__))
  191. elif os.path.isfile('exec_command.py'):
  192. exec_dir = os.path.abspath('.')
  193. else:
  194. exec_dir = os.path.abspath(sys.argv[0])
  195. if os.path.isfile(exec_dir):
  196. exec_dir = os.path.dirname(exec_dir)
  197. if oldcwd!=execute_in:
  198. os.chdir(execute_in)
  199. log.debug('New cwd: %s' % execute_in)
  200. else:
  201. log.debug('Retaining cwd: %s' % oldcwd)
  202. oldenv = _preserve_environment( list(env.keys()) )
  203. _update_environment( **env )
  204. try:
  205. st = _exec_command(command,
  206. use_shell=use_shell,
  207. use_tee=use_tee,
  208. **env)
  209. finally:
  210. if oldcwd!=execute_in:
  211. os.chdir(oldcwd)
  212. log.debug('Restored cwd to %s' % oldcwd)
  213. _update_environment(**oldenv)
  214. return st
  215. def _exec_command(command, use_shell=None, use_tee = None, **env):
  216. """
  217. Internal workhorse for exec_command().
  218. """
  219. if use_shell is None:
  220. use_shell = os.name=='posix'
  221. if use_tee is None:
  222. use_tee = os.name=='posix'
  223. if os.name == 'posix' and use_shell:
  224. # On POSIX, subprocess always uses /bin/sh, override
  225. sh = os.environ.get('SHELL', '/bin/sh')
  226. if is_sequence(command):
  227. command = [sh, '-c', ' '.join(command)]
  228. else:
  229. command = [sh, '-c', command]
  230. use_shell = False
  231. elif os.name == 'nt' and is_sequence(command):
  232. # On Windows, join the string for CreateProcess() ourselves as
  233. # subprocess does it a bit differently
  234. command = ' '.join(_quote_arg(arg) for arg in command)
  235. # Inherit environment by default
  236. env = env or None
  237. try:
  238. # universal_newlines is set to False so that communicate()
  239. # will return bytes. We need to decode the output ourselves
  240. # so that Python will not raise a UnicodeDecodeError when
  241. # it encounters an invalid character; rather, we simply replace it
  242. proc = subprocess.Popen(command, shell=use_shell, env=env,
  243. stdout=subprocess.PIPE,
  244. stderr=subprocess.STDOUT,
  245. universal_newlines=False)
  246. except EnvironmentError:
  247. # Return 127, as os.spawn*() and /bin/sh do
  248. return 127, ''
  249. text, err = proc.communicate()
  250. mylocale = locale.getpreferredencoding(False)
  251. if mylocale is None:
  252. mylocale = 'ascii'
  253. text = text.decode(mylocale, errors='replace')
  254. text = text.replace('\r\n', '\n')
  255. # Another historical oddity
  256. if text[-1:] == '\n':
  257. text = text[:-1]
  258. # stdio uses bytes in python 2, so to avoid issues, we simply
  259. # remove all non-ascii characters
  260. if sys.version_info < (3, 0):
  261. text = text.encode('ascii', errors='replace')
  262. if use_tee and text:
  263. print(text)
  264. return proc.returncode, text
  265. def _quote_arg(arg):
  266. """
  267. Quote the argument for safe use in a shell command line.
  268. """
  269. # If there is a quote in the string, assume relevants parts of the
  270. # string are already quoted (e.g. '-I"C:\\Program Files\\..."')
  271. if '"' not in arg and ' ' in arg:
  272. return '"%s"' % arg
  273. return arg
  274. ############################################################