noseclasses.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366
  1. # These classes implement a doctest runner plugin for nose, a "known failure"
  2. # error class, and a customized TestProgram for NumPy.
  3. # Because this module imports nose directly, it should not
  4. # be used except by nosetester.py to avoid a general NumPy
  5. # dependency on nose.
  6. from __future__ import division, absolute_import, print_function
  7. import os
  8. import sys
  9. import doctest
  10. import inspect
  11. import numpy
  12. import nose
  13. from nose.plugins import doctests as npd
  14. from nose.plugins.errorclass import ErrorClass, ErrorClassPlugin
  15. from nose.plugins.base import Plugin
  16. from nose.util import src
  17. from .nosetester import get_package_name
  18. from .utils import KnownFailureException, KnownFailureTest
  19. # Some of the classes in this module begin with 'Numpy' to clearly distinguish
  20. # them from the plethora of very similar names from nose/unittest/doctest
  21. #-----------------------------------------------------------------------------
  22. # Modified version of the one in the stdlib, that fixes a python bug (doctests
  23. # not found in extension modules, https://bugs.python.org/issue3158)
  24. class NumpyDocTestFinder(doctest.DocTestFinder):
  25. def _from_module(self, module, object):
  26. """
  27. Return true if the given object is defined in the given
  28. module.
  29. """
  30. if module is None:
  31. return True
  32. elif inspect.isfunction(object):
  33. return module.__dict__ is object.__globals__
  34. elif inspect.isbuiltin(object):
  35. return module.__name__ == object.__module__
  36. elif inspect.isclass(object):
  37. return module.__name__ == object.__module__
  38. elif inspect.ismethod(object):
  39. # This one may be a bug in cython that fails to correctly set the
  40. # __module__ attribute of methods, but since the same error is easy
  41. # to make by extension code writers, having this safety in place
  42. # isn't such a bad idea
  43. return module.__name__ == object.__self__.__class__.__module__
  44. elif inspect.getmodule(object) is not None:
  45. return module is inspect.getmodule(object)
  46. elif hasattr(object, '__module__'):
  47. return module.__name__ == object.__module__
  48. elif isinstance(object, property):
  49. return True # [XX] no way not be sure.
  50. else:
  51. raise ValueError("object must be a class or function")
  52. def _find(self, tests, obj, name, module, source_lines, globs, seen):
  53. """
  54. Find tests for the given object and any contained objects, and
  55. add them to `tests`.
  56. """
  57. doctest.DocTestFinder._find(self, tests, obj, name, module,
  58. source_lines, globs, seen)
  59. # Below we re-run pieces of the above method with manual modifications,
  60. # because the original code is buggy and fails to correctly identify
  61. # doctests in extension modules.
  62. # Local shorthands
  63. from inspect import (
  64. isroutine, isclass, ismodule, isfunction, ismethod
  65. )
  66. # Look for tests in a module's contained objects.
  67. if ismodule(obj) and self._recurse:
  68. for valname, val in obj.__dict__.items():
  69. valname1 = '%s.%s' % (name, valname)
  70. if ( (isroutine(val) or isclass(val))
  71. and self._from_module(module, val)):
  72. self._find(tests, val, valname1, module, source_lines,
  73. globs, seen)
  74. # Look for tests in a class's contained objects.
  75. if isclass(obj) and self._recurse:
  76. for valname, val in obj.__dict__.items():
  77. # Special handling for staticmethod/classmethod.
  78. if isinstance(val, staticmethod):
  79. val = getattr(obj, valname)
  80. if isinstance(val, classmethod):
  81. val = getattr(obj, valname).__func__
  82. # Recurse to methods, properties, and nested classes.
  83. if ((isfunction(val) or isclass(val) or
  84. ismethod(val) or isinstance(val, property)) and
  85. self._from_module(module, val)):
  86. valname = '%s.%s' % (name, valname)
  87. self._find(tests, val, valname, module, source_lines,
  88. globs, seen)
  89. # second-chance checker; if the default comparison doesn't
  90. # pass, then see if the expected output string contains flags that
  91. # tell us to ignore the output
  92. class NumpyOutputChecker(doctest.OutputChecker):
  93. def check_output(self, want, got, optionflags):
  94. ret = doctest.OutputChecker.check_output(self, want, got,
  95. optionflags)
  96. if not ret:
  97. if "#random" in want:
  98. return True
  99. # it would be useful to normalize endianness so that
  100. # bigendian machines don't fail all the tests (and there are
  101. # actually some bigendian examples in the doctests). Let's try
  102. # making them all little endian
  103. got = got.replace("'>", "'<")
  104. want = want.replace("'>", "'<")
  105. # try to normalize out 32 and 64 bit default int sizes
  106. for sz in [4, 8]:
  107. got = got.replace("'<i%d'" % sz, "int")
  108. want = want.replace("'<i%d'" % sz, "int")
  109. ret = doctest.OutputChecker.check_output(self, want,
  110. got, optionflags)
  111. return ret
  112. # Subclass nose.plugins.doctests.DocTestCase to work around a bug in
  113. # its constructor that blocks non-default arguments from being passed
  114. # down into doctest.DocTestCase
  115. class NumpyDocTestCase(npd.DocTestCase):
  116. def __init__(self, test, optionflags=0, setUp=None, tearDown=None,
  117. checker=None, obj=None, result_var='_'):
  118. self._result_var = result_var
  119. self._nose_obj = obj
  120. doctest.DocTestCase.__init__(self, test,
  121. optionflags=optionflags,
  122. setUp=setUp, tearDown=tearDown,
  123. checker=checker)
  124. print_state = numpy.get_printoptions()
  125. class NumpyDoctest(npd.Doctest):
  126. name = 'numpydoctest' # call nosetests with --with-numpydoctest
  127. score = 1000 # load late, after doctest builtin
  128. # always use whitespace and ellipsis options for doctests
  129. doctest_optflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
  130. # files that should be ignored for doctests
  131. doctest_ignore = ['generate_numpy_api.py',
  132. 'setup.py']
  133. # Custom classes; class variables to allow subclassing
  134. doctest_case_class = NumpyDocTestCase
  135. out_check_class = NumpyOutputChecker
  136. test_finder_class = NumpyDocTestFinder
  137. # Don't use the standard doctest option handler; hard-code the option values
  138. def options(self, parser, env=os.environ):
  139. Plugin.options(self, parser, env)
  140. # Test doctests in 'test' files / directories. Standard plugin default
  141. # is False
  142. self.doctest_tests = True
  143. # Variable name; if defined, doctest results stored in this variable in
  144. # the top-level namespace. None is the standard default
  145. self.doctest_result_var = None
  146. def configure(self, options, config):
  147. # parent method sets enabled flag from command line --with-numpydoctest
  148. Plugin.configure(self, options, config)
  149. self.finder = self.test_finder_class()
  150. self.parser = doctest.DocTestParser()
  151. if self.enabled:
  152. # Pull standard doctest out of plugin list; there's no reason to run
  153. # both. In practice the Unplugger plugin above would cover us when
  154. # run from a standard numpy.test() call; this is just in case
  155. # someone wants to run our plugin outside the numpy.test() machinery
  156. config.plugins.plugins = [p for p in config.plugins.plugins
  157. if p.name != 'doctest']
  158. def set_test_context(self, test):
  159. """ Configure `test` object to set test context
  160. We set the numpy / scipy standard doctest namespace
  161. Parameters
  162. ----------
  163. test : test object
  164. with ``globs`` dictionary defining namespace
  165. Returns
  166. -------
  167. None
  168. Notes
  169. -----
  170. `test` object modified in place
  171. """
  172. # set the namespace for tests
  173. pkg_name = get_package_name(os.path.dirname(test.filename))
  174. # Each doctest should execute in an environment equivalent to
  175. # starting Python and executing "import numpy as np", and,
  176. # for SciPy packages, an additional import of the local
  177. # package (so that scipy.linalg.basic.py's doctests have an
  178. # implicit "from scipy import linalg" as well.
  179. #
  180. # Note: __file__ allows the doctest in NoseTester to run
  181. # without producing an error
  182. test.globs = {'__builtins__':__builtins__,
  183. '__file__':'__main__',
  184. '__name__':'__main__',
  185. 'np':numpy}
  186. # add appropriate scipy import for SciPy tests
  187. if 'scipy' in pkg_name:
  188. p = pkg_name.split('.')
  189. p2 = p[-1]
  190. test.globs[p2] = __import__(pkg_name, test.globs, {}, [p2])
  191. # Override test loading to customize test context (with set_test_context
  192. # method), set standard docstring options, and install our own test output
  193. # checker
  194. def loadTestsFromModule(self, module):
  195. if not self.matches(module.__name__):
  196. npd.log.debug("Doctest doesn't want module %s", module)
  197. return
  198. try:
  199. tests = self.finder.find(module)
  200. except AttributeError:
  201. # nose allows module.__test__ = False; doctest does not and
  202. # throws AttributeError
  203. return
  204. if not tests:
  205. return
  206. tests.sort()
  207. module_file = src(module.__file__)
  208. for test in tests:
  209. if not test.examples:
  210. continue
  211. if not test.filename:
  212. test.filename = module_file
  213. # Set test namespace; test altered in place
  214. self.set_test_context(test)
  215. yield self.doctest_case_class(test,
  216. optionflags=self.doctest_optflags,
  217. checker=self.out_check_class(),
  218. result_var=self.doctest_result_var)
  219. # Add an afterContext method to nose.plugins.doctests.Doctest in order
  220. # to restore print options to the original state after each doctest
  221. def afterContext(self):
  222. numpy.set_printoptions(**print_state)
  223. # Ignore NumPy-specific build files that shouldn't be searched for tests
  224. def wantFile(self, file):
  225. bn = os.path.basename(file)
  226. if bn in self.doctest_ignore:
  227. return False
  228. return npd.Doctest.wantFile(self, file)
  229. class Unplugger(object):
  230. """ Nose plugin to remove named plugin late in loading
  231. By default it removes the "doctest" plugin.
  232. """
  233. name = 'unplugger'
  234. enabled = True # always enabled
  235. score = 4000 # load late in order to be after builtins
  236. def __init__(self, to_unplug='doctest'):
  237. self.to_unplug = to_unplug
  238. def options(self, parser, env):
  239. pass
  240. def configure(self, options, config):
  241. # Pull named plugin out of plugins list
  242. config.plugins.plugins = [p for p in config.plugins.plugins
  243. if p.name != self.to_unplug]
  244. class KnownFailurePlugin(ErrorClassPlugin):
  245. '''Plugin that installs a KNOWNFAIL error class for the
  246. KnownFailureClass exception. When KnownFailure is raised,
  247. the exception will be logged in the knownfail attribute of the
  248. result, 'K' or 'KNOWNFAIL' (verbose) will be output, and the
  249. exception will not be counted as an error or failure.'''
  250. enabled = True
  251. knownfail = ErrorClass(KnownFailureException,
  252. label='KNOWNFAIL',
  253. isfailure=False)
  254. def options(self, parser, env=os.environ):
  255. env_opt = 'NOSE_WITHOUT_KNOWNFAIL'
  256. parser.add_option('--no-knownfail', action='store_true',
  257. dest='noKnownFail', default=env.get(env_opt, False),
  258. help='Disable special handling of KnownFailure '
  259. 'exceptions')
  260. def configure(self, options, conf):
  261. if not self.can_configure:
  262. return
  263. self.conf = conf
  264. disable = getattr(options, 'noKnownFail', False)
  265. if disable:
  266. self.enabled = False
  267. KnownFailure = KnownFailurePlugin # backwards compat
  268. class FPUModeCheckPlugin(Plugin):
  269. """
  270. Plugin that checks the FPU mode before and after each test,
  271. raising failures if the test changed the mode.
  272. """
  273. def prepareTestCase(self, test):
  274. from numpy.core._multiarray_tests import get_fpu_mode
  275. def run(result):
  276. old_mode = get_fpu_mode()
  277. test.test(result)
  278. new_mode = get_fpu_mode()
  279. if old_mode != new_mode:
  280. try:
  281. raise AssertionError(
  282. "FPU mode changed from {0:#x} to {1:#x} during the "
  283. "test".format(old_mode, new_mode))
  284. except AssertionError:
  285. result.addFailure(test, sys.exc_info())
  286. return run
  287. # Class allows us to save the results of the tests in runTests - see runTests
  288. # method docstring for details
  289. class NumpyTestProgram(nose.core.TestProgram):
  290. def runTests(self):
  291. """Run Tests. Returns true on success, false on failure, and
  292. sets self.success to the same value.
  293. Because nose currently discards the test result object, but we need
  294. to return it to the user, override TestProgram.runTests to retain
  295. the result
  296. """
  297. if self.testRunner is None:
  298. self.testRunner = nose.core.TextTestRunner(stream=self.config.stream,
  299. verbosity=self.config.verbosity,
  300. config=self.config)
  301. plug_runner = self.config.plugins.prepareTestRunner(self.testRunner)
  302. if plug_runner is not None:
  303. self.testRunner = plug_runner
  304. self.result = self.testRunner.run(self.test)
  305. self.success = self.result.wasSuccessful()
  306. return self.success