nosetester.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562
  1. """
  2. Nose test running.
  3. This module implements ``test()`` and ``bench()`` functions for NumPy modules.
  4. """
  5. from __future__ import division, absolute_import, print_function
  6. import os
  7. import sys
  8. import warnings
  9. from numpy.compat import basestring
  10. import numpy as np
  11. from .utils import import_nose, suppress_warnings
  12. __all__ = ['get_package_name', 'run_module_suite', 'NoseTester',
  13. '_numpy_tester', 'get_package_name', 'import_nose',
  14. 'suppress_warnings']
  15. def get_package_name(filepath):
  16. """
  17. Given a path where a package is installed, determine its name.
  18. Parameters
  19. ----------
  20. filepath : str
  21. Path to a file. If the determination fails, "numpy" is returned.
  22. Examples
  23. --------
  24. >>> np.testing.nosetester.get_package_name('nonsense')
  25. 'numpy'
  26. """
  27. fullpath = filepath[:]
  28. pkg_name = []
  29. while 'site-packages' in filepath or 'dist-packages' in filepath:
  30. filepath, p2 = os.path.split(filepath)
  31. if p2 in ('site-packages', 'dist-packages'):
  32. break
  33. pkg_name.append(p2)
  34. # if package name determination failed, just default to numpy/scipy
  35. if not pkg_name:
  36. if 'scipy' in fullpath:
  37. return 'scipy'
  38. else:
  39. return 'numpy'
  40. # otherwise, reverse to get correct order and return
  41. pkg_name.reverse()
  42. # don't include the outer egg directory
  43. if pkg_name[0].endswith('.egg'):
  44. pkg_name.pop(0)
  45. return '.'.join(pkg_name)
  46. def run_module_suite(file_to_run=None, argv=None):
  47. """
  48. Run a test module.
  49. Equivalent to calling ``$ nosetests <argv> <file_to_run>`` from
  50. the command line
  51. Parameters
  52. ----------
  53. file_to_run : str, optional
  54. Path to test module, or None.
  55. By default, run the module from which this function is called.
  56. argv : list of strings
  57. Arguments to be passed to the nose test runner. ``argv[0]`` is
  58. ignored. All command line arguments accepted by ``nosetests``
  59. will work. If it is the default value None, sys.argv is used.
  60. .. versionadded:: 1.9.0
  61. Examples
  62. --------
  63. Adding the following::
  64. if __name__ == "__main__" :
  65. run_module_suite(argv=sys.argv)
  66. at the end of a test module will run the tests when that module is
  67. called in the python interpreter.
  68. Alternatively, calling::
  69. >>> run_module_suite(file_to_run="numpy/tests/test_matlib.py") # doctest: +SKIP
  70. from an interpreter will run all the test routine in 'test_matlib.py'.
  71. """
  72. if file_to_run is None:
  73. f = sys._getframe(1)
  74. file_to_run = f.f_locals.get('__file__', None)
  75. if file_to_run is None:
  76. raise AssertionError
  77. if argv is None:
  78. argv = sys.argv + [file_to_run]
  79. else:
  80. argv = argv + [file_to_run]
  81. nose = import_nose()
  82. from .noseclasses import KnownFailurePlugin
  83. nose.run(argv=argv, addplugins=[KnownFailurePlugin()])
  84. class NoseTester(object):
  85. """
  86. Nose test runner.
  87. This class is made available as numpy.testing.Tester, and a test function
  88. is typically added to a package's __init__.py like so::
  89. from numpy.testing import Tester
  90. test = Tester().test
  91. Calling this test function finds and runs all tests associated with the
  92. package and all its sub-packages.
  93. Attributes
  94. ----------
  95. package_path : str
  96. Full path to the package to test.
  97. package_name : str
  98. Name of the package to test.
  99. Parameters
  100. ----------
  101. package : module, str or None, optional
  102. The package to test. If a string, this should be the full path to
  103. the package. If None (default), `package` is set to the module from
  104. which `NoseTester` is initialized.
  105. raise_warnings : None, str or sequence of warnings, optional
  106. This specifies which warnings to configure as 'raise' instead
  107. of being shown once during the test execution. Valid strings are:
  108. - "develop" : equals ``(Warning,)``
  109. - "release" : equals ``()``, don't raise on any warnings.
  110. Default is "release".
  111. depth : int, optional
  112. If `package` is None, then this can be used to initialize from the
  113. module of the caller of (the caller of (...)) the code that
  114. initializes `NoseTester`. Default of 0 means the module of the
  115. immediate caller; higher values are useful for utility routines that
  116. want to initialize `NoseTester` objects on behalf of other code.
  117. """
  118. def __init__(self, package=None, raise_warnings="release", depth=0,
  119. check_fpu_mode=False):
  120. # Back-compat: 'None' used to mean either "release" or "develop"
  121. # depending on whether this was a release or develop version of
  122. # numpy. Those semantics were fine for testing numpy, but not so
  123. # helpful for downstream projects like scipy that use
  124. # numpy.testing. (They want to set this based on whether *they* are a
  125. # release or develop version, not whether numpy is.) So we continue to
  126. # accept 'None' for back-compat, but it's now just an alias for the
  127. # default "release".
  128. if raise_warnings is None:
  129. raise_warnings = "release"
  130. package_name = None
  131. if package is None:
  132. f = sys._getframe(1 + depth)
  133. package_path = f.f_locals.get('__file__', None)
  134. if package_path is None:
  135. raise AssertionError
  136. package_path = os.path.dirname(package_path)
  137. package_name = f.f_locals.get('__name__', None)
  138. elif isinstance(package, type(os)):
  139. package_path = os.path.dirname(package.__file__)
  140. package_name = getattr(package, '__name__', None)
  141. else:
  142. package_path = str(package)
  143. self.package_path = package_path
  144. # Find the package name under test; this name is used to limit coverage
  145. # reporting (if enabled).
  146. if package_name is None:
  147. package_name = get_package_name(package_path)
  148. self.package_name = package_name
  149. # Set to "release" in constructor in maintenance branches.
  150. self.raise_warnings = raise_warnings
  151. # Whether to check for FPU mode changes
  152. self.check_fpu_mode = check_fpu_mode
  153. def _test_argv(self, label, verbose, extra_argv):
  154. ''' Generate argv for nosetest command
  155. Parameters
  156. ----------
  157. label : {'fast', 'full', '', attribute identifier}, optional
  158. see ``test`` docstring
  159. verbose : int, optional
  160. Verbosity value for test outputs, in the range 1-10. Default is 1.
  161. extra_argv : list, optional
  162. List with any extra arguments to pass to nosetests.
  163. Returns
  164. -------
  165. argv : list
  166. command line arguments that will be passed to nose
  167. '''
  168. argv = [__file__, self.package_path, '-s']
  169. if label and label != 'full':
  170. if not isinstance(label, basestring):
  171. raise TypeError('Selection label should be a string')
  172. if label == 'fast':
  173. label = 'not slow'
  174. argv += ['-A', label]
  175. argv += ['--verbosity', str(verbose)]
  176. # When installing with setuptools, and also in some other cases, the
  177. # test_*.py files end up marked +x executable. Nose, by default, does
  178. # not run files marked with +x as they might be scripts. However, in
  179. # our case nose only looks for test_*.py files under the package
  180. # directory, which should be safe.
  181. argv += ['--exe']
  182. if extra_argv:
  183. argv += extra_argv
  184. return argv
  185. def _show_system_info(self):
  186. nose = import_nose()
  187. import numpy
  188. print("NumPy version %s" % numpy.__version__)
  189. relaxed_strides = numpy.ones((10, 1), order="C").flags.f_contiguous
  190. print("NumPy relaxed strides checking option:", relaxed_strides)
  191. npdir = os.path.dirname(numpy.__file__)
  192. print("NumPy is installed in %s" % npdir)
  193. if 'scipy' in self.package_name:
  194. import scipy
  195. print("SciPy version %s" % scipy.__version__)
  196. spdir = os.path.dirname(scipy.__file__)
  197. print("SciPy is installed in %s" % spdir)
  198. pyversion = sys.version.replace('\n', '')
  199. print("Python version %s" % pyversion)
  200. print("nose version %d.%d.%d" % nose.__versioninfo__)
  201. def _get_custom_doctester(self):
  202. """ Return instantiated plugin for doctests
  203. Allows subclassing of this class to override doctester
  204. A return value of None means use the nose builtin doctest plugin
  205. """
  206. from .noseclasses import NumpyDoctest
  207. return NumpyDoctest()
  208. def prepare_test_args(self, label='fast', verbose=1, extra_argv=None,
  209. doctests=False, coverage=False, timer=False):
  210. """
  211. Run tests for module using nose.
  212. This method does the heavy lifting for the `test` method. It takes all
  213. the same arguments, for details see `test`.
  214. See Also
  215. --------
  216. test
  217. """
  218. # fail with nice error message if nose is not present
  219. import_nose()
  220. # compile argv
  221. argv = self._test_argv(label, verbose, extra_argv)
  222. # our way of doing coverage
  223. if coverage:
  224. argv += ['--cover-package=%s' % self.package_name, '--with-coverage',
  225. '--cover-tests', '--cover-erase']
  226. if timer:
  227. if timer is True:
  228. argv += ['--with-timer']
  229. elif isinstance(timer, int):
  230. argv += ['--with-timer', '--timer-top-n', str(timer)]
  231. # construct list of plugins
  232. import nose.plugins.builtin
  233. from nose.plugins import EntryPointPluginManager
  234. from .noseclasses import (KnownFailurePlugin, Unplugger,
  235. FPUModeCheckPlugin)
  236. plugins = [KnownFailurePlugin()]
  237. plugins += [p() for p in nose.plugins.builtin.plugins]
  238. if self.check_fpu_mode:
  239. plugins += [FPUModeCheckPlugin()]
  240. argv += ["--with-fpumodecheckplugin"]
  241. try:
  242. # External plugins (like nose-timer)
  243. entrypoint_manager = EntryPointPluginManager()
  244. entrypoint_manager.loadPlugins()
  245. plugins += [p for p in entrypoint_manager.plugins]
  246. except ImportError:
  247. # Relies on pkg_resources, not a hard dependency
  248. pass
  249. # add doctesting if required
  250. doctest_argv = '--with-doctest' in argv
  251. if doctests == False and doctest_argv:
  252. doctests = True
  253. plug = self._get_custom_doctester()
  254. if plug is None:
  255. # use standard doctesting
  256. if doctests and not doctest_argv:
  257. argv += ['--with-doctest']
  258. else: # custom doctesting
  259. if doctest_argv: # in fact the unplugger would take care of this
  260. argv.remove('--with-doctest')
  261. plugins += [Unplugger('doctest'), plug]
  262. if doctests:
  263. argv += ['--with-' + plug.name]
  264. return argv, plugins
  265. def test(self, label='fast', verbose=1, extra_argv=None,
  266. doctests=False, coverage=False, raise_warnings=None,
  267. timer=False):
  268. """
  269. Run tests for module using nose.
  270. Parameters
  271. ----------
  272. label : {'fast', 'full', '', attribute identifier}, optional
  273. Identifies the tests to run. This can be a string to pass to
  274. the nosetests executable with the '-A' option, or one of several
  275. special values. Special values are:
  276. * 'fast' - the default - which corresponds to the ``nosetests -A``
  277. option of 'not slow'.
  278. * 'full' - fast (as above) and slow tests as in the
  279. 'no -A' option to nosetests - this is the same as ''.
  280. * None or '' - run all tests.
  281. * attribute_identifier - string passed directly to nosetests as '-A'.
  282. verbose : int, optional
  283. Verbosity value for test outputs, in the range 1-10. Default is 1.
  284. extra_argv : list, optional
  285. List with any extra arguments to pass to nosetests.
  286. doctests : bool, optional
  287. If True, run doctests in module. Default is False.
  288. coverage : bool, optional
  289. If True, report coverage of NumPy code. Default is False.
  290. (This requires the
  291. `coverage module <https://nedbatchelder.com/code/modules/coveragehtml>`_).
  292. raise_warnings : None, str or sequence of warnings, optional
  293. This specifies which warnings to configure as 'raise' instead
  294. of being shown once during the test execution. Valid strings are:
  295. * "develop" : equals ``(Warning,)``
  296. * "release" : equals ``()``, do not raise on any warnings.
  297. timer : bool or int, optional
  298. Timing of individual tests with ``nose-timer`` (which needs to be
  299. installed). If True, time tests and report on all of them.
  300. If an integer (say ``N``), report timing results for ``N`` slowest
  301. tests.
  302. Returns
  303. -------
  304. result : object
  305. Returns the result of running the tests as a
  306. ``nose.result.TextTestResult`` object.
  307. Notes
  308. -----
  309. Each NumPy module exposes `test` in its namespace to run all tests for it.
  310. For example, to run all tests for numpy.lib:
  311. >>> np.lib.test() #doctest: +SKIP
  312. Examples
  313. --------
  314. >>> result = np.lib.test() #doctest: +SKIP
  315. Running unit tests for numpy.lib
  316. ...
  317. Ran 976 tests in 3.933s
  318. OK
  319. >>> result.errors #doctest: +SKIP
  320. []
  321. >>> result.knownfail #doctest: +SKIP
  322. []
  323. """
  324. # cap verbosity at 3 because nose becomes *very* verbose beyond that
  325. verbose = min(verbose, 3)
  326. from . import utils
  327. utils.verbose = verbose
  328. argv, plugins = self.prepare_test_args(
  329. label, verbose, extra_argv, doctests, coverage, timer)
  330. if doctests:
  331. print("Running unit tests and doctests for %s" % self.package_name)
  332. else:
  333. print("Running unit tests for %s" % self.package_name)
  334. self._show_system_info()
  335. # reset doctest state on every run
  336. import doctest
  337. doctest.master = None
  338. if raise_warnings is None:
  339. raise_warnings = self.raise_warnings
  340. _warn_opts = dict(develop=(Warning,),
  341. release=())
  342. if isinstance(raise_warnings, basestring):
  343. raise_warnings = _warn_opts[raise_warnings]
  344. with suppress_warnings("location") as sup:
  345. # Reset the warning filters to the default state,
  346. # so that running the tests is more repeatable.
  347. warnings.resetwarnings()
  348. # Set all warnings to 'warn', this is because the default 'once'
  349. # has the bad property of possibly shadowing later warnings.
  350. warnings.filterwarnings('always')
  351. # Force the requested warnings to raise
  352. for warningtype in raise_warnings:
  353. warnings.filterwarnings('error', category=warningtype)
  354. # Filter out annoying import messages.
  355. sup.filter(message='Not importing directory')
  356. sup.filter(message="numpy.dtype size changed")
  357. sup.filter(message="numpy.ufunc size changed")
  358. sup.filter(category=np.ModuleDeprecationWarning)
  359. # Filter out boolean '-' deprecation messages. This allows
  360. # older versions of scipy to test without a flood of messages.
  361. sup.filter(message=".*boolean negative.*")
  362. sup.filter(message=".*boolean subtract.*")
  363. # Filter out distutils cpu warnings (could be localized to
  364. # distutils tests). ASV has problems with top level import,
  365. # so fetch module for suppression here.
  366. with warnings.catch_warnings():
  367. warnings.simplefilter("always")
  368. from ...distutils import cpuinfo
  369. sup.filter(category=UserWarning, module=cpuinfo)
  370. # See #7949: Filter out deprecation warnings due to the -3 flag to
  371. # python 2
  372. if sys.version_info.major == 2 and sys.py3kwarning:
  373. # This is very specific, so using the fragile module filter
  374. # is fine
  375. import threading
  376. sup.filter(DeprecationWarning,
  377. r"sys\.exc_clear\(\) not supported in 3\.x",
  378. module=threading)
  379. sup.filter(DeprecationWarning, message=r"in 3\.x, __setslice__")
  380. sup.filter(DeprecationWarning, message=r"in 3\.x, __getslice__")
  381. sup.filter(DeprecationWarning, message=r"buffer\(\) not supported in 3\.x")
  382. sup.filter(DeprecationWarning, message=r"CObject type is not supported in 3\.x")
  383. sup.filter(DeprecationWarning, message=r"comparing unequal types not supported in 3\.x")
  384. # Filter out some deprecation warnings inside nose 1.3.7 when run
  385. # on python 3.5b2. See
  386. # https://github.com/nose-devs/nose/issues/929
  387. # Note: it is hard to filter based on module for sup (lineno could
  388. # be implemented).
  389. warnings.filterwarnings("ignore", message=".*getargspec.*",
  390. category=DeprecationWarning,
  391. module=r"nose\.")
  392. from .noseclasses import NumpyTestProgram
  393. t = NumpyTestProgram(argv=argv, exit=False, plugins=plugins)
  394. return t.result
  395. def bench(self, label='fast', verbose=1, extra_argv=None):
  396. """
  397. Run benchmarks for module using nose.
  398. Parameters
  399. ----------
  400. label : {'fast', 'full', '', attribute identifier}, optional
  401. Identifies the benchmarks to run. This can be a string to pass to
  402. the nosetests executable with the '-A' option, or one of several
  403. special values. Special values are:
  404. * 'fast' - the default - which corresponds to the ``nosetests -A``
  405. option of 'not slow'.
  406. * 'full' - fast (as above) and slow benchmarks as in the
  407. 'no -A' option to nosetests - this is the same as ''.
  408. * None or '' - run all tests.
  409. * attribute_identifier - string passed directly to nosetests as '-A'.
  410. verbose : int, optional
  411. Verbosity value for benchmark outputs, in the range 1-10. Default is 1.
  412. extra_argv : list, optional
  413. List with any extra arguments to pass to nosetests.
  414. Returns
  415. -------
  416. success : bool
  417. Returns True if running the benchmarks works, False if an error
  418. occurred.
  419. Notes
  420. -----
  421. Benchmarks are like tests, but have names starting with "bench" instead
  422. of "test", and can be found under the "benchmarks" sub-directory of the
  423. module.
  424. Each NumPy module exposes `bench` in its namespace to run all benchmarks
  425. for it.
  426. Examples
  427. --------
  428. >>> success = np.lib.bench() #doctest: +SKIP
  429. Running benchmarks for numpy.lib
  430. ...
  431. using 562341 items:
  432. unique:
  433. 0.11
  434. unique1d:
  435. 0.11
  436. ratio: 1.0
  437. nUnique: 56230 == 56230
  438. ...
  439. OK
  440. >>> success #doctest: +SKIP
  441. True
  442. """
  443. print("Running benchmarks for %s" % self.package_name)
  444. self._show_system_info()
  445. argv = self._test_argv(label, verbose, extra_argv)
  446. argv += ['--match', r'(?:^|[\\b_\\.%s-])[Bb]ench' % os.sep]
  447. # import nose or make informative error
  448. nose = import_nose()
  449. # get plugin to disable doctests
  450. from .noseclasses import Unplugger
  451. add_plugins = [Unplugger('doctest')]
  452. return nose.run(argv=argv, addplugins=add_plugins)
  453. def _numpy_tester():
  454. if hasattr(np, "__version__") and ".dev0" in np.__version__:
  455. mode = "develop"
  456. else:
  457. mode = "release"
  458. return NoseTester(raise_warnings=mode, depth=1,
  459. check_fpu_mode=True)