Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Nose test running.
4This module implements ``test()`` and ``bench()`` functions for NumPy modules.
6"""
7import os
8import sys
9import warnings
10import numpy as np
12from .utils import import_nose, suppress_warnings
15__all__ = ['get_package_name', 'run_module_suite', 'NoseTester',
16 '_numpy_tester', 'get_package_name', 'import_nose',
17 'suppress_warnings']
20def get_package_name(filepath):
21 """
22 Given a path where a package is installed, determine its name.
24 Parameters
25 ----------
26 filepath : str
27 Path to a file. If the determination fails, "numpy" is returned.
29 Examples
30 --------
31 >>> np.testing.nosetester.get_package_name('nonsense')
32 'numpy'
34 """
36 fullpath = filepath[:]
37 pkg_name = []
38 while 'site-packages' in filepath or 'dist-packages' in filepath:
39 filepath, p2 = os.path.split(filepath)
40 if p2 in ('site-packages', 'dist-packages'):
41 break
42 pkg_name.append(p2)
44 # if package name determination failed, just default to numpy/scipy
45 if not pkg_name:
46 if 'scipy' in fullpath:
47 return 'scipy'
48 else:
49 return 'numpy'
51 # otherwise, reverse to get correct order and return
52 pkg_name.reverse()
54 # don't include the outer egg directory
55 if pkg_name[0].endswith('.egg'):
56 pkg_name.pop(0)
58 return '.'.join(pkg_name)
61def run_module_suite(file_to_run=None, argv=None):
62 """
63 Run a test module.
65 Equivalent to calling ``$ nosetests <argv> <file_to_run>`` from
66 the command line
68 Parameters
69 ----------
70 file_to_run : str, optional
71 Path to test module, or None.
72 By default, run the module from which this function is called.
73 argv : list of strings
74 Arguments to be passed to the nose test runner. ``argv[0]`` is
75 ignored. All command line arguments accepted by ``nosetests``
76 will work. If it is the default value None, sys.argv is used.
78 .. versionadded:: 1.9.0
80 Examples
81 --------
82 Adding the following::
84 if __name__ == "__main__" :
85 run_module_suite(argv=sys.argv)
87 at the end of a test module will run the tests when that module is
88 called in the python interpreter.
90 Alternatively, calling::
92 >>> run_module_suite(file_to_run="numpy/tests/test_matlib.py") # doctest: +SKIP
94 from an interpreter will run all the test routine in 'test_matlib.py'.
95 """
96 if file_to_run is None:
97 f = sys._getframe(1)
98 file_to_run = f.f_locals.get('__file__', None)
99 if file_to_run is None:
100 raise AssertionError
102 if argv is None:
103 argv = sys.argv + [file_to_run]
104 else:
105 argv = argv + [file_to_run]
107 nose = import_nose()
108 from .noseclasses import KnownFailurePlugin
109 nose.run(argv=argv, addplugins=[KnownFailurePlugin()])
112class NoseTester:
113 """
114 Nose test runner.
116 This class is made available as numpy.testing.Tester, and a test function
117 is typically added to a package's __init__.py like so::
119 from numpy.testing import Tester
120 test = Tester().test
122 Calling this test function finds and runs all tests associated with the
123 package and all its sub-packages.
125 Attributes
126 ----------
127 package_path : str
128 Full path to the package to test.
129 package_name : str
130 Name of the package to test.
132 Parameters
133 ----------
134 package : module, str or None, optional
135 The package to test. If a string, this should be the full path to
136 the package. If None (default), `package` is set to the module from
137 which `NoseTester` is initialized.
138 raise_warnings : None, str or sequence of warnings, optional
139 This specifies which warnings to configure as 'raise' instead
140 of being shown once during the test execution. Valid strings are:
142 - "develop" : equals ``(Warning,)``
143 - "release" : equals ``()``, don't raise on any warnings.
145 Default is "release".
146 depth : int, optional
147 If `package` is None, then this can be used to initialize from the
148 module of the caller of (the caller of (...)) the code that
149 initializes `NoseTester`. Default of 0 means the module of the
150 immediate caller; higher values are useful for utility routines that
151 want to initialize `NoseTester` objects on behalf of other code.
153 """
154 def __init__(self, package=None, raise_warnings="release", depth=0,
155 check_fpu_mode=False):
156 # Back-compat: 'None' used to mean either "release" or "develop"
157 # depending on whether this was a release or develop version of
158 # numpy. Those semantics were fine for testing numpy, but not so
159 # helpful for downstream projects like scipy that use
160 # numpy.testing. (They want to set this based on whether *they* are a
161 # release or develop version, not whether numpy is.) So we continue to
162 # accept 'None' for back-compat, but it's now just an alias for the
163 # default "release".
164 if raise_warnings is None:
165 raise_warnings = "release"
167 package_name = None
168 if package is None:
169 f = sys._getframe(1 + depth)
170 package_path = f.f_locals.get('__file__', None)
171 if package_path is None:
172 raise AssertionError
173 package_path = os.path.dirname(package_path)
174 package_name = f.f_locals.get('__name__', None)
175 elif isinstance(package, type(os)):
176 package_path = os.path.dirname(package.__file__)
177 package_name = getattr(package, '__name__', None)
178 else:
179 package_path = str(package)
181 self.package_path = package_path
183 # Find the package name under test; this name is used to limit coverage
184 # reporting (if enabled).
185 if package_name is None:
186 package_name = get_package_name(package_path)
187 self.package_name = package_name
189 # Set to "release" in constructor in maintenance branches.
190 self.raise_warnings = raise_warnings
192 # Whether to check for FPU mode changes
193 self.check_fpu_mode = check_fpu_mode
195 def _test_argv(self, label, verbose, extra_argv):
196 ''' Generate argv for nosetest command
198 Parameters
199 ----------
200 label : {'fast', 'full', '', attribute identifier}, optional
201 see ``test`` docstring
202 verbose : int, optional
203 Verbosity value for test outputs, in the range 1-10. Default is 1.
204 extra_argv : list, optional
205 List with any extra arguments to pass to nosetests.
207 Returns
208 -------
209 argv : list
210 command line arguments that will be passed to nose
211 '''
212 argv = [__file__, self.package_path, '-s']
213 if label and label != 'full':
214 if not isinstance(label, str):
215 raise TypeError('Selection label should be a string')
216 if label == 'fast':
217 label = 'not slow'
218 argv += ['-A', label]
219 argv += ['--verbosity', str(verbose)]
221 # When installing with setuptools, and also in some other cases, the
222 # test_*.py files end up marked +x executable. Nose, by default, does
223 # not run files marked with +x as they might be scripts. However, in
224 # our case nose only looks for test_*.py files under the package
225 # directory, which should be safe.
226 argv += ['--exe']
228 if extra_argv:
229 argv += extra_argv
230 return argv
232 def _show_system_info(self):
233 nose = import_nose()
235 import numpy
236 print(f'NumPy version {numpy.__version__}')
237 relaxed_strides = numpy.ones((10, 1), order="C").flags.f_contiguous
238 print("NumPy relaxed strides checking option:", relaxed_strides)
239 npdir = os.path.dirname(numpy.__file__)
240 print(f'NumPy is installed in {npdir}')
242 if 'scipy' in self.package_name:
243 import scipy
244 print(f'SciPy version {scipy.__version__}')
245 spdir = os.path.dirname(scipy.__file__)
246 print(f'SciPy is installed in {spdir}')
248 pyversion = sys.version.replace('\n', '')
249 print(f'Python version {pyversion}')
250 print("nose version %d.%d.%d" % nose.__versioninfo__)
252 def _get_custom_doctester(self):
253 """ Return instantiated plugin for doctests
255 Allows subclassing of this class to override doctester
257 A return value of None means use the nose builtin doctest plugin
258 """
259 from .noseclasses import NumpyDoctest
260 return NumpyDoctest()
262 def prepare_test_args(self, label='fast', verbose=1, extra_argv=None,
263 doctests=False, coverage=False, timer=False):
264 """
265 Run tests for module using nose.
267 This method does the heavy lifting for the `test` method. It takes all
268 the same arguments, for details see `test`.
270 See Also
271 --------
272 test
274 """
275 # fail with nice error message if nose is not present
276 import_nose()
277 # compile argv
278 argv = self._test_argv(label, verbose, extra_argv)
279 # our way of doing coverage
280 if coverage:
281 argv += [f'--cover-package={self.package_name}', '--with-coverage',
282 '--cover-tests', '--cover-erase']
284 if timer:
285 if timer is True:
286 argv += ['--with-timer']
287 elif isinstance(timer, int):
288 argv += ['--with-timer', '--timer-top-n', str(timer)]
290 # construct list of plugins
291 import nose.plugins.builtin
292 from nose.plugins import EntryPointPluginManager
293 from .noseclasses import (KnownFailurePlugin, Unplugger,
294 FPUModeCheckPlugin)
295 plugins = [KnownFailurePlugin()]
296 plugins += [p() for p in nose.plugins.builtin.plugins]
297 if self.check_fpu_mode:
298 plugins += [FPUModeCheckPlugin()]
299 argv += ["--with-fpumodecheckplugin"]
300 try:
301 # External plugins (like nose-timer)
302 entrypoint_manager = EntryPointPluginManager()
303 entrypoint_manager.loadPlugins()
304 plugins += [p for p in entrypoint_manager.plugins]
305 except ImportError:
306 # Relies on pkg_resources, not a hard dependency
307 pass
309 # add doctesting if required
310 doctest_argv = '--with-doctest' in argv
311 if doctests == False and doctest_argv:
312 doctests = True
313 plug = self._get_custom_doctester()
314 if plug is None:
315 # use standard doctesting
316 if doctests and not doctest_argv:
317 argv += ['--with-doctest']
318 else: # custom doctesting
319 if doctest_argv: # in fact the unplugger would take care of this
320 argv.remove('--with-doctest')
321 plugins += [Unplugger('doctest'), plug]
322 if doctests:
323 argv += ['--with-' + plug.name]
324 return argv, plugins
326 def test(self, label='fast', verbose=1, extra_argv=None,
327 doctests=False, coverage=False, raise_warnings=None,
328 timer=False):
329 """
330 Run tests for module using nose.
332 Parameters
333 ----------
334 label : {'fast', 'full', '', attribute identifier}, optional
335 Identifies the tests to run. This can be a string to pass to
336 the nosetests executable with the '-A' option, or one of several
337 special values. Special values are:
339 * 'fast' - the default - which corresponds to the ``nosetests -A``
340 option of 'not slow'.
341 * 'full' - fast (as above) and slow tests as in the
342 'no -A' option to nosetests - this is the same as ''.
343 * None or '' - run all tests.
344 * attribute_identifier - string passed directly to nosetests as '-A'.
346 verbose : int, optional
347 Verbosity value for test outputs, in the range 1-10. Default is 1.
348 extra_argv : list, optional
349 List with any extra arguments to pass to nosetests.
350 doctests : bool, optional
351 If True, run doctests in module. Default is False.
352 coverage : bool, optional
353 If True, report coverage of NumPy code. Default is False.
354 (This requires the
355 `coverage module <https://pypi.org/project/coverage/>`_).
356 raise_warnings : None, str or sequence of warnings, optional
357 This specifies which warnings to configure as 'raise' instead
358 of being shown once during the test execution. Valid strings are:
360 * "develop" : equals ``(Warning,)``
361 * "release" : equals ``()``, do not raise on any warnings.
362 timer : bool or int, optional
363 Timing of individual tests with ``nose-timer`` (which needs to be
364 installed). If True, time tests and report on all of them.
365 If an integer (say ``N``), report timing results for ``N`` slowest
366 tests.
368 Returns
369 -------
370 result : object
371 Returns the result of running the tests as a
372 ``nose.result.TextTestResult`` object.
374 Notes
375 -----
376 Each NumPy module exposes `test` in its namespace to run all tests for it.
377 For example, to run all tests for numpy.lib:
379 >>> np.lib.test() #doctest: +SKIP
381 Examples
382 --------
383 >>> result = np.lib.test() #doctest: +SKIP
384 Running unit tests for numpy.lib
385 ...
386 Ran 976 tests in 3.933s
388 OK
390 >>> result.errors #doctest: +SKIP
391 []
392 >>> result.knownfail #doctest: +SKIP
393 []
394 """
396 # cap verbosity at 3 because nose becomes *very* verbose beyond that
397 verbose = min(verbose, 3)
399 from . import utils
400 utils.verbose = verbose
402 argv, plugins = self.prepare_test_args(
403 label, verbose, extra_argv, doctests, coverage, timer)
405 if doctests:
406 print(f'Running unit tests and doctests for {self.package_name}')
407 else:
408 print(f'Running unit tests for {self.package_name}')
410 self._show_system_info()
412 # reset doctest state on every run
413 import doctest
414 doctest.master = None
416 if raise_warnings is None:
417 raise_warnings = self.raise_warnings
419 _warn_opts = dict(develop=(Warning,),
420 release=())
421 if isinstance(raise_warnings, str):
422 raise_warnings = _warn_opts[raise_warnings]
424 with suppress_warnings("location") as sup:
425 # Reset the warning filters to the default state,
426 # so that running the tests is more repeatable.
427 warnings.resetwarnings()
428 # Set all warnings to 'warn', this is because the default 'once'
429 # has the bad property of possibly shadowing later warnings.
430 warnings.filterwarnings('always')
431 # Force the requested warnings to raise
432 for warningtype in raise_warnings:
433 warnings.filterwarnings('error', category=warningtype)
434 # Filter out annoying import messages.
435 sup.filter(message='Not importing directory')
436 sup.filter(message="numpy.dtype size changed")
437 sup.filter(message="numpy.ufunc size changed")
438 sup.filter(category=np.ModuleDeprecationWarning)
439 # Filter out boolean '-' deprecation messages. This allows
440 # older versions of scipy to test without a flood of messages.
441 sup.filter(message=".*boolean negative.*")
442 sup.filter(message=".*boolean subtract.*")
443 # Filter out distutils cpu warnings (could be localized to
444 # distutils tests). ASV has problems with top level import,
445 # so fetch module for suppression here.
446 with warnings.catch_warnings():
447 warnings.simplefilter("always")
448 from ...distutils import cpuinfo
449 sup.filter(category=UserWarning, module=cpuinfo)
450 # Filter out some deprecation warnings inside nose 1.3.7 when run
451 # on python 3.5b2. See
452 # https://github.com/nose-devs/nose/issues/929
453 # Note: it is hard to filter based on module for sup (lineno could
454 # be implemented).
455 warnings.filterwarnings("ignore", message=".*getargspec.*",
456 category=DeprecationWarning,
457 module=r"nose\.")
459 from .noseclasses import NumpyTestProgram
461 t = NumpyTestProgram(argv=argv, exit=False, plugins=plugins)
463 return t.result
465 def bench(self, label='fast', verbose=1, extra_argv=None):
466 """
467 Run benchmarks for module using nose.
469 Parameters
470 ----------
471 label : {'fast', 'full', '', attribute identifier}, optional
472 Identifies the benchmarks to run. This can be a string to pass to
473 the nosetests executable with the '-A' option, or one of several
474 special values. Special values are:
476 * 'fast' - the default - which corresponds to the ``nosetests -A``
477 option of 'not slow'.
478 * 'full' - fast (as above) and slow benchmarks as in the
479 'no -A' option to nosetests - this is the same as ''.
480 * None or '' - run all tests.
481 * attribute_identifier - string passed directly to nosetests as '-A'.
483 verbose : int, optional
484 Verbosity value for benchmark outputs, in the range 1-10. Default is 1.
485 extra_argv : list, optional
486 List with any extra arguments to pass to nosetests.
488 Returns
489 -------
490 success : bool
491 Returns True if running the benchmarks works, False if an error
492 occurred.
494 Notes
495 -----
496 Benchmarks are like tests, but have names starting with "bench" instead
497 of "test", and can be found under the "benchmarks" sub-directory of the
498 module.
500 Each NumPy module exposes `bench` in its namespace to run all benchmarks
501 for it.
503 Examples
504 --------
505 >>> success = np.lib.bench() #doctest: +SKIP
506 Running benchmarks for numpy.lib
507 ...
508 using 562341 items:
509 unique:
510 0.11
511 unique1d:
512 0.11
513 ratio: 1.0
514 nUnique: 56230 == 56230
515 ...
516 OK
518 >>> success #doctest: +SKIP
519 True
521 """
523 print(f'Running benchmarks for {self.package_name}')
524 self._show_system_info()
526 argv = self._test_argv(label, verbose, extra_argv)
527 argv += ['--match', r'(?:^|[\\b_\\.%s-])[Bb]ench' % os.sep]
529 # import nose or make informative error
530 nose = import_nose()
532 # get plugin to disable doctests
533 from .noseclasses import Unplugger
534 add_plugins = [Unplugger('doctest')]
536 return nose.run(argv=argv, addplugins=add_plugins)
539def _numpy_tester():
540 if hasattr(np, "__version__") and ".dev0" in np.__version__:
541 mode = "develop"
542 else:
543 mode = "release"
544 return NoseTester(raise_warnings=mode, depth=1,
545 check_fpu_mode=True)