4
4
# Copyright (c) 2001-2004 Twisted Matrix Laboratories.
5
5
# See LICENSE for details.
7
# Maintainer: Jonathan Lange <jml@twistedmatrix.com>
8
A miscellany of code used to run Trial tests.
10
Maintainer: Jonathan Lange <jml@twistedmatrix.com>
9
14
from __future__ import generators
11
import os, glob, types, warnings, sys, inspect, imp
12
import fnmatch, random, doctest, time
13
from os.path import join as opj
15
import pdb, shutil, sets
16
import os, types, warnings, sys, inspect, imp
17
import random, doctest, time
19
from twisted.python import reflect, log, failure, modules
20
from twisted.python.util import dsu
15
22
from twisted.internet import defer, interfaces
16
from twisted.python import reflect, log, failure
17
from twisted.python.util import dsu
18
23
from twisted.trial import util, unittest
19
24
from twisted.trial.itrial import ITestCase
43
48
def samefile(filename1, filename2):
44
# hacky implementation of os.path.samefile
50
A hacky implementation of C{os.path.samefile}. Used by L{filenameToModule}
51
when the platform doesn't provide C{os.path.samefile}. Do not use this.
45
53
return os.path.abspath(filename1) == os.path.abspath(filename2)
47
55
def filenameToModule(fn):
57
Given a filename, do whatever possible to return a module object matching
60
If the file in question is a module in Python path, properly import and
61
return that module. Otherwise, load the source manually.
63
@param fn: A filename.
64
@return: A module object.
65
@raise ValueError: If C{fn} does not exist.
48
67
if not os.path.exists(fn):
49
68
raise ValueError("%r doesn't exist" % (fn,))
89
def visit_suite(suite, visitor):
108
def suiteVisit(suite, visitor):
110
Visit each test in C{suite} with C{visitor}.
112
@param visitor: A callable which takes a single argument, the L{TestCase}
90
116
for case in suite._tests:
91
117
visit = getattr(case, 'visit', None)
92
118
if visit is not None:
120
elif isinstance(case, pyunit.TestCase):
121
case = PyUnitTestCase(case)
93
122
case.visit(visitor)
123
elif isinstance(case, pyunit.TestSuite):
124
suiteVisit(case, visitor)
95
if isinstance(case, pyunit.TestCase):
96
case = PyUnitTestCase(case)
98
elif isinstance(case, pyunit.TestSuite):
99
visit_suite(case, visitor)
105
129
class TestSuite(pyunit.TestSuite):
131
Extend the standard library's C{TestSuite} with support for the visitor
132
pattern and a consistently overrideable C{run} method.
108
137
def __call__(self, result):
109
138
return self.run(result)
111
140
def run(self, result):
142
Call C{run} on every member of the suite.
112
144
# we implement this because Python 2.3 unittest defines this code
113
145
# in __call__, whereas 2.4 defines the code in run.
114
146
for test in self._tests:
115
147
if result.shouldStop:
154
# When an error occurs outside of any test, the user will see this string
155
# in place of a test's name.
156
NOT_IN_TEST = "<not in test>"
160
class LoggedSuite(TestSuite):
162
Any errors logged in this suite will be reported to the L{TestResult}
166
def run(self, result):
168
Run the suite, storing all errors in C{result}. If an error is logged
169
while no tests are running, then it will be added as an error to
172
@param result: A L{TestResult} object.
174
observer = unittest._logObserver
176
super(LoggedSuite, self).run(result)
178
for error in observer.getErrors():
179
result.addError(TestHolder(NOT_IN_TEST), error)
180
observer.flushErrors()
121
184
class DocTestSuite(TestSuite):
186
Behaves like doctest.DocTestSuite, but decorates individual TestCases so
187
they support visit and so that id() behaviour is meaningful and consistent
188
between Python versions.
122
191
def __init__(self, testModule):
123
192
TestSuite.__init__(self)
124
193
suite = doctest.DocTestSuite(testModule)
125
194
for test in suite._tests: #yay encapsulation
126
self.addTest(PyUnitTestCase(test))
195
self.addTest(DocTestCase(test))
129
198
class PyUnitTestCase(object):
131
200
This class decorates the pyunit.TestCase class, mainly to work around the
132
differences between unittest in Python 2.3 and unittest in Python 2.4 These
201
differences between unittest in Python 2.3, 2.4, and 2.5. These
133
202
differences are::
135
204
- The way doctest unittests describe themselves
136
205
- Where the implementation of TestCase.run is (used to be in __call__)
206
- Where the test method name is kept (mangled-private or non-mangled
138
209
It also implements visit, which we like.
143
214
test.id = self.id
146
return self._test.shortDescription()
217
cls = self._test.__class__
218
tmn = getattr(self._test, '_TestCase__testMethodName', None)
220
# python2.5's 'unittest' module is more sensible; but different.
221
tmn = self._test._testMethodName
222
return (cls.__module__ + '.' + cls.__name__ + '.' +
226
return 'PyUnitTestCase<%r>'%(self.id(),)
148
228
def __call__(self, results):
149
229
return self._test(results)
151
232
def visit(self, visitor):
152
"""Call visitor.visitCase(self)."""
153
visitor.visitCase(self)
234
Call the given visitor with the original, standard library, test case
235
that C{self} wraps. See L{unittest.TestCase.visit}.
155
240
def __getattr__(self, name):
156
241
return getattr(self._test, name)
244
class DocTestCase(PyUnitTestCase):
247
In Python 2.4, doctests have correct id() behaviour. In Python 2.3,
248
id() returns 'runit'.
250
Here we override id() so that at least it will always contain the
251
fully qualified Python name of the doctest.
253
return self._test.shortDescription()
159
256
class TrialSuite(TestSuite):
258
Suite to wrap around every single test in a C{trial} run. Used internally
259
by Trial to set up things necessary for Trial tests to work, regardless of
260
what context they are run in.
263
def __init__(self, tests=()):
264
suite = LoggedSuite(tests)
265
super(TrialSuite, self).__init__([suite])
162
269
from twisted.internet import reactor
176
283
def run(self, result):
178
log.startKeepingErrors()
179
285
TestSuite.run(self, result)
185
if isinstance(thing, str):
187
if hasattr(thing, '__name__'):
188
return thing.__name__
292
@param thing: an object from modules (instance of PythonModule,
293
PythonAttribute), a TestCase subclass, or an instance of a TestCase.
295
if isTestCase(thing):
297
theName = reflect.qual(thing)
299
# thing from trial, or thing from modules.
300
# this monstrosity exists so that modules' objects do not have to
301
# implement id(). -jml
304
except AttributeError:
192
309
def isTestCase(obj):
311
Returns C{True} if C{obj} is a class that contains test cases, C{False}
312
otherwise. Used to find all the tests in a module.
194
315
return ITestCase.implementedBy(obj)
195
316
except TypeError:
204
class ErrorHolder(object):
326
class TestHolder(object):
328
Placeholder for a L{TestCase} inside a reporter. As far as a L{TestResult}
329
is concerned, this looks exactly like a unit test.
332
def __init__(self, description):
334
@param description: A string to be displayed L{TestResult}.
336
self.description = description
340
return self.description
343
def shortDescription(self):
344
return self.description
348
class ErrorHolder(TestHolder):
350
Used to insert arbitrary errors into a test suite run. Provides enough
351
methods to look like a C{TestCase}, however, when it is run, it simply adds
352
an error to the C{TestResult}. The most common use-case is for when a
353
module fails to import.
205
356
def __init__(self, description, error):
206
self.description = description
358
@param description: A string used by C{TestResult}s to identify this
359
error. Generally, this is the name of a module that failed to import.
361
@param error: The error to be added to the result. Can be an exc_info
362
tuple or a L{twisted.python.failure.Failure}.
364
super(ErrorHolder, self).__init__(description)
207
365
self.error = error
210
return self.description
212
def shortDescription(self):
213
return self.description
215
368
def __repr__(self):
216
369
return "<ErrorHolder description=%r error=%r>" % (self.description,
219
373
def run(self, result):
220
374
result.addError(self, self.error)
222
377
def __call__(self, result):
223
378
return self.run(result)
225
381
def countTestCases(self):
385
def visit(self, visitor):
387
See L{unittest.TestCase.visit}.
229
393
class TestLoader(object):
395
I find tests inside function, modules, files -- whatever -- then return
396
them wrapped inside a Test (either a L{TestSuite} or a L{TestCase}).
398
@ivar methodPrefix: A string prefix. C{TestLoader} will assume that all the
399
methods in a class that begin with C{methodPrefix} are test cases.
401
@ivar modulePrefix: A string prefix. Every module in a package that begins
402
with C{modulePrefix} is considered a module full of tests.
404
@ivar forceGarbageCollection: A flag applied to each C{TestCase} loaded.
405
See L{unittest.TestCase} for more information.
407
@ivar sorter: A key function used to sort C{TestCase}s, test classes,
408
modules and packages.
410
@ivar suiteFactory: A callable which is passed a list of tests (which
411
themselves may be suites of tests). Must return a test suite.
230
414
methodPrefix = 'test'
231
moduleGlob = 'test_*.py'
415
modulePrefix = 'test_'
233
417
def __init__(self):
234
418
self.suiteFactory = TestSuite
235
419
self.sorter = name
236
420
self._importErrors = []
421
self.forceGarbageCollection = False
238
423
def sort(self, xs):
425
Sort the given things using L{sorter}.
427
@param xs: A list of test cases, class or modules.
239
429
return dsu(xs, self.sorter)
241
def _findTestClasses(self, module):
242
"""Given a module, return all trial Test classes"""
431
def findTestClasses(self, module):
432
"""Given a module, return all Trial test classes"""
244
434
for name, val in inspect.getmembers(module):
245
435
if isTestCase(val):
247
437
return self.sort(classes)
249
439
def findByName(self, name):
441
Return a Python object given a string describing it.
443
@param name: a string which may be either a filename or a
444
fully-qualified Python name.
446
@return: If C{name} is a filename, return the module. If C{name} is a
447
fully-qualified Python name, return the object it refers to.
250
449
if os.path.exists(name):
251
450
return filenameToModule(name)
252
451
return reflect.namedAny(name)
254
453
def loadModule(self, module):
455
Return a test suite with all the tests from a module.
457
Included are TestCase subclasses and doctests listed in the module's
458
__doctests__ module. If that's not good for you, put a function named
459
either C{testSuite} or C{test_suite} in your module that returns a
460
TestSuite, and I'll use the results of that instead.
462
If C{testSuite} and C{test_suite} are both present, then I'll use
465
## XXX - should I add an optional parameter to disable the check for
467
## OR, should I add another method
255
468
if not isinstance(module, types.ModuleType):
256
469
raise TypeError("%r is not a module" % (module,))
470
if hasattr(module, 'testSuite'):
471
return module.testSuite()
472
elif hasattr(module, 'test_suite'):
473
return module.test_suite()
257
474
suite = self.suiteFactory()
258
for testClass in self._findTestClasses(module):
475
for testClass in self.findTestClasses(module):
259
476
suite.addTest(self.loadClass(testClass))
260
477
if not hasattr(module, '__doctests__'):
266
483
loadTestsFromModule = loadModule
268
485
def loadClass(self, klass):
487
Given a class which contains test cases, return a sorted list of
488
C{TestCase} instances.
269
490
if not (isinstance(klass, type) or isinstance(klass, types.ClassType)):
270
491
raise TypeError("%r is not a class" % (klass,))
271
492
if not isTestCase(klass):
272
493
raise ValueError("%r is not a test case" % (klass,))
273
494
names = self.getTestCaseNames(klass)
274
tests = self.sort([ klass(self.methodPrefix+name) for name in names ])
495
tests = self.sort([self._makeCase(klass, self.methodPrefix+name)
275
497
return self.suiteFactory(tests)
276
498
loadTestsFromTestCase = loadClass
278
500
def getTestCaseNames(self, klass):
502
Given a class that contains C{TestCase}s, return a list of names of
503
methods that probably contain tests.
279
505
return reflect.prefixedMethodNames(klass, self.methodPrefix)
281
507
def loadMethod(self, method):
509
Given a method of a C{TestCase} that represents a test, return a
510
C{TestCase} instance for that test.
282
512
if not isinstance(method, types.MethodType):
283
513
raise TypeError("%r not a method" % (method,))
284
return method.im_class(method.__name__)
286
def _findTestModules(self, package):
287
modGlob = os.path.join(os.path.dirname(package.__file__),
289
return [ reflect.filenameToModuleName(filename)
290
for filename in glob.glob(modGlob) ]
514
return self._makeCase(method.im_class, method.__name__)
516
def _makeCase(self, klass, methodName):
517
test = klass(methodName)
518
test.forceGarbageCollection = self.forceGarbageCollection
292
521
def loadPackage(self, package, recurse=False):
523
Load tests from a module object representing a package, and return a
524
TestSuite containing those tests.
526
Tests are only loaded from modules whose name begins with 'test_'
527
(or whatever C{modulePrefix} is set to).
529
@param package: a types.ModuleType object (or reasonable facsimilie
530
obtained by importing) which may contain tests.
532
@param recurse: A boolean. If True, inspect modules within packages
533
within the given package (and so on), otherwise, only inspect modules
534
in the package itself.
536
@raise: TypeError if 'package' is not a package.
538
@return: a TestSuite created with my suiteFactory, containing all the
293
541
if not isPackage(package):
294
542
raise TypeError("%r is not a package" % (package,))
543
pkgobj = modules.getModule(package.__name__)
296
return self.loadPackageRecursive(package)
297
suite = self.suiteFactory()
298
for moduleName in self.sort(self._findTestModules(package)):
299
suite.addTest(self.loadByName(moduleName))
302
def _packageRecurse(self, suite, dirname, names):
303
if not isPackageDirectory(dirname):
306
moduleNames = [reflect.filenameToModuleName(opj(dirname, filename))
307
for filename in fnmatch.filter(names, self.moduleGlob)]
308
for moduleName in self.sort(moduleNames):
309
suite.addTest(self.loadByName(moduleName))
311
def loadPackageRecursive(self, package):
312
packageDir = os.path.dirname(package.__file__)
313
suite = self.suiteFactory()
314
os.path.walk(packageDir, self._packageRecurse, suite)
545
discovery = pkgobj.walkModules()
547
discovery = pkgobj.iterModules()
549
for disco in discovery:
550
if disco.name.split(".")[-1].startswith(self.modulePrefix):
551
discovered.append(disco)
552
suite = self.suiteFactory()
553
for modinfo in self.sort(discovered):
555
module = modinfo.load()
557
thingToAdd = ErrorHolder(modinfo.name, failure.Failure())
559
thingToAdd = self.loadModule(module)
560
suite.addTest(thingToAdd)
317
563
def loadDoctests(self, module):
565
Return a suite of tests for all the doctests defined in C{module}.
567
@param module: A module object or a module name.
318
569
if isinstance(module, str):
320
571
module = reflect.namedAny(module)
346
617
return self.loadAnything(thing, recurse)
347
618
loadTestsFromName = loadByName
349
def loadTestsFromNames(self, names, module=None):
620
def loadByNames(self, names, recurse=False):
622
Construct a TestSuite containing all the tests found in 'names', where
623
names is a list of fully qualified python names and/or filenames. The
624
suite returned will have no duplicate tests, even if the same object
351
629
for name in names:
352
suites.append(self.loadTestsFromName(name, module))
353
return self.suiteClass(suites)
356
class DryRunVisitor(unittest.TestVisitor):
631
things.append(self.findByName(name))
633
errors.append(ErrorHolder(name, failure.Failure()))
634
suites = [self.loadAnything(thing, recurse)
635
for thing in sets.Set(things)]
636
suites.extend(errors)
637
return self.suiteFactory(suites)
641
class DryRunVisitor(object):
643
A visitor that makes a reporter think that every test visited has run
358
647
def __init__(self, reporter):
649
@param reporter: A C{TestResult} object.
359
651
self.reporter = reporter
361
def visitSuite(self, testSuite):
362
self.reporter.startSuite(testSuite.name())
364
def visitSuiteAfter(self, testSuite):
365
self.reporter.endSuite(testSuite.name())
367
def visitCase(self, testCase):
654
def markSuccessful(self, testCase):
656
Convince the reporter that this test has been run successfully.
368
658
self.reporter.startTest(testCase)
369
659
self.reporter.addSuccess(testCase)
370
660
self.reporter.stopTest(testCase)
373
664
class TrialRunner(object):
374
"""A specialised runner that the trial front end uses."""
666
A specialised runner that the trial front end uses.
377
670
DRY_RUN = 'dry-run'
473
766
self._logWarnings = True
475
768
def run(self, test):
476
"""Run the test or suite and return a result object."""
770
Run the test or suite and return a result object.
477
772
result = self._makeResult()
478
773
# decorate the suite with reactor cleanup and log starting
479
# This should move out of the runner and be presumed to be
774
# This should move out of the runner and be presumed to be
481
776
suite = TrialSuite([test])
482
777
startTime = time.time()
483
778
result.write("Running %d tests.\n", suite.countTestCases())
484
779
if self.mode == self.DRY_RUN:
485
suite.visit(DryRunVisitor(result))
780
suite.visit(DryRunVisitor(result).markSuccessful)
486
781
elif self.mode == self.DEBUG:
487
782
# open question - should this be self.debug() instead.
488
783
debugger = self._getDebugger()
added
removed
twisted/trial/runner.py
