3
# Copyright 2009 Google Inc. All Rights Reserved.
5
# Redistribution and use in source and binary forms, with or without
6
# modification, are permitted provided that the following conditions are
9
# * Redistributions of source code must retain the above copyright
10
# notice, this list of conditions and the following disclaimer.
11
# * Redistributions in binary form must reproduce the above
12
# copyright notice, this list of conditions and the following disclaimer
13
# in the documentation and/or other materials provided with the
15
# * Neither the name of Google Inc. nor the names of its
16
# contributors may be used to endorse or promote products derived from
17
# this software without specific prior written permission.
19
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
"""Tests for run_tests_util.py test runner script."""
33
__author__ = 'vladl@google.com (Vlad Losev)'
43
GTEST_DBG_DIR = 'scons/build/dbg/gtest/scons'
44
GTEST_OPT_DIR = 'scons/build/opt/gtest/scons'
45
GTEST_OTHER_DIR = 'scons/build/other/gtest/scons'
48
def AddExeExtension(path):
49
"""Appends .exe to the path on Windows or Cygwin."""
51
if run_tests_util.IS_WINDOWS or run_tests_util.IS_CYGWIN:
57
class FakePath(object):
58
"""A fake os.path module for testing."""
60
def __init__(self, current_dir=os.getcwd(), known_paths=None):
61
self.current_dir = current_dir
63
self.path_separator = os.sep
65
# known_paths contains either absolute or relative paths. Relative paths
66
# are absolutized with self.current_dir.
68
self._AddPaths(known_paths)
70
def _AddPath(self, path):
71
ends_with_slash = path.endswith('/')
72
path = self.abspath(path)
74
path += self.path_separator
75
name_list = path.split(self.path_separator)
77
for name in name_list[:-1]:
89
assert tree[name] == 1
93
def _AddPaths(self, paths):
97
def PathElement(self, path):
98
"""Returns an internal representation of directory tree entry for path."""
100
name_list = self.abspath(path).split(self.path_separator)
101
for name in name_list:
104
tree = tree.get(name, None)
110
# Silences pylint warning about using standard names.
111
# pylint: disable-msg=C6409
112
def normpath(self, path):
113
return os.path.normpath(path)
115
def abspath(self, path):
116
return self.normpath(os.path.join(self.current_dir, path))
118
def isfile(self, path):
119
return self.PathElement(self.abspath(path)) == 1
121
def isdir(self, path):
122
return type(self.PathElement(self.abspath(path))) == type(dict())
124
def basename(self, path):
125
return os.path.basename(path)
127
def dirname(self, path):
128
return os.path.dirname(path)
130
def join(self, *kargs):
131
return os.path.join(*kargs)
134
class FakeOs(object):
135
"""A fake os module for testing."""
138
def __init__(self, fake_path_module):
139
self.path = fake_path_module
141
# Some methods/attributes are delegated to the real os module.
142
self.environ = os.environ
144
# pylint: disable-msg=C6409
145
def listdir(self, path):
146
assert self.path.isdir(path)
147
return self.path.PathElement(path).iterkeys()
149
def spawnv(self, wait, executable, *kargs):
150
assert wait == FakeOs.P_WAIT
151
return self.spawn_impl(executable, kargs)
154
class GetTestsToRunTest(unittest.TestCase):
155
"""Exercises TestRunner.GetTestsToRun."""
157
def NormalizeGetTestsToRunResults(self, results):
158
"""Normalizes path data returned from GetTestsToRun for comparison."""
160
def NormalizePythonTestPair(pair):
161
"""Normalizes path data in the (directory, python_script) pair."""
163
return (os.path.normpath(pair[0]), os.path.normpath(pair[1]))
165
def NormalizeBinaryTestPair(pair):
166
"""Normalizes path data in the (directory, binary_executable) pair."""
168
directory, executable = map(os.path.normpath, pair)
170
# On Windows and Cygwin, the test file names have the .exe extension, but
171
# they can be invoked either by name or by name+extension. Our test must
172
# accommodate both situations.
173
if run_tests_util.IS_WINDOWS or run_tests_util.IS_CYGWIN:
174
executable = re.sub(r'\.exe$', '', executable)
175
return (directory, executable)
177
python_tests = sets.Set(map(NormalizePythonTestPair, results[0]))
178
binary_tests = sets.Set(map(NormalizeBinaryTestPair, results[1]))
179
return (python_tests, binary_tests)
181
def AssertResultsEqual(self, results, expected):
182
"""Asserts results returned by GetTestsToRun equal to expected results."""
184
self.assertEqual(self.NormalizeGetTestsToRunResults(results),
185
self.NormalizeGetTestsToRunResults(expected),
186
'Incorrect set of tests returned:\n%s\nexpected:\n%s' %
190
self.fake_os = FakeOs(FakePath(
191
current_dir=os.path.abspath(os.path.dirname(run_tests_util.__file__)),
192
known_paths=[AddExeExtension(GTEST_DBG_DIR + '/gtest_unittest'),
193
AddExeExtension(GTEST_OPT_DIR + '/gtest_unittest'),
194
'test/gtest_color_test.py']))
195
self.fake_configurations = ['dbg', 'opt']
196
self.test_runner = run_tests_util.TestRunner(script_dir='.',
197
injected_os=self.fake_os,
198
injected_subprocess=None)
200
def testBinaryTestsOnly(self):
201
"""Exercises GetTestsToRun with parameters designating binary tests only."""
204
self.AssertResultsEqual(
205
self.test_runner.GetTestsToRun(
209
available_configurations=self.fake_configurations),
211
[(GTEST_DBG_DIR, GTEST_DBG_DIR + '/gtest_unittest')]))
213
# An explicitly specified directory.
214
self.AssertResultsEqual(
215
self.test_runner.GetTestsToRun(
216
[GTEST_DBG_DIR, 'gtest_unittest'],
219
available_configurations=self.fake_configurations),
221
[(GTEST_DBG_DIR, GTEST_DBG_DIR + '/gtest_unittest')]))
223
# A particular configuration.
224
self.AssertResultsEqual(
225
self.test_runner.GetTestsToRun(
229
available_configurations=self.fake_configurations),
231
[(GTEST_OTHER_DIR, GTEST_OTHER_DIR + '/gtest_unittest')]))
233
# All available configurations
234
self.AssertResultsEqual(
235
self.test_runner.GetTestsToRun(
239
available_configurations=self.fake_configurations),
241
[(GTEST_DBG_DIR, GTEST_DBG_DIR + '/gtest_unittest'),
242
(GTEST_OPT_DIR, GTEST_OPT_DIR + '/gtest_unittest')]))
244
# All built configurations (unbuilt don't cause failure).
245
self.AssertResultsEqual(
246
self.test_runner.GetTestsToRun(
250
available_configurations=self.fake_configurations + ['unbuilt']),
252
[(GTEST_DBG_DIR, GTEST_DBG_DIR + '/gtest_unittest'),
253
(GTEST_OPT_DIR, GTEST_OPT_DIR + '/gtest_unittest')]))
255
# A combination of an explicit directory and a configuration.
256
self.AssertResultsEqual(
257
self.test_runner.GetTestsToRun(
258
[GTEST_DBG_DIR, 'gtest_unittest'],
261
available_configurations=self.fake_configurations),
263
[(GTEST_DBG_DIR, GTEST_DBG_DIR + '/gtest_unittest'),
264
(GTEST_OPT_DIR, GTEST_OPT_DIR + '/gtest_unittest')]))
266
# Same test specified in an explicit directory and via a configuration.
267
self.AssertResultsEqual(
268
self.test_runner.GetTestsToRun(
269
[GTEST_DBG_DIR, 'gtest_unittest'],
272
available_configurations=self.fake_configurations),
274
[(GTEST_DBG_DIR, GTEST_DBG_DIR + '/gtest_unittest')]))
276
# All built configurations + explicit directory + explicit configuration.
277
self.AssertResultsEqual(
278
self.test_runner.GetTestsToRun(
279
[GTEST_DBG_DIR, 'gtest_unittest'],
282
available_configurations=self.fake_configurations),
284
[(GTEST_DBG_DIR, GTEST_DBG_DIR + '/gtest_unittest'),
285
(GTEST_OPT_DIR, GTEST_OPT_DIR + '/gtest_unittest')]))
287
def testPythonTestsOnly(self):
288
"""Exercises GetTestsToRun with parameters designating Python tests only."""
291
self.AssertResultsEqual(
292
self.test_runner.GetTestsToRun(
293
['gtest_color_test.py'],
296
available_configurations=self.fake_configurations),
297
([(GTEST_DBG_DIR, 'test/gtest_color_test.py')],
300
# An explicitly specified directory.
301
self.AssertResultsEqual(
302
self.test_runner.GetTestsToRun(
303
[GTEST_DBG_DIR, 'test/gtest_color_test.py'],
306
available_configurations=self.fake_configurations),
307
([(GTEST_DBG_DIR, 'test/gtest_color_test.py')],
310
# A particular configuration.
311
self.AssertResultsEqual(
312
self.test_runner.GetTestsToRun(
313
['gtest_color_test.py'],
316
available_configurations=self.fake_configurations),
317
([(GTEST_OTHER_DIR, 'test/gtest_color_test.py')],
320
# All available configurations
321
self.AssertResultsEqual(
322
self.test_runner.GetTestsToRun(
323
['test/gtest_color_test.py'],
326
available_configurations=self.fake_configurations),
327
([(GTEST_DBG_DIR, 'test/gtest_color_test.py'),
328
(GTEST_OPT_DIR, 'test/gtest_color_test.py')],
331
# All built configurations (unbuilt don't cause failure).
332
self.AssertResultsEqual(
333
self.test_runner.GetTestsToRun(
334
['gtest_color_test.py'],
337
available_configurations=self.fake_configurations + ['unbuilt']),
338
([(GTEST_DBG_DIR, 'test/gtest_color_test.py'),
339
(GTEST_OPT_DIR, 'test/gtest_color_test.py')],
342
# A combination of an explicit directory and a configuration.
343
self.AssertResultsEqual(
344
self.test_runner.GetTestsToRun(
345
[GTEST_DBG_DIR, 'gtest_color_test.py'],
348
available_configurations=self.fake_configurations),
349
([(GTEST_DBG_DIR, 'test/gtest_color_test.py'),
350
(GTEST_OPT_DIR, 'test/gtest_color_test.py')],
353
# Same test specified in an explicit directory and via a configuration.
354
self.AssertResultsEqual(
355
self.test_runner.GetTestsToRun(
356
[GTEST_DBG_DIR, 'gtest_color_test.py'],
359
available_configurations=self.fake_configurations),
360
([(GTEST_DBG_DIR, 'test/gtest_color_test.py')],
363
# All built configurations + explicit directory + explicit configuration.
364
self.AssertResultsEqual(
365
self.test_runner.GetTestsToRun(
366
[GTEST_DBG_DIR, 'gtest_color_test.py'],
369
available_configurations=self.fake_configurations),
370
([(GTEST_DBG_DIR, 'test/gtest_color_test.py'),
371
(GTEST_OPT_DIR, 'test/gtest_color_test.py')],
374
def testCombinationOfBinaryAndPythonTests(self):
375
"""Exercises GetTestsToRun with mixed binary/Python tests."""
377
# Use only default configuration for this test.
379
# Neither binary nor Python tests are specified so find all.
380
self.AssertResultsEqual(
381
self.test_runner.GetTestsToRun(
385
available_configurations=self.fake_configurations),
386
([(GTEST_DBG_DIR, 'test/gtest_color_test.py')],
387
[(GTEST_DBG_DIR, GTEST_DBG_DIR + '/gtest_unittest')]))
389
# Specifying both binary and Python tests.
390
self.AssertResultsEqual(
391
self.test_runner.GetTestsToRun(
392
['gtest_unittest', 'gtest_color_test.py'],
395
available_configurations=self.fake_configurations),
396
([(GTEST_DBG_DIR, 'test/gtest_color_test.py')],
397
[(GTEST_DBG_DIR, GTEST_DBG_DIR + '/gtest_unittest')]))
399
# Specifying binary tests suppresses Python tests.
400
self.AssertResultsEqual(
401
self.test_runner.GetTestsToRun(
405
available_configurations=self.fake_configurations),
407
[(GTEST_DBG_DIR, GTEST_DBG_DIR + '/gtest_unittest')]))
409
# Specifying Python tests suppresses binary tests.
410
self.AssertResultsEqual(
411
self.test_runner.GetTestsToRun(
412
['gtest_color_test.py'],
415
available_configurations=self.fake_configurations),
416
([(GTEST_DBG_DIR, 'test/gtest_color_test.py')],
419
def testIgnoresNonTestFiles(self):
420
"""Verifies that GetTestsToRun ignores non-test files in the filesystem."""
422
self.fake_os = FakeOs(FakePath(
423
current_dir=os.path.abspath(os.path.dirname(run_tests_util.__file__)),
424
known_paths=[AddExeExtension(GTEST_DBG_DIR + '/gtest_nontest'),
426
self.test_runner = run_tests_util.TestRunner(script_dir='.',
427
injected_os=self.fake_os,
428
injected_subprocess=None)
429
self.AssertResultsEqual(
430
self.test_runner.GetTestsToRun(
434
available_configurations=self.fake_configurations),
437
def testWorksFromDifferentDir(self):
438
"""Exercises GetTestsToRun from a directory different from run_test.py's."""
440
# Here we simulate an test script in directory /d/ called from the
442
self.fake_os = FakeOs(FakePath(
443
current_dir=os.path.abspath('/a/b/c'),
446
AddExeExtension('/d/' + GTEST_DBG_DIR + '/gtest_unittest'),
447
AddExeExtension('/d/' + GTEST_OPT_DIR + '/gtest_unittest'),
448
'/d/test/gtest_color_test.py']))
449
self.fake_configurations = ['dbg', 'opt']
450
self.test_runner = run_tests_util.TestRunner(script_dir='/d/',
451
injected_os=self.fake_os,
452
injected_subprocess=None)
454
self.AssertResultsEqual(
455
self.test_runner.GetTestsToRun(
459
available_configurations=self.fake_configurations),
461
[('/d/' + GTEST_DBG_DIR, '/d/' + GTEST_DBG_DIR + '/gtest_unittest')]))
464
self.AssertResultsEqual(
465
self.test_runner.GetTestsToRun(
466
['gtest_color_test.py'],
469
available_configurations=self.fake_configurations),
470
([('/d/' + GTEST_DBG_DIR, '/d/test/gtest_color_test.py')], []))
472
def testNonTestBinary(self):
473
"""Exercises GetTestsToRun with a non-test parameter."""
476
not self.test_runner.GetTestsToRun(
477
['gtest_unittest_not_really'],
480
available_configurations=self.fake_configurations))
482
def testNonExistingPythonTest(self):
483
"""Exercises GetTestsToRun with a non-existent Python test parameter."""
486
not self.test_runner.GetTestsToRun(
487
['nonexistent_test.py'],
490
available_configurations=self.fake_configurations))
492
if run_tests_util.IS_WINDOWS or run_tests_util.IS_CYGWIN:
494
def testDoesNotPickNonExeFilesOnWindows(self):
495
"""Verifies that GetTestsToRun does not find _test files on Windows."""
497
self.fake_os = FakeOs(FakePath(
498
current_dir=os.path.abspath(os.path.dirname(run_tests_util.__file__)),
499
known_paths=['/d/' + GTEST_DBG_DIR + '/gtest_test', 'test/']))
500
self.test_runner = run_tests_util.TestRunner(script_dir='.',
501
injected_os=self.fake_os,
502
injected_subprocess=None)
503
self.AssertResultsEqual(
504
self.test_runner.GetTestsToRun(
508
available_configurations=self.fake_configurations),
512
class RunTestsTest(unittest.TestCase):
513
"""Exercises TestRunner.RunTests."""
515
def SpawnSuccess(self, unused_executable, unused_argv):
516
"""Fakes test success by returning 0 as an exit code."""
518
self.num_spawn_calls += 1
521
def SpawnFailure(self, unused_executable, unused_argv):
522
"""Fakes test success by returning 1 as an exit code."""
524
self.num_spawn_calls += 1
528
self.fake_os = FakeOs(FakePath(
529
current_dir=os.path.abspath(os.path.dirname(run_tests_util.__file__)),
531
AddExeExtension(GTEST_DBG_DIR + '/gtest_unittest'),
532
AddExeExtension(GTEST_OPT_DIR + '/gtest_unittest'),
533
'test/gtest_color_test.py']))
534
self.fake_configurations = ['dbg', 'opt']
535
self.test_runner = run_tests_util.TestRunner(
536
script_dir=os.path.dirname(__file__) or '.',
537
injected_os=self.fake_os,
538
injected_subprocess=None)
539
self.num_spawn_calls = 0 # A number of calls to spawn.
541
def testRunPythonTestSuccess(self):
542
"""Exercises RunTests to handle a Python test success."""
544
self.fake_os.spawn_impl = self.SpawnSuccess
546
self.test_runner.RunTests(
547
[(GTEST_DBG_DIR, 'test/gtest_color_test.py')],
550
self.assertEqual(self.num_spawn_calls, 1)
552
def testRunBinaryTestSuccess(self):
553
"""Exercises RunTests to handle a binary test success."""
555
self.fake_os.spawn_impl = self.SpawnSuccess
557
self.test_runner.RunTests(
559
[(GTEST_DBG_DIR, GTEST_DBG_DIR + '/gtest_unittest')]),
561
self.assertEqual(self.num_spawn_calls, 1)
563
def testRunPythonTestFauilure(self):
564
"""Exercises RunTests to handle a Python test failure."""
566
self.fake_os.spawn_impl = self.SpawnFailure
568
self.test_runner.RunTests(
569
[(GTEST_DBG_DIR, 'test/gtest_color_test.py')],
572
self.assertEqual(self.num_spawn_calls, 1)
574
def testRunBinaryTestFailure(self):
575
"""Exercises RunTests to handle a binary test failure."""
577
self.fake_os.spawn_impl = self.SpawnFailure
579
self.test_runner.RunTests(
581
[(GTEST_DBG_DIR, GTEST_DBG_DIR + '/gtest_unittest')]),
583
self.assertEqual(self.num_spawn_calls, 1)
585
def testCombinedTestSuccess(self):
586
"""Exercises RunTests to handle a success of both Python and binary test."""
588
self.fake_os.spawn_impl = self.SpawnSuccess
590
self.test_runner.RunTests(
591
[(GTEST_DBG_DIR, GTEST_DBG_DIR + '/gtest_unittest')],
592
[(GTEST_DBG_DIR, GTEST_DBG_DIR + '/gtest_unittest')]),
594
self.assertEqual(self.num_spawn_calls, 2)
596
def testCombinedTestSuccessAndFailure(self):
597
"""Exercises RunTests to handle a success of both Python and binary test."""
599
def SpawnImpl(executable, argv):
600
self.num_spawn_calls += 1
601
# Simulates failure of a Python test and success of a binary test.
602
if '.py' in executable or '.py' in argv[0]:
607
self.fake_os.spawn_impl = SpawnImpl
609
self.test_runner.RunTests(
610
[(GTEST_DBG_DIR, GTEST_DBG_DIR + '/gtest_unittest')],
611
[(GTEST_DBG_DIR, GTEST_DBG_DIR + '/gtest_unittest')]),
613
self.assertEqual(self.num_spawn_calls, 2)
616
class ParseArgsTest(unittest.TestCase):
617
"""Exercises ParseArgs."""
619
def testNoOptions(self):
620
options, args = run_tests_util.ParseArgs('gtest', argv=['script.py'])
621
self.assertEqual(args, ['script.py'])
622
self.assert_(options.configurations is None)
623
self.assertFalse(options.built_configurations)
625
def testOptionC(self):
626
options, args = run_tests_util.ParseArgs(
627
'gtest', argv=['script.py', '-c', 'dbg'])
628
self.assertEqual(args, ['script.py'])
629
self.assertEqual(options.configurations, 'dbg')
630
self.assertFalse(options.built_configurations)
632
def testOptionA(self):
633
options, args = run_tests_util.ParseArgs('gtest', argv=['script.py', '-a'])
634
self.assertEqual(args, ['script.py'])
635
self.assertEqual(options.configurations, 'all')
636
self.assertFalse(options.built_configurations)
638
def testOptionB(self):
639
options, args = run_tests_util.ParseArgs('gtest', argv=['script.py', '-b'])
640
self.assertEqual(args, ['script.py'])
641
self.assert_(options.configurations is None)
642
self.assertTrue(options.built_configurations)
644
def testOptionCAndOptionB(self):
645
options, args = run_tests_util.ParseArgs(
646
'gtest', argv=['script.py', '-c', 'dbg', '-b'])
647
self.assertEqual(args, ['script.py'])
648
self.assertEqual(options.configurations, 'dbg')
649
self.assertTrue(options.built_configurations)
651
def testOptionH(self):
652
help_called = [False]
654
# Suppresses lint warning on unused arguments. These arguments are
655
# required by optparse, even though they are unused.
656
# pylint: disable-msg=W0613
657
def VerifyHelp(option, opt, value, parser):
658
help_called[0] = True
660
# Verifies that -h causes the help callback to be called.
661
help_called[0] = False
662
_, args = run_tests_util.ParseArgs(
663
'gtest', argv=['script.py', '-h'], help_callback=VerifyHelp)
664
self.assertEqual(args, ['script.py'])
665
self.assertTrue(help_called[0])
667
# Verifies that --help causes the help callback to be called.
668
help_called[0] = False
669
_, args = run_tests_util.ParseArgs(
670
'gtest', argv=['script.py', '--help'], help_callback=VerifyHelp)
671
self.assertEqual(args, ['script.py'])
672
self.assertTrue(help_called[0])
675
if __name__ == '__main__':