« back to all changes in this revision
Viewing changes to QMTest/TestCommon.py
- browse files at revision 1
- compare with another revision
- download diff
- download tarball
- view history from revision 1
- Committer: Dirk Baechle
- Date: 2010-05-27 22:56:26 UTC
- Revision ID: dl9obn@darc.de-20100527225626-bkgihswua4luj9qk
Imported original version from the SCons trunk
added
removed
QMTest/TestCommon.py
1
"""
2
TestCommon.py: a testing framework for commands and scripts
3
with commonly useful error handling
4
5
The TestCommon module provides a simple, high-level interface for writing
6
tests of executable commands and scripts, especially commands and scripts
7
that interact with the file system. All methods throw exceptions and
8
exit on failure, with useful error messages. This makes a number of
9
explicit checks unnecessary, making the test scripts themselves simpler
10
to write and easier to read.
11
12
The TestCommon class is a subclass of the TestCmd class. In essence,
13
TestCommon is a wrapper that handles common TestCmd error conditions in
14
useful ways. You can use TestCommon directly, or subclass it for your
15
program and add additional (or override) methods to tailor it to your
16
program's specific needs. Alternatively, the TestCommon class serves
17
as a useful example of how to define your own TestCmd subclass.
18
19
As a subclass of TestCmd, TestCommon provides access to all of the
20
variables and methods from the TestCmd module. Consequently, you can
21
use any variable or method documented in the TestCmd module without
22
having to explicitly import TestCmd.
23
24
A TestCommon environment object is created via the usual invocation:
25
26
import TestCommon
27
test = TestCommon.TestCommon()
28
29
You can use all of the TestCmd keyword arguments when instantiating a
30
TestCommon object; see the TestCmd documentation for details.
31
32
Here is an overview of the methods and keyword arguments that are
33
provided by the TestCommon class:
34
35
test.must_be_writable('file1', ['file2', ...])
36
37
test.must_contain('file', 'required text\n')
38
39
test.must_contain_all_lines(output, lines, ['title', find])
40
41
test.must_contain_any_line(output, lines, ['title', find])
42
43
test.exactly_contain_all_lines(output, lines, ['title', find])
44
45
test.must_exist('file1', ['file2', ...])
46
47
test.must_match('file', "expected contents\n")
48
49
test.must_not_be_writable('file1', ['file2', ...])
50
51
test.must_not_contain('file', 'banned text\n')
52
53
test.must_not_contain_any_line(output, lines, ['title', find])
54
55
test.must_not_exist('file1', ['file2', ...])
56
57
test.run(options = "options to be prepended to arguments",
58
stdout = "expected standard output from the program",
59
stderr = "expected error output from the program",
60
status = expected_status,
61
match = match_function)
62
63
The TestCommon module also provides the following variables
64
65
TestCommon.python
66
TestCommon._python_
67
TestCommon.exe_suffix
68
TestCommon.obj_suffix
69
TestCommon.shobj_prefix
70
TestCommon.shobj_suffix
71
TestCommon.lib_prefix
72
TestCommon.lib_suffix
73
TestCommon.dll_prefix
74
TestCommon.dll_suffix
75
76
"""
77
78
# Copyright 2000-2010 Steven Knight
79
# This module is free software, and you may redistribute it and/or modify
80
# it under the same terms as Python itself, so long as this copyright message
81
# and disclaimer are retained in their original form.
82
#
83
# IN NO EVENT SHALL THE AUTHOR BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT,
84
# SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OF
85
# THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
86
# DAMAGE.
87
#
88
# THE AUTHOR SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT
89
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
90
# PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS,
91
# AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
92
# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
93
94
__author__ = "Steven Knight <knight at baldmt dot com>"
95
__revision__ = "TestCommon.py 0.37.D001 2010/01/11 16:55:50 knight"
96
__version__ = "0.37"
97
98
import copy
99
import os
100
import stat
101
import sys
102
try:
103
from collections import UserList
104
except ImportError:
105
# no 'collections' module or no UserList in collections
106
exec('from UserList import UserList')
107
108
from TestCmd import *
109
from TestCmd import __all__
110
111
__all__.extend([ 'TestCommon',
112
'exe_suffix',
113
'obj_suffix',
114
'shobj_prefix',
115
'shobj_suffix',
116
'lib_prefix',
117
'lib_suffix',
118
'dll_prefix',
119
'dll_suffix',
120
])
121
122
# Variables that describe the prefixes and suffixes on this system.
123
if sys.platform == 'win32':
124
exe_suffix = '.exe'
125
obj_suffix = '.obj'
126
shobj_suffix = '.obj'
127
shobj_prefix = ''
128
lib_prefix = ''
129
lib_suffix = '.lib'
130
dll_prefix = ''
131
dll_suffix = '.dll'
132
elif sys.platform == 'cygwin':
133
exe_suffix = '.exe'
134
obj_suffix = '.o'
135
shobj_suffix = '.os'
136
shobj_prefix = ''
137
lib_prefix = 'lib'
138
lib_suffix = '.a'
139
dll_prefix = ''
140
dll_suffix = '.dll'
141
elif sys.platform.find('irix') != -1:
142
exe_suffix = ''
143
obj_suffix = '.o'
144
shobj_suffix = '.o'
145
shobj_prefix = ''
146
lib_prefix = 'lib'
147
lib_suffix = '.a'
148
dll_prefix = 'lib'
149
dll_suffix = '.so'
150
elif sys.platform.find('darwin') != -1:
151
exe_suffix = ''
152
obj_suffix = '.o'
153
shobj_suffix = '.os'
154
shobj_prefix = ''
155
lib_prefix = 'lib'
156
lib_suffix = '.a'
157
dll_prefix = 'lib'
158
dll_suffix = '.dylib'
159
elif sys.platform.find('sunos') != -1:
160
exe_suffix = ''
161
obj_suffix = '.o'
162
shobj_suffix = '.os'
163
shobj_prefix = 'so_'
164
lib_prefix = 'lib'
165
lib_suffix = '.a'
166
dll_prefix = 'lib'
167
dll_suffix = '.dylib'
168
else:
169
exe_suffix = ''
170
obj_suffix = '.o'
171
shobj_suffix = '.os'
172
shobj_prefix = ''
173
lib_prefix = 'lib'
174
lib_suffix = '.a'
175
dll_prefix = 'lib'
176
dll_suffix = '.so'
177
178
def is_List(e):
179
return isinstance(e, (list,UserList))
180
181
def is_writable(f):
182
mode = os.stat(f)[stat.ST_MODE]
183
return mode & stat.S_IWUSR
184
185
def separate_files(flist):
186
existing = []
187
missing = []
188
for f in flist:
189
if os.path.exists(f):
190
existing.append(f)
191
else:
192
missing.append(f)
193
return existing, missing
194
195
if os.name == 'posix':
196
def _failed(self, status = 0):
197
if self.status is None or status is None:
198
return None
199
return _status(self) != status
200
def _status(self):
201
return self.status
202
elif os.name == 'nt':
203
def _failed(self, status = 0):
204
return not (self.status is None or status is None) and \
205
self.status != status
206
def _status(self):
207
return self.status
208
209
class TestCommon(TestCmd):
210
211
# Additional methods from the Perl Test::Cmd::Common module
212
# that we may wish to add in the future:
213
#
214
# $test->subdir('subdir', ...);
215
#
216
# $test->copy('src_file', 'dst_file');
217
218
def __init__(self, **kw):
219
"""Initialize a new TestCommon instance. This involves just
220
calling the base class initialization, and then changing directory
221
to the workdir.
222
"""
223
TestCmd.__init__(self, **kw)
224
os.chdir(self.workdir)
225
226
def must_be_writable(self, *files):
227
"""Ensures that the specified file(s) exist and are writable.
228
An individual file can be specified as a list of directory names,
229
in which case the pathname will be constructed by concatenating
230
them. Exits FAILED if any of the files does not exist or is
231
not writable.
232
"""
233
files = [is_List(x) and os.path.join(*x) or x for x in files]
234
existing, missing = separate_files(files)
235
unwritable = [x for x in existing if not is_writable(x)]
236
if missing:
237
print "Missing files: `%s'" % "', `".join(missing)
238
if unwritable:
239
print "Unwritable files: `%s'" % "', `".join(unwritable)
240
self.fail_test(missing + unwritable)
241
242
def must_contain(self, file, required, mode = 'rb'):
243
"""Ensures that the specified file contains the required text.
244
"""
245
file_contents = self.read(file, mode)
246
contains = (file_contents.find(required) != -1)
247
if not contains:
248
print "File `%s' does not contain required string." % file
249
print self.banner('Required string ')
250
print required
251
print self.banner('%s contents ' % file)
252
print file_contents
253
self.fail_test(not contains)
254
255
def must_contain_all_lines(self, output, lines, title=None, find=None):
256
"""Ensures that the specified output string (first argument)
257
contains all of the specified lines (second argument).
258
259
An optional third argument can be used to describe the type
260
of output being searched, and only shows up in failure output.
261
262
An optional fourth argument can be used to supply a different
263
function, of the form "find(line, output), to use when searching
264
for lines in the output.
265
"""
266
if find is None:
267
find = lambda o, l: o.find(l) != -1
268
missing = []
269
for line in lines:
270
if not find(output, line):
271
missing.append(line)
272
273
if missing:
274
if title is None:
275
title = 'output'
276
sys.stdout.write("Missing expected lines from %s:\n" % title)
277
for line in missing:
278
sys.stdout.write(' ' + repr(line) + '\n')
279
sys.stdout.write(self.banner(title + ' '))
280
sys.stdout.write(output)
281
self.fail_test()
282
283
def must_contain_any_line(self, output, lines, title=None, find=None):
284
"""Ensures that the specified output string (first argument)
285
contains at least one of the specified lines (second argument).
286
287
An optional third argument can be used to describe the type
288
of output being searched, and only shows up in failure output.
289
290
An optional fourth argument can be used to supply a different
291
function, of the form "find(line, output), to use when searching
292
for lines in the output.
293
"""
294
if find is None:
295
find = lambda o, l: o.find(l) != -1
296
for line in lines:
297
if find(output, line):
298
return
299
300
if title is None:
301
title = 'output'
302
sys.stdout.write("Missing any expected line from %s:\n" % title)
303
for line in lines:
304
sys.stdout.write(' ' + repr(line) + '\n')
305
sys.stdout.write(self.banner(title + ' '))
306
sys.stdout.write(output)
307
self.fail_test()
308
309
def exactly_contain_all_lines(self, output, expect, title=None, find=None):
310
"""Ensures that the specified output string (first argument)
311
contains all of the lines in the expected string (second argument)
312
with none left over.
313
314
An optional third argument can be used to describe the type
315
of output being searched, and only shows up in failure output.
316
317
An optional fourth argument can be used to supply a different
318
function, of the form "find(line, output), to use when searching
319
for lines in the output.
320
"""
321
out = output.splitlines()
322
exp = expect.splitlines()
323
if sorted(out) == sorted(exp):
324
# early out for exact match
325
return
326
if find is None:
327
def find(o, l):
328
try:
329
return o.index(l)
330
except:
331
return None
332
missing = []
333
for line in exp:
334
found = find(out, line)
335
if found is None:
336
missing.append(line)
337
else:
338
out.pop(found)
339
340
if not missing and not out:
341
# all lines were matched
342
return
343
344
if title is None:
345
title = 'output'
346
if missing:
347
sys.stdout.write("Missing expected lines from %s:\n" % title)
348
for line in missing:
349
sys.stdout.write(' ' + repr(line) + '\n')
350
sys.stdout.write(self.banner('missing %s ' % title))
351
if out:
352
sys.stdout.write("Extra unexpected lines from %s:\n" % title)
353
for line in out:
354
sys.stdout.write(' ' + repr(line) + '\n')
355
sys.stdout.write(self.banner('extra %s ' % title))
356
sys.stdout.flush()
357
self.fail_test()
358
359
def must_contain_lines(self, lines, output, title=None):
360
# Deprecated; retain for backwards compatibility.
361
return self.must_contain_all_lines(output, lines, title)
362
363
def must_exist(self, *files):
364
"""Ensures that the specified file(s) must exist. An individual
365
file be specified as a list of directory names, in which case the
366
pathname will be constructed by concatenating them. Exits FAILED
367
if any of the files does not exist.
368
"""
369
files = [is_List(x) and os.path.join(*x) or x for x in files]
370
missing = [x for x in files if not os.path.exists(x)]
371
if missing:
372
print "Missing files: `%s'" % "', `".join(missing)
373
self.fail_test(missing)
374
375
def must_match(self, file, expect, mode = 'rb'):
376
"""Matches the contents of the specified file (first argument)
377
against the expected contents (second argument). The expected
378
contents are a list of lines or a string which will be split
379
on newlines.
380
"""
381
file_contents = self.read(file, mode)
382
try:
383
self.fail_test(not self.match(file_contents, expect))
384
except KeyboardInterrupt:
385
raise
386
except:
387
print "Unexpected contents of `%s'" % file
388
self.diff(expect, file_contents, 'contents ')
389
raise
390
391
def must_not_contain(self, file, banned, mode = 'rb'):
392
"""Ensures that the specified file doesn't contain the banned text.
393
"""
394
file_contents = self.read(file, mode)
395
contains = (file_contents.find(banned) != -1)
396
if contains:
397
print "File `%s' contains banned string." % file
398
print self.banner('Banned string ')
399
print banned
400
print self.banner('%s contents ' % file)
401
print file_contents
402
self.fail_test(contains)
403
404
def must_not_contain_any_line(self, output, lines, title=None, find=None):
405
"""Ensures that the specified output string (first argument)
406
does not contain any of the specified lines (second argument).
407
408
An optional third argument can be used to describe the type
409
of output being searched, and only shows up in failure output.
410
411
An optional fourth argument can be used to supply a different
412
function, of the form "find(line, output), to use when searching
413
for lines in the output.
414
"""
415
if find is None:
416
find = lambda o, l: o.find(l) != -1
417
unexpected = []
418
for line in lines:
419
if find(output, line):
420
unexpected.append(line)
421
422
if unexpected:
423
if title is None:
424
title = 'output'
425
sys.stdout.write("Unexpected lines in %s:\n" % title)
426
for line in unexpected:
427
sys.stdout.write(' ' + repr(line) + '\n')
428
sys.stdout.write(self.banner(title + ' '))
429
sys.stdout.write(output)
430
self.fail_test()
431
432
def must_not_contain_lines(self, lines, output, title=None):
433
return self.must_not_contain_any_line(output, lines, title)
434
435
def must_not_exist(self, *files):
436
"""Ensures that the specified file(s) must not exist.
437
An individual file be specified as a list of directory names, in
438
which case the pathname will be constructed by concatenating them.
439
Exits FAILED if any of the files exists.
440
"""
441
files = [is_List(x) and os.path.join(*x) or x for x in files]
442
existing = list(filter(os.path.exists, files))
443
if existing:
444
print "Unexpected files exist: `%s'" % "', `".join(existing)
445
self.fail_test(existing)
446
447
448
def must_not_be_writable(self, *files):
449
"""Ensures that the specified file(s) exist and are not writable.
450
An individual file can be specified as a list of directory names,
451
in which case the pathname will be constructed by concatenating
452
them. Exits FAILED if any of the files does not exist or is
453
writable.
454
"""
455
files = [is_List(x) and os.path.join(*x) or x for x in files]
456
existing, missing = separate_files(files)
457
writable = list(filter(is_writable, existing))
458
if missing:
459
print "Missing files: `%s'" % "', `".join(missing)
460
if writable:
461
print "Writable files: `%s'" % "', `".join(writable)
462
self.fail_test(missing + writable)
463
464
def _complete(self, actual_stdout, expected_stdout,
465
actual_stderr, expected_stderr, status, match):
466
"""
467
Post-processes running a subcommand, checking for failure
468
status and displaying output appropriately.
469
"""
470
if _failed(self, status):
471
expect = ''
472
if status != 0:
473
expect = " (expected %s)" % str(status)
474
print "%s returned %s%s" % (self.program, str(_status(self)), expect)
475
print self.banner('STDOUT ')
476
print actual_stdout
477
print self.banner('STDERR ')
478
print actual_stderr
479
self.fail_test()
480
if not expected_stdout is None and not match(actual_stdout, expected_stdout):
481
self.diff(expected_stdout, actual_stdout, 'STDOUT ')
482
if actual_stderr:
483
print self.banner('STDERR ')
484
print actual_stderr
485
self.fail_test()
486
if not expected_stderr is None and not match(actual_stderr, expected_stderr):
487
print self.banner('STDOUT ')
488
print actual_stdout
489
self.diff(expected_stderr, actual_stderr, 'STDERR ')
490
self.fail_test()
491
492
def start(self, program = None,
493
interpreter = None,
494
arguments = None,
495
universal_newlines = None,
496
**kw):
497
"""
498
Starts a program or script for the test environment.
499
500
This handles the "options" keyword argument and exceptions.
501
"""
502
try:
503
options = kw['options']
504
del kw['options']
505
except KeyError:
506
pass
507
else:
508
if options:
509
if arguments is None:
510
arguments = options
511
else:
512
arguments = options + " " + arguments
513
try:
514
return TestCmd.start(self, program, interpreter, arguments, universal_newlines,
515
**kw)
516
except KeyboardInterrupt:
517
raise
518
except Exception, e:
519
print self.banner('STDOUT ')
520
try:
521
print self.stdout()
522
except IndexError:
523
pass
524
print self.banner('STDERR ')
525
try:
526
print self.stderr()
527
except IndexError:
528
pass
529
cmd_args = self.command_args(program, interpreter, arguments)
530
sys.stderr.write('Exception trying to execute: %s\n' % cmd_args)
531
raise e
532
533
def finish(self, popen, stdout = None, stderr = '', status = 0, **kw):
534
"""
535
Finishes and waits for the process being run under control of
536
the specified popen argument. Additional arguments are similar
537
to those of the run() method:
538
539
stdout The expected standard output from
540
the command. A value of None means
541
don't test standard output.
542
543
stderr The expected error output from
544
the command. A value of None means
545
don't test error output.
546
547
status The expected exit status from the
548
command. A value of None means don't
549
test exit status.
550
"""
551
TestCmd.finish(self, popen, **kw)
552
match = kw.get('match', self.match)
553
self._complete(self.stdout(), stdout,
554
self.stderr(), stderr, status, match)
555
556
def run(self, options = None, arguments = None,
557
stdout = None, stderr = '', status = 0, **kw):
558
"""Runs the program under test, checking that the test succeeded.
559
560
The arguments are the same as the base TestCmd.run() method,
561
with the addition of:
562
563
options Extra options that get appended to the beginning
564
of the arguments.
565
566
stdout The expected standard output from
567
the command. A value of None means
568
don't test standard output.
569
570
stderr The expected error output from
571
the command. A value of None means
572
don't test error output.
573
574
status The expected exit status from the
575
command. A value of None means don't
576
test exit status.
577
578
By default, this expects a successful exit (status = 0), does
579
not test standard output (stdout = None), and expects that error
580
output is empty (stderr = "").
581
"""
582
if options:
583
if arguments is None:
584
arguments = options
585
else:
586
arguments = options + " " + arguments
587
kw['arguments'] = arguments
588
try:
589
match = kw['match']
590
del kw['match']
591
except KeyError:
592
match = self.match
593
TestCmd.run(self, **kw)
594
self._complete(self.stdout(), stdout,
595
self.stderr(), stderr, status, match)
596
597
def skip_test(self, message="Skipping test.\n"):
598
"""Skips a test.
599
600
Proper test-skipping behavior is dependent on the external
601
TESTCOMMON_PASS_SKIPS environment variable. If set, we treat
602
the skip as a PASS (exit 0), and otherwise treat it as NO RESULT.
603
In either case, we print the specified message as an indication
604
that the substance of the test was skipped.
605
606
(This was originally added to support development under Aegis.
607
Technically, skipping a test is a NO RESULT, but Aegis would
608
treat that as a test failure and prevent the change from going to
609
the next step. Since we ddn't want to force anyone using Aegis
610
to have to install absolutely every tool used by the tests, we
611
would actually report to Aegis that a skipped test has PASSED
612
so that the workflow isn't held up.)
613
"""
614
if message:
615
sys.stdout.write(message)
616
sys.stdout.flush()
617
pass_skips = os.environ.get('TESTCOMMON_PASS_SKIPS')
618
if pass_skips in [None, 0, '0']:
619
# skip=1 means skip this function when showing where this
620
# result came from. They only care about the line where the
621
# script called test.skip_test(), not the line number where
622
# we call test.no_result().
623
self.no_result(skip=1)
624
else:
625
# We're under the development directory for this change,
626
# so this is an Aegis invocation; pass the test (exit 0).
627
self.pass_test()
628
629
# Local Variables:
630
# tab-width:4
631
# indent-tabs-mode:nil
632
# End:
633
# vim: set expandtab tabstop=4 shiftwidth=4:
Loggerhead is a web-based interface for Breezy
Version: 2.0.1