33
33
# --all: run tests in all subdirs
34
# --valgrind: valgrind to use. Default is one built from this source tree.
34
# --valgrind: valgrind launcher to use. Default is ./coregrind/valgrind.
35
# (This option should probably only be used in conjunction with
37
# --valgrind-lib: valgrind libraries to use. Default is $tests_dir/.in_place.
38
# (This option should probably only be used in conjunction with
36
41
# The easiest way is to run all tests in valgrind/ with (assuming you installed
50
55
# - stdout_filter: <filter to run stdout through> (default: none)
51
56
# - stderr_filter: <filter to run stderr through> (default: ./filter_stderr)
52
57
# - prereq: <prerequisite command> (default: none)
53
# - posttest: <post-test check command> (default: none)
54
# - cleanup: <post-test cleanup cmd to run> (default: none)
58
# - post: <post-test check command> (default: none)
59
# - cleanup: <post-test cleanup cmd> (default: none)
56
61
# Note that filters are necessary for stderr results to filter out things that
57
62
# always change, eg. process id numbers.
59
64
# Expected stdout (filtered) is kept in <test>.stdout.exp[0-9]* (can be more
60
65
# than one expected output). It can be missing if it would be empty. Expected
61
# stderr (filtered) is kept in <test>.stderr.exp[0-9]*.
66
# stderr (filtered) is kept in <test>.stderr.exp*. There must be at least
67
# one stderr.exp* file.
63
69
# The prerequisite command, if present, must return 0 otherwise the test is
64
70
# skipped. The post-test command, if present, must return 0 and its stdout
65
# must match the expected stdout which is kept in <test>.posttest.exp[0-9]*.
71
# must match the expected stdout which is kept in <test>.post.exp[0-9]*.
67
73
# If results don't match, the output can be found in <test>.std<strm>.out,
68
74
# and the diff between expected and actual in <test>.std<strm>.diff[0-9]*.
73
79
# and handed to valgrind prior to any other flags specified by the
76
# Notes on adding regression tests for a new tool are in
77
# coregrind/docs/coregrind_tools.html.
82
# Some more notes on adding regression tests for a new tool are in
83
# docs/xml/manual-writing-tools.xml.
78
84
#----------------------------------------------------------------------------
83
89
#----------------------------------------------------------------------------
85
91
#----------------------------------------------------------------------------
86
my $usage="vg_regtest [--all, --valgrind]\n";
94
. " vg_regtest [--all, --valgrind, --valgrind-lib]\n"
95
. " Use EXTRA_REGTEST_OPTS to supply extra args for all tests\n"
88
98
my $tmp="vg_regtest.tmp.$$";
94
104
my $stdout_filter; # filter program to run stdout results file through
95
105
my $stderr_filter; # filter program to run stderr results file through
96
106
my $prereq; # prerequisite test to satisfy before running test
97
my $posttest; # check command after running test
107
my $post; # check command after running test
98
108
my $cleanup; # cleanup command to run
100
110
my @failures; # List of failed tests
102
112
my $num_tests_done = 0;
103
my %num_failures = (stderr => 0, stdout => 0, posttest => 0);
113
my %num_failures = (stderr => 0, stdout => 0, post => 0);
105
115
# Default valgrind to use is this build tree's (uninstalled) one
106
116
my $valgrind = "./coregrind/valgrind";
108
118
chomp(my $tests_dir = `pwd`);
120
my $valgrind_lib = "$tests_dir/.in_place";
110
122
# default filter is the one named "filter_stderr" in the test's directory
111
123
my $default_stderr_filter = "filter_stderr";
184
198
($vgopts, $prog, $args) = ("", undef, "");
185
199
($stdout_filter, $stderr_filter) = (undef, undef);
186
($prereq, $posttest, $cleanup) = (undef, undef, undef);
200
($prereq, $post, $cleanup) = (undef, undef, undef);
188
202
# Every test directory must have a "filter_stderr"
189
203
$stderr_filter = validate_program(".", $default_stderr_filter, 1, 1);
205
219
$stderr_filter = validate_program(".", $1, 1, 1);
206
220
} elsif ($line =~ /^\s*prereq:\s*(.*)$/) {
208
} elsif ($line =~ /^\s*posttest:\s*(.*)$/) {
222
} elsif ($line =~ /^\s*post:\s*(.*)$/) {
210
224
} elsif ($line =~ /^\s*cleanup:\s*(.*)$/) {
227
241
# propagate a Ctrl-C enabling us to quit.
230
(system($_[0]) != 2) or exit 1; # 2 is SIGINT
244
my $exit_code = system($_[0]);
245
($exit_code == 2) and exit 1; # 2 is SIGINT
233
249
# from a directory name like "/foo/cachesim/tests/" determine the tool name
255
271
($f_exp eq "/dev/null") or die "Unexpected .exp file: $f_exp\n";
258
#print("diff -C0 $f_exp $name.$mid.out > $name.$mid.diff$n\n");
259
mysystem("diff -C0 $f_exp $name.$mid.out > $name.$mid.diff$n");
274
#print("diff $f_exp $name.$mid.out > $name.$mid.diff$n\n");
275
mysystem("diff $f_exp $name.$mid.out > $name.$mid.diff$n");
261
277
if (not -s "$name.$mid.diff$n") {
262
278
# A match; remove .out and any previously created .diff files.
299
315
# VALGRIND_LIB_INNER in case this Valgrind was configured with
300
316
# --enable-inner.
301
317
my $tool=determine_tool();
302
mysystem("VALGRIND_LIB=$tests_dir/.in_place VALGRIND_LIB_INNER=$tests_dir/.in_place "
318
mysystem("VALGRIND_LIB=$valgrind_lib VALGRIND_LIB_INNER=$valgrind_lib "
303
319
. "$valgrind --command-line-only=yes --memcheck:leak-check=no "
304
320
. "--tool=$tool $extraopts $vgopts "
305
321
. "$prog $args > $name.stdout.out 2> $name.stderr.out");
318
334
mysystem("$stderr_filter < $name.stderr.out > $tmp");
319
335
rename($tmp, "$name.stderr.out");
320
# Find all the .stderr.exp files. $name.stderr.exp must exist.
336
# Find all the .stderr.exp files. At least one must exist.
321
337
my @stderr_exps = <$name.stderr.exp*>;
322
(-r "$name.stderr.exp") or die "Could not read `$name.stderr.exp'\n";
338
(0 != scalar @stderr_exps) or die "Could not find `$name.stderr.exp*'\n";
323
339
do_diffs($fullname, $name, "stderr", \@stderr_exps);
325
341
# Maybe do post-test check
326
if (defined $posttest) {
327
if (mysystem("$posttest > $name.posttest.out") != 0) {
328
print("posttest failed: $posttest\n");
329
$num_failures{"posttest"}++;
343
if (mysystem("$post > $name.post.out") != 0) {
344
print("post check failed: $post\n");
345
$num_failures{"post"}++;
331
# Find all the .posttest.exp files. If none, use /dev/null.
332
my @posttest_exps = <$name.posttest.exp*>;
333
@posttest_exps = ( "/dev/null" ) if (0 == scalar @posttest_exps);
334
do_diffs($fullname, $name, "posttest", \@posttest_exps);
347
# Find all the .post.exp files. If none, use /dev/null.
348
my @post_exps = <$name.post.exp*>;
349
@post_exps = ( "/dev/null" ) if (0 == scalar @post_exps);
350
do_diffs($fullname, $name, "post", \@post_exps);
404
420
my $x = ( $num_tests_done == 1 ? "test" : "tests" );
406
422
printf("\n== %d test%s, %d stderr failure%s, %d stdout failure%s, "
407
. "%d posttest failure%s ==\n",
423
. "%d post failure%s ==\n",
408
424
$num_tests_done, plural($num_tests_done),
409
425
$num_failures{"stderr"}, plural($num_failures{"stderr"}),
410
426
$num_failures{"stdout"}, plural($num_failures{"stdout"}),
411
$num_failures{"posttest"}, plural($num_failures{"posttest"}));
427
$num_failures{"post"}, plural($num_failures{"post"}));
413
429
foreach my $failure (@failures) {
414
430
print "$failure\n";
added
removed
tests/vg_regtest.in
