2
# Bootstrap Samba and run a number of tests against it.
3
# Copyright (C) 2005-2008 Jelmer Vernooij <jelmer@samba.org>
4
# Copyright (C) 2007-2009 Stefan Metzmacher <metze@samba.org>
5
# Published under the GNU GPL, v3 or later.
11
selftest - Samba test runner
17
selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--immediate] [--testlist=FILE] [TESTS]
21
A simple test runner. TESTS is a regular expression with tests to run.
29
Show list of available options.
35
=item I<--builddir=DIR>
45
Change directory to run tests in. Default is 'st'.
49
Show errors as soon as they happen rather than at the end of the test run.
51
=item I<--target samba4|samba3|win|kvm>
53
Specify test target against which to run. Default is 'samba4'.
57
Run only a limited number of tests. Intended to run in about 30 seconds on
58
moderately recent systems.
60
=item I<--socket-wrapper>
62
Use socket wrapper library for communication with server. Only works
63
when the server is running locally.
65
Will prevent TCP and UDP ports being opened on the local host but
66
(transparently) redirects these calls to use unix domain sockets.
68
=item I<--expected-failures>
70
Specify a file containing a list of tests that are expected to fail. Failures for
71
these tests will be counted as successes, successes will be counted as failures.
73
The format for the file is, one entry per line:
75
TESTSUITE-NAME.TEST-NAME
77
The reason for a test can also be specified, by adding a hash sign (#) and the reason
82
Specify a file containing a list of tests that should be skipped. Possible
83
candidates are tests that segfault the server, flip or don't end. The format of this file is the same as
84
for the --expected-failures flag.
88
Specify a file containing a list of tests that should be run. Same format
89
as the --exclude flag.
91
Not includes specified means all tests will be run.
95
Abort as soon as one test fails.
99
Load a list of tests from the specified location.
107
=item I<SMBD_VALGRIND>
109
=item I<TORTURE_MAXTIME>
121
selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
131
use FindBin qw($RealBin $Script);
135
use Cwd qw(abs_path);
137
use Subunit qw(parse_results);
141
my $opt_target = "samba4";
143
my $opt_socket_wrapper = 0;
144
my $opt_socket_wrapper_pcap = undef;
145
my $opt_socket_wrapper_keep_pcap = undef;
147
my $opt_immediate = 0;
148
my $opt_expected_failures = undef;
149
my @opt_exclude = ();
150
my @opt_include = ();
152
my $opt_image = undef;
155
my $opt_analyse_cmd = undef;
156
my $opt_resetup_env = undef;
157
my $opt_bindir = undef;
158
my $opt_no_lazy_setup = undef;
159
my $opt_format = "plain";
167
my @expected_failures = ();
174
TESTS_UNEXPECTED_OK => 0,
175
TESTS_EXPECTED_OK => 0,
176
TESTS_UNEXPECTED_FAIL => 0,
177
TESTS_EXPECTED_FAIL => 0,
184
my ($list, $fullname) = @_;
187
if ($fullname =~ /$$_[0]/) {
188
return ($$_[1]) if ($$_[1]);
189
return "NO REASON SPECIFIED";
196
sub expecting_failure($)
199
return find_in_list(\@expected_failures, $name);
206
return find_in_list(\@excludes, $name);
215
return unless ($opt_socket_wrapper_pcap);
216
return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
219
$fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
221
my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
223
SocketWrapper::setup_pcap($pcap_file);
228
sub cleanup_pcap($$$)
230
my ($pcap_file, $expected_ret, $ret) = @_;
232
return unless ($opt_socket_wrapper_pcap);
233
return if ($opt_socket_wrapper_keep_pcap);
234
return unless ($expected_ret == $ret);
235
return unless defined($pcap_file);
240
sub run_testsuite($$$$$$)
242
my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
243
my $pcap_file = setup_pcap($name);
245
$msg_ops->start_test([], $name);
247
unless (open(RESULT, "$cmd 2>&1|")) {
248
$statistics->{TESTS_ERROR}++;
249
$msg_ops->end_test([], $name, "error", 1, "Unable to run $cmd: $!");
250
$statistics->{SUITES_FAIL}++;
254
my $expected_ret = parse_results(
255
$msg_ops, $statistics, *RESULT, \&expecting_failure, [$name]);
257
my $envlog = getlog_env($envname);
258
$msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
260
$msg_ops->output_msg("CMD: $cmd\n");
262
my $ret = close(RESULT);
263
$ret = 0 unless $ret == 1;
265
my $exitcode = $? >> 8;
268
$msg_ops->end_test([], $name, "success", $expected_ret != $ret, undef);
270
$msg_ops->end_test([], $name, "failure", $expected_ret != $ret, "Exit code was $exitcode");
273
cleanup_pcap($pcap_file, $expected_ret, $ret);
275
if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
276
$msg_ops->output_msg("PCAP FILE: $pcap_file\n");
279
if ($ret != $expected_ret) {
280
$statistics->{SUITES_FAIL}++;
281
exit(1) if ($opt_one);
284
return ($ret == $expected_ret);
289
print "Samba test runner
290
Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
291
Copyright (C) Stefan Metzmacher <metze\@samba.org>
293
Usage: $Script [OPTIONS] TESTNAME-REGEX
296
--help this help page
297
--target=samba[34]|win|kvm Samba version to target
298
--testlist=FILE file to read available tests from
301
--prefix=DIR prefix to run tests in [st]
302
--srcdir=DIR source directory [.]
303
--builddir=DIR output directory [.]
304
--exeext=EXT executable extention []
307
--socket-wrapper-pcap save traffic to pcap directories
308
--socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
310
--socket-wrapper enable socket wrapper
311
--bindir=PATH path to target binaries
312
--expected-failures=FILE specify list of tests that is guaranteed to fail
315
--ldap=openldap|fedora-ds back samba onto specified ldap server
318
--image=PATH path to KVM image
321
--quick run quick overall test
322
--one abort when the first test fails
323
--immediate print test output for failed tests during run
325
--analyse-cmd CMD command to run after each test
330
my $result = GetOptions (
331
'help|h|?' => \$opt_help,
332
'target=s' => \$opt_target,
333
'prefix=s' => \$prefix,
334
'socket-wrapper' => \$opt_socket_wrapper,
335
'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
336
'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
337
'quick' => \$opt_quick,
339
'immediate' => \$opt_immediate,
340
'expected-failures=s' => \$opt_expected_failures,
341
'exclude=s' => \@opt_exclude,
342
'include=s' => \@opt_include,
343
'srcdir=s' => \$srcdir,
344
'builddir=s' => \$builddir,
345
'exeext=s' => \$exeext,
346
'verbose' => \$opt_verbose,
347
'testenv' => \$opt_testenv,
349
'analyse-cmd=s' => \$opt_analyse_cmd,
350
'no-lazy-setup' => \$opt_no_lazy_setup,
351
'resetup-environment' => \$opt_resetup_env,
352
'bindir:s' => \$opt_bindir,
353
'format=s' => \$opt_format,
354
'image=s' => \$opt_image,
355
'testlist=s' => \@testlists
358
exit(1) if (not $result);
360
ShowHelp() if ($opt_help);
364
# quick hack to disable rpc validation when using valgrind - its way too slow
365
unless (defined($ENV{VALGRIND})) {
366
$ENV{VALIDATE} = "validate";
367
$ENV{MALLOC_CHECK_} = 2;
370
my $bindir = ($opt_bindir or "$builddir/bin");
371
my $bindir_abs = abs_path($bindir);
373
# Backwards compatibility:
374
if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
375
if (defined($ENV{FEDORA_DS_ROOT})) {
382
my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
385
$torture_maxtime *= 2;
392
die("using an empty prefix isn't allowed") unless $prefix ne "";
394
#Ensure we have the test prefix around
395
mkdir($prefix, 0777) unless -d $prefix;
397
my $prefix_abs = abs_path($prefix);
398
my $srcdir_abs = abs_path($srcdir);
399
my $builddir_abs = abs_path($builddir);
401
die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
402
die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
404
$ENV{PREFIX} = $prefix;
405
$ENV{KRB5CCNAME} = "$prefix/krb5ticket";
406
$ENV{PREFIX_ABS} = $prefix_abs;
407
$ENV{SRCDIR} = $srcdir;
408
$ENV{SRCDIR_ABS} = $srcdir_abs;
409
$ENV{BUILDDIR} = $builddir;
410
$ENV{BUILDDIR_ABS} = $builddir_abs;
411
$ENV{EXEEXT} = $exeext;
413
if (defined($ENV{RUN_FROM_BUILD_FARM}) and
414
($ENV{RUN_FROM_BUILD_FARM} eq "yes")) {
415
$opt_format = "buildfarm";
418
my $tls_enabled = not $opt_quick;
419
$ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
420
$ENV{LDB_MODULES_PATH} = "$bindir_abs/modules/ldb";
421
$ENV{LD_SAMBA_MODULE_PATH} = "$bindir_abs/modules";
422
sub prefix_pathvar($$)
424
my ($name, $newpath) = @_;
425
if (defined($ENV{$name})) {
426
$ENV{$name} = "$newpath:$ENV{$name}";
428
$ENV{$name} = $newpath;
431
prefix_pathvar("PKG_CONFIG_PATH", "$bindir_abs/pkgconfig");
432
prefix_pathvar("PYTHONPATH", "$bindir_abs/python");
434
if ($opt_socket_wrapper_keep_pcap) {
435
# Socket wrapper keep pcap implies socket wrapper pcap
436
$opt_socket_wrapper_pcap = 1;
439
if ($opt_socket_wrapper_pcap) {
440
# Socket wrapper pcap implies socket wrapper
441
$opt_socket_wrapper = 1;
444
my $socket_wrapper_dir;
445
if ($opt_socket_wrapper) {
446
$socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
447
print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
450
print "WARNING: Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports\n";
455
my $testenv_default = "none";
457
if ($opt_target eq "samba4") {
458
$testenv_default = "member";
459
require target::Samba4;
460
$target = new Samba4($bindir, $ldap, "$srcdir/setup", $exeext);
461
} elsif ($opt_target eq "samba3") {
462
if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
463
die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
465
$testenv_default = "member";
466
require target::Samba3;
467
$target = new Samba3($bindir);
468
} elsif ($opt_target eq "win") {
469
die("Windows tests will not run with socket wrapper enabled.")
470
if ($opt_socket_wrapper);
471
$testenv_default = "dc";
472
require target::Windows;
473
$target = new Windows();
474
} elsif ($opt_target eq "kvm") {
475
die("Kvm tests will not run with socket wrapper enabled.")
476
if ($opt_socket_wrapper);
478
die("No image specified") unless ($opt_image);
479
$target = new Kvm($opt_image, undef);
483
# Start a Virtual Distributed Ethernet Switch
484
# Returns the pid of the switch.
486
sub start_vde_switch($)
490
system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
492
open(PID, "$path/vde.pid");
500
# Stop a Virtual Distributed Ethernet Switch
501
sub stop_vde_switch($)
507
sub read_test_regexes($)
511
open(LF, "<$name") or die("unable to read $name: $!");
515
if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
516
push (@ret, [$1, $4]);
518
s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
519
push (@ret, [$_, undef]);
526
if (defined($opt_expected_failures)) {
527
@expected_failures = read_test_regexes($opt_expected_failures);
530
foreach (@opt_exclude) {
531
push (@excludes, read_test_regexes($_));
534
foreach (@opt_include) {
535
push (@includes, read_test_regexes($_));
538
my $interfaces = join(',', ("127.0.0.6/8",
545
my $conffile = "$prefix_abs/client/client.conf";
546
$ENV{SMB_CONF_PATH} = $conffile;
548
sub write_clientconf($$)
550
my ($conffile, $vars) = @_;
552
mkdir("$prefix/client", 0777) unless -d "$prefix/client";
554
if ( -d "$prefix/client/private" ) {
555
unlink <$prefix/client/private/*>;
557
mkdir("$prefix/client/private", 0777);
560
if ( -d "$prefix/client/lock" ) {
561
unlink <$prefix/client/lockdir/*>;
563
mkdir("$prefix/client/lockdir", 0777);
566
open(CF, ">$conffile");
567
print CF "[global]\n";
568
if (defined($ENV{VALGRIND})) {
569
print CF "\ticonv:native = true\n";
571
print CF "\ticonv:native = false\n";
573
print CF "\tnetbios name = client\n";
574
if (defined($vars->{DOMAIN})) {
575
print CF "\tworkgroup = $vars->{DOMAIN}\n";
577
if (defined($vars->{REALM})) {
578
print CF "\trealm = $vars->{REALM}\n";
580
if ($opt_socket_wrapper) {
581
print CF "\tinterfaces = $interfaces\n";
584
private dir = $prefix_abs/client/private
585
lock dir = $prefix_abs/client/lockdir
586
name resolve order = bcast
587
panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
589
notify:inotify = false
591
system:anonymous = true
592
client lanman auth = Yes
593
torture:basedir = $prefix_abs/client
594
#We don't want to pass our self-tests if the PAC code is wrong
595
gensec:require_pac = true
596
modules dir = $ENV{LD_SAMBA_MODULE_PATH}
603
my $testsdir = "$srcdir/selftest";
605
my %required_envs = ();
612
open(IN, $filename) or die("Unable to open $filename: $!");
615
if ($_ eq "-- TEST --\n") {
622
if (not defined($tests) or $name =~ /$tests/) {
623
$required_envs{$env} = 1;
624
push (@ret, [$name, $env, $cmdline]);
630
close(IN) or die("Error creating recipe");
634
if ($#testlists == -1) {
635
die("No testlists specified");
638
$ENV{SELFTEST_PREFIX} = "$prefix_abs";
639
if ($opt_socket_wrapper) {
640
$ENV{SELFTEST_INTERFACES} = $interfaces;
642
$ENV{SELFTEST_INTERFACES} = "";
645
$ENV{SELFTEST_VERBOSE} = "1";
647
$ENV{SELFTEST_VERBOSE} = "";
650
$ENV{SELFTEST_QUICK} = "1";
652
$ENV{SELFTEST_QUICK} = "";
654
$ENV{SELFTEST_TARGET} = $opt_target;
655
$ENV{SELFTEST_MAXTIME} = $torture_maxtime;
658
foreach my $fn (@testlists) {
659
foreach (read_testlist($fn)) {
661
next if (@includes and not find_in_list(\@includes, $name));
662
push (@available, $_);
667
if ($opt_format eq "buildfarm") {
668
require output::buildfarm;
669
$msg_ops = new output::buildfarm($statistics);
670
} elsif ($opt_format eq "plain") {
671
require output::plain;
672
$msg_ops = new output::plain("$prefix/summary", $opt_verbose, $opt_immediate, $statistics, $#available+1);
673
} elsif ($opt_format eq "html") {
674
require output::html;
675
mkdir("test-results", 0777);
676
$msg_ops = new output::html("test-results", $statistics);
678
die("Invalid output format '$opt_format'");
682
foreach (@available) {
684
my $skipreason = skip($name);
686
$msg_ops->skip_testsuite($name, $skipreason);
693
print STDERR "No tests to run\n";
697
my $suitestotal = $#todo + 1;
701
my %running_envs = ();
703
sub get_running_env($)
711
return $running_envs{$envname};
714
my @exported_envvars = (
719
# domain controller stuff
739
"WINBINDD_SOCKET_DIR",
740
"WINBINDD_PRIV_PIPE_DIR"
743
$SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
745
teardown_env($_) foreach(keys %running_envs);
746
die("Received signal $signame");
753
my $testenv_vars = undef;
759
$option =~ s/^[^:]*//;
762
$option = "client" if $option eq "";
764
if ($envname eq "none") {
766
} elsif (defined(get_running_env($envname))) {
767
$testenv_vars = get_running_env($envname);
768
if (not $target->check_env($testenv_vars)) {
769
$testenv_vars = undef;
772
$testenv_vars = $target->setup_env($envname, $prefix);
775
return undef unless defined($testenv_vars);
777
$running_envs{$envname} = $testenv_vars;
779
if ($option eq "local") {
780
SocketWrapper::set_default_iface($testenv_vars->{SOCKET_WRAPPER_DEFAULT_IFACE});
781
$ENV{SMB_CONF_PATH} = $testenv_vars->{SERVERCONFFILE};
782
} elsif ($option eq "client") {
783
SocketWrapper::set_default_iface(6);
784
write_clientconf($conffile, $testenv_vars);
785
$ENV{SMB_CONF_PATH} = $conffile;
787
die("Unknown option[$option] for envname[$envname]");
790
foreach (@exported_envvars) {
791
if (defined($testenv_vars->{$_})) {
792
$ENV{$_} = $testenv_vars->{$_};
798
return $testenv_vars;
801
sub exported_envvars_str($)
803
my ($testenv_vars) = @_;
806
foreach (@exported_envvars) {
807
next unless defined($testenv_vars->{$_});
808
$out .= $_."=".$testenv_vars->{$_}."\n";
817
return "" if ($envname eq "none");
818
return $target->getlog_env(get_running_env($envname));
824
return 1 if ($envname eq "none");
825
return $target->check_env(get_running_env($envname));
831
return if ($envname eq "none");
832
$target->teardown_env(get_running_env($envname));
833
delete $running_envs{$envname};
836
if ($opt_no_lazy_setup) {
837
setup_env($_) foreach (keys %required_envs);
841
my $testenv_name = $ENV{SELFTEST_TESTENV};
842
$testenv_name = $testenv_default unless defined($testenv_name);
844
my $testenv_vars = setup_env($testenv_name);
846
$ENV{PIDDIR} = $testenv_vars->{PIDDIR};
848
my $envvarstr = exported_envvars_str($testenv_vars);
850
my $term = ($ENV{TERM} or "xterm");
851
system("$term -e 'echo -e \"
852
Welcome to the Samba4 Test environment '$testenv_name'
854
This matches the client environment used in make test
855
server is pid `cat \$PIDDIR/samba.pid`
857
Some useful environment variables:
858
TORTURE_OPTIONS=\$TORTURE_OPTIONS
859
SMB_CONF_PATH=\$SMB_CONF_PATH
862
\" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
863
teardown_env($testenv_name);
868
$cmd =~ s/([\(\)])/\\$1/g;
870
my $envname = $$_[1];
872
my $envvars = setup_env($envname);
873
if (not defined($envvars)) {
874
$msg_ops->skip_testsuite($name, "unable to set up environment $envname");
878
run_testsuite($envname, $name, $cmd, $i, $suitestotal,
881
if (defined($opt_analyse_cmd)) {
882
system("$opt_analyse_cmd \"$name\"");
885
teardown_env($envname) if ($opt_resetup_env);
891
teardown_env($_) foreach (keys %running_envs);
899
# if there were any valgrind failures, show them
900
foreach (<$prefix/valgrind.log*>) {
902
system("grep DWARF2.CFI.reader $_ > /dev/null");
904
print "VALGRIND FAILURE\n";
910
if ($opt_format eq "buildfarm") {
911
print "TEST STATUS: $statistics->{SUITES_FAIL}\n";
914
exit $statistics->{SUITES_FAIL};