diff options
Diffstat (limited to 'source4/selftest')
| -rw-r--r-- | source4/selftest/README | 81 | ||||
| -rw-r--r-- | source4/selftest/Subunit.pm | 15 | ||||
| -rw-r--r-- | source4/selftest/output/plain.pm | 3 | ||||
| -rwxr-xr-x | source4/selftest/samba4_tests.sh | 21 | ||||
| -rwxr-xr-x | source4/selftest/selftest.pl | 34 | ||||
| -rw-r--r-- | source4/selftest/target/Samba4.pm | 8 |
6 files changed, 128 insertions, 34 deletions
diff --git a/source4/selftest/README b/source4/selftest/README index e8e87c8b3f..f8be20a569 100644 --- a/source4/selftest/README +++ b/source4/selftest/README @@ -3,15 +3,70 @@ This directory contains test scripts that are useful for running a bunch of tests all at once. -=============== -Available tests -=============== -The available tests are obtained from a script, usually -selftest/samba{3,4}_tests.sh. This script should for each test output +Available testsuites +==================== +The available testsuites are obtained from a script, usually +selftest/samba{3,4}_tests.sh. This script should for each testsuite output the name of the test, the command to run and the environment that should be -provided. +provided. Use the included "plantest" function to generate the required output. + +Testsuite behaviour +================================ + +Exit code +------------ +The testsuites should exit with a non-zero exit code if at least one +test failed. Skipped tests should not influence the exit code. + +Output format +------------- +Testsuites can simply use the exit code to indicate whether all of their +tests have succeeded or one or more have failed. It is also possible to +provide more granular information using the Subunit protocol. + +This protocol works by writing simple messages to standard output. Any +messages that can not be interpreted by this protocol are considered comments +for the last announced test. + +Accepted commands are: + +test +~~~~~~~~~~~~ +test: <NAME> + +Announce that a new test with the specified name is starting + +success +~~~~~~~~~~~~~~~ +success: <NAME> + +Announce that the test with the specified name is done and ran successfully. + +failure +~~~~~~~~~~~~~~~ +failure: <NAME> +failure: <NAME> [ REASON ] + +Announce that the test with the specified name failed. Optionally, it is +possible to specify a reason it failed. + +skip +~~~~~~~~~~~~ +skip: <NAME> +skip: <NAME> [ REASON ] + +Announce that the test with the specified name was skipped. Optionally a +reason can be specified. + +knownfail +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +knownfail: <NAME> +knownfail: <NAME> [ REASON ] + +Announce that the test with the specified name was run and failed as expected. +Alternatively it is also possible to simply return "failure:" here but +specify in the samba4-knownfailures file that it is failing. -============ Environments ============ Tests often need to run against a server with particular things set up, @@ -23,6 +78,7 @@ The following environments are currently available: - none: No server set up, no variables set. - dc: Domain controller set up. The following environment variables will be set: + * USERNAME: Administrator user name * PASSWORD: Administrator password * DOMAIN: Domain name @@ -34,6 +90,7 @@ The following environments are currently available: - member: Domain controller and member server that is joined to it set up. The following environment variables will be set: + * USERNAME: Domain administrator user name * PASSWORD: Domain administrator password * DOMAIN: Domain name @@ -41,22 +98,22 @@ The following environments are currently available: * SERVER: Name of the member server -============= Running tests ============= -To run all the tests use: +To run all the tests use:: make test -To run a quick subset (aiming for about 1 minute of testing) run: +To run a quick subset (aiming for about 1 minute of testing) run:: make quicktest -To run a specific test, use this syntax +To run a specific test, use this syntax:: make test TESTS=testname -for example +for example:: make test TESTS=samba4.BASE-DELETE + diff --git a/source4/selftest/Subunit.pm b/source4/selftest/Subunit.pm index e5c61ca9ba..05e51da541 100644 --- a/source4/selftest/Subunit.pm +++ b/source4/selftest/Subunit.pm @@ -20,15 +20,22 @@ sub parse_results($$$$$) $msg_ops->control_msg($_); $msg_ops->start_test($open_tests, $1); push (@$open_tests, $1); - } elsif (/^(success|successful|failure|skip|error): (.*?)( \[)?([ \t]*)\n/) { + } elsif (/^(success|successful|failure|skip|knownfail|error): (.*?)( \[)?([ \t]*)\n/) { $msg_ops->control_msg($_); my $reason = undef; if ($3) { $reason = ""; # reason may be specified in next lines + my $terminated = 0; while(<$fh>) { $msg_ops->control_msg($_); - if ($_ eq "]\n") { last; } else { $reason .= $_; } + if ($_ eq "]\n") { $terminated = 1; last; } else { $reason .= $_; } + } + + unless ($terminated) { + $statistics->{TESTS_ERROR}++; + $msg_ops->end_test($open_tests, $2, $1, 1, "reason interrupted"); + return 1; } } my $result = $1; @@ -53,6 +60,10 @@ sub parse_results($$$$$) $msg_ops->end_test($open_tests, $2, $1, 1, $reason); $unexpected_fail++; } + } elsif ($1 eq "knownfail") { + pop(@$open_tests); #FIXME: Check that popped value == $2 + $statistics->{TESTS_EXPECTED_FAIL}++; + $msg_ops->end_test($open_tests, $2, $1, 0, $reason); } elsif ($1 eq "skip") { $statistics->{TESTS_SKIP}++; pop(@$open_tests); #FIXME: Check that popped value == $2 diff --git a/source4/selftest/output/plain.pm b/source4/selftest/output/plain.pm index 25ff74792e..f14e26b38d 100644 --- a/source4/selftest/output/plain.pm +++ b/source4/selftest/output/plain.pm @@ -81,6 +81,9 @@ sub end_testsuite($$$$$) my $out = ""; if ($unexpected) { + if ($result eq "success" and not defined($reason)) { + $reason = "Expected negative exit code, got positive exit code"; + } $self->output_msg("ERROR: $reason\n"); push (@{$self->{suitesfailed}}, $name); } else { diff --git a/source4/selftest/samba4_tests.sh b/source4/selftest/samba4_tests.sh index 3a3e1f91b1..36cf2adccb 100755 --- a/source4/selftest/samba4_tests.sh +++ b/source4/selftest/samba4_tests.sh @@ -64,8 +64,7 @@ SCRIPTDIR=$samba4srcdir/../testprogs/ejs smb4torture="$samba4bindir/smbtorture $TORTURE_OPTIONS" plantest "js.base" dc "$SCRIPTDIR/base.js" $CONFIGURATION -plantest "js.samr" dc "$SCRIPTDIR/samr.js" $CONFIGURATION ncalrpc: -U\$USERNAME%\$PASSWORD -plantest "js.echo" dc "$SCRIPTDIR/echo.js" $CONFIGURATION ncalrpc: -U\$USERNAME%\$PASSWORD +plantest "samr.python" dc "$samba4bindir/../scripting/bin/samr.py" ncalrpc: #plantest "ejsnet.js" dc "$SCRIPTDIR/ejsnet.js" $CONFIGURATION -U\$USERNAME%\$PASSWORD \$DOMAIN ejstestuser plantest "js.ldb" none "$SCRIPTDIR/ldb.js" `pwd` $CONFIGURATION -d 10 plantest "js.winreg" dc $samba4srcdir/scripting/bin/winreg $CONFIGURATION ncalrpc: 'HKLM' -U\$USERNAME%\$PASSWORD @@ -219,13 +218,16 @@ plantest "rpc.echo on ncacn_np over smb2" dc $smb4torture ncacn_np:"\$SERVER[smb # Tests against the NTVFS POSIX backend NTVFSARGS="--option=torture:sharedelay=100000 --option=torture:oplocktimeout=3" smb2=`$smb4torture --list | grep "^SMB2-" | xargs` -raw=`$smb4torture --list | grep "^RAW-" | xargs` +#The QFILEINFO-IPC test needs to be on ipc$ +raw=`$smb4torture --list | grep "^RAW-" | grep -v "RAW-QFILEINFO-IPC"| xargs` base=`$smb4torture --list | grep "^BASE-" | xargs` for t in $base $raw $smb2; do plansmbtorturetest "$t" dc $ADDARGS //\$SERVER/tmp -U"\$USERNAME"%"\$PASSWORD" $NTVFSARGS done +plansmbtorturetest "RAW-QFILEINFO-IPC" dc $ADDARGS //\$SERVER/ipc$ -U"\$USERNAME"%"\$PASSWORD" + rap=`$smb4torture --list | grep "^RAP-" | xargs` for t in $rap; do plansmbtorturetest "$t" dc $ADDARGS //\$SERVER/IPC\\\$ -U"\$USERNAME"%"\$PASSWORD" @@ -274,6 +276,8 @@ plantest "blackbox.nmblookup" member $samba4srcdir/utils/tests/test_nmblookup.sh plantest "blackbox.locktest" dc $bbdir/test_locktest.sh "\$SERVER" "\$USERNAME" "\$PASSWORD" "\$DOMAIN" "$PREFIX" plantest "blackbox.masktest" dc $bbdir/test_masktest.sh "\$SERVER" "\$USERNAME" "\$PASSWORD" "\$DOMAIN" "$PREFIX" plantest "blackbox.gentest" dc $bbdir/test_gentest.sh "\$SERVER" "\$USERNAME" "\$PASSWORD" "\$DOMAIN" "$PREFIX" +plantest "blackbox.wbinfo" dc $bbdir/test_wbinfo.sh "\$DOMAIN" "\$USERNAME" "\$PASSWORD" "dc" +plantest "blackbox.wbinfo" member $bbdir/test_wbinfo.sh "\$DOMAIN" "\$DC_USERNAME" "\$DC_PASSWORD" "member" # Tests using the "Simple" NTVFS backend @@ -283,7 +287,7 @@ done DATADIR=$samba4srcdir/../testdata -plantest "js.samba3sam" none $SCRIPTDIR/samba3sam.js $CONFIGURATION `pwd` $DATADIR/samba3/ +plantest "js.samba3sam" none $samba4bindir/smbscript $SCRIPTDIR/samba3sam.js $CONFIGURATION `pwd` $DATADIR/samba3/ # Domain Member Tests @@ -292,7 +296,6 @@ plantest "rpc.echo against member server with domain creds" member $VALGRIND $sm plantest "rpc.samr against member server with local creds" member $VALGRIND $smb4torture ncacn_np:"\$NETBIOSNAME" -U"\$NETBIOSNAME/\$USERNAME"%"\$PASSWORD" "RPC-SAMR" "$*" plantest "rpc.samr.users against member server with local creds" member $VALGRIND $smb4torture ncacn_np:"\$NETBIOSNAME" -U"\$NETBIOSNAME/\$USERNAME"%"\$PASSWORD" "RPC-SAMR-USERS" "$*" plantest "rpc.samr.passwords against member server with local creds" member $VALGRIND $smb4torture ncacn_np:"\$NETBIOSNAME" -U"\$NETBIOSNAME/\$USERNAME"%"\$PASSWORD" "RPC-SAMR-PASSWORDS" "$*" -plantest "wbinfo -a against member server with domain creds" member $VALGRIND $samba4bindir/wbinfo -a "\$DOMAIN/\$DC_USERNAME"%"\$DC_PASSWORD" NBT_TESTS=`$smb4torture --list | grep "^NBT-" | xargs` @@ -300,7 +303,7 @@ for t in $NBT_TESTS; do plansmbtorturetest "$t" dc //\$SERVER/_none_ -U\$USERNAME%\$PASSWORD done -WB_OPTS="--option=\"torture:strict mode=yes\"" +WB_OPTS="--option=\"torture:strict mode=no\"" WB_OPTS="${WB_OPTS} --option=\"torture:timelimit=1\"" WB_OPTS="${WB_OPTS} --option=\"torture:winbindd separator=/\"" WB_OPTS="${WB_OPTS} --option=\"torture:winbindd private pipe dir=\$WINBINDD_PRIV_PIPE_DIR\"" @@ -324,7 +327,7 @@ then plantest "nss.test using winbind" member $VALGRIND $samba4bindir/nsstest $samba4bindir/shared/libnss_winbind.so fi -PYTHON=bin/smbpython +PYTHON=/usr/bin/python SUBUNITRUN="$PYTHON ./scripting/bin/subunitrun" plantest "ldb.python" none PYTHONPATH="$PYTHONPATH:lib/ldb/tests/python/" $SUBUNITRUN api plantest "credentials.python" none PYTHONPATH="$PYTHONPATH:auth/credentials/tests" $SUBUNITRUN bindings @@ -347,7 +350,9 @@ plantest "winreg.python" dc $SUBUNITRUN -U\$USERNAME%\$PASSWORD samba.tests.dcer plantest "ldap.python" dc $PYTHON $samba4srcdir/lib/ldb/tests/python/ldap.py $CONFIGURATION \$SERVER -U\$USERNAME%\$PASSWORD -W \$DOMAIN plantest "blackbox.samba3dump" none $PYTHON scripting/bin/samba3dump $samba4srcdir/../testdata/samba3 rm -rf $PREFIX/upgrade -plantest "blackbox.upgrade" none $PYTHON setup/upgrade.py $CONFIGURATION --targetdir=$PREFIX/upgrade ../testdata/samba3 ../testdata/samba3/smb.conf +plantest "blackbox.upgrade" none $PYTHON setup/upgrade $CONFIGURATION --targetdir=$PREFIX/upgrade ../testdata/samba3 ../testdata/samba3/smb.conf rm -rf $PREFIX/provision mkdir $PREFIX/provision plantest "blackbox.provision.py" none PYTHON="$PYTHON" $samba4srcdir/setup/tests/blackbox_provision.sh "$PREFIX/provision" "$CONFIGURATION" +plantest "blackbox.setpassword.py" none PYTHON="$PYTHON" $samba4srcdir/setup/tests/blackbox_setpassword.sh "$PREFIX/provision" "$CONFIGURATION" +plantest "blackbox.newuser.py" none PYTHON="$PYTHON" $samba4srcdir/setup/tests/blackbox_newuser.sh "$PREFIX/provision" "$CONFIGURATION" diff --git a/source4/selftest/selftest.pl b/source4/selftest/selftest.pl index 39a1b5a450..5854a94b8d 100755 --- a/source4/selftest/selftest.pl +++ b/source4/selftest/selftest.pl @@ -238,7 +238,13 @@ sub run_testsuite($$$$$$) $msg_ops->start_test([], $name); - open(RESULT, "$cmd 2>&1|"); + unless (open(RESULT, "$cmd 2>&1|")) { + $statistics->{TESTS_ERROR}++; + $msg_ops->end_test([], $name, "error", 1, "Unable to run $cmd: $!"); + $statistics->{SUITES_FAIL}++; + return 0; + } + my $expected_ret = parse_results( $msg_ops, $statistics, *RESULT, \&expecting_failure, [$name]); @@ -250,17 +256,17 @@ sub run_testsuite($$$$$$) my $ret = close(RESULT); $ret = 0 unless $ret == 1; + my $exitcode = $? >> 8; + if ($ret == 1) { - $msg_ops->end_test([], $name, "success", $expected_ret != $ret, undef); + $msg_ops->end_test([], $name, "success", $expected_ret != $ret, undef); } else { - $msg_ops->end_test([], $name, "failure", $expected_ret != $ret, - "Returned $ret"); + $msg_ops->end_test([], $name, "failure", $expected_ret != $ret, "Exit code was $exitcode"); } cleanup_pcap($pcap_file, $expected_ret, $ret); - if (not $opt_socket_wrapper_keep_pcap and - defined($pcap_file)) { + if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) { $msg_ops->output_msg("PCAP FILE: $pcap_file\n"); } @@ -401,13 +407,19 @@ my $tls_enabled = not $opt_quick; $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no"); $ENV{LDB_MODULES_PATH} = "$old_pwd/bin/modules/ldb"; $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules"; -if (defined($ENV{PKG_CONFIG_PATH})) { - $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}"; -} else { - $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig"; +sub prefix_pathvar($$) +{ + my ($name, $newpath) = @_; + if (defined($ENV{$name})) { + $ENV{$name} = "$newpath:$ENV{$name}"; + } else { + $ENV{$name} = $newpath; + } } +prefix_pathvar("PKG_CONFIG_PATH", "$old_pwd/bin/pkgconfig"); # Required for smbscript: -$ENV{PATH} = "$old_pwd/bin:$old_pwd:$ENV{PATH}"; +prefix_pathvar("PATH", "$old_pwd/bin"); +prefix_pathvar("PYTHONPATH", "$old_pwd/bin/python"); if ($opt_socket_wrapper_keep_pcap) { # Socket wrapper keep pcap implies socket wrapper pcap diff --git a/source4/selftest/target/Samba4.pm b/source4/selftest/target/Samba4.pm index 069aff73cf..a12939b0a1 100644 --- a/source4/selftest/target/Samba4.pm +++ b/source4/selftest/target/Samba4.pm @@ -695,11 +695,17 @@ nogroup:x:65534:nobody my @provision_options = (); push (@provision_options, "NSS_WRAPPER_PASSWD=\"$nsswrap_passwd\""); push (@provision_options, "NSS_WRAPPER_GROUP=\"$nsswrap_group\""); + if (defined($ENV{GDB_PROVISION})) { + push (@provision_options, "gdb --args"); + } + if (defined($ENV{VALGRIND_PROVISION})) { + push (@provision_options, "valgrind"); + } if (defined($ENV{PROVISION_EJS})) { push (@provision_options, "$self->{bindir}/smbscript"); push (@provision_options, "$self->{setupdir}/provision.js"); } else { - push (@provision_options, "$self->{bindir}/smbpython"); +# push (@provision_options, "$self->{bindir}/smbpython"); push (@provision_options, "$self->{setupdir}/provision"); } push (@provision_options, split(' ', $configuration)); |