diff --git a/bin/tests/system/Makefile.in b/bin/tests/system/Makefile.in index 7c780595c8..bb6b4d7c7d 100644 --- a/bin/tests/system/Makefile.in +++ b/bin/tests/system/Makefile.in @@ -66,24 +66,19 @@ parallel.mk: echo " @$(SHELL) ./run.sh -p $$port $$directory" >> $@ ; \ done -# Targets to run the tests that can be done in parallel (which have unique -# ports assigned to them) and the tests which must be run sequentially (because -# they all use query port 5300 and control port 9953). +# Targets to run the tests. -parallel: parallel.mk - @$(MAKE) -f parallel.mk - -sequential: - @if test -f ./runall.sh; then $(SHELL) ./runall.sh; fi - -# Standard targets. +test: parallel.mk + @$(MAKE) -f parallel.mk check 2>&1 | tee systests.output + @$(SHELL) ./runsequential.sh 2>&1 | tee -a systests.output + @$(SHELL) ./testsummary.sh check: test -test: parallel sequential +# Other targets. testclean clean distclean:: - if test -f ./cleanall.sh; then sh ./cleanall.sh; fi + if test -f ./cleanall.sh; then $(SHELL) ./cleanall.sh; fi rm -f systests.output rm -f random.data rm -f parallel.mk diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in index 515783c0eb..27516421d0 100644 --- a/bin/tests/system/conf.sh.in +++ b/bin/tests/system/conf.sh.in @@ -158,7 +158,7 @@ if test -t 1 && type tput > /dev/null 2>&1 ; then COLOR_FAIL=`tput setaf 1` # red COLOR_WARN=`tput setaf 3` # yellow COLOR_PASS=`tput setaf 2` # green - COLOR_STEN=`tput setaf 4` # blue + COLOR_STEN=`tput setaf 4` # blue - for start/end messages COLOR_INFO=`tput bold` # bold COLOR_NONE=`tput sgr0` else @@ -207,10 +207,6 @@ fi TESTNAME="`basename $PWD`" echo_i() { - # Without formatting - echo "I:$TESTNAME:$@" >> test.output - - # With formatting echoinfo "I:$TESTNAME:$@" } diff --git a/bin/tests/system/getopts.sh b/bin/tests/system/getopts.sh index 24d1b36731..e2e8455f09 100644 --- a/bin/tests/system/getopts.sh +++ b/bin/tests/system/getopts.sh @@ -42,7 +42,7 @@ OPTIND=1 # the upper of the 10 ports notionally assigned does not exceed 65535. if [ "$((${port}+0))" != "${port}" ] || [ "${port}" -le 1024 ] || [ "${port}" -gt 65520 ]; then - echo "Specified port '$port' must be numeric and in the range 1025 to 65520" >&2 + echo "Base of port range ($port) must be numeric and in the range 1025 to 65520" >&2 exit 1 fi @@ -56,6 +56,11 @@ aport7=$(($port + 7)) aport8=$(($port + 8)) controlport=$(($port + 9)) +# Two more symbols that denote the limits of the range. + +portlow=$port +porthigh=$controlport + # copy_setports - Copy Configuration File and Replace Ports # diff --git a/bin/tests/system/run.sh b/bin/tests/system/run.sh index 99dcdbccb0..68bc86c187 100644 --- a/bin/tests/system/run.sh +++ b/bin/tests/system/run.sh @@ -43,8 +43,7 @@ test -d $test || { echofail "$0: $test: no such test" >&2; exit 1; } echoinfo "S:$test:`date $dateargs`" >&2 echoinfo "T:$test:1:A" >&2 echoinfo "A:$test:System test $test" >&2 -echoinfo "I:$test:PORT:${port}" >&2 -echoinfo "I:$test:CONTROLPORT:${controlport}" >&2 +echoinfo "I:$test:PORTRANGE:${portlow} - ${porthigh}" if [ x${PERL:+set} = x ] then @@ -113,7 +112,7 @@ $PERL stop.pl $test status=`expr $status + $?` if [ $status != 0 ]; then - echofail "R:$test:$FAIL" + echofail "R:$test:FAIL" # Don't clean up - we need the evidence. find . -name core -exec chmod 0644 '{}' \; else diff --git a/bin/tests/system/runall.sh b/bin/tests/system/runall.sh index 315caf67c0..f9e1e47a3c 100644 --- a/bin/tests/system/runall.sh +++ b/bin/tests/system/runall.sh @@ -6,38 +6,29 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# # Run all the system tests. # -# Note: Use "make check" to run all the system tests. This script will just -# run those tests that require that each of their nameservers is the only one -# running on an IP address. +# Usage: +# runall.sh [numprocesses] # +# ...where numprocess is the number of processes to use. The default is 1, +# which runs the tests sequentially. SYSTEMTESTTOP=. . $SYSTEMTESTTOP/conf.sh -status=0 +numproc= +if [ $# -eq 0 ]; then + numproc=1 -{ - for d in $SEQUENTIALDIRS - do - $SHELL run.sh "${@}" $d || status=1 - done -} 2>&1 | tee "systests.output" +elif [ $# -gt 1 ] || "$(($1 + 0))" -ne "$1" ]; then + echo "Usage: ./runall.sh [numprocesses]" -$PERL testsock.pl || { - cat <&2 -I: -I:NOTE: System tests were skipped because they require that the -I: IP addresses 10.53.0.1 through 10.53.0.8 be configured -I: as alias addresses on the loopback interface. Please run -I: "bin/tests/system/ifconfig.sh up" as root to configure them. -EOF -} +else + numproc=$1 -echo "I:System test result summary:" -grep '^R:' systests.output | sed -e 's/^/I: /' -e 's/R:[^:]*//' | sort | uniq -c -grep '^R:[^:]*:FAIL' systests.output > /dev/null && status=1 +fi -exit $status +make -j $numproc check + +exit $? diff --git a/bin/tests/system/runsequential.sh b/bin/tests/system/runsequential.sh new file mode 100644 index 0000000000..caa4ed7b93 --- /dev/null +++ b/bin/tests/system/runsequential.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# +# Copyright (C) 2000, 2001, 2004, 2007, 2010-2012, 2014-2016 Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Run system tests that must be run sequentially +# +# Note: Use "make check" (or runall.sh) to run all the system tests. This +# script will just run those tests that require that each of their nameservers +# is the only one running on an IP address. +# + +SYSTEMTESTTOP=. +. $SYSTEMTESTTOP/conf.sh + + for d in $SEQUENTIALDIRS + do + $SHELL run.sh "${@}" $d + done diff --git a/bin/tests/system/testsummary.sh b/bin/tests/system/testsummary.sh new file mode 100644 index 0000000000..11cc79f12c --- /dev/null +++ b/bin/tests/system/testsummary.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# +# Copyright (C) 2000, 2001, 2004, 2007, 2010-2012, 2014-2016 Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Searches the system test output file (systests.output) and prints a summary +# of tests passed, failed, not run. It also checks whether the IP addresses +# 10.53.0.[1-8] were set up and, if not, prints a warning. +# +# Usage: +# testsummary.sh +# +# Status return: +# 0 - no tests failed +# 1 - one or more tests failed + +SYSTEMTESTTOP=. +. $SYSTEMTESTTOP/conf.sh + +$PERL testsock.pl || { + cat <&2 +I: +I:NOTE: System tests were skipped because they require that the +I: IP addresses 10.53.0.1 through 10.53.0.8 be configured +I: as alias addresses on the loopback interface. Please run +I: "bin/tests/system/ifconfig.sh up" as root to configure them. +EOF +} + +status=0 +echo "I:System test result summary:" +grep '^R:' systests.output | cut -d':' -f3 | sort | uniq -c | sed -e 's/^/I:/' +grep '^R:[^:]*:FAIL' systests.output > /dev/null && status=1 + +exit $status