diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0245affc04..b0662fc1de 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,6 +31,8 @@ variables: TARBALL_COMPRESSOR: xz TARBALL_EXTENSION: xz + AM_COLOR_TESTS: always + stages: - autoconf - precheck @@ -255,8 +257,10 @@ stages: - *setup_interfaces - *setup_softhsm script: - - ( cd bin/tests/system && make -j${TEST_PARALLEL_JOBS:-1} -k check V=1 ) || cat bin/tests/system/test-suite.log - - test -s bin/tests/system/test-suite.log + - cd bin/tests/system + - make -j${TEST_PARALLEL_JOBS:-1} -k check V=1 || make -j${TEST_PARALLEL_JOBS:-1} -k recheck V=1 + after_script: + - cat bin/tests/system/test-suite.log .system_test: &system_test_job <<: *system_test_common @@ -269,6 +273,7 @@ stages: <<: *system_test_common allow_failure: true after_script: + - cat bin/tests/system/test-suite.log - find bin -name 'tsan.*' -exec python3 util/parse_tsan.py {} \; artifacts: expire_in: "1 day" @@ -315,7 +320,7 @@ stages: before_script: - *setup_softhsm script: - - cd lib && make -j${TEST_PARALLEL_JOBS:-1} -k check V=1 + - make -j${TEST_PARALLEL_JOBS:-1} -k unit V=1 .unit_test: &unit_test_job <<: *unit_test_common diff --git a/.pylintrc b/.pylintrc index 62cbfae3e6..750e50e37c 100644 --- a/.pylintrc +++ b/.pylintrc @@ -4,3 +4,4 @@ disable= C0115, # missing-class-docstring C0116, # missing-function-docstring R0801, # duplicate-code + C0103, # invalid-name diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore index 998718e758..0c66e8d2cc 100644 --- a/bin/tests/system/.gitignore +++ b/bin/tests/system/.gitignore @@ -12,10 +12,10 @@ named.run parallel.mk /*.log /*.trs -/get_base_port.state -/get_base_port.lock +/get_ports.state +/get_ports.lock /run.sh +/run.log /start.sh /stop.sh -/system-test-driver.sh /ifconfig.sh diff --git a/bin/tests/system/Makefile.am b/bin/tests/system/Makefile.am index 17fa9d1205..ea4d13a934 100644 --- a/bin/tests/system/Makefile.am +++ b/bin/tests/system/Makefile.am @@ -43,7 +43,7 @@ tkey_keycreate_CPPFLAGS = \ $(AM_CPPFLAGS) \ $(LIBDNS_CFLAGS) -tkey_keycreate_LDADD = \ +tkey_keycreate_LDADD = \ $(LDADD) \ $(LIBDNS_LIBS) @@ -51,10 +51,22 @@ tkey_keydelete_CPPFLAGS = \ $(AM_CPPFLAGS) \ $(LIBDNS_CFLAGS) -tkey_keydelete_LDADD = \ +tkey_keydelete_LDADD = \ $(LDADD) \ $(LIBDNS_LIBS) +if HAVE_DNSTAP +check_PROGRAMS += wire_test + +wire_test_CPPFLAGS = \ + $(AM_CPPFLAGS) \ + $(LIBDNS_CFLAGS) + +wire_test_LDADD = \ + $(LDADD) \ + $(LIBDNS_LIBS) +endif HAVE_DNSTAP + TESTS = \ acl additional addzone allow-query auth autosign \ builtin cacheclean case catz cds \ @@ -149,7 +161,20 @@ check: exit 1 endif !HAVE_PERL -LOG_COMPILER = $(builddir)/system-test-driver.sh +LOG_DRIVER_V = $(LOG_DRIVER_V_@AM_V@) +LOG_DRIVER_V_ = $(LOG_DRIVER_V_@AM_DEFAULT_V@) +LOG_DRIVER_V_0 = --verbose no +LOG_DRIVER_V_1 = --verbose yes + +LOG_DRIVER = $(top_srcdir)/custom-test-driver +AM_LOG_DRIVER_FLAGS = $(LOG_DRIVER_V) + +LOG_COMPILER = $(builddir)/run.sh +AM_LOG_FLAGS = -r + +$(TESTS): run.sh clean-local: - -rm -f get_base_port.state get_base_port.lock + -rm -f get_ports.state get_ports.lock + +test-local: check diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in index 65dbdeddb2..85f8df69e4 100644 --- a/bin/tests/system/conf.sh.in +++ b/bin/tests/system/conf.sh.in @@ -61,7 +61,7 @@ SETTIME=$TOP_BUILDDIR/bin/dnssec/dnssec-settime SIGNER=$TOP_BUILDDIR/bin/dnssec/dnssec-signzone TSIGKEYGEN=$TOP_BUILDDIR/bin/confgen/tsig-keygen VERIFY=$TOP_BUILDDIR/bin/dnssec/dnssec-verify -WIRETEST=$TOP_BUILDDIR/bin/tests/wire_test +WIRETEST=$TOP_BUILDDIR/bin/tests/system/wire_test BIGKEY=$TOP_BUILDDIR/bin/tests/system/rsabigexponent/bigkey GENCHECK=$TOP_BUILDDIR/bin/tests/system/rndc/gencheck @@ -106,12 +106,12 @@ XMLLINT=@XMLLINT@ XSLTPROC=@XSLTPROC@ # PERL will be an empty string if no perl interpreter was found. -PERL=@PERL@ +PERL=$(command -v "@PERL@") # Windows process management leave empty PSSUSPEND= -PYTHON=@PYTHON@ +PYTHON=$(command -v "@PYTHON@") # # Determine if we support various optional features. diff --git a/bin/tests/system/digdelv/tests.sh b/bin/tests/system/digdelv/tests.sh index bd8aab4c44..2d2c63e285 100644 --- a/bin/tests/system/digdelv/tests.sh +++ b/bin/tests/system/digdelv/tests.sh @@ -62,7 +62,7 @@ KEYDATA="$(< ns2/keydata sed -e 's/+/[+]/g')" NOSPLIT="$(< ns2/keydata sed -e 's/+/[+]/g' -e 's/ //g')" HAS_PYYAML=0 -if [ -n "$PYTHON" ] ; then +if [ -x "$PYTHON" ] ; then $PYTHON -c "import yaml" 2> /dev/null && HAS_PYYAML=1 fi @@ -789,11 +789,14 @@ if [ -x "$DIG" ] ; then echo_i "check dig +yaml output ($n)" ret=0 dig_with_opts +qr +yaml @10.53.0.3 any ns2.example > dig.out.test$n 2>&1 || ret=1 - value=$($PYTHON yamlget.py dig.out.test$n 0 message query_message_data status || ret=1) + $PYTHON yamlget.py dig.out.test$n 0 message query_message_data status > yamlget.out.test$n 2>&1 || ret=1 + read -r value < yamlget.out.test$n [ "$value" = "NOERROR" ] || ret=1 - value=$($PYTHON yamlget.py dig.out.test$n 1 message response_message_data status || ret=1) + $PYTHON yamlget.py dig.out.test$n 1 message response_message_data status > yamlget.out.test$n 2>&1 || ret=1 + read -r value < yamlget.out.test$n [ "$value" = "NOERROR" ] || ret=1 - value=$($PYTHON yamlget.py dig.out.test$n 1 message response_message_data QUESTION_SECTION 0 || ret=1) + $PYTHON yamlget.py dig.out.test$n 1 message response_message_data QUESTION_SECTION 0 > yamlget.out.test$n 2>&1 || ret=1 + read -r value < yamlget.out.test$n [ "$value" = "ns2.example. IN ANY" ] || ret=1 if [ $ret -ne 0 ]; then echo_i "failed"; fi status=$((status+ret)) @@ -860,9 +863,11 @@ if [ -x "$MDIG" ] ; then echo_i "check mdig +yaml output ($n)" ret=0 mdig_with_opts +yaml @10.53.0.3 -t any ns2.example > dig.out.test$n 2>&1 || ret=1 - value=$($PYTHON yamlget.py dig.out.test$n 0 message response_message_data status || ret=1) + $PYTHON yamlget.py dig.out.test$n 0 message response_message_data status > yamlget.out.test$n 2>&1 || ret=1 + read -r value < yamlget.out.test$n [ "$value" = "NOERROR" ] || ret=1 - value=$($PYTHON yamlget.py dig.out.test$n 0 message response_message_data QUESTION_SECTION 0 || ret=1) + $PYTHON yamlget.py dig.out.test$n 0 message response_message_data QUESTION_SECTION 0 > yamlget.out.test$n 2>&1 || ret=1 + read -r value < yamlget.out.test$n [ "$value" = "ns2.example. IN ANY" ] || ret=1 if [ $ret -ne 0 ]; then echo_i "failed"; fi status=$((status+ret)) @@ -1110,11 +1115,14 @@ if [ -x "$DELV" ] ; then echo_i "check delv +yaml output ($n)" ret=0 delv_with_opts +yaml @10.53.0.3 any ns2.example > delv.out.test$n 2>&1 || ret=1 - value=$($PYTHON yamlget.py delv.out.test$n status || ret=1) + $PYTHON yamlget.py delv.out.test$n status > yamlget.out.test$n 2>&1 || ret=1 + read -r value < yamlget.out.test$n [ "$value" = "success" ] || ret=1 - value=$($PYTHON yamlget.py delv.out.test$n query_name || ret=1) + $PYTHON yamlget.py delv.out.test$n query_name > yamlget.out.test$n 2>&1 || ret=1 + read -r value < yamlget.out.test$n [ "$value" = "ns2.example" ] || ret=1 - value=$($PYTHON yamlget.py delv.out.test$n records 0 answer_not_validated 0 || ret=1) + $PYTHON yamlget.py delv.out.test$n records 0 answer_not_validated 0 > yamlget.out.test$n 2>&1 || ret=1 + read -r value < yamlget.out.test$n count=$(echo $value | wc -w ) [ ${count:-0} -eq 5 ] || ret=1 if [ $ret -ne 0 ]; then echo_i "failed"; fi diff --git a/bin/tests/system/dnstap/tests.sh b/bin/tests/system/dnstap/tests.sh index d7d80a6134..a666b7e01f 100644 --- a/bin/tests/system/dnstap/tests.sh +++ b/bin/tests/system/dnstap/tests.sh @@ -499,7 +499,7 @@ if [ $ret != 0 ]; then echo_i "failed"; fi status=`expr $status + $ret` HAS_PYYAML=0 -if [ -n "$PYTHON" ] ; then +if [ -x "$PYTHON" ] ; then $PYTHON -c "import yaml" 2> /dev/null && HAS_PYYAML=1 fi diff --git a/bin/tests/system/dnstap/ydump.py b/bin/tests/system/dnstap/ydump.py index 6788b054b8..e199effca4 100644 --- a/bin/tests/system/dnstap/ydump.py +++ b/bin/tests/system/dnstap/ydump.py @@ -25,5 +25,5 @@ DATAFILE = sys.argv[2] ARGS = [DNSTAP_READ, '-y', DATAFILE] with subprocess.Popen(ARGS, stdout=subprocess.PIPE) as f: - for l in yaml.load_all(f.stdout): + for l in yaml.load_all(f.stdout, Loader=yaml.SafeLoader): pprint.pprint(l) diff --git a/bin/tests/system/get_base_port.sh b/bin/tests/system/get_base_port.sh deleted file mode 100755 index 7c1b2d3887..0000000000 --- a/bin/tests/system/get_base_port.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh -# -# Copyright (C) Internet Systems Consortium, Inc. ("ISC") -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# -# See the COPYRIGHT file distributed with this work for additional -# information regarding copyright ownership. - -# This script is a 'port' broker. It keeps track of ports given to the -# individual system subtests, so every test is given a unique port range. - -lockfile=get_base_port.lock -statefile=get_base_port.state - -ephemeral_port_min=49152 -ephemeral_port_max=65535 - -get_base_port() { - if ( set -o noclobber; echo "$$" > "${lockfile}" ) 2> /dev/null; then - trap 'rm -f "${lockfile}"; exit $?' INT TERM EXIT - - base_port=$(cat "${statefile}" 2>/dev/null) - - if [ -z "${base_port}" ]; then - base_port="${ephemeral_port_min}" - fi - - if [ "$((base_port+100))" -gt "${ephemeral_port_max}" ]; then - base_port="${ephemeral_port_min}" - fi - - echo $((base_port+100)) > get_base_port.state - - # clean up after yourself, and release your trap - rm -f "${lockfile}" - trap - INT TERM EXIT - echo "${base_port}" - else - echo 0 - fi -} - -tries=10 - -while [ "${tries}" -gt 0 ]; do - base_port=$(get_base_port) - if [ "${base_port}" -gt 0 ]; then - echo "${base_port}" - exit 0 - fi - sleep 1 - tries=$((tries-1)) -done - -exit 1 diff --git a/bin/tests/system/get_ports.sh b/bin/tests/system/get_ports.sh new file mode 100755 index 0000000000..30c066b2b4 --- /dev/null +++ b/bin/tests/system/get_ports.sh @@ -0,0 +1,94 @@ +#!/bin/sh +# +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +# This script is a 'port' broker. It keeps track of ports given to the +# individual system subtests, so every test is given a unique port range. + +lockfile=get_ports.lock +statefile=get_ports.state + +port_min=5001 +port_max=32767 + +get_random() { + dd if=/dev/urandom bs=1 count=2 2>/dev/null | od -tu2 -An +} + +get_port() { + tries=10 + port=0 + while [ "${tries}" -gt 0 ]; do + if ( set -o noclobber; echo "$$" > "${lockfile}" ) 2> /dev/null; then + trap 'rm -f "${lockfile}"; exit $?' INT TERM EXIT + + port=$(cat "${statefile}" 2>/dev/null) + + if [ -z "${port}" ]; then + if [ "$1" -gt 0 ]; then + port="$1" + else + port_range=$((port_max-port_min)) + port=$(($(get_random)%port_range+port_range)) + fi + fi + + if [ "$((port+1))" -gt "${port_max}" ]; then + port="${port_min}" + fi + + echo $((port+1)) > get_ports.state + + # clean up after yourself, and release your trap + rm -f "${lockfile}" + trap - INT TERM EXIT + + # we have our port + break + fi + sleep 1 + tries=$((tries-1)) + done + if [ "$port" -eq 0 ]; then + exit 1 + fi + echo "$port" +} + +baseport=0 +while getopts "p:-:" OPT; do + if [ "$OPT" = "-" ] && [ -n "$OPTARG" ]; then + OPT="${OPTARG%%=*}" + OPTARG="${OPTARG#$OPT}" + OPTARG="${OPTARG#=}" + fi + + # shellcheck disable=SC2214 + case "$OPT" in + p | port) baseport=$OPTARG ;; + -) break ;; + *) echo "invalid option" >&2; exit 1 ;; + esac +done + +echo "export PORT=$(get_port "$baseport")" +echo "export EXTRAPORT1=$(get_port)" +echo "export EXTRAPORT2=$(get_port)" +echo "export EXTRAPORT3=$(get_port)" +echo "export EXTRAPORT4=$(get_port)" +echo "export EXTRAPORT5=$(get_port)" +echo "export EXTRAPORT6=$(get_port)" +echo "export EXTRAPORT7=$(get_port)" +echo "export EXTRAPORT8=$(get_port)" +echo "export CONTROLPORT=$(get_port)" + +# Local Variables: +# sh-basic-offset: 4 +# End: diff --git a/bin/tests/system/rrsetorder/tests.sh b/bin/tests/system/rrsetorder/tests.sh index 08266af6f1..00c6e3bb2b 100644 --- a/bin/tests/system/rrsetorder/tests.sh +++ b/bin/tests/system/rrsetorder/tests.sh @@ -19,7 +19,7 @@ status=0 GOOD_RANDOM="1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24" GOOD_RANDOM_NO=24 -if grep "^#define DNS_RDATASET_FIXED" $TOP/config.h > /dev/null 2>&1 ; then +if grep "^#define DNS_RDATASET_FIXED" "$TOP_BUILDDIR/config.h" > /dev/null 2>&1 ; then test_fixed=true else echo_i "Order 'fixed' disabled at compile time" diff --git a/bin/tests/system/run.sh.in b/bin/tests/system/run.sh.in index 1f38dc45a3..5e4095ea67 100644 --- a/bin/tests/system/run.sh.in +++ b/bin/tests/system/run.sh.in @@ -17,6 +17,7 @@ top_builddir=@top_builddir@ builddir=@abs_builddir@ srcdir=@abs_srcdir@ +# shellcheck source=conf.sh . ${builddir}/conf.sh SYSTEMTESTTOP="$(cd -P -- "${builddir}" && pwd -P)" @@ -29,7 +30,8 @@ date_with_args() ( ) stopservers=true -baseport=5300 +# baseport == 0 means random +baseport=0 if [ "${SYSTEMTEST_NO_CLEAN:-0}" -eq 1 ]; then clean=false @@ -37,24 +39,41 @@ else clean=true fi -while getopts "knp:r-:" flag; do - case "$flag" in - -) case "${OPTARG}" in - keep) stopservers=false ;; - noclean) clean=false ;; - esac - ;; - k) stopservers=false ;; - n) clean=false ;; - p) baseport=$OPTARG ;; - *) echo "invalid option" >&2; exit 1 ;; - esac +do_run=false +log_flags="-r" +while getopts "sknp:r-:" OPT; do + log_flags="$log_flags -$OPT$OPTARG" + if [ "$OPT" = "-" ] && [ -n "$OPTARG" ]; then + OPT="${OPTARG%%=*}" + OPTARG="${OPTARG#$OPT}" + OPTARG="${OPTARG#=}" + fi + + # shellcheck disable=SC2214 + case "$OPT" in + k | keep) stopservers=false ;; + n | noclean) clean=false ;; + p | port) baseport=$OPTARG ;; + r | run) do_run=true ;; + s | skip) exit 77 ;; + -) break ;; + *) echo "invalid option" >&2; exit 1 ;; + esac done + shift $((OPTIND-1)) +if ! $do_run; then + if [ "$baseport" -eq 0 ]; then + log_flags="$log_flags -p 5300" + fi + env - TESTS="$*" TEST_SUITE_LOG=run.log LOG_DRIVER_FLAGS="--verbose yes --color-tests yes" LOG_FLAGS="$log_flags" make -e check + exit $? +fi + if [ $# -eq 0 ]; then - echofail "Usage: $0 [-k] [-n] [-p ] test-directory [test-options]" >&2; - exit 1 + echofail "Usage: $0 [-k] [-n] [-p ] test-directory [test-options]" >&2; + exit 1 fi systest=$(basename "${1%%/}") @@ -81,64 +100,18 @@ if [ ! -d "${systest}" ]; then exit 1 fi -# Define the number of ports allocated for each test, and the lowest and -# highest valid values for the "-p" option. -# -# The lowest valid value is one more than the highest privileged port number -# (1024). -# -# The highest valid value is calculated by noting that the value passed on the -# command line is the lowest port number in a block of "numports" consecutive -# ports and that the highest valid port number is 65,535. -numport=100 -minvalid=$((1024 + 1)) -maxvalid=$((65535 - numport + 1)) -if ! [ "$baseport" -eq "$baseport" ] > /dev/null 2>&1; then - echofail "$0: $systest: must specify a numeric value for the port" >&2 - exit 1 -elif [ "$baseport" -lt "$minvalid" ] || [ "$baseport" -gt "$maxvalid" ]; then - echofail "$0: $systest: the specified port must be in the range $minvalid to $maxvalid" >&2 - exit 1 -fi - -# Name the first 10 ports in the set (it is assumed that each test has access +# Get the first 10 ports in the set (it is assumed that each test has access # to ten or more ports): the query port, the control port and eight extra # ports. Since the lowest numbered port (specified in the command line) # will usually be a multiple of 10, the names are chosen so that if this is # true, the last digit of EXTRAPORTn is "n". -PORT=$baseport -EXTRAPORT1=$((baseport + 1)) -EXTRAPORT2=$((baseport + 2)) -EXTRAPORT3=$((baseport + 3)) -EXTRAPORT4=$((baseport + 4)) -EXTRAPORT5=$((baseport + 5)) -EXTRAPORT6=$((baseport + 6)) -EXTRAPORT7=$((baseport + 7)) -EXTRAPORT8=$((baseport + 8)) -CONTROLPORT=$((baseport + 9)) - -LOWPORT=$baseport -HIGHPORT=$((baseport + numport - 1)) - -export PORT -export EXTRAPORT1 -export EXTRAPORT2 -export EXTRAPORT3 -export EXTRAPORT4 -export EXTRAPORT5 -export EXTRAPORT6 -export EXTRAPORT7 -export EXTRAPORT8 -export CONTROLPORT - -export LOWPORT -export HIGHPORT +eval "$(${srcdir}/get_ports.sh -p "$baseport")" echostart "S:$systest:$(date_with_args)" echoinfo "T:$systest:1:A" echoinfo "A:$systest:System test $systest" -echoinfo "I:$systest:PORTRANGE:${LOWPORT} - ${HIGHPORT}" +echoinfo "I:$systest:PORTS:${PORT},${EXTRAPORT1},${EXTRAPORT2},${EXTRAPORT3},${EXTRAPORT4},${EXTRAPORT5},${EXTRAPORT6},${EXTRAPORT7},${EXTRAPORT8},${CONTROLPORT}" $PERL ${srcdir}/testsock.pl -p "$PORT" || { echowarn "I:$systest:Network interface aliases not set up. Skipping test." @@ -223,7 +196,7 @@ else echofail "R:$systest:FAIL" # Do not clean up - we need the evidence. find "$systest/" -name 'core*' -or -name '*.core' | while read -r coredump; do - SYSTESTDIR="$systest" + export SYSTESTDIR="$systest" echoinfo "D:$systest:backtrace from $coredump start" binary=$(gdb --batch --core="$coredump" | sed -ne "s/Core was generated by \`//;s/ .*'.$//p;") "${top_builddir}/libtool" --mode=execute gdb \ diff --git a/bin/tests/system/runtime/tests.sh b/bin/tests/system/runtime/tests.sh index 00162731d4..f46924acdd 100644 --- a/bin/tests/system/runtime/tests.sh +++ b/bin/tests/system/runtime/tests.sh @@ -203,7 +203,7 @@ if [ "$(id -u)" -eq 0 ] && [ -z "$CYGWIN" ]; then if [ "$?" -eq 0 ]; then copy_setports ns2/named-alt9.conf.in "${TEMP_NAMED_DIR}/named-alt9.conf" export SOFTHSM2_CONF="${TEMP_NAMED_DIR}/softhsm2.conf" - sh "$TOP/bin/tests/prepare-softhsm2.sh" + sh "$TOP_SRCDIR/bin/tests/prepare-softhsm2.sh" chown -R nobody: "${TEMP_NAMED_DIR}" chmod 0700 "${TEMP_NAMED_DIR}" ( cd "${TEMP_NAMED_DIR}" && $NAMED -u nobody -c named-alt9.conf -d 99 -g -U 4 >> named$n.run 2>&1 & ) || ret=1 diff --git a/bin/tests/system/system-test-driver.sh.in b/bin/tests/system/system-test-driver.sh.in deleted file mode 100644 index 6ad8380643..0000000000 --- a/bin/tests/system/system-test-driver.sh.in +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/sh -# shellcheck disable=SC2181 -# shellcheck disable=SC2034 - -builddir=@abs_builddir@ -srcdir=@abs_srcdir@ - -usage() { - echo "$0 --test-name=NAME --log-file=PATH.log --trs-file=PATH.trs --color-tests={yes|no} --expect-failure={yes|no} --enable-hard-errors={yes|no}" -} - -# -# This requires GNU getopt -# -getopt --test >/dev/null -if [ "$?" -ne 4 ]; then - echo "fatal: GNU getopt is required" - exit 1 -fi - -OPTS=$(getopt --shell sh --name "$(basename "$0")" --options '' --longoptions test-name:,log-file:,trs-file:,color-tests:,expect-failure:,enable-hard-errors: -- "$@") - -if [ "$?" -ne 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi - -eval set -- "$OPTS" - -TEST_NAME= -LOG_FILE= -TRS_FILE= -COLOR_TESTS=yes -EXPECT_FAILURE=no -HARD_ERRORS=yes - -while true; do - case "$1" in - --test-name ) TEST_NAME="$2"; shift; shift ;; - --log-file ) LOG_FILE="$2"; shift; shift ;; - --trs-file ) TRS_FILE="$2"; shift; shift ;; - --color-tests ) COLOR_TESTS="$2"; shift; shift ;; - --expect-failure ) EXPECT_FAILURE="$2"; shift; shift ;; - --hard-errors ) HARD_ERRORS="$2"; shift; shift ;; - -- ) shift; break ;; - *) break ;; - esac -done - -if [ -z "$1" ]; then - echo "fatal: test name required" - usage - exit 1 -fi - -TEST_PROGRAM="$1" -shift - -if [ -z "$TEST_NAME" ]; then - TEST_NAME="$(basename "$TEST_PROGRAM")" -fi -if [ -z "$LOG_FILE" ]; then - LOG_FILE="$TEST_PROGRAM.log" -fi -if [ -z "$TRS_FILE" ]; then - TRS_FILE="$TEST_PROGRAM.trs" -fi - -echo "Running $TEST_PROGRAM" - -"${builddir}/run.sh" -p "$("${srcdir}/get_base_port.sh")" "$@" "$TEST_PROGRAM" - -exit $? diff --git a/bin/tests/wire_test.c b/bin/tests/system/wire_test.c similarity index 100% rename from bin/tests/wire_test.c rename to bin/tests/system/wire_test.c diff --git a/configure.ac b/configure.ac index 592f40f169..8ad62afb27 100644 --- a/configure.ac +++ b/configure.ac @@ -58,6 +58,7 @@ AC_DEFINE([PACKAGE_BUILDER], ["make"], [make or Visual Studio]) AC_CONFIG_SRCDIR([bin/named/main.c]) AM_INIT_AUTOMAKE([foreign subdir-objects dist-xz -Wall -Werror]) AM_SILENT_RULES([yes]) +AM_EXTRA_RECURSIVE_TARGETS([test unit]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) @@ -1248,7 +1249,8 @@ AC_ARG_ENABLE([dnstap], AS_IF([test "$enable_dnstap" != "no"], [PKG_CHECK_MODULES([DNSTAP], [libfstrm libprotobuf-c], [], [AC_MSG_FAILURE([Required libraries (fstrm, protobuf-c) were not found, please install them.])]) - AC_PATH_PROG([PROTOC_C], [protoc-c], []) + AC_PATH_PROG([FSTRM_CAPTURE], [fstrm_capture]) + AC_PATH_PROG([PROTOC_C], [protoc-c]) AS_IF([test -z "$PROTOC_C"], [AC_MSG_ERROR([protoc-c compiler not found])]) AC_DEFINE([HAVE_DNSTAP], 1, [Define to 1 to enable dnstap support]) @@ -1594,8 +1596,6 @@ AC_CONFIG_FILES([bin/tests/system/start.sh], [chmod +x bin/tests/system/start.sh]) AC_CONFIG_FILES([bin/tests/system/stop.sh], [chmod +x bin/tests/system/stop.sh]) -AC_CONFIG_FILES([bin/tests/system/system-test-driver.sh], - [chmod +x bin/tests/system/system-test-driver.sh]) # Misc diff --git a/custom-test-driver b/custom-test-driver new file mode 100755 index 0000000000..e11acd5685 --- /dev/null +++ b/custom-test-driver @@ -0,0 +1,156 @@ +#! /bin/sh +# test-driver - basic testsuite driver script. + +scriptversion=2020-04-24.14; # UTC + +# Copyright (C) 2011-2020 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# This file is maintained in Automake, please report +# bugs to or send patches to +# . + +# Make unconditional expansion of undefined variables an error. This +# helps a lot in preventing typo-related bugs. +set -u + +usage_error () +{ + echo "$0: $*" >&2 + print_usage >&2 + exit 2 +} + +print_usage () +{ + cat <&1; echo $? > "$status_file") | tee $log_file +else + "$@" >$log_file 2>&1; echo $? > "$status_file" +fi +read -r estatus < "$status_file" + +if test $enable_hard_errors = no && test $estatus -eq 99; then + tweaked_estatus=1 +else + tweaked_estatus=$estatus +fi + +case $tweaked_estatus:$expect_failure in + 0:yes) col=$red res=XPASS recheck=yes gcopy=yes;; + 0:*) col=$grn res=PASS recheck=no gcopy=no;; + 77:*) col=$blu res=SKIP recheck=no gcopy=yes;; + 99:*) col=$mgn res=ERROR recheck=yes gcopy=yes;; + *:yes) col=$lgn res=XFAIL recheck=no gcopy=yes;; + *:*) col=$red res=FAIL recheck=yes gcopy=yes;; +esac + +# Report the test outcome and exit status in the logs, so that one can +# know whether the test passed or failed simply by looking at the '.log' +# file, without the need of also peaking into the corresponding '.trs' +# file (automake bug#11814). +echo "$res $test_name (exit status: $estatus)" >>$log_file + +# Report outcome to console. +echo "${col}${res}${std}: $test_name" + +# Register the test result, and other relevant metadata. +echo ":test-result: $res" > $trs_file +echo ":global-test-result: $res" >> $trs_file +echo ":recheck: $recheck" >> $trs_file +echo ":copy-in-global-log: $gcopy" >> $trs_file + +# Local Variables: +# mode: shell-script +# sh-indentation: 2 +# eval: (add-hook 'before-save-hook 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC0" +# time-stamp-end: "; # UTC" +# End: diff --git a/lib/dns/tests/Makefile.am b/lib/dns/tests/Makefile.am index 177cadec1a..0d658db251 100644 --- a/lib/dns/tests/Makefile.am +++ b/lib/dns/tests/Makefile.am @@ -13,7 +13,7 @@ LDADD += \ $(LIBISC_LIBS) \ $(LIBDNS_LIBS) -noinst_LTLIBRARIES = libdnstest.la +check_LTLIBRARIES = libdnstest.la libdnstest_la_SOURCES = dnstest.c dnstest.h check_PROGRAMS = \ @@ -106,3 +106,5 @@ dst_test_CPPFLAGS = \ rsa_test_CPPFLAGS = \ $(AM_CPPFLAGS) \ $(OPENSSL_CFLAGS) + +unit-local: check diff --git a/lib/irs/tests/Makefile.am b/lib/irs/tests/Makefile.am index 5dc892cf39..0246f34cc7 100644 --- a/lib/irs/tests/Makefile.am +++ b/lib/irs/tests/Makefile.am @@ -13,3 +13,5 @@ check_PROGRAMS = \ resconf_test TESTS = $(check_PROGRAMS) + +unit-local: check diff --git a/lib/isc/tests/Makefile.am b/lib/isc/tests/Makefile.am index 9a0dc87e29..c7ae206d78 100644 --- a/lib/isc/tests/Makefile.am +++ b/lib/isc/tests/Makefile.am @@ -8,7 +8,7 @@ LDADD += \ libisctest.la \ $(LIBISC_LIBS) -noinst_LTLIBRARIES = libisctest.la +check_LTLIBRARIES = libisctest.la libisctest_la_SOURCES = isctest.c isctest.h check_PROGRAMS = \ @@ -63,3 +63,5 @@ md_test_LDADD = \ random_test_LDADD = \ $(LDADD) \ -lm + +unit-local: check diff --git a/lib/isccc/tests/Makefile.am b/lib/isccc/tests/Makefile.am index 000e5c8d42..a0b5abd3b8 100644 --- a/lib/isccc/tests/Makefile.am +++ b/lib/isccc/tests/Makefile.am @@ -13,3 +13,5 @@ check_PROGRAMS = \ result_test TESTS = $(check_PROGRAMS) + +unit-local: check diff --git a/lib/isccfg/tests/Makefile.am b/lib/isccfg/tests/Makefile.am index 9f31441bb2..42242ad070 100644 --- a/lib/isccfg/tests/Makefile.am +++ b/lib/isccfg/tests/Makefile.am @@ -15,3 +15,5 @@ check_PROGRAMS = \ parser_test TESTS = $(check_PROGRAMS) + +unit-local: check diff --git a/lib/ns/tests/Makefile.am b/lib/ns/tests/Makefile.am index 76145db571..c7ff73cd31 100644 --- a/lib/ns/tests/Makefile.am +++ b/lib/ns/tests/Makefile.am @@ -43,3 +43,5 @@ query_test_LDFLAGS = \ -Wl,--wrap=isc_nmhandle_unref endif + +unit-local: check diff --git a/util/copyrights b/util/copyrights index 293a843c85..e13441fee7 100644 --- a/util/copyrights +++ b/util/copyrights @@ -523,7 +523,7 @@ ./bin/tests/system/geoip2/prereq.sh SH 2019,2020 ./bin/tests/system/geoip2/setup.sh SH 2019,2020 ./bin/tests/system/geoip2/tests.sh SH 2019,2020 -./bin/tests/system/get_base_port.sh SH 2020 +./bin/tests/system/get_ports.sh SH 2020 ./bin/tests/system/glue/clean.sh SH 2000,2001,2004,2007,2012,2014,2015,2016,2018,2019,2020 ./bin/tests/system/glue/fi.good X 2000,2001,2018,2019,2020 ./bin/tests/system/glue/noglue.good X 2000,2001,2018,2019,2020 @@ -888,7 +888,6 @@ ./bin/tests/system/synthfromdnssec/ns1/sign.sh SH 2017,2018,2019,2020 ./bin/tests/system/synthfromdnssec/setup.sh SH 2017,2018,2019,2020 ./bin/tests/system/synthfromdnssec/tests.sh SH 2017,2018,2019,2020 -./bin/tests/system/system-test-driver.sh.in X 2020 ./bin/tests/system/tcp/ans6/ans.py PYTHON 2019,2020 ./bin/tests/system/tcp/clean.sh SH 2014,2016,2018,2019,2020 ./bin/tests/system/tcp/setup.sh SH 2018,2019,2020 @@ -972,6 +971,7 @@ ./bin/tests/system/win32/pipequeries.vcxproj.filters.in X 2016,2018,2019,2020 ./bin/tests/system/win32/pipequeries.vcxproj.in X 2016,2017,2018,2019,2020 ./bin/tests/system/win32/pipequeries.vcxproj.user X 2016,2018,2019,2020 +./bin/tests/system/wire_test.c C 2020 ./bin/tests/system/xfer/ans5/badkeydata X 2011,2018,2019,2020 ./bin/tests/system/xfer/ans5/badmessageid X 2020 ./bin/tests/system/xfer/ans5/goodaxfr X 2011,2018,2019,2020 @@ -1023,7 +1023,6 @@ ./bin/tests/win32/timer_test.vcxproj.filters.in X 2013,2015,2018,2019,2020 ./bin/tests/win32/timer_test.vcxproj.in X 2013,2015,2016,2017,2018,2019,2020 ./bin/tests/win32/timer_test.vcxproj.user X 2013,2018,2019,2020 -./bin/tests/wire_test.c C 1999,2000,2001,2004,2005,2007,2015,2016,2018,2019,2020 ./bin/tools/arpaname.1 MAN DOCBOOK ./bin/tools/arpaname.c C 2009,2015,2016,2018,2019,2020 ./bin/tools/arpaname.docbook SGML 2009,2014,2015,2016,2018,2019,2020 @@ -1191,6 +1190,7 @@ ./contrib/scripts/named-bootconf.sh SH.PORTION 1999,2000,2001,2004,2006,2007,2012,2014,2016,2018,2019,2020 ./contrib/scripts/nanny.pl PERL 2000,2001,2004,2007,2012,2014,2016,2018,2019,2020 ./contrib/scripts/zone-edit.sh.in SH 2010,2012,2014,2016,2018,2019,2020 +./custom-test-driver X 2020 ./doc/arm/Bv9ARM-book.xml SGML 2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020 ./doc/arm/Bv9ARM.ch01.html X 2000,2001,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020 ./doc/arm/Bv9ARM.ch02.html X 2000,2001,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020