From 626409aac9cafb7ecaf1f5dbd3783f717ad4c280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Wed, 22 Apr 2020 14:58:55 +0200 Subject: [PATCH 01/16] Properly fail the GitLab CI system test job when any system test fails --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0245affc04..6c74136089 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -255,8 +255,9 @@ stages: - *setup_interfaces - *setup_softhsm script: - - ( cd bin/tests/system && make -j${TEST_PARALLEL_JOBS:-1} -k check V=1 ) || cat bin/tests/system/test-suite.log - - test -s bin/tests/system/test-suite.log + - cd bin/tests/system && make -j${TEST_PARALLEL_JOBS:-1} -k check V=1 + after_script: + - cat bin/tests/system/test-suite.log .system_test: &system_test_job <<: *system_test_common @@ -269,6 +270,7 @@ stages: <<: *system_test_common allow_failure: true after_script: + - cat bin/tests/system/test-suite.log - find bin -name 'tsan.*' -exec python3 util/parse_tsan.py {} \; artifacts: expire_in: "1 day" From e6d277777100de83c30b0cc21ccad2c04b2e96ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Wed, 22 Apr 2020 15:17:45 +0200 Subject: [PATCH 02/16] Use $TOP_BUILDDIR instead of $TOP of checking config.h values --- bin/tests/system/rrsetorder/tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/tests/system/rrsetorder/tests.sh b/bin/tests/system/rrsetorder/tests.sh index 08266af6f1..00c6e3bb2b 100644 --- a/bin/tests/system/rrsetorder/tests.sh +++ b/bin/tests/system/rrsetorder/tests.sh @@ -19,7 +19,7 @@ status=0 GOOD_RANDOM="1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24" GOOD_RANDOM_NO=24 -if grep "^#define DNS_RDATASET_FIXED" $TOP/config.h > /dev/null 2>&1 ; then +if grep "^#define DNS_RDATASET_FIXED" "$TOP_BUILDDIR/config.h" > /dev/null 2>&1 ; then test_fixed=true else echo_i "Order 'fixed' disabled at compile time" From 0bc1b4804eddd416ec66342dc891fe464b0eaa24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Wed, 22 Apr 2020 15:39:45 +0200 Subject: [PATCH 03/16] Use $TOP_SRCDIR instead of $TOP of calling prepare-softhsm2 script --- bin/tests/system/runtime/tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/tests/system/runtime/tests.sh b/bin/tests/system/runtime/tests.sh index 00162731d4..f46924acdd 100644 --- a/bin/tests/system/runtime/tests.sh +++ b/bin/tests/system/runtime/tests.sh @@ -203,7 +203,7 @@ if [ "$(id -u)" -eq 0 ] && [ -z "$CYGWIN" ]; then if [ "$?" -eq 0 ]; then copy_setports ns2/named-alt9.conf.in "${TEMP_NAMED_DIR}/named-alt9.conf" export SOFTHSM2_CONF="${TEMP_NAMED_DIR}/softhsm2.conf" - sh "$TOP/bin/tests/prepare-softhsm2.sh" + sh "$TOP_SRCDIR/bin/tests/prepare-softhsm2.sh" chown -R nobody: "${TEMP_NAMED_DIR}" chmod 0700 "${TEMP_NAMED_DIR}" ( cd "${TEMP_NAMED_DIR}" && $NAMED -u nobody -c named-alt9.conf -d 99 -g -U 4 >> named$n.run 2>&1 & ) || ret=1 From ba0e9cb56c66e26793d7c23614f4fc3d3f10ff98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Thu, 23 Apr 2020 08:07:07 +0200 Subject: [PATCH 04/16] Silence PyYAML warning Make yaml.load_all() use yaml.SafeLoader to address a warning currently emitted when bin/tests/system/dnstap/ydump.py is run: ydump.py:28: YAMLLoadWarning: calling yaml.load_all() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details. for l in yaml.load_all(f.stdout): --- bin/tests/system/dnstap/ydump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/tests/system/dnstap/ydump.py b/bin/tests/system/dnstap/ydump.py index 6788b054b8..e199effca4 100644 --- a/bin/tests/system/dnstap/ydump.py +++ b/bin/tests/system/dnstap/ydump.py @@ -25,5 +25,5 @@ DATAFILE = sys.argv[2] ARGS = [DNSTAP_READ, '-y', DATAFILE] with subprocess.Popen(ARGS, stdout=subprocess.PIPE) as f: - for l in yaml.load_all(f.stdout): + for l in yaml.load_all(f.stdout, Loader=yaml.SafeLoader): pprint.pprint(l) From 32247dc997d4776170faba5a2f3dd0be857c155e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Thu, 23 Apr 2020 08:07:07 +0200 Subject: [PATCH 05/16] Build wire_test The bin/tests/wire_test helper program is currently not included in any Makefile.am file. Move its source code to bin/tests/system and build it along other helper tools when dnstap support is requested as the "dnstap" system test needs this tool in order to pass. --- bin/tests/system/Makefile.am | 16 ++++++++++++++-- bin/tests/system/conf.sh.in | 2 +- bin/tests/{ => system}/wire_test.c | 0 util/copyrights | 1 - 4 files changed, 15 insertions(+), 4 deletions(-) rename bin/tests/{ => system}/wire_test.c (100%) diff --git a/bin/tests/system/Makefile.am b/bin/tests/system/Makefile.am index 17fa9d1205..cca65fa19e 100644 --- a/bin/tests/system/Makefile.am +++ b/bin/tests/system/Makefile.am @@ -43,7 +43,7 @@ tkey_keycreate_CPPFLAGS = \ $(AM_CPPFLAGS) \ $(LIBDNS_CFLAGS) -tkey_keycreate_LDADD = \ +tkey_keycreate_LDADD = \ $(LDADD) \ $(LIBDNS_LIBS) @@ -51,10 +51,22 @@ tkey_keydelete_CPPFLAGS = \ $(AM_CPPFLAGS) \ $(LIBDNS_CFLAGS) -tkey_keydelete_LDADD = \ +tkey_keydelete_LDADD = \ $(LDADD) \ $(LIBDNS_LIBS) +if HAVE_DNSTAP +check_PROGRAMS += wire_test + +wire_test_CPPFLAGS = \ + $(AM_CPPFLAGS) \ + $(LIBDNS_CFLAGS) + +wire_test_LDADD = \ + $(LDADD) \ + $(LIBDNS_LIBS) +endif HAVE_DNSTAP + TESTS = \ acl additional addzone allow-query auth autosign \ builtin cacheclean case catz cds \ diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in index 65dbdeddb2..dc9b73be7c 100644 --- a/bin/tests/system/conf.sh.in +++ b/bin/tests/system/conf.sh.in @@ -61,7 +61,7 @@ SETTIME=$TOP_BUILDDIR/bin/dnssec/dnssec-settime SIGNER=$TOP_BUILDDIR/bin/dnssec/dnssec-signzone TSIGKEYGEN=$TOP_BUILDDIR/bin/confgen/tsig-keygen VERIFY=$TOP_BUILDDIR/bin/dnssec/dnssec-verify -WIRETEST=$TOP_BUILDDIR/bin/tests/wire_test +WIRETEST=$TOP_BUILDDIR/bin/tests/system/wire_test BIGKEY=$TOP_BUILDDIR/bin/tests/system/rsabigexponent/bigkey GENCHECK=$TOP_BUILDDIR/bin/tests/system/rndc/gencheck diff --git a/bin/tests/wire_test.c b/bin/tests/system/wire_test.c similarity index 100% rename from bin/tests/wire_test.c rename to bin/tests/system/wire_test.c diff --git a/util/copyrights b/util/copyrights index 293a843c85..2aa21cca2b 100644 --- a/util/copyrights +++ b/util/copyrights @@ -1023,7 +1023,6 @@ ./bin/tests/win32/timer_test.vcxproj.filters.in X 2013,2015,2018,2019,2020 ./bin/tests/win32/timer_test.vcxproj.in X 2013,2015,2016,2017,2018,2019,2020 ./bin/tests/win32/timer_test.vcxproj.user X 2013,2018,2019,2020 -./bin/tests/wire_test.c C 1999,2000,2001,2004,2005,2007,2015,2016,2018,2019,2020 ./bin/tools/arpaname.1 MAN DOCBOOK ./bin/tools/arpaname.c C 2009,2015,2016,2018,2019,2020 ./bin/tools/arpaname.docbook SGML 2009,2014,2015,2016,2018,2019,2020 From 51188b33f801ade652211e5751bdecb9d5dd44b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Thu, 23 Apr 2020 08:34:11 +0200 Subject: [PATCH 06/16] Ensure fstrm_capture is looked for fstrm_capture is not an essential utility, but its corresponding Makefile token needs to substituted even if it is not found in PATH or else the "dnstap" system test will consistently fail. --- configure.ac | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 592f40f169..36fdcfb3b0 100644 --- a/configure.ac +++ b/configure.ac @@ -1248,7 +1248,8 @@ AC_ARG_ENABLE([dnstap], AS_IF([test "$enable_dnstap" != "no"], [PKG_CHECK_MODULES([DNSTAP], [libfstrm libprotobuf-c], [], [AC_MSG_FAILURE([Required libraries (fstrm, protobuf-c) were not found, please install them.])]) - AC_PATH_PROG([PROTOC_C], [protoc-c], []) + AC_PATH_PROG([FSTRM_CAPTURE], [fstrm_capture]) + AC_PATH_PROG([PROTOC_C], [protoc-c]) AS_IF([test -z "$PROTOC_C"], [AC_MSG_ERROR([protoc-c compiler not found])]) AC_DEFINE([HAVE_DNSTAP], 1, [Define to 1 to enable dnstap support]) From d10294acb37f7683b2032ef2300cf5644d33a79c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Fri, 24 Apr 2020 13:34:40 +0200 Subject: [PATCH 07/16] Remove the requirement for GNU getopt and parse long options using getopts --- bin/tests/system/system-test-driver.sh.in | 42 +++++++++-------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/bin/tests/system/system-test-driver.sh.in b/bin/tests/system/system-test-driver.sh.in index 6ad8380643..edffd7be22 100644 --- a/bin/tests/system/system-test-driver.sh.in +++ b/bin/tests/system/system-test-driver.sh.in @@ -9,21 +9,6 @@ usage() { echo "$0 --test-name=NAME --log-file=PATH.log --trs-file=PATH.trs --color-tests={yes|no} --expect-failure={yes|no} --enable-hard-errors={yes|no}" } -# -# This requires GNU getopt -# -getopt --test >/dev/null -if [ "$?" -ne 4 ]; then - echo "fatal: GNU getopt is required" - exit 1 -fi - -OPTS=$(getopt --shell sh --name "$(basename "$0")" --options '' --longoptions test-name:,log-file:,trs-file:,color-tests:,expect-failure:,enable-hard-errors: -- "$@") - -if [ "$?" -ne 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi - -eval set -- "$OPTS" - TEST_NAME= LOG_FILE= TRS_FILE= @@ -31,18 +16,23 @@ COLOR_TESTS=yes EXPECT_FAILURE=no HARD_ERRORS=yes -while true; do - case "$1" in - --test-name ) TEST_NAME="$2"; shift; shift ;; - --log-file ) LOG_FILE="$2"; shift; shift ;; - --trs-file ) TRS_FILE="$2"; shift; shift ;; - --color-tests ) COLOR_TESTS="$2"; shift; shift ;; - --expect-failure ) EXPECT_FAILURE="$2"; shift; shift ;; - --hard-errors ) HARD_ERRORS="$2"; shift; shift ;; - -- ) shift; break ;; - *) break ;; - esac +while getopts -: OPT; do + if [ "$OPT" = "-" ] && [ -n "$OPTARG" ]; then + OPT="${OPTARG%%=*}" + OPTARG="${OPTARG#$OPT}" + OPTARG="${OPTARG#=}" + fi + case "$OPT" in + test-name) TEST_NAME="$OPTARG" ;; + log-file) LOG_FILE="$OPTARG" ;; + trs-file) TRS_FILE="$OPTARG" ;; + color-tests) COLOR_TESTS="$OPTARG" ;; + expect-failure) EXPECT_FAILURE="$OPTARG" ;; + hard-errors) HARD_ERRORS="$OPTARG" ;; + *) break ;; + esac done +shift $((OPTIND-1)) if [ -z "$1" ]; then echo "fatal: test name required" From 5813818036ba531e68d021eb6c6ea53ef68f855e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Fri, 24 Apr 2020 15:29:54 +0200 Subject: [PATCH 08/16] Improve the LOG_DRIVER and LOG_COMPILER for the system tests There are several improvements over the default/previous behaviour of the test log driver and log compiler: * The system-test-driver.sh was dropped (it was used incorrectly) * The run.sh script is now both log compiler and cli script to run individual tests * The custom-test-driver was added as extended version of the automake test-driver with capability to tee the test output to stdout when `--verbose yes` is passed to it (you can use LOG_DRIVER_FLAGS to add the option by default) * Makefile.am has been extended to honor V=1 for the system tests test-driver (e.g. V=1 adds `--verbose yes` to AM_LOG_DRIVER_FLAGS) --- .gitlab-ci.yml | 3 +- bin/tests/system/.gitignore | 2 +- bin/tests/system/Makefile.am | 13 +- bin/tests/system/run.sh.in | 41 ++++-- bin/tests/system/system-test-driver.sh.in | 60 --------- configure.ac | 2 - custom-test-driver | 156 ++++++++++++++++++++++ util/copyrights | 3 +- 8 files changed, 200 insertions(+), 80 deletions(-) delete mode 100644 bin/tests/system/system-test-driver.sh.in create mode 100755 custom-test-driver diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6c74136089..103ca7569e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -255,7 +255,8 @@ stages: - *setup_interfaces - *setup_softhsm script: - - cd bin/tests/system && make -j${TEST_PARALLEL_JOBS:-1} -k check V=1 + - cd bin/tests/system + - make -j${TEST_PARALLEL_JOBS:-1} -k check V=1 after_script: - cat bin/tests/system/test-suite.log diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore index 998718e758..8205de1dbc 100644 --- a/bin/tests/system/.gitignore +++ b/bin/tests/system/.gitignore @@ -15,7 +15,7 @@ parallel.mk /get_base_port.state /get_base_port.lock /run.sh +/run.log /start.sh /stop.sh -/system-test-driver.sh /ifconfig.sh diff --git a/bin/tests/system/Makefile.am b/bin/tests/system/Makefile.am index cca65fa19e..43e00b9486 100644 --- a/bin/tests/system/Makefile.am +++ b/bin/tests/system/Makefile.am @@ -161,7 +161,18 @@ check: exit 1 endif !HAVE_PERL -LOG_COMPILER = $(builddir)/system-test-driver.sh +LOG_DRIVER_V = $(LOG_DRIVER_V_@AM_V@) +LOG_DRIVER_V_ = $(LOG_DRIVER_V_@AM_DEFAULT_V@) +LOG_DRIVER_V_0 = --verbose no +LOG_DRIVER_V_1 = --verbose yes + +LOG_DRIVER = $(top_srcdir)/custom-test-driver +AM_LOG_DRIVER_FLAGS = $(LOG_DRIVER_V) + +LOG_COMPILER = $(builddir)/run.sh +AM_LOG_FLAGS = -r -p "$$("$(srcdir)/get_base_port.sh")" + +$(TESTS): run.sh clean-local: -rm -f get_base_port.state get_base_port.lock diff --git a/bin/tests/system/run.sh.in b/bin/tests/system/run.sh.in index 1f38dc45a3..f7d9735a73 100644 --- a/bin/tests/system/run.sh.in +++ b/bin/tests/system/run.sh.in @@ -37,24 +37,37 @@ else clean=true fi -while getopts "knp:r-:" flag; do - case "$flag" in - -) case "${OPTARG}" in - keep) stopservers=false ;; - noclean) clean=false ;; - esac - ;; - k) stopservers=false ;; - n) clean=false ;; - p) baseport=$OPTARG ;; - *) echo "invalid option" >&2; exit 1 ;; - esac +do_run=false +log_flags="-r" +while getopts "knp:r-:" OPT; do + log_flags="$log_flags -$OPT$OPTARG" + if [ "$OPT" = "-" ] && [ -n "$OPTARG" ]; then + OPT="${OPTARG%%=*}" + OPTARG="${OPTARG#$OPT}" + OPTARG="${OPTARG#=}" + fi + + case "$OPT" in + k | keep) stopservers=false ;; + n | noclean) clean=false ;; + p | port) baseport=$OPTARG ;; + r | run) do_run=true ;; + s | skip) exit 77 ;; + -) break ;; + *) echo "invalid option" >&2; exit 1 ;; + esac done + shift $((OPTIND-1)) +if ! $do_run; then + env - TESTS="$1" TEST_SUITE_LOG=run.log LOG_DRIVER_FLAGS="--verbose yes --color-tests yes" LOG_FLAGS="$log_flags" make -e check + exit $? +fi + if [ $# -eq 0 ]; then - echofail "Usage: $0 [-k] [-n] [-p ] test-directory [test-options]" >&2; - exit 1 + echofail "Usage: $0 [-k] [-n] [-p ] test-directory [test-options]" >&2; + exit 1 fi systest=$(basename "${1%%/}") diff --git a/bin/tests/system/system-test-driver.sh.in b/bin/tests/system/system-test-driver.sh.in deleted file mode 100644 index edffd7be22..0000000000 --- a/bin/tests/system/system-test-driver.sh.in +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/sh -# shellcheck disable=SC2181 -# shellcheck disable=SC2034 - -builddir=@abs_builddir@ -srcdir=@abs_srcdir@ - -usage() { - echo "$0 --test-name=NAME --log-file=PATH.log --trs-file=PATH.trs --color-tests={yes|no} --expect-failure={yes|no} --enable-hard-errors={yes|no}" -} - -TEST_NAME= -LOG_FILE= -TRS_FILE= -COLOR_TESTS=yes -EXPECT_FAILURE=no -HARD_ERRORS=yes - -while getopts -: OPT; do - if [ "$OPT" = "-" ] && [ -n "$OPTARG" ]; then - OPT="${OPTARG%%=*}" - OPTARG="${OPTARG#$OPT}" - OPTARG="${OPTARG#=}" - fi - case "$OPT" in - test-name) TEST_NAME="$OPTARG" ;; - log-file) LOG_FILE="$OPTARG" ;; - trs-file) TRS_FILE="$OPTARG" ;; - color-tests) COLOR_TESTS="$OPTARG" ;; - expect-failure) EXPECT_FAILURE="$OPTARG" ;; - hard-errors) HARD_ERRORS="$OPTARG" ;; - *) break ;; - esac -done -shift $((OPTIND-1)) - -if [ -z "$1" ]; then - echo "fatal: test name required" - usage - exit 1 -fi - -TEST_PROGRAM="$1" -shift - -if [ -z "$TEST_NAME" ]; then - TEST_NAME="$(basename "$TEST_PROGRAM")" -fi -if [ -z "$LOG_FILE" ]; then - LOG_FILE="$TEST_PROGRAM.log" -fi -if [ -z "$TRS_FILE" ]; then - TRS_FILE="$TEST_PROGRAM.trs" -fi - -echo "Running $TEST_PROGRAM" - -"${builddir}/run.sh" -p "$("${srcdir}/get_base_port.sh")" "$@" "$TEST_PROGRAM" - -exit $? diff --git a/configure.ac b/configure.ac index 36fdcfb3b0..36d038b810 100644 --- a/configure.ac +++ b/configure.ac @@ -1595,8 +1595,6 @@ AC_CONFIG_FILES([bin/tests/system/start.sh], [chmod +x bin/tests/system/start.sh]) AC_CONFIG_FILES([bin/tests/system/stop.sh], [chmod +x bin/tests/system/stop.sh]) -AC_CONFIG_FILES([bin/tests/system/system-test-driver.sh], - [chmod +x bin/tests/system/system-test-driver.sh]) # Misc diff --git a/custom-test-driver b/custom-test-driver new file mode 100755 index 0000000000..e11acd5685 --- /dev/null +++ b/custom-test-driver @@ -0,0 +1,156 @@ +#! /bin/sh +# test-driver - basic testsuite driver script. + +scriptversion=2020-04-24.14; # UTC + +# Copyright (C) 2011-2020 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# This file is maintained in Automake, please report +# bugs to or send patches to +# . + +# Make unconditional expansion of undefined variables an error. This +# helps a lot in preventing typo-related bugs. +set -u + +usage_error () +{ + echo "$0: $*" >&2 + print_usage >&2 + exit 2 +} + +print_usage () +{ + cat <&1; echo $? > "$status_file") | tee $log_file +else + "$@" >$log_file 2>&1; echo $? > "$status_file" +fi +read -r estatus < "$status_file" + +if test $enable_hard_errors = no && test $estatus -eq 99; then + tweaked_estatus=1 +else + tweaked_estatus=$estatus +fi + +case $tweaked_estatus:$expect_failure in + 0:yes) col=$red res=XPASS recheck=yes gcopy=yes;; + 0:*) col=$grn res=PASS recheck=no gcopy=no;; + 77:*) col=$blu res=SKIP recheck=no gcopy=yes;; + 99:*) col=$mgn res=ERROR recheck=yes gcopy=yes;; + *:yes) col=$lgn res=XFAIL recheck=no gcopy=yes;; + *:*) col=$red res=FAIL recheck=yes gcopy=yes;; +esac + +# Report the test outcome and exit status in the logs, so that one can +# know whether the test passed or failed simply by looking at the '.log' +# file, without the need of also peaking into the corresponding '.trs' +# file (automake bug#11814). +echo "$res $test_name (exit status: $estatus)" >>$log_file + +# Report outcome to console. +echo "${col}${res}${std}: $test_name" + +# Register the test result, and other relevant metadata. +echo ":test-result: $res" > $trs_file +echo ":global-test-result: $res" >> $trs_file +echo ":recheck: $recheck" >> $trs_file +echo ":copy-in-global-log: $gcopy" >> $trs_file + +# Local Variables: +# mode: shell-script +# sh-indentation: 2 +# eval: (add-hook 'before-save-hook 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC0" +# time-stamp-end: "; # UTC" +# End: diff --git a/util/copyrights b/util/copyrights index 2aa21cca2b..a16a02e5ec 100644 --- a/util/copyrights +++ b/util/copyrights @@ -888,7 +888,6 @@ ./bin/tests/system/synthfromdnssec/ns1/sign.sh SH 2017,2018,2019,2020 ./bin/tests/system/synthfromdnssec/setup.sh SH 2017,2018,2019,2020 ./bin/tests/system/synthfromdnssec/tests.sh SH 2017,2018,2019,2020 -./bin/tests/system/system-test-driver.sh.in X 2020 ./bin/tests/system/tcp/ans6/ans.py PYTHON 2019,2020 ./bin/tests/system/tcp/clean.sh SH 2014,2016,2018,2019,2020 ./bin/tests/system/tcp/setup.sh SH 2018,2019,2020 @@ -972,6 +971,7 @@ ./bin/tests/system/win32/pipequeries.vcxproj.filters.in X 2016,2018,2019,2020 ./bin/tests/system/win32/pipequeries.vcxproj.in X 2016,2017,2018,2019,2020 ./bin/tests/system/win32/pipequeries.vcxproj.user X 2016,2018,2019,2020 +./bin/tests/system/wire_test.c C 2020 ./bin/tests/system/xfer/ans5/badkeydata X 2011,2018,2019,2020 ./bin/tests/system/xfer/ans5/badmessageid X 2020 ./bin/tests/system/xfer/ans5/goodaxfr X 2011,2018,2019,2020 @@ -1190,6 +1190,7 @@ ./contrib/scripts/named-bootconf.sh SH.PORTION 1999,2000,2001,2004,2006,2007,2012,2014,2016,2018,2019,2020 ./contrib/scripts/nanny.pl PERL 2000,2001,2004,2007,2012,2014,2016,2018,2019,2020 ./contrib/scripts/zone-edit.sh.in SH 2010,2012,2014,2016,2018,2019,2020 +./custom-test-driver X 2020 ./doc/arm/Bv9ARM-book.xml SGML 2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020 ./doc/arm/Bv9ARM.ch01.html X 2000,2001,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020 ./doc/arm/Bv9ARM.ch02.html X 2000,2001,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020 From 7286d4d6340ef54404164ecf5ce73a58db37b3e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Fri, 24 Apr 2020 15:37:36 +0200 Subject: [PATCH 09/16] Colorize the test-driver output by default --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 103ca7569e..a0b410cc5c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,6 +31,8 @@ variables: TARBALL_COMPRESSOR: xz TARBALL_EXTENSION: xz + AM_COLOR_TESTS: always + stages: - autoconf - precheck From 85d0b4bcf05d6b9d8ceaf8da8cbdb5b10f633426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Fri, 24 Apr 2020 17:00:39 +0200 Subject: [PATCH 10/16] Run recheck in case some system test fails --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a0b410cc5c..0565580d99 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -258,7 +258,7 @@ stages: - *setup_softhsm script: - cd bin/tests/system - - make -j${TEST_PARALLEL_JOBS:-1} -k check V=1 + - make -j${TEST_PARALLEL_JOBS:-1} -k check V=1 || make -j${TEST_PARALLEL_JOBS:-1} -k recheck V=1 after_script: - cat bin/tests/system/test-suite.log From 52683934cdd956aa0b30b869203e3f46fd1cd7da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Mon, 27 Apr 2020 10:47:08 +0200 Subject: [PATCH 11/16] Refactor the script that gets/sets the ports for system tests The current script used ephemeral port range which clashed with the ports used by the tools (dig, ...), and the range always started with the first port and there was 100 ports allocated for each system test. In this commit, the first port has been randomized, the get_ports.sh script outputs the variables (the output has to be eval'ed from run.sh) and there's less waste in the port range. --- bin/tests/system/.gitignore | 4 +- bin/tests/system/Makefile.am | 4 +- bin/tests/system/get_base_port.sh | 58 ------------------- bin/tests/system/get_ports.sh | 94 +++++++++++++++++++++++++++++++ bin/tests/system/run.sh.in | 68 +++++----------------- util/copyrights | 2 +- 6 files changed, 113 insertions(+), 117 deletions(-) delete mode 100755 bin/tests/system/get_base_port.sh create mode 100755 bin/tests/system/get_ports.sh diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore index 8205de1dbc..0c66e8d2cc 100644 --- a/bin/tests/system/.gitignore +++ b/bin/tests/system/.gitignore @@ -12,8 +12,8 @@ named.run parallel.mk /*.log /*.trs -/get_base_port.state -/get_base_port.lock +/get_ports.state +/get_ports.lock /run.sh /run.log /start.sh diff --git a/bin/tests/system/Makefile.am b/bin/tests/system/Makefile.am index 43e00b9486..0dda39f679 100644 --- a/bin/tests/system/Makefile.am +++ b/bin/tests/system/Makefile.am @@ -170,9 +170,9 @@ LOG_DRIVER = $(top_srcdir)/custom-test-driver AM_LOG_DRIVER_FLAGS = $(LOG_DRIVER_V) LOG_COMPILER = $(builddir)/run.sh -AM_LOG_FLAGS = -r -p "$$("$(srcdir)/get_base_port.sh")" +AM_LOG_FLAGS = -r $(TESTS): run.sh clean-local: - -rm -f get_base_port.state get_base_port.lock + -rm -f get_ports.state get_ports.lock diff --git a/bin/tests/system/get_base_port.sh b/bin/tests/system/get_base_port.sh deleted file mode 100755 index 7c1b2d3887..0000000000 --- a/bin/tests/system/get_base_port.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh -# -# Copyright (C) Internet Systems Consortium, Inc. ("ISC") -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# -# See the COPYRIGHT file distributed with this work for additional -# information regarding copyright ownership. - -# This script is a 'port' broker. It keeps track of ports given to the -# individual system subtests, so every test is given a unique port range. - -lockfile=get_base_port.lock -statefile=get_base_port.state - -ephemeral_port_min=49152 -ephemeral_port_max=65535 - -get_base_port() { - if ( set -o noclobber; echo "$$" > "${lockfile}" ) 2> /dev/null; then - trap 'rm -f "${lockfile}"; exit $?' INT TERM EXIT - - base_port=$(cat "${statefile}" 2>/dev/null) - - if [ -z "${base_port}" ]; then - base_port="${ephemeral_port_min}" - fi - - if [ "$((base_port+100))" -gt "${ephemeral_port_max}" ]; then - base_port="${ephemeral_port_min}" - fi - - echo $((base_port+100)) > get_base_port.state - - # clean up after yourself, and release your trap - rm -f "${lockfile}" - trap - INT TERM EXIT - echo "${base_port}" - else - echo 0 - fi -} - -tries=10 - -while [ "${tries}" -gt 0 ]; do - base_port=$(get_base_port) - if [ "${base_port}" -gt 0 ]; then - echo "${base_port}" - exit 0 - fi - sleep 1 - tries=$((tries-1)) -done - -exit 1 diff --git a/bin/tests/system/get_ports.sh b/bin/tests/system/get_ports.sh new file mode 100755 index 0000000000..30c066b2b4 --- /dev/null +++ b/bin/tests/system/get_ports.sh @@ -0,0 +1,94 @@ +#!/bin/sh +# +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +# This script is a 'port' broker. It keeps track of ports given to the +# individual system subtests, so every test is given a unique port range. + +lockfile=get_ports.lock +statefile=get_ports.state + +port_min=5001 +port_max=32767 + +get_random() { + dd if=/dev/urandom bs=1 count=2 2>/dev/null | od -tu2 -An +} + +get_port() { + tries=10 + port=0 + while [ "${tries}" -gt 0 ]; do + if ( set -o noclobber; echo "$$" > "${lockfile}" ) 2> /dev/null; then + trap 'rm -f "${lockfile}"; exit $?' INT TERM EXIT + + port=$(cat "${statefile}" 2>/dev/null) + + if [ -z "${port}" ]; then + if [ "$1" -gt 0 ]; then + port="$1" + else + port_range=$((port_max-port_min)) + port=$(($(get_random)%port_range+port_range)) + fi + fi + + if [ "$((port+1))" -gt "${port_max}" ]; then + port="${port_min}" + fi + + echo $((port+1)) > get_ports.state + + # clean up after yourself, and release your trap + rm -f "${lockfile}" + trap - INT TERM EXIT + + # we have our port + break + fi + sleep 1 + tries=$((tries-1)) + done + if [ "$port" -eq 0 ]; then + exit 1 + fi + echo "$port" +} + +baseport=0 +while getopts "p:-:" OPT; do + if [ "$OPT" = "-" ] && [ -n "$OPTARG" ]; then + OPT="${OPTARG%%=*}" + OPTARG="${OPTARG#$OPT}" + OPTARG="${OPTARG#=}" + fi + + # shellcheck disable=SC2214 + case "$OPT" in + p | port) baseport=$OPTARG ;; + -) break ;; + *) echo "invalid option" >&2; exit 1 ;; + esac +done + +echo "export PORT=$(get_port "$baseport")" +echo "export EXTRAPORT1=$(get_port)" +echo "export EXTRAPORT2=$(get_port)" +echo "export EXTRAPORT3=$(get_port)" +echo "export EXTRAPORT4=$(get_port)" +echo "export EXTRAPORT5=$(get_port)" +echo "export EXTRAPORT6=$(get_port)" +echo "export EXTRAPORT7=$(get_port)" +echo "export EXTRAPORT8=$(get_port)" +echo "export CONTROLPORT=$(get_port)" + +# Local Variables: +# sh-basic-offset: 4 +# End: diff --git a/bin/tests/system/run.sh.in b/bin/tests/system/run.sh.in index f7d9735a73..5e4095ea67 100644 --- a/bin/tests/system/run.sh.in +++ b/bin/tests/system/run.sh.in @@ -17,6 +17,7 @@ top_builddir=@top_builddir@ builddir=@abs_builddir@ srcdir=@abs_srcdir@ +# shellcheck source=conf.sh . ${builddir}/conf.sh SYSTEMTESTTOP="$(cd -P -- "${builddir}" && pwd -P)" @@ -29,7 +30,8 @@ date_with_args() ( ) stopservers=true -baseport=5300 +# baseport == 0 means random +baseport=0 if [ "${SYSTEMTEST_NO_CLEAN:-0}" -eq 1 ]; then clean=false @@ -39,7 +41,7 @@ fi do_run=false log_flags="-r" -while getopts "knp:r-:" OPT; do +while getopts "sknp:r-:" OPT; do log_flags="$log_flags -$OPT$OPTARG" if [ "$OPT" = "-" ] && [ -n "$OPTARG" ]; then OPT="${OPTARG%%=*}" @@ -47,6 +49,7 @@ while getopts "knp:r-:" OPT; do OPTARG="${OPTARG#=}" fi + # shellcheck disable=SC2214 case "$OPT" in k | keep) stopservers=false ;; n | noclean) clean=false ;; @@ -61,8 +64,11 @@ done shift $((OPTIND-1)) if ! $do_run; then - env - TESTS="$1" TEST_SUITE_LOG=run.log LOG_DRIVER_FLAGS="--verbose yes --color-tests yes" LOG_FLAGS="$log_flags" make -e check - exit $? + if [ "$baseport" -eq 0 ]; then + log_flags="$log_flags -p 5300" + fi + env - TESTS="$*" TEST_SUITE_LOG=run.log LOG_DRIVER_FLAGS="--verbose yes --color-tests yes" LOG_FLAGS="$log_flags" make -e check + exit $? fi if [ $# -eq 0 ]; then @@ -94,64 +100,18 @@ if [ ! -d "${systest}" ]; then exit 1 fi -# Define the number of ports allocated for each test, and the lowest and -# highest valid values for the "-p" option. -# -# The lowest valid value is one more than the highest privileged port number -# (1024). -# -# The highest valid value is calculated by noting that the value passed on the -# command line is the lowest port number in a block of "numports" consecutive -# ports and that the highest valid port number is 65,535. -numport=100 -minvalid=$((1024 + 1)) -maxvalid=$((65535 - numport + 1)) -if ! [ "$baseport" -eq "$baseport" ] > /dev/null 2>&1; then - echofail "$0: $systest: must specify a numeric value for the port" >&2 - exit 1 -elif [ "$baseport" -lt "$minvalid" ] || [ "$baseport" -gt "$maxvalid" ]; then - echofail "$0: $systest: the specified port must be in the range $minvalid to $maxvalid" >&2 - exit 1 -fi - -# Name the first 10 ports in the set (it is assumed that each test has access +# Get the first 10 ports in the set (it is assumed that each test has access # to ten or more ports): the query port, the control port and eight extra # ports. Since the lowest numbered port (specified in the command line) # will usually be a multiple of 10, the names are chosen so that if this is # true, the last digit of EXTRAPORTn is "n". -PORT=$baseport -EXTRAPORT1=$((baseport + 1)) -EXTRAPORT2=$((baseport + 2)) -EXTRAPORT3=$((baseport + 3)) -EXTRAPORT4=$((baseport + 4)) -EXTRAPORT5=$((baseport + 5)) -EXTRAPORT6=$((baseport + 6)) -EXTRAPORT7=$((baseport + 7)) -EXTRAPORT8=$((baseport + 8)) -CONTROLPORT=$((baseport + 9)) - -LOWPORT=$baseport -HIGHPORT=$((baseport + numport - 1)) - -export PORT -export EXTRAPORT1 -export EXTRAPORT2 -export EXTRAPORT3 -export EXTRAPORT4 -export EXTRAPORT5 -export EXTRAPORT6 -export EXTRAPORT7 -export EXTRAPORT8 -export CONTROLPORT - -export LOWPORT -export HIGHPORT +eval "$(${srcdir}/get_ports.sh -p "$baseport")" echostart "S:$systest:$(date_with_args)" echoinfo "T:$systest:1:A" echoinfo "A:$systest:System test $systest" -echoinfo "I:$systest:PORTRANGE:${LOWPORT} - ${HIGHPORT}" +echoinfo "I:$systest:PORTS:${PORT},${EXTRAPORT1},${EXTRAPORT2},${EXTRAPORT3},${EXTRAPORT4},${EXTRAPORT5},${EXTRAPORT6},${EXTRAPORT7},${EXTRAPORT8},${CONTROLPORT}" $PERL ${srcdir}/testsock.pl -p "$PORT" || { echowarn "I:$systest:Network interface aliases not set up. Skipping test." @@ -236,7 +196,7 @@ else echofail "R:$systest:FAIL" # Do not clean up - we need the evidence. find "$systest/" -name 'core*' -or -name '*.core' | while read -r coredump; do - SYSTESTDIR="$systest" + export SYSTESTDIR="$systest" echoinfo "D:$systest:backtrace from $coredump start" binary=$(gdb --batch --core="$coredump" | sed -ne "s/Core was generated by \`//;s/ .*'.$//p;") "${top_builddir}/libtool" --mode=execute gdb \ diff --git a/util/copyrights b/util/copyrights index a16a02e5ec..e13441fee7 100644 --- a/util/copyrights +++ b/util/copyrights @@ -523,7 +523,7 @@ ./bin/tests/system/geoip2/prereq.sh SH 2019,2020 ./bin/tests/system/geoip2/setup.sh SH 2019,2020 ./bin/tests/system/geoip2/tests.sh SH 2019,2020 -./bin/tests/system/get_base_port.sh SH 2020 +./bin/tests/system/get_ports.sh SH 2020 ./bin/tests/system/glue/clean.sh SH 2000,2001,2004,2007,2012,2014,2015,2016,2018,2019,2020 ./bin/tests/system/glue/fi.good X 2000,2001,2018,2019,2020 ./bin/tests/system/glue/noglue.good X 2000,2001,2018,2019,2020 From dd0faa15565a51e662bed93b668a589079e02d26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Mon, 27 Apr 2020 11:43:22 +0200 Subject: [PATCH 12/16] Store the output from yamlget.py into intermediate files for easier debugging --- bin/tests/system/digdelv/tests.sh | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/bin/tests/system/digdelv/tests.sh b/bin/tests/system/digdelv/tests.sh index bd8aab4c44..6aaf0bdf54 100644 --- a/bin/tests/system/digdelv/tests.sh +++ b/bin/tests/system/digdelv/tests.sh @@ -789,11 +789,14 @@ if [ -x "$DIG" ] ; then echo_i "check dig +yaml output ($n)" ret=0 dig_with_opts +qr +yaml @10.53.0.3 any ns2.example > dig.out.test$n 2>&1 || ret=1 - value=$($PYTHON yamlget.py dig.out.test$n 0 message query_message_data status || ret=1) + $PYTHON yamlget.py dig.out.test$n 0 message query_message_data status > yamlget.out.test$n 2>&1 || ret=1 + read -r value < yamlget.out.test$n [ "$value" = "NOERROR" ] || ret=1 - value=$($PYTHON yamlget.py dig.out.test$n 1 message response_message_data status || ret=1) + $PYTHON yamlget.py dig.out.test$n 1 message response_message_data status > yamlget.out.test$n 2>&1 || ret=1 + read -r value < yamlget.out.test$n [ "$value" = "NOERROR" ] || ret=1 - value=$($PYTHON yamlget.py dig.out.test$n 1 message response_message_data QUESTION_SECTION 0 || ret=1) + $PYTHON yamlget.py dig.out.test$n 1 message response_message_data QUESTION_SECTION 0 > yamlget.out.test$n 2>&1 || ret=1 + read -r value < yamlget.out.test$n [ "$value" = "ns2.example. IN ANY" ] || ret=1 if [ $ret -ne 0 ]; then echo_i "failed"; fi status=$((status+ret)) @@ -860,9 +863,11 @@ if [ -x "$MDIG" ] ; then echo_i "check mdig +yaml output ($n)" ret=0 mdig_with_opts +yaml @10.53.0.3 -t any ns2.example > dig.out.test$n 2>&1 || ret=1 - value=$($PYTHON yamlget.py dig.out.test$n 0 message response_message_data status || ret=1) + $PYTHON yamlget.py dig.out.test$n 0 message response_message_data status > yamlget.out.test$n 2>&1 || ret=1 + read -r value < yamlget.out.test$n [ "$value" = "NOERROR" ] || ret=1 - value=$($PYTHON yamlget.py dig.out.test$n 0 message response_message_data QUESTION_SECTION 0 || ret=1) + $PYTHON yamlget.py dig.out.test$n 0 message response_message_data QUESTION_SECTION 0 > yamlget.out.test$n 2>&1 || ret=1 + read -r value < yamlget.out.test$n [ "$value" = "ns2.example. IN ANY" ] || ret=1 if [ $ret -ne 0 ]; then echo_i "failed"; fi status=$((status+ret)) @@ -1110,11 +1115,14 @@ if [ -x "$DELV" ] ; then echo_i "check delv +yaml output ($n)" ret=0 delv_with_opts +yaml @10.53.0.3 any ns2.example > delv.out.test$n 2>&1 || ret=1 - value=$($PYTHON yamlget.py delv.out.test$n status || ret=1) + $PYTHON yamlget.py delv.out.test$n status > yamlget.out.test$n 2>&1 || ret=1 + read -r value < yamlget.out.test$n [ "$value" = "success" ] || ret=1 - value=$($PYTHON yamlget.py delv.out.test$n query_name || ret=1) + $PYTHON yamlget.py delv.out.test$n query_name > yamlget.out.test$n 2>&1 || ret=1 + read -r value < yamlget.out.test$n [ "$value" = "ns2.example" ] || ret=1 - value=$($PYTHON yamlget.py delv.out.test$n records 0 answer_not_validated 0 || ret=1) + $PYTHON yamlget.py delv.out.test$n records 0 answer_not_validated 0 > yamlget.out.test$n 2>&1 || ret=1 + read -r value < yamlget.out.test$n count=$(echo $value | wc -w ) [ ${count:-0} -eq 5 ] || ret=1 if [ $ret -ne 0 ]; then echo_i "failed"; fi From 968523dd8dba4a64a8b4be19f26ea15481c89aeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Mon, 27 Apr 2020 12:23:44 +0200 Subject: [PATCH 13/16] Use absolute path to PYTHON and PERL, so test -x works properly in system tests --- bin/tests/system/conf.sh.in | 4 ++-- bin/tests/system/digdelv/tests.sh | 2 +- bin/tests/system/dnstap/tests.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in index dc9b73be7c..85f8df69e4 100644 --- a/bin/tests/system/conf.sh.in +++ b/bin/tests/system/conf.sh.in @@ -106,12 +106,12 @@ XMLLINT=@XMLLINT@ XSLTPROC=@XSLTPROC@ # PERL will be an empty string if no perl interpreter was found. -PERL=@PERL@ +PERL=$(command -v "@PERL@") # Windows process management leave empty PSSUSPEND= -PYTHON=@PYTHON@ +PYTHON=$(command -v "@PYTHON@") # # Determine if we support various optional features. diff --git a/bin/tests/system/digdelv/tests.sh b/bin/tests/system/digdelv/tests.sh index 6aaf0bdf54..2d2c63e285 100644 --- a/bin/tests/system/digdelv/tests.sh +++ b/bin/tests/system/digdelv/tests.sh @@ -62,7 +62,7 @@ KEYDATA="$(< ns2/keydata sed -e 's/+/[+]/g')" NOSPLIT="$(< ns2/keydata sed -e 's/+/[+]/g' -e 's/ //g')" HAS_PYYAML=0 -if [ -n "$PYTHON" ] ; then +if [ -x "$PYTHON" ] ; then $PYTHON -c "import yaml" 2> /dev/null && HAS_PYYAML=1 fi diff --git a/bin/tests/system/dnstap/tests.sh b/bin/tests/system/dnstap/tests.sh index d7d80a6134..a666b7e01f 100644 --- a/bin/tests/system/dnstap/tests.sh +++ b/bin/tests/system/dnstap/tests.sh @@ -499,7 +499,7 @@ if [ $ret != 0 ]; then echo_i "failed"; fi status=`expr $status + $ret` HAS_PYYAML=0 -if [ -n "$PYTHON" ] ; then +if [ -x "$PYTHON" ] ; then $PYTHON -c "import yaml" 2> /dev/null && HAS_PYYAML=1 fi From 7fe5f670b5e3b7bcea22cbcedcd5c668b71fc2ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Mon, 27 Apr 2020 14:15:10 +0200 Subject: [PATCH 14/16] Disable pylint: 'Constant name dname doesn't conform to UPPER_CASE naming style (invalid-name)' --- .pylintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.pylintrc b/.pylintrc index 62cbfae3e6..750e50e37c 100644 --- a/.pylintrc +++ b/.pylintrc @@ -4,3 +4,4 @@ disable= C0115, # missing-class-docstring C0116, # missing-function-docstring R0801, # duplicate-code + C0103, # invalid-name From 17a1bafc08b23f182131808771c09c1ebeed4ab1 Mon Sep 17 00:00:00 2001 From: Evan Hunt Date: Thu, 23 Apr 2020 16:22:21 -0700 Subject: [PATCH 15/16] Restore behaviour of "make test" and "make unit" Add recursive "test" and "unit" rules, which execute "make check" in specific directories - "make test" runs the system tests, and "make unit" runs the unit tests. --- .gitlab-ci.yml | 2 +- bin/tests/system/Makefile.am | 2 ++ configure.ac | 1 + lib/dns/tests/Makefile.am | 2 ++ lib/irs/tests/Makefile.am | 2 ++ lib/isc/tests/Makefile.am | 2 ++ lib/isccc/tests/Makefile.am | 2 ++ lib/isccfg/tests/Makefile.am | 2 ++ lib/ns/tests/Makefile.am | 2 ++ 9 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0565580d99..b0662fc1de 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -320,7 +320,7 @@ stages: before_script: - *setup_softhsm script: - - cd lib && make -j${TEST_PARALLEL_JOBS:-1} -k check V=1 + - make -j${TEST_PARALLEL_JOBS:-1} -k unit V=1 .unit_test: &unit_test_job <<: *unit_test_common diff --git a/bin/tests/system/Makefile.am b/bin/tests/system/Makefile.am index 0dda39f679..ea4d13a934 100644 --- a/bin/tests/system/Makefile.am +++ b/bin/tests/system/Makefile.am @@ -176,3 +176,5 @@ $(TESTS): run.sh clean-local: -rm -f get_ports.state get_ports.lock + +test-local: check diff --git a/configure.ac b/configure.ac index 36d038b810..8ad62afb27 100644 --- a/configure.ac +++ b/configure.ac @@ -58,6 +58,7 @@ AC_DEFINE([PACKAGE_BUILDER], ["make"], [make or Visual Studio]) AC_CONFIG_SRCDIR([bin/named/main.c]) AM_INIT_AUTOMAKE([foreign subdir-objects dist-xz -Wall -Werror]) AM_SILENT_RULES([yes]) +AM_EXTRA_RECURSIVE_TARGETS([test unit]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/lib/dns/tests/Makefile.am b/lib/dns/tests/Makefile.am index 177cadec1a..1ce7163773 100644 --- a/lib/dns/tests/Makefile.am +++ b/lib/dns/tests/Makefile.am @@ -106,3 +106,5 @@ dst_test_CPPFLAGS = \ rsa_test_CPPFLAGS = \ $(AM_CPPFLAGS) \ $(OPENSSL_CFLAGS) + +unit-local: check diff --git a/lib/irs/tests/Makefile.am b/lib/irs/tests/Makefile.am index 5dc892cf39..0246f34cc7 100644 --- a/lib/irs/tests/Makefile.am +++ b/lib/irs/tests/Makefile.am @@ -13,3 +13,5 @@ check_PROGRAMS = \ resconf_test TESTS = $(check_PROGRAMS) + +unit-local: check diff --git a/lib/isc/tests/Makefile.am b/lib/isc/tests/Makefile.am index 9a0dc87e29..554e806453 100644 --- a/lib/isc/tests/Makefile.am +++ b/lib/isc/tests/Makefile.am @@ -63,3 +63,5 @@ md_test_LDADD = \ random_test_LDADD = \ $(LDADD) \ -lm + +unit-local: check diff --git a/lib/isccc/tests/Makefile.am b/lib/isccc/tests/Makefile.am index 000e5c8d42..a0b5abd3b8 100644 --- a/lib/isccc/tests/Makefile.am +++ b/lib/isccc/tests/Makefile.am @@ -13,3 +13,5 @@ check_PROGRAMS = \ result_test TESTS = $(check_PROGRAMS) + +unit-local: check diff --git a/lib/isccfg/tests/Makefile.am b/lib/isccfg/tests/Makefile.am index 9f31441bb2..42242ad070 100644 --- a/lib/isccfg/tests/Makefile.am +++ b/lib/isccfg/tests/Makefile.am @@ -15,3 +15,5 @@ check_PROGRAMS = \ parser_test TESTS = $(check_PROGRAMS) + +unit-local: check diff --git a/lib/ns/tests/Makefile.am b/lib/ns/tests/Makefile.am index 76145db571..c7ff73cd31 100644 --- a/lib/ns/tests/Makefile.am +++ b/lib/ns/tests/Makefile.am @@ -43,3 +43,5 @@ query_test_LDFLAGS = \ -Wl,--wrap=isc_nmhandle_unref endif + +unit-local: check From 86f322133f0ad9c5af9d4981009e1b2a4a2cd588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Mon, 27 Apr 2020 15:18:31 +0200 Subject: [PATCH 16/16] Fix the noinst_LTLIBRARIES -> check_LTLIBRARIES in libisc and libdns unit tests --- lib/dns/tests/Makefile.am | 2 +- lib/isc/tests/Makefile.am | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/dns/tests/Makefile.am b/lib/dns/tests/Makefile.am index 1ce7163773..0d658db251 100644 --- a/lib/dns/tests/Makefile.am +++ b/lib/dns/tests/Makefile.am @@ -13,7 +13,7 @@ LDADD += \ $(LIBISC_LIBS) \ $(LIBDNS_LIBS) -noinst_LTLIBRARIES = libdnstest.la +check_LTLIBRARIES = libdnstest.la libdnstest_la_SOURCES = dnstest.c dnstest.h check_PROGRAMS = \ diff --git a/lib/isc/tests/Makefile.am b/lib/isc/tests/Makefile.am index 554e806453..c7ae206d78 100644 --- a/lib/isc/tests/Makefile.am +++ b/lib/isc/tests/Makefile.am @@ -8,7 +8,7 @@ LDADD += \ libisctest.la \ $(LIBISC_LIBS) -noinst_LTLIBRARIES = libisctest.la +check_LTLIBRARIES = libisctest.la libisctest_la_SOURCES = isctest.c isctest.h check_PROGRAMS = \