From 986b364fe634ad164b8d9b0d9ed1100ab5047e81 Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Mon, 14 Dec 2020 20:40:58 +0100 Subject: [PATCH 1/5] Drop testsummary.sh testsummary.sh was not updated after build system rewrite to Autotools, and needs to be fixed to produce test summary and core dump, assertion failures, and ThreadSanitizer reports. Given that all of this is provided by Autotools and run.sh already, there's little use to testsummary.sh script and should be dropped. --- bin/tests/system/testsummary.sh | 91 --------------------------------- 1 file changed, 91 deletions(-) delete mode 100644 bin/tests/system/testsummary.sh diff --git a/bin/tests/system/testsummary.sh b/bin/tests/system/testsummary.sh deleted file mode 100644 index 52c79b94db..0000000000 --- a/bin/tests/system/testsummary.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/sh - -# Copyright (C) Internet Systems Consortium, Inc. ("ISC") -# -# SPDX-License-Identifier: MPL-2.0 -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, you can obtain one at https://mozilla.org/MPL/2.0/. -# -# See the COPYRIGHT file distributed with this work for additional -# information regarding copyright ownership. - -# Creates the system tests output file from the various test.output.* files. It -# then searches that file and prints the number of tests passed, failed, not -# run. It also checks whether the IP addresses 10.53.0.[1-8] were set up and, -# if not, prints a warning. -# -# Usage: -# testsummary.sh [-n] -# -# -n Do NOT delete the individual test.output.* files after concatenating -# them into systests.output. -# -# Status return: -# 0 - no tests failed -# 1 - one or more tests failed - -. ./conf.sh - -keepfile=0 - -while getopts "n" flag; do - case $flag in - n) keepfile=1 ;; - *) exit 1 ;; - esac -done - -if [ "$(find . -name 'test.output.*' 2>/dev/null | wc -l)" -eq 0 ]; then - echowarn "I:No 'test.output.*' files were found." - echowarn "I:Printing summary from pre-existing 'systests.output'." -else - cat test.output.* > systests.output - if [ $keepfile -eq 0 ]; then - rm -f test.output.* - fi -fi - -if [ ! -f systests.output ]; then - echowarn "I:No 'systests.output' file found." - exit 1 -fi - -status=0 -echoinfo "I:System test result summary:" -echoinfo "$(grep 'R:[a-z0-9_-][a-z0-9_-]*:[A-Z][A-Z]*' systests.output | cut -d':' -f3 | sort | uniq -c | sed -e 's/^/I:/')" - -FAILED_TESTS=$(grep 'R:[a-z0-9_-][a-z0-9_-]*:FAIL' systests.output | cut -d':' -f2 | sort | sed -e 's/^/I: /') -if [ -n "${FAILED_TESTS}" ]; then - echoinfo "I:The following system tests failed:" - echoinfo "${FAILED_TESTS}" - status=1 -fi - -CRASHED_TESTS=$(find . \( -name 'core' -or -name 'core.*' -or -name '*.core' \) ! -name '*.txt' | cut -d'/' -f2 | sort -u | sed -e 's/^/I: /') -if [ -n "${CRASHED_TESTS}" ]; then - echoinfo "I:Core dumps were found for the following system tests:" - echoinfo "${CRASHED_TESTS}" -fi - -ASSERTION_FAILED_TESTS=$(find . -name named.run -print0 | xargs -0 grep "assertion failure" | cut -d'/' -f2 | sort -u | sed -e 's/^/I: /') -if [ -n "${ASSERTION_FAILED_TESTS}" ]; then - echoinfo "I:Assertion failures were detected for the following system tests:" - echoinfo "${ASSERTION_FAILED_TESTS}" -fi - -TSAN_REPORT_TESTS=$(find . -name 'tsan.*' | cut -d'/' -f2 | sort -u | sed -e 's/^/I: /') -if [ -n "${TSAN_REPORT_TESTS}" ]; then - echoinfo "I:ThreadSanitizer reported issues for the following system tests:" - echoinfo "${TSAN_REPORT_TESTS}" -fi - -RESULTS_FOUND=$(grep -c 'R:[a-z0-9_-][a-z0-9_-]*:[A-Z][A-Z]*' systests.output) -TESTS_RUN=$(echo "${SUBDIRS}" | wc -w) -if [ "${RESULTS_FOUND}" -ne "${TESTS_RUN}" ]; then - echofail "I:Found ${RESULTS_FOUND} test results, but ${TESTS_RUN} tests were run" - status=1 -fi - -exit $status From 9d398572f00d5794f95b27589391e133557cfb65 Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Thu, 20 Jan 2022 13:11:49 +0100 Subject: [PATCH 2/5] Drop bin/tests/system/parallel.sh "parallel.sh" script was used on Windows to run system tests in parallel. Since Windows support was removed from BIND 9, the script is not needed anymore. --- bin/tests/system/.gitignore | 1 - bin/tests/system/README | 11 ++--------- bin/tests/system/parallel.sh | 34 ---------------------------------- 3 files changed, 2 insertions(+), 44 deletions(-) delete mode 100644 bin/tests/system/parallel.sh diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore index 3cde7cdb6b..846ada8373 100644 --- a/bin/tests/system/.gitignore +++ b/bin/tests/system/.gitignore @@ -12,7 +12,6 @@ named.run /makejournal /systests.output /random.data -parallel.mk /*.log /*.trs /resolve diff --git a/bin/tests/system/README b/bin/tests/system/README index f4248a38d5..6b0e789ff2 100644 --- a/bin/tests/system/README +++ b/bin/tests/system/README @@ -632,15 +632,8 @@ test, the running of all tests in the test suite is controlled by the Makefile. All system tests are capable of being run in parallel. For this to work, each test needs to use a unique set of ports. To avoid the need to define which tests use which ports (and so risk port clashes as further tests are added), -the ports are assigned when the tests are run. This is achieved by having the -"test" target in the Makefile depend on "parallel.mk". That file is created -when "make check" is run, and contains a target for each test of the form: - - : - @$(SHELL) run.sh -p - -The is unique and the values of for each test are -separated by at least 100 ports. +the ports are determined by "get_ports.sh", a port broker script which keeps +track of ports given to each individual system test. Cleaning Up From Tests diff --git a/bin/tests/system/parallel.sh b/bin/tests/system/parallel.sh deleted file mode 100644 index 16b0b49c94..0000000000 --- a/bin/tests/system/parallel.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh - -# Copyright (C) Internet Systems Consortium, Inc. ("ISC") -# -# SPDX-License-Identifier: MPL-2.0 -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, you can obtain one at https://mozilla.org/MPL/2.0/. -# -# See the COPYRIGHT file distributed with this work for additional -# information regarding copyright ownership. - -. ./conf.sh - -PARALLELS=$(echo "$PARALLELDIRS" | sed "s|\([^ ][^ ]*\)|test-\1|g;" | tr _ - | tr "\n" " ") - -echo ".PHONY: $PARALLELS" -echo -echo "check_interfaces:" -echo " @${PERL} testsock.pl > /dev/null 2>&1 || { \\" -echo " echo \"I:NOTE: System tests were skipped because they require the\"; \\" -echo " echo \"I: test IP addresses 10.53.0.* to be configured as alias\"; \\" -echo " echo \"I: addresses on the loopback interface. Please run\"; \\" -echo " echo \"I: \"bin/tests/system/ifconfig.sh up\" as root to configure them.\"; \\" -echo " exit 1; \\" -echo " }" -echo -echo "test check: $PARALLELS" -for directory in $PARALLELDIRS ; do - echo - echo "test-$(echo "$directory" | tr _ -): check_interfaces" - echo " @${SHELL} ./run.sh -r $directory 2>&1 | tee test.output.$directory" -done From 8109e924b557720af1cc71aa982d52071e8b5cd8 Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Wed, 26 Jan 2022 13:49:32 +0100 Subject: [PATCH 3/5] Drop support for sequential system tests System test used to have sequential system tests, which can't run in parallel with the rest of system tests. As there are no such tests anymore the underlying infrastructure can be dropped. --- bin/tests/system/README | 9 +++------ bin/tests/system/conf.sh.common | 7 ------- bin/tests/system/conf.sh.in | 6 +----- bin/tests/system/runsequential.sh | 26 -------------------------- 4 files changed, 4 insertions(+), 44 deletions(-) delete mode 100755 bin/tests/system/runsequential.sh diff --git a/bin/tests/system/README b/bin/tests/system/README index 6b0e789ff2..8db1a05848 100644 --- a/bin/tests/system/README +++ b/bin/tests/system/README @@ -598,13 +598,10 @@ Adding a Test to the System Test Suite --- Once a test has been created, the following files should be edited: -* conf.sh.in The name of the test should be added to the PARALLELDIRS or -SEQUENTIALDIRS variables as appropriate. The former is used for tests that -can run in parallel with other tests, the latter for tests that are unable to -do so. +* conf.sh.common The name of the test should be added to the PARALLEL_COMMON +variable. -* Makefile.in The name of the test should be added to one of the the PARALLEL -or SEQUENTIAL variables. +* Makefile.am The name of the test should be added to the TESTS variable. (It is likely that a future iteration of the system test suite will remove the need to edit multiple files to add a test.) diff --git a/bin/tests/system/conf.sh.common b/bin/tests/system/conf.sh.common index 4f6066e468..10a252c837 100644 --- a/bin/tests/system/conf.sh.common +++ b/bin/tests/system/conf.sh.common @@ -31,13 +31,6 @@ export LANG=C # The "dialup", "delzone", and "dupsigs" tests are also not run by # default because they take a very long time to complete. # -# The following tests are hard-coded to use ports 5300 and 9953. For -# this reason, these must be run sequentially. -# -# Sequential tests that only run on unix/linux should be added to -# SEQUENTIAL_UNIX in conf.sh.in -# -SEQUENTIAL_COMMON="" # # These tests can use ports assigned by the caller (other than 5300 diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in index df364b562f..817f1b976a 100644 --- a/bin/tests/system/conf.sh.in +++ b/bin/tests/system/conf.sh.in @@ -73,8 +73,6 @@ KRB5_KTNAME=dns.keytab # # Construct the lists of tests to run # -SEQUENTIAL_UNIX="" -SEQUENTIALDIRS="$SEQUENTIAL_COMMON $SEQUENTIAL_UNIX" PARALLEL_UNIX="chain checkds @@ -93,9 +91,7 @@ pipelined qmin shutdown tcp" -PARALLELDIRS="$PARALLEL_COMMON $PARALLEL_UNIX" - -SUBDIRS="$SEQUENTIALDIRS $PARALLELDIRS" +SUBDIRS="$PARALLEL_COMMON $PARALLEL_UNIX" # Use the CONFIG_SHELL detected by configure for tests SHELL=@SHELL@ diff --git a/bin/tests/system/runsequential.sh b/bin/tests/system/runsequential.sh deleted file mode 100755 index b3c4b00e7c..0000000000 --- a/bin/tests/system/runsequential.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -# Copyright (C) Internet Systems Consortium, Inc. ("ISC") -# -# SPDX-License-Identifier: MPL-2.0 -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, you can obtain one at https://mozilla.org/MPL/2.0/. -# -# See the COPYRIGHT file distributed with this work for additional -# information regarding copyright ownership. - -# Run system tests that must be run sequentially -# -# Note: Use "make check" to run all the system tests. This script will just -# run those tests that require that each of their nameservers is the only one -# running on an IP address. -# - -. ./conf.sh - -for d in $SEQUENTIALDIRS -do - $SHELL run.sh "${@}" "$d" 2>&1 | tee "test.output.$d" -done From f6b996f6fc7f5f52df3d774123adcd69889a651a Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Thu, 20 Jan 2022 13:43:43 +0100 Subject: [PATCH 4/5] Drop systests.output references from system test Since "runall.sh" script removal systests.output file is not being created and its references are useless. --- bin/tests/system/.gitignore | 1 - bin/tests/system/README | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore index 846ada8373..d41de04dd4 100644 --- a/bin/tests/system/.gitignore +++ b/bin/tests/system/.gitignore @@ -10,7 +10,6 @@ named.run /feature-test /test.output.* /makejournal -/systests.output /random.data /*.log /*.trs diff --git a/bin/tests/system/README b/bin/tests/system/README index 8db1a05848..b4d64dd7ff 100644 --- a/bin/tests/system/README +++ b/bin/tests/system/README @@ -126,9 +126,7 @@ new tests being started as tests finish. Each test will get a unique set of ports, so there is no danger of tests interfering with one another. Parallel running will reduce the total time taken to run the BIND system tests, but will mean that the output from all the tests sent to the screen will be mixed up -with one another. However, the systests.output file produced at the end of the -run (in the bin/tests/system directory) will contain the output from each test -in sequential order. +with one another. In this case, retention of the output files after a test completes successfully is specified by setting the environment variable SYSTEMTEST_NO_CLEAN to 1 prior From e97ed8d9b621440e4555a813da231e829a99fa94 Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Wed, 26 Jan 2022 14:02:22 +0100 Subject: [PATCH 5/5] Clean up test.output.* references test.output.* files are no longer created by the system test framework. Remove all references to these files from the source tree. --- bin/tests/system/.gitignore | 1 - bin/tests/system/README | 8 -------- bin/tests/system/cleanall.sh | 1 - 3 files changed, 10 deletions(-) diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore index d41de04dd4..cff1aa39fd 100644 --- a/bin/tests/system/.gitignore +++ b/bin/tests/system/.gitignore @@ -8,7 +8,6 @@ named.lock named.pid named.run /feature-test -/test.output.* /makejournal /random.data /*.log diff --git a/bin/tests/system/README b/bin/tests/system/README index b4d64dd7ff..ffc3a900cb 100644 --- a/bin/tests/system/README +++ b/bin/tests/system/README @@ -641,10 +641,6 @@ stored in the test directory. 2. Files produced by named which may not be cleaned up if named exits abnormally, e.g. core files, PID files etc., are stored in the test directory. -3. A file "test.output." containing the text written to stdout by the -test is written to bin/tests/system/. This file is only produced when the test -is run as part of the entire test suite (e.g. via make). - If the test fails, all these files are retained. But if the test succeeds, they are cleaned up at different times: @@ -653,7 +649,3 @@ they are cleaned up at different times: 2. Files that may not be cleaned up if named exits abnormally can be removed using the "cleanall.sh" script. - -3. "test.output.*" files are deleted when the test suite ends. At this point, -the file "testsummary.sh" is called which concatenates all the "test.output.*" -files into a single "systests.output" file before deleting them. diff --git a/bin/tests/system/cleanall.sh b/bin/tests/system/cleanall.sh index 79a8434b3e..5fafffd203 100644 --- a/bin/tests/system/cleanall.sh +++ b/bin/tests/system/cleanall.sh @@ -31,6 +31,5 @@ rm -f ../random.data for d in $SUBDIRS do test ! -f $d/clean.sh || ( cd $d && $SHELL clean.sh ) - rm -f test.output.$d test -d $d && find $d -type d -exec rmdir '{}' \; 2> /dev/null done