From b0916bba41b2898185c91a9ddc07dea42d1bdfaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Fri, 6 Dec 2019 14:11:01 +0100 Subject: [PATCH] Store system test output in bin/tests/system/ Some clean.sh scripts contain overly broad file deletion wildcards which cause the test.output file (used by the system test framework for collecting output) in a given system test's directory to be erroneously removed immediately after the test is started (due to setup.sh scripts calling clean.sh at the beginning). This prevents the test's output from being placed in bin/tests/system/systests.output at the end of a test suite run and thus can lead to test failures being ignored. Fix by storing each test's output in a test.output. file in bin/tests/system/, which prevents clean.sh scripts from removing it (as they should only ever affect files contained in a given system test's directory). --- bin/tests/system/.gitignore | 2 +- bin/tests/system/README | 20 ++++++++++---------- bin/tests/system/cleanall.sh | 2 +- bin/tests/system/parallel.sh | 2 +- bin/tests/system/runsequential.sh | 2 +- bin/tests/system/testsummary.sh | 12 ++++++------ 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore index c85b5b92da..f51188bc28 100644 --- a/bin/tests/system/.gitignore +++ b/bin/tests/system/.gitignore @@ -5,7 +5,7 @@ named.lock named.pid named.run /feature-test -**/test.output +/test.output.* /systests.output /random.data parallel.mk diff --git a/bin/tests/system/README b/bin/tests/system/README index 13286c054e..552344ba87 100644 --- a/bin/tests/system/README +++ b/bin/tests/system/README @@ -717,17 +717,17 @@ separated by at least 100 ports. Cleaning Up From Tests --- -When a test is run, files are created in the test directory. These files fall -into three categories: +When a test is run, up to three different types of files are created: -1. Files generated by the test itself, e.g. output from "dig" and "rndc". +1. Files generated by the test itself, e.g. output from "dig" and "rndc", are +stored in the test directory. 2. Files produced by named which may not be cleaned up if named exits -abnormally, e.g. core files, PID files etc. +abnormally, e.g. core files, PID files etc., are stored in the test directory. -3. The file "test.output" containing the text written to stdout by the test. -This is only produced when the test is run as part of the entire test suite -(e.g. via "runall.sh"). +3. A file "test.output." containing the text written to stdout by the +test is written to bin/tests/system/. This file is only produced when the test +is run as part of the entire test suite (e.g. via "runall.sh"). If the test fails, all these files are retained. But if the test succeeds, they are cleaned up at different times: @@ -739,9 +739,9 @@ called from "run.sh". 2. Files that may not be cleaned up if named exits abnormally are removed by the system's "clean.sh". -3. "test.output" files are deleted when the test suite ends. At this point, -the file "testsummary.sh" is called which concatenates all the "test.output" -files into a single "systest.output" file before deleting them. +3. "test.output.*" files are deleted when the test suite ends. At this point, +the file "testsummary.sh" is called which concatenates all the "test.output.*" +files into a single "systests.output" file before deleting them. A complication arises with the "test.output" file however: diff --git a/bin/tests/system/cleanall.sh b/bin/tests/system/cleanall.sh index d57d450bf2..d4b3c665cc 100644 --- a/bin/tests/system/cleanall.sh +++ b/bin/tests/system/cleanall.sh @@ -30,6 +30,6 @@ rm -f $SYSTEMTESTTOP/random.data for d in $SUBDIRS do test ! -f $d/clean.sh || ( cd $d && $SHELL clean.sh ) - test -f $d/test.output && rm $d/test.output + rm -f test.output.$d test -d $d && find $d -type d -exec rmdir '{}' \; 2> /dev/null done diff --git a/bin/tests/system/parallel.sh b/bin/tests/system/parallel.sh index 24b465e105..2eab88c444 100644 --- a/bin/tests/system/parallel.sh +++ b/bin/tests/system/parallel.sh @@ -29,6 +29,6 @@ port=${STARTPORT:-5000} for directory in $PARALLELDIRS ; do echo echo "test-`echo $directory | tr _ -`: check_interfaces" - echo " @${SHELL} ./run.sh -r -p $port $directory 2>&1 | tee $directory/test.output" + echo " @${SHELL} ./run.sh -r -p $port $directory 2>&1 | tee test.output.$directory" port=`expr $port + 100` done diff --git a/bin/tests/system/runsequential.sh b/bin/tests/system/runsequential.sh index 5e352e9bc5..6380f4c2f2 100755 --- a/bin/tests/system/runsequential.sh +++ b/bin/tests/system/runsequential.sh @@ -21,5 +21,5 @@ SYSTEMTESTTOP=. for d in $SEQUENTIALDIRS do - $SHELL run.sh "${@}" $d 2>&1 | tee $d/test.output + $SHELL run.sh "${@}" $d 2>&1 | tee test.output.$d done diff --git a/bin/tests/system/testsummary.sh b/bin/tests/system/testsummary.sh index 1c008cdbad..a6f2bc03f9 100644 --- a/bin/tests/system/testsummary.sh +++ b/bin/tests/system/testsummary.sh @@ -9,7 +9,7 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. -# Creates the system tests output file from the various test.output files. It +# Creates the system tests output file from the various test.output.* files. It # then searches that file and prints the number of tests passed, failed, not # run. It also checks whether the IP addresses 10.53.0.[1-8] were set up and, # if not, prints a warning. @@ -17,7 +17,7 @@ # Usage: # testsummary.sh [-n] # -# -n Do NOT delete the individual test.output files after concatenating +# -n Do NOT delete the individual test.output.* files after concatenating # them into systests.output. # # Status return: @@ -35,13 +35,13 @@ while getopts "n" flag; do esac done -if [ `ls */test.output 2> /dev/null | wc -l` -eq 0 ]; then - echowarn "I:No 'test.output' files were found." +if [ `ls test.output.* 2> /dev/null | wc -l` -eq 0 ]; then + echowarn "I:No 'test.output.*' files were found." echowarn "I:Printing summary from pre-existing 'systests.output'." else - cat */test.output > systests.output + cat test.output.* > systests.output if [ $keepfile -eq 0 ]; then - rm -f */test.output + rm -f test.output.* fi fi