From 3862043879534542a75e40d5e6c0cc09f37f8d6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Wed, 21 Feb 2018 14:59:33 +0100 Subject: [PATCH 1/3] Allow retaining system test output using an environment variable Instead of exporting an environment variable containing a command line argument (NOCLEAN="-n"), extend run.sh to handle a "boolean" environment variable (SYSTEMTEST_NO_CLEAN) itself. The former method is buggy because the value of NOCLEAN is set in parallel.mk when that file is first created, but it is not subsequently updated upon each test run (because make considers parallel.mk to be up to date). To retain backward compatibility, the "-n" command line argument for run.sh is still supported (and has a higher priority than the relevant environment variable). The SYSTEMTEST_NO_CLEAN environment variable can also be used directly to prevent cleanup when using "make test" instead of runall.sh. Apart from fixing a bug, this simplifies the way runall.sh controls run.sh behavior due to the Makefile being bypassed. Direct processing of environment variables in run.sh is more scalable in the long run, given that the previously utilized technique, even with its implementation fixed, would still require Makefile.in to be modified in two places each time a new flag needed to be passed from runall.sh to run.sh. --- bin/tests/system/Makefile.in | 4 ++-- bin/tests/system/README | 8 ++++---- bin/tests/system/run.sh | 7 ++++++- bin/tests/system/runall.sh | 9 ++++++--- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/bin/tests/system/Makefile.in b/bin/tests/system/Makefile.in index 2ab0d3c586..6f086f491c 100644 --- a/bin/tests/system/Makefile.in +++ b/bin/tests/system/Makefile.in @@ -65,7 +65,7 @@ parallel.mk: for directory in $(PARALLEL) ; do \ echo "" >> $@ ; \ echo "test-`echo $$directory | tr _ -`:" >> $@ ; \ - echo " @$(SHELL) ./run.sh $$NOCLEAN -r -p $$port $$directory 2>&1 | tee $$directory/test.output" >> $@ ; \ + echo " @$(SHELL) ./run.sh -r -p $$port $$directory 2>&1 | tee $$directory/test.output" >> $@ ; \ port=`expr $$port + 100` ; \ done @@ -73,7 +73,7 @@ parallel.mk: test: parallel.mk subdirs @$(MAKE) -f parallel.mk check - @$(SHELL) ./runsequential.sh $$NOCLEAN -r + @$(SHELL) ./runsequential.sh -r @$(SHELL) ./testsummary.sh check: test diff --git a/bin/tests/system/README b/bin/tests/system/README index 4e59cf5aa5..ad5d1dce63 100644 --- a/bin/tests/system/README +++ b/bin/tests/system/README @@ -151,10 +151,10 @@ A run of all the system tests can also be initiated via make: make [-j numproc] test In this case, retention of the output files after a test completes successfully -is specified by setting the environment variable NOCLEAN to "-n" prior to -running make, e.g. +is specified by setting the environment variable SYSTEMTEST_NO_CLEAN to 1 prior +to running make, e.g. - NOCLEAN=-n make [-j numproc] test + SYSTEMTEST_NO_CLEAN=1 make [-j numproc] test @@ -712,7 +712,7 @@ the ports are assigned when the tests are run. This is achieved by having the when "make check" is run, and contains a target for each test of the form: : - @$(SHELL) run.sh $$NOCLEAN -r -p + @$(SHELL) run.sh -r -p The is unique and the values of for each test are separated by at least 100 ports. diff --git a/bin/tests/system/run.sh b/bin/tests/system/run.sh index 38ee8623f0..3b9b709097 100644 --- a/bin/tests/system/run.sh +++ b/bin/tests/system/run.sh @@ -14,9 +14,14 @@ SYSTEMTESTTOP=. . $SYSTEMTESTTOP/conf.sh stopservers=true -clean=true baseport=5300 +if [ ${SYSTEMTEST_NO_CLEAN:-0} -eq 1 ]; then + clean=false +else + clean=true +fi + while getopts "knp:r" flag; do case "$flag" in k) stopservers=false ;; diff --git a/bin/tests/system/runall.sh b/bin/tests/system/runall.sh index ad2a825c2e..a4a477214c 100644 --- a/bin/tests/system/runall.sh +++ b/bin/tests/system/runall.sh @@ -24,12 +24,13 @@ SYSTEMTESTTOP=. usage="Usage: ./runall.sh [-n] [numprocesses]" +SYSTEMTEST_NO_CLEAN=0 + # Handle "-n" switch if present. -NOCLEAN="" while getopts "n" flag; do case "$flag" in - n) NOCLEAN="-n" ;; + n) SYSTEMTEST_NO_CLEAN=1 ;; esac done export NOCLEAN @@ -54,6 +55,8 @@ fi # Run the tests. +export SYSTEMTEST_NO_CLEAN + status=0 if [ "$CYGWIN" = "" ]; then # Running on Unix, use "make" to run tests in parallel. @@ -66,7 +69,7 @@ else # used, the tests would be run sequentially anyway.) { for testdir in $SUBDIRS; do - $SHELL run.sh $NOCLEAN $testdir || status=1 + $SHELL run.sh $testdir || status=1 done } 2>&1 | tee "systests.output" fi From a324031a82a2b312d8f803040dd81eadb554785f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Wed, 21 Feb 2018 14:59:35 +0100 Subject: [PATCH 2/3] Allow forcing colored system test output As parallel.mk and runsequential.sh both pipe system test output through "tee" (for the purpose of creating test.output), run.sh invoked from these two files detects it is not writing to a terminal, which prevents colored output from being generated. Allow forcing colored output using a new command line argument for runall.sh, "-c", which sets an environment variable (SYSTEMTEST_FORCE_COLOR) causing conf.sh to unconditionally enable colored output. The same environment variable can also be used directly to force colored output when using "make test" instead of runall.sh. --- bin/tests/system/README | 7 ++++++- bin/tests/system/conf.sh.in | 2 +- bin/tests/system/runall.sh | 13 +++++++++---- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/bin/tests/system/README b/bin/tests/system/README index ad5d1dce63..2ccf040058 100644 --- a/bin/tests/system/README +++ b/bin/tests/system/README @@ -127,7 +127,10 @@ Running All The System Tests --- To run all the system tests, enter the command: - sh runall.sh [-n] [numproc] + sh runall.sh [-c] [-n] [numproc] + +The optional flag "-c" forces colored output (by default system test output is +not printed in color due to run.sh being piped through "tee"). The optional flag "-n" has the same effect as it does for "run.sh" - it causes the retention of all output files from all tests. @@ -156,6 +159,8 @@ to running make, e.g. SYSTEMTEST_NO_CLEAN=1 make [-j numproc] test +while setting environment variable SYSTEMTEST_FORCE_COLOR to 1 forces system +test output to be printed in color. Running Multiple System Test Suites Simultaneously diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in index 1d0d55fbe1..6dacfd8d10 100644 --- a/bin/tests/system/conf.sh.in +++ b/bin/tests/system/conf.sh.in @@ -156,7 +156,7 @@ NZD=@NZD_TOOLS@ # # Set up color-coded test output # -if test -t 1 && type tput > /dev/null 2>&1 ; then +if [ ${SYSTEMTEST_FORCE_COLOR:-0} -eq 1 ] || test -t 1 && type tput > /dev/null 2>&1 ; then COLOR_END=`tput setaf 4` # blue COLOR_FAIL=`tput setaf 1` # red COLOR_INFO=`tput bold` # bold diff --git a/bin/tests/system/runall.sh b/bin/tests/system/runall.sh index a4a477214c..284a6a50d4 100644 --- a/bin/tests/system/runall.sh +++ b/bin/tests/system/runall.sh @@ -9,7 +9,9 @@ # Run all the system tests. # # Usage: -# runall.sh [-n] [numprocesses] +# runall.sh [-c] [-n] [numprocesses] +# +# -c Force colored output. # # -n Noclean. Keep all output files produced by all tests. These # can later be removed by running "cleanall.sh". @@ -22,14 +24,16 @@ SYSTEMTESTTOP=. . $SYSTEMTESTTOP/conf.sh -usage="Usage: ./runall.sh [-n] [numprocesses]" +usage="Usage: ./runall.sh [-c] [-n] [numprocesses]" +SYSTEMTEST_FORCE_COLOR=0 SYSTEMTEST_NO_CLEAN=0 -# Handle "-n" switch if present. +# Handle command line switches if present. -while getopts "n" flag; do +while getopts "cn" flag; do case "$flag" in + c) SYSTEMTEST_FORCE_COLOR=1 ;; n) SYSTEMTEST_NO_CLEAN=1 ;; esac done @@ -55,6 +59,7 @@ fi # Run the tests. +export SYSTEMTEST_FORCE_COLOR export SYSTEMTEST_NO_CLEAN status=0 From d989d20fe526716c774357ff372b38995e1e5065 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Wed, 21 Feb 2018 14:59:38 +0100 Subject: [PATCH 3/3] Preserve values of environment variables which are already set This enables the environment variables controlling run.sh behavior to be permanently set in a working environment (e.g. to automatically force colored output without using "-c" in each runall.sh invocation). Relevant runall.sh command line arguments still have a higher priority. --- bin/tests/system/runall.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/tests/system/runall.sh b/bin/tests/system/runall.sh index 284a6a50d4..5bb0b589ac 100644 --- a/bin/tests/system/runall.sh +++ b/bin/tests/system/runall.sh @@ -26,8 +26,10 @@ SYSTEMTESTTOP=. usage="Usage: ./runall.sh [-c] [-n] [numprocesses]" -SYSTEMTEST_FORCE_COLOR=0 -SYSTEMTEST_NO_CLEAN=0 +# Preserve values of environment variables which are already set. + +SYSTEMTEST_FORCE_COLOR=${SYSTEMTEST_FORCE_COLOR:-0} +SYSTEMTEST_NO_CLEAN=${SYSTEMTEST_NO_CLEAN:-0} # Handle command line switches if present.