From 931357d8015d8a3ef703dae2d4053284f943af9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= <michal@isc.org>
Date: Tue, 18 Jun 2019 09:14:07 +0200
Subject: [PATCH] Always fail a system test if crashes are detected

In certain situations (e.g. a named instance crashing upon shutdown in a
system test which involves shutting down a server and restarting it
afterwards), a system test may succeed despite a named crash being
triggered.  This must never be the case.  Extend run.sh to mark a test
as failed if core dumps or log lines indicating assertion failures are
detected (the latter is only an extra measure aimed at test environments
in which core dumps are not generated; note that some types of crashes,
e.g. segmentation faults, will not be detected using this method alone).

(cherry picked from commit 7706f22924dac215e22d4be4322f89814041a3a8)
---
 bin/tests/system/run.sh | 35 +++++++++++++++++++++++------------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/bin/tests/system/run.sh b/bin/tests/system/run.sh
index 6754a4252d..4344b4412e 100755
--- a/bin/tests/system/run.sh
+++ b/bin/tests/system/run.sh
@@ -188,19 +188,30 @@ status=`expr $status + $?`
 if [ $status != 0 ]; then
     echofail "R:$systest:FAIL"
     # Do not clean up - we need the evidence.
-    find . -name core -exec chmod 0644 '{}' \;
 else
-    echopass "R:$systest:PASS"
-    if $clean
-    then
-	$SHELL clean.sh $runall $systest "$@"
-	if test -d ../../../.git
-	then
-	    git status -su --ignored $systest 2>/dev/null | \
-	    sed -n -e 's|^?? \(.*\)|I:file \1 not removed|p' \
-	    -e 's|^!! \(.*/named.run\)$|I:file \1 not removed|p' \
-	    -e 's|^!! \(.*/named.memstats\)$|I:file \1 not removed|p'
-	fi
+    core_dumps="$(find $systest/ -name 'core*' | sort | tr '\n' ' ')"
+    assertion_failures=$(find $systest/ -name named.run | xargs grep "assertion failure" | wc -l)
+    if [ -n "$core_dumps" ]; then
+        echoinfo "I:$systest:Test claims success despite crashes: $core_dumps"
+        echofail "R:$systest:FAIL"
+        # Do not clean up - we need the evidence.
+    elif [ $assertion_failures -ne 0 ]; then
+        echoinfo "I:$systest:Test claims success despite $assertion_failures assertion failure(s)"
+        echofail "R:$systest:FAIL"
+        # Do not clean up - we need the evidence.
+    else
+        echopass "R:$systest:PASS"
+        if $clean
+        then
+            $SHELL clean.sh $runall $systest "$@"
+            if test -d ../../../.git
+            then
+                git status -su --ignored $systest 2>/dev/null | \
+                sed -n -e 's|^?? \(.*\)|I:file \1 not removed|p' \
+                -e 's|^!! \(.*/named.run\)$|I:file \1 not removed|p' \
+                -e 's|^!! \(.*/named.memstats\)$|I:file \1 not removed|p'
+            fi
+        fi
     fi
 fi