From bea5bcff8d8da63db3a424624a1d899fc4286b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Thu, 5 Dec 2019 18:09:21 +0100 Subject: [PATCH] Bail-out early if dig fails to finish successfully or takes too long Before, the zero system test could get stuck almost infinitely, because the first test sends > 300 queries with 5 seconds timeout on each in each pass. If named crashed early, it would took the test more than 4 hours to properly timeout. This commit introduces a "watchdog" on the dig commands running in the background and failing the test on timeout, failing any test if any dig command fails to return successfully, and making the tests.sh script shellcheck clean. (cherry picked from commit 2a65a47f39db732c05c628a6463cdd5c672463b3) --- bin/tests/system/zero/tests.sh | 130 ++++++++++++++++++++------------- 1 file changed, 80 insertions(+), 50 deletions(-) diff --git a/bin/tests/system/zero/tests.sh b/bin/tests/system/zero/tests.sh index b89aee223d..681ba118db 100644 --- a/bin/tests/system/zero/tests.sh +++ b/bin/tests/system/zero/tests.sh @@ -1,3 +1,5 @@ +#!/bin/sh +# # Copyright (C) Internet Systems Consortium, Inc. ("ISC") # # This Source Code Form is subject to the terms of the Mozilla Public @@ -7,84 +9,112 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. +#shellcheck source=conf.sh SYSTEMTESTTOP=.. -. $SYSTEMTESTTOP/conf.sh +. "$SYSTEMTESTTOP/conf.sh" -DIGOPTS="-p ${PORT}" +dig_with_opts() { + "$DIG" -p "${PORT}" "$@" +} + +wait_for_pid() ( + for pid in "$@"; do + kill -0 "$pid" 2>/dev/null && return 1 + done + return 0 +) status=0 n=0 -n=`expr $n + 1` +n=$((n+1)) echo_i "check lookups against TTL=0 records ($n)" i=0 +ret=0 passes=10 -$DIG $DIGOPTS @10.53.0.2 axfr example | grep -v "^ds0" | +dig_with_opts @10.53.0.2 axfr example | grep -v "^ds0" | \ awk '$2 == "0" { print "-q", $1, $4; print "-q", "zzz"$1, $4;}' > query.list + +# add 1/5 second per query +timeout=$(($(wc -l < query.list) / 5)) while [ $i -lt $passes ] do - ret=0 - $DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.1.test$n & - $DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.2.test$n & - $DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.3.test$n & - $DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.4.test$n & - $DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.5.test$n & - $DIG $DIGOPTS @10.53.0.3 -f query.list > dig.out$i.6.test$n & - wait - grep "status: SERVFAIL" dig.out$i.1.test$n > /dev/null && ret=1 - grep "status: SERVFAIL" dig.out$i.2.test$n > /dev/null && ret=1 - grep "status: SERVFAIL" dig.out$i.3.test$n > /dev/null && ret=1 - grep "status: SERVFAIL" dig.out$i.4.test$n > /dev/null && ret=1 - grep "status: SERVFAIL" dig.out$i.5.test$n > /dev/null && ret=1 - grep "status: SERVFAIL" dig.out$i.6.test$n > /dev/null && ret=1 + (dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.1.test$n") & pid1="$!" + (dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.2.test$n") & pid2="$!" + (dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.3.test$n") & pid3="$!" + (dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.4.test$n") & pid4="$!" + (dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.5.test$n") & pid5="$!" + (dig_with_opts @10.53.0.3 -f query.list > "dig.out$i.6.test$n") & pid6="$!" + + retry_quiet "$timeout" wait_for_pid "$pid1" "$pid2" "$pid3" "$pid4" "$pid5" "$pid6" || ret=1 + kill -TERM "$pid1" "$pid2" "$pid3" "$pid4" "$pid5" "$pid6" 2>/dev/null + + wait "$pid1" || ret=1 + wait "$pid2" || ret=1 + wait "$pid3" || ret=1 + wait "$pid4" || ret=1 + wait "$pid5" || ret=1 + wait "$pid6" || ret=1 + + grep "status: SERVFAIL" "dig.out$i.1.test$n" > /dev/null && ret=1 + grep "status: SERVFAIL" "dig.out$i.2.test$n" > /dev/null && ret=1 + grep "status: SERVFAIL" "dig.out$i.3.test$n" > /dev/null && ret=1 + grep "status: SERVFAIL" "dig.out$i.4.test$n" > /dev/null && ret=1 + grep "status: SERVFAIL" "dig.out$i.5.test$n" > /dev/null && ret=1 + grep "status: SERVFAIL" "dig.out$i.6.test$n" > /dev/null && ret=1 [ $ret = 1 ] && break - i=`expr $i + 1` + i=$((i+1)) echo_i "successfully completed pass $i of $passes" done if [ $ret != 0 ]; then echo_i "failed"; fi -status=`expr $status + $ret` +status=$((status+ret)) -n=`expr $n + 1` +repeat_query() ( + i=0 + while [ "$i" -lt "$1" ]; do + dig_with_opts +short "@$2" "$3" | tee "dig.out$i.test$n" || return 1 + i=$((i+1)) + done +) + +count_unique() ( + repeat_query "$@" | sort -u | wc -l +) + +n=$((n+1)) echo_i "check repeated recursive lookups of non recurring TTL=0 responses get new values ($n)" ret=0 -count=`( -$DIG $DIGOPTS +short @10.53.0.3 foo.increment -$DIG $DIGOPTS +short @10.53.0.3 foo.increment -$DIG $DIGOPTS +short @10.53.0.3 foo.increment -$DIG $DIGOPTS +short @10.53.0.3 foo.increment -$DIG $DIGOPTS +short @10.53.0.3 foo.increment -$DIG $DIGOPTS +short @10.53.0.3 foo.increment -$DIG $DIGOPTS +short @10.53.0.3 foo.increment -) | sort -u | wc -l ` -if [ $count -ne 7 ] ; then echo_i "failed (count=$count)"; ret=1; fi -status=`expr $status + $ret` +repeats=9 +count=$(count_unique "$repeats" 10.53.0.3 foo.increment) +if [ "$count" -ne "$repeats" ] ; then echo_i "failed (count=$count, repeats=$repeats)"; ret=1; fi +status=$((status+ret)) -n=`expr $n + 1` +n=$((n+1)) echo_i "check lookups against TTL=1 records ($n)" i=0 passes=10 +ret=0 while [ $i -lt $passes ] do - ret=0 - $DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.1.test$n - $DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.2.test$n - $DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.3.test$n - $DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.4.test$n - $DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.5.test$n - $DIG $DIGOPTS @10.53.0.3 www.one.tld > dig.out$i.6.test$n - grep "status: SERVFAIL" dig.out$i.1.test$n > /dev/null && ret=1 - grep "status: SERVFAIL" dig.out$i.2.test$n > /dev/null && ret=1 - grep "status: SERVFAIL" dig.out$i.3.test$n > /dev/null && ret=1 - grep "status: SERVFAIL" dig.out$i.4.test$n > /dev/null && ret=1 - grep "status: SERVFAIL" dig.out$i.5.test$n > /dev/null && ret=1 - grep "status: SERVFAIL" dig.out$i.6.test$n > /dev/null && ret=1 + dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.1.test$n" || ret=1 + dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.2.test$n" || ret=1 + dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.3.test$n" || ret=1 + dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.4.test$n" || ret=1 + dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.5.test$n" || ret=1 + dig_with_opts @10.53.0.3 www.one.tld > "dig.out$i.6.test$n" || ret=1 + grep "status: SERVFAIL" "dig.out$i.1.test$n" > /dev/null && ret=1 + grep "status: SERVFAIL" "dig.out$i.2.test$n" > /dev/null && ret=1 + grep "status: SERVFAIL" "dig.out$i.3.test$n" > /dev/null && ret=1 + grep "status: SERVFAIL" "dig.out$i.4.test$n" > /dev/null && ret=1 + grep "status: SERVFAIL" "dig.out$i.5.test$n" > /dev/null && ret=1 + grep "status: SERVFAIL" "dig.out$i.6.test$n" > /dev/null && ret=1 [ $ret = 1 ] && break - i=`expr $i + 1` + i=$((i+1)) echo_i "successfully completed pass $i of $passes" - $PERL -e 'select(undef, undef, undef, 0.3);' + sleep 1 done if [ $ret != 0 ]; then echo_i "failed"; fi -status=`expr $status + $ret` +status=$((status+ret)) echo_i "exit status: $status" -[ $status -eq 0 ] || exit 1 +[ "$status" -eq 0 ] || exit 1