From bd2941fc72dc622ba28506e73c6c700099e3fb73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Mon, 7 Aug 2023 11:26:58 +0200 Subject: [PATCH 1/3] Capture all fstrm_capture output The fstrm_capture.out file is overwritten when the fstrm_capture utility is restarted during the "dnstap" system test. Use a separate output file for each fstrm_capture instance to ensure all output produced by that tool during the "dnstap" system test is preserved for forensic purposes. --- bin/tests/system/dnstap/clean.sh | 2 +- bin/tests/system/dnstap/tests.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/tests/system/dnstap/clean.sh b/bin/tests/system/dnstap/clean.sh index 0a0081fd94..b9e476f661 100644 --- a/bin/tests/system/dnstap/clean.sh +++ b/bin/tests/system/dnstap/clean.sh @@ -18,7 +18,7 @@ rm -f */named.run.prev rm -f */named.stats rm -f dig.out* rm -f dnstap.* -rm -f fstrm_capture.out +rm -f fstrm_capture.out.* rm -f ns*/dnstap.out rm -f ns*/dnstap.out.save rm -f ns*/dnstap.out.save.? diff --git a/bin/tests/system/dnstap/tests.sh b/bin/tests/system/dnstap/tests.sh index 3336a9fe0c..c6bad88475 100644 --- a/bin/tests/system/dnstap/tests.sh +++ b/bin/tests/system/dnstap/tests.sh @@ -79,7 +79,7 @@ mv ns2/dnstap.out ns2/dnstap.out.save if [ -n "$FSTRM_CAPTURE" ] ; then $FSTRM_CAPTURE -t protobuf:dnstap.Dnstap -u ns4/dnstap.out \ - -w dnstap.out > fstrm_capture.out 2>&1 & + -w dnstap.out > fstrm_capture.out.1 2>&1 & fstrm_capture_pid=$! fi @@ -674,7 +674,7 @@ EOF mv dnstap.out dnstap.out.save $FSTRM_CAPTURE -t protobuf:dnstap.Dnstap -u ns4/dnstap.out \ - -w dnstap.out > fstrm_capture.out 2>&1 & + -w dnstap.out > fstrm_capture.out.2 2>&1 & fstrm_capture_pid=$! sleep 1 $RNDCCMD -s 10.53.0.4 dnstap -reopen | sed 's/^/ns4 /' | cat_i From 26d3d97f12db29dbad5bcdf7f698af0498c77e02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Mon, 7 Aug 2023 11:26:58 +0200 Subject: [PATCH 2/3] Wait until fstrm_capture is ready The fstrm_capture utility is started in the background during the "dnstap" system test. Consequently, "rndc dnstap-reopen" and similar commands may be executed before fstrm_capture starts listening on the Unix domain socket it is configured to receive dnstap data on. This results in the dnstap data sent to that socket in the meantime to be lost; while the fstrm writer thread is able to recover from such a scenario within a couple of seconds (by reopening the configured dnstap destination itself), only one write attempt is made for data successfully queued to the writer thread, so dnstap frames can still be lost in the process. This may happen during the "dnstap" system test, leading to the dnstap output file being empty, which in turn causes the test to fail. Fix by waiting until fstrm_capture starts listening on the Unix domain socket it is configured to use before asking named to reopen the configured dnstap destination. Since various fstrm_capture versions log different messages when the listening socket is set up, wait for a common string that works for all fstrm_capture versions released to date. Add a few extra debug messages indicating test progress and make the test fail if the expected fstrm_capture log message is not generated within 10 seconds. --- bin/tests/system/dnstap/tests.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bin/tests/system/dnstap/tests.sh b/bin/tests/system/dnstap/tests.sh index c6bad88475..42189e1486 100644 --- a/bin/tests/system/dnstap/tests.sh +++ b/bin/tests/system/dnstap/tests.sh @@ -78,9 +78,14 @@ mv ns1/dnstap.out ns1/dnstap.out.save mv ns2/dnstap.out ns2/dnstap.out.save if [ -n "$FSTRM_CAPTURE" ] ; then + ret=0 + echo_i "starting fstrm_capture" $FSTRM_CAPTURE -t protobuf:dnstap.Dnstap -u ns4/dnstap.out \ -w dnstap.out > fstrm_capture.out.1 2>&1 & fstrm_capture_pid=$! + wait_for_log 10 "socket path ns4/dnstap.out" fstrm_capture.out.1 || ret=1 + if [ $ret != 0 ]; then echo_i "failed"; fi + status=$((status + ret)) fi $RNDCCMD -s 10.53.0.1 dnstap-reopen | sed 's/^/ns1 /' | cat_i @@ -673,10 +678,14 @@ EOF mv dnstap.out dnstap.out.save + echo_i "restarting fstrm_capture" $FSTRM_CAPTURE -t protobuf:dnstap.Dnstap -u ns4/dnstap.out \ -w dnstap.out > fstrm_capture.out.2 2>&1 & fstrm_capture_pid=$! - sleep 1 + wait_for_log 10 "socket path ns4/dnstap.out" fstrm_capture.out.2 || { + echo_i "failed" + ret=1 + } $RNDCCMD -s 10.53.0.4 dnstap -reopen | sed 's/^/ns4 /' | cat_i $DIG $DIGOPTS @10.53.0.4 a.example > dig.out From 67df78e5189575fc551b97f06c72da0126b3c6c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Mon, 7 Aug 2023 11:26:58 +0200 Subject: [PATCH 3/3] Lower the minimum expected dnstap output file size Lower the size requirement for the dnstap output file produced during the "dnstap" system test from 454 to 450 bytes; while files of that size are not generated in any GitLab CI job, they are in other environments where the test passes. --- bin/tests/system/dnstap/tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/tests/system/dnstap/tests.sh b/bin/tests/system/dnstap/tests.sh index 42189e1486..36202a28ed 100644 --- a/bin/tests/system/dnstap/tests.sh +++ b/bin/tests/system/dnstap/tests.sh @@ -569,7 +569,7 @@ EOF echo_i "checking unix socket message counts" sleep 2 - retry_quiet 5 dnstap_data_ready $fstrm_capture_pid dnstap.out 454 || { + retry_quiet 5 dnstap_data_ready $fstrm_capture_pid dnstap.out 450 || { echo_i "dnstap output file smaller than expected" ret=1 }