[9.20] chg: test: Properly process JSON and XML in tests

Processing JSON and XML using `grep` and `sed` is error-prone, overly
lax in some ways, overly strict in others, and neither accurate nor
expressive.  Use `jq` and `xmllint` with XPath expressions to make
things right in system tests.

See #3304

Backport of MR !10942

Merge branch 'backport-3304-properly-process-json-and-xml-in-tests-9.20' into 'bind-9.20'

See merge request isc-projects/bind9!11153

.gitlab-ci.yml

@@ -673,7 +673,6 @@ misc:
     - sh util/check-trailing-whitespace.sh
     - if git grep SYSTEMTESTTOP -- ':!.gitlab-ci.yml'; then echo 'Please use relative paths instead of $SYSTEMTESTTOP.'; exit 1; fi
     - bash util/unused-headers.sh
-    - bash util/xmllint-html.sh
     # Check dangling symlinks in the repository
     - if find . -xtype l | grep .; then exit 1; fi
   needs: []

bin/tests/system/isctest/vars/.ac_vars/JQ.in

@@ -0,0 +1 @@
+@JQ@

bin/tests/system/isctest/vars/.ac_vars/XMLLINT.in

@@ -0,0 +1 @@
+@XMLLINT@

bin/tests/system/statistics/tests.sh

@@ -150,11 +150,11 @@ n=$((n + 1))
 
 ret=0
 echo_i "checking that zones with slash are properly shown in XML output ($n)"
-if $FEATURETEST --have-libxml2 && [ -x ${CURL} ]; then
+if $FEATURETEST --have-libxml2 && [ -x "${CURL}" ] && [ -x "${XMLLINT}" ]; then
   ${CURL} http://10.53.0.1:${EXTRAPORT1}/xml/v3/zones >curl.out.${n} 2>/dev/null || ret=1
-  grep '<zone name="32/1.0.0.127-in-addr.example" rdataclass="IN">' curl.out.${n} >/dev/null || ret=1
+  test -n "$("$XMLLINT" --xpath '/statistics/views/view[@name="_default"]/zones/zone[@name="32/1.0.0.127-in-addr.example"]' curl.out.${n})" || ret=1
 else
-  echo_i "skipping test as libxml2 and/or curl was not found"
+  echo_i "skipping test as libxml2 and/or curl and/or xmllint was not found"
 fi
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=$((status + ret))
@@ -162,11 +162,11 @@ n=$((n + 1))
 
 ret=0
 echo_i "checking that zones return their type ($n)"
-if $FEATURETEST --have-libxml2 && [ -x ${CURL} ]; then
+if $FEATURETEST --have-libxml2 && [ -x "${CURL}" ] && [ -x "${XMLLINT}" ]; then
   ${CURL} http://10.53.0.1:${EXTRAPORT1}/xml/v3/zones >curl.out.${n} 2>/dev/null || ret=1
-  grep '<zone name="32/1.0.0.127-in-addr.example" rdataclass="IN"><type>primary</type>' curl.out.${n} >/dev/null || ret=1
+  test -n "$("$XMLLINT" --xpath '/statistics/views/view[@name="_default"]/zones/zone[@name="32/1.0.0.127-in-addr.example"]/type[text()="primary"]' curl.out.${n})" || ret=1
 else
-  echo_i "skipping test as libxml2 and/or curl was not found"
+  echo_i "skipping test as libxml2 and/or curl and/or xmllint was not found"
 fi
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=$((status + ret))
@@ -230,23 +230,23 @@ n=$((n + 1))
 
 ret=0
 echo_i "checking bind9.xml socket statistics ($n)"
-if $FEATURETEST --have-libxml2 && [ -e stats.xml.out ] && [ -x "${XSLTPROC}" ]; then
+if $FEATURETEST --have-libxml2 && [ -e stats.xml.out ] && [ -x "${XSLTPROC}" ] && [ -x "${XMLLINT}" ]; then
   # Socket statistics (expect no errors)
-  grep "<counter name=\"TCP4AcceptFail\">0</counter>" stats.xml.out >/dev/null || ret=1
-  grep "<counter name=\"TCP4BindFail\">0</counter>" stats.xml.out >/dev/null || ret=1
-  grep "<counter name=\"TCP4ConnFail\">0</counter>" stats.xml.out >/dev/null || ret=1
-  grep "<counter name=\"TCP4OpenFail\">0</counter>" stats.xml.out >/dev/null || ret=1
-  grep "<counter name=\"TCP4RecvErr\">0</counter>" stats.xml.out >/dev/null || ret=1
-  # grep "<counter name=\"TCP4SendErr\">0</counter>" stats.xml.out >/dev/null || ret=1
+  [ "$("$XMLLINT" --xpath 'count(/statistics/server/counters[@type="sockstat"]/counter[@name="TCP4AcceptFail" and text()="0"])' stats.xml.out)" -eq 1 ] || ret=1
+  [ "$("$XMLLINT" --xpath 'count(/statistics/server/counters[@type="sockstat"]/counter[@name="TCP4BindFail" and text()="0"])' stats.xml.out)" -eq 1 ] || ret=1
+  [ "$("$XMLLINT" --xpath 'count(/statistics/server/counters[@type="sockstat"]/counter[@name="TCP4ConnFail" and text()="0"])' stats.xml.out)" -eq 1 ] || ret=1
+  [ "$("$XMLLINT" --xpath 'count(/statistics/server/counters[@type="sockstat"]/counter[@name="TCP4OpenFail" and text()="0"])' stats.xml.out)" -eq 1 ] || ret=1
+  [ "$("$XMLLINT" --xpath 'count(/statistics/server/counters[@type="sockstat"]/counter[@name="TCP4RecvErr" and text()="0"])' stats.xml.out)" -eq 1 ] || ret=1
+  # [ "$("$XMLLINT" --xpath 'count(/statistics/server/counters[@type="sockstat"]/counter[@name="TCP4SendErr" and text()="0"])' stats.xml.out)" -eq 1 ] || ret=1
 
-  grep "<counter name=\"TCP6AcceptFail\">0</counter>" stats.xml.out >/dev/null || ret=1
-  grep "<counter name=\"TCP6BindFail\">0</counter>" stats.xml.out >/dev/null || ret=1
-  grep "<counter name=\"TCP6ConnFail\">0</counter>" stats.xml.out >/dev/null || ret=1
-  grep "<counter name=\"TCP6OpenFail\">0</counter>" stats.xml.out >/dev/null || ret=1
-  grep "<counter name=\"TCP6RecvErr\">0</counter>" stats.xml.out >/dev/null || ret=1
-  grep "<counter name=\"TCP6SendErr\">0</counter>" stats.xml.out >/dev/null || ret=1
+  [ "$("$XMLLINT" --xpath 'count(/statistics/server/counters[@type="sockstat"]/counter[@name="TCP6AcceptFail" and text()="0"])' stats.xml.out)" -eq 1 ] || ret=1
+  [ "$("$XMLLINT" --xpath 'count(/statistics/server/counters[@type="sockstat"]/counter[@name="TCP6BindFail" and text()="0"])' stats.xml.out)" -eq 1 ] || ret=1
+  [ "$("$XMLLINT" --xpath 'count(/statistics/server/counters[@type="sockstat"]/counter[@name="TCP6ConnFail" and text()="0"])' stats.xml.out)" -eq 1 ] || ret=1
+  [ "$("$XMLLINT" --xpath 'count(/statistics/server/counters[@type="sockstat"]/counter[@name="TCP6OpenFail" and text()="0"])' stats.xml.out)" -eq 1 ] || ret=1
+  [ "$("$XMLLINT" --xpath 'count(/statistics/server/counters[@type="sockstat"]/counter[@name="TCP6RecvErr" and text()="0"])' stats.xml.out)" -eq 1 ] || ret=1
+  [ "$("$XMLLINT" --xpath 'count(/statistics/server/counters[@type="sockstat"]/counter[@name="TCP6SendErr" and text()="0"])' stats.xml.out)" -eq 1 ] || ret=1
 else
-  echo_i "skipping test as libxml2 and/or stats.xml.out file and/or xsltproc was not found"
+  echo_i "skipping test as libxml2 and/or stats.xml.out file and/or xsltproc and/or xmllint was not found"
 fi
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=$((status + ret))

bin/tests/system/statschannel/tests.sh

@@ -715,38 +715,35 @@ status=$((status + ret))
 n=$((n + 1))
 
 _wait_for_transfers() {
-  if [ "$PERL_XML" ]; then
+  if [ "$PERL_XML" ] && [ -x "$XMLLINT" ]; then
     getxfrins xml x$n || return 1
 
-    # XML is encoded in one line, use awk to separate each transfer
-    # with a newline
-
     # We expect 4 transfers
-    count=$(awk '{ gsub("<xfrin ", "\n<xfrin ") } 1' xfrins.xml.x$n | grep -c -E '<state>(Zone Transfer Request|First Data|Receiving AXFR Data)</state>')
+    count=$("$XMLLINT" --xpath 'count(/statistics/views/view[@name="_default"]/xfrins/xfrin)' xfrins.xml.x$n)
     if [ $count != 4 ]; then return 1; fi
 
     # We expect 3 of 4 to be retransfers
-    count=$(awk '{ gsub("<xfrin ", "\n<xfrin ") } 1' xfrins.xml.x$n | grep -c -F '<firstrefresh>No</firstrefresh>')
+    count=$("$XMLLINT" --xpath 'count(/statistics/views/view[@name="_default"]/xfrins/xfrin[firstrefresh[text()="No"]])' xfrins.xml.x$n)
     if [ $count != 3 ]; then return 1; fi
 
     # We expect 1 of 4 to be a new transfer
-    count=$(awk '{ gsub("<xfrin ", "\n<xfrin ") } 1' xfrins.xml.x$n | grep -c -F '<firstrefresh>Yes</firstrefresh>')
+    count=$("$XMLLINT" --xpath 'count(/statistics/views/view[@name="_default"]/xfrins/xfrin[firstrefresh[text()="Yes"]])' xfrins.xml.x$n)
     if [ $count != 1 ]; then return 1; fi
   fi
 
-  if [ "$PERL_JSON" ]; then
+  if [ "$PERL_JSON" ] && [ -x "$JQ" ]; then
     getxfrins json j$n || return 1
 
     # We expect 4 transfers
-    count=$(grep -c -E '"state":"(Zone Transfer Request|First Data|Receiving AXFR Data)"' xfrins.json.j$n)
+    count=$("$JQ" '.views._default.xfrins | length' <xfrins.json.j$n)
     if [ $count != 4 ]; then return 1; fi
 
     # We expect 3 of 4 to be retransfers
-    count=$(grep -c -F '"firstrefresh":"No"' xfrins.json.j$n)
+    count=$("$JQ" '.views._default.xfrins | map(select(.firstrefresh == "No")) | length' <xfrins.json.j$n)
     if [ $count != 3 ]; then return 1; fi
 
     # We expect 1 of 4 to be a new transfer
-    count=$(grep -c -F '"firstrefresh":"Yes"' xfrins.json.j$n)
+    count=$("$JQ" '.views._default.xfrins | map(select(.firstrefresh == "Yes")) | length' <xfrins.json.j$n)
     if [ $count != 1 ]; then return 1; fi
   fi
 }
@@ -766,7 +763,7 @@ if [ $ret != 0 ]; then echo_i "failed"; fi
 status=$((status + ret))
 n=$((n + 1))
 
-if [ "$PERL_JSON" ]; then
+if [ "$PERL_JSON" ] && [ -x "$JQ" ]; then
   echo_i "Checking zone transfer transports ($n)"
   ret=0
   cp xfrins.json.j$((n - 2)) xfrins.json.j$n

bin/tests/system/statschannel/traffic-json.pl

@@ -1,49 +0,0 @@
-#!/usr/bin/perl
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-# traffic-json.pl:
-# Parses the JSON version of the RSSAC002 traffic stats into a
-# normalized format.
-
-use JSON;
-
-my $file = $ARGV[0];
-open(INPUT, "<$file");
-my $text = do{local$/;<INPUT>};
-close(INPUT);
-
-my $ref = decode_json($text);
-
-my $tcprcvd = $ref->{traffic}->{"dns-tcp-requests-sizes-received-ipv4"};
-my $type = "tcp request-size ";
-foreach $key (keys %{$tcprcvd}) {
-    print $type . $key . ": ". $tcprcvd->{$key} ."\n";
-}
-
-my $tcpsent = $ref->{traffic}->{"dns-tcp-responses-sizes-sent-ipv4"};
-my $type = "tcp response-size ";
-foreach $key (keys %{$tcpsent}) {
-    print $type . $key . ": ". $tcpsent->{$key} ."\n";
-}
-
-my $udprcvd = $ref->{traffic}->{"dns-udp-requests-sizes-received-ipv4"};
-my $type = "udp request-size ";
-foreach $key (keys %{$udprcvd}) {
-    print $type . $key . ": ". $udprcvd->{$key} ."\n";
-}
-
-my $udpsent = $ref->{traffic}->{"dns-udp-responses-sizes-sent-ipv4"};
-my $type = "udp response-size ";
-foreach $key (keys %{$udpsent}) {
-    print $type . $key . ": ". $udpsent->{$key} ."\n";
-}

bin/tests/system/statschannel/traffic-xml.pl

@@ -1,46 +0,0 @@
-#!/usr/bin/perl
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-# traffic-xml.pl:
-# Parses the XML version of the RSSAC002 traffic stats into a
-# normalized format.
-
-use XML::Simple;
-
-my $file = $ARGV[0];
-
-my $ref = XMLin($file);
-
-my $udp = $ref->{traffic}->{ipv4}->{udp}->{counters};
-foreach $group (@$udp) {
-    my $type = "udp " . $group->{type} . " ";
-    if (exists $group->{counter}->{name}) {
-        print $type . $group->{counter}->{name} . ": " . $group->{counter}->{content} . "\n";
-    } else {
-        foreach $key (keys %{$group->{counter}}) {
-            print $type . $key . ": ". $group->{counter}->{$key}->{content} ."\n";
-        }
-    }
-}
-
-my $tcp = $ref->{traffic}->{ipv4}->{tcp}->{counters};
-foreach $group (@$tcp) {
-    my $type = "tcp " . $group->{type} . " ";
-    if (exists $group->{counter}->{name}) {
-        print $type . $group->{counter}->{name} . ": " . $group->{counter}->{content} . "\n";
-    } else {
-        foreach $key (keys %{$group->{counter}}) {
-            print $type . $key . ": ". $group->{counter}->{$key}->{content} ."\n";
-        }
-    }
-}

bin/tests/system/synthfromdnssec/tests.sh

@@ -715,7 +715,7 @@ for ns in 2 4 5 6; do
     status=$((status + ret))
   done
 
-  if ${FEATURETEST} --have-libxml2 && [ -x "${CURL}" ]; then
+  if ${FEATURETEST} --have-libxml2 && [ -x "${CURL}" ] && [ -x "${XMLLINT}" ]; then
     echo_i "getting XML statisistcs for (synth-from-dnssec ${description};) ($n)"
     ret=0
     xml=xml.out$n
@@ -726,10 +726,9 @@ for ns in 2 4 5 6; do
 
     echo_i "check XML for 'CoveringNSEC' with (synth-from-dnssec ${description};) ($n)"
     ret=0
-    counter=$(sed -n 's;.*<view name="_default">.*\(<counter name="CoveringNSEC">[0-9]*</counter>\).*</view><view.*;\1;gp' $xml)
-    count=$(echo "$counter" | grep CoveringNSEC | wc -l)
+    count=$("${XMLLINT}" --xpath 'count(/statistics/views/view[@name="_default"]/counters[@type="cachestats"]/counter[@name="CoveringNSEC"])' $xml)
     test $count = 1 || ret=1
-    zero=$(echo "$counter" | grep ">0<" | wc -l)
+    zero=$("${XMLLINT}" --xpath 'count(/statistics/views/view[@name="_default"]/counters[@type="cachestats"]/counter[@name="CoveringNSEC" and text()="0"])' $xml)
     if [ ${synth} = yes ]; then
       test $zero = 0 || ret=1
     else
@@ -741,10 +740,9 @@ for ns in 2 4 5 6; do
 
     echo_i "check XML for 'CacheNSECNodes' with (synth-from-dnssec ${description};) ($n)"
     ret=0
-    counter=$(sed -n 's;.*<view name="_default">.*\(<counter name="CacheNSECNodes">[0-9]*</counter>\).*</view><view.*;\1;gp' $xml)
-    count=$(echo "$counter" | grep CacheNSECNodes | wc -l)
+    count=$("${XMLLINT}" --xpath 'count(/statistics/views/view[@name="_default"]/counters[@type="cachestats"]/counter[@name="CacheNSECNodes"])' $xml)
     test $count = 1 || ret=1
-    zero=$(echo "$counter" | grep ">0<" | wc -l)
+    zero=$("${XMLLINT}" --xpath 'count(/statistics/views/view[@name="_default"]/counters[@type="cachestats"]/counter[@name="CacheNSECNodes" and text()="0"])' $xml)
     if [ ${ad} = yes ]; then
       test $zero = 0 || ret=1
     else
@@ -763,11 +761,10 @@ for ns in 2 4 5 6; do
 
       echo_i "check XML for '$synthesized}' with (synth-from-dnssec ${description};) ($n)"
       ret=0
-      if [ ${synth} = yes ]; then
-        grep '<counter name="'$synthesized'">'$count'</counter>' $xml >/dev/null || ret=1
-      else
-        grep '<counter name="'$synthesized'">'0'</counter>' $xml >/dev/null || ret=1
+      if [ ${synth} != yes ]; then
+        count=0
       fi
+      test $("${XMLLINT}" --xpath '/statistics/server/counters[@type="nsstat"]/counter[@name="'"${synthesized}"'"]/text()' $xml) -eq $count || ret=1
       n=$((n + 1))
       if [ $ret != 0 ]; then echo_i "failed"; fi
       status=$((status + ret))
@@ -776,7 +773,7 @@ for ns in 2 4 5 6; do
     echo_i "Skipping XML statistics checks"
   fi
 
-  if $FEATURETEST --have-json-c && [ -x "${CURL}" ]; then
+  if $FEATURETEST --have-json-c && [ -x "${CURL}" ] && [ -x "${JQ}" ]; then
     echo_i "getting JSON statisistcs for (synth-from-dnssec ${description};) ($n)"
     ret=0
     json=json.out$n
@@ -787,9 +784,9 @@ for ns in 2 4 5 6; do
 
     echo_i "check JSON for 'CoveringNSEC' with (synth-from-dnssec ${description};) ($n)"
     ret=0
-    count=$(grep '"CoveringNSEC":' $json | wc -l)
+    count=$("${JQ}" '.views | map(select(.resolver.cachestats | has("CoveringNSEC"))) | length' <$json)
     test $count = 2 || ret=1
-    zero=$(grep '"CoveringNSEC":0' $json | wc -l)
+    zero=$("${JQ}" '.views | map(select(.resolver.cachestats.CoveringNSEC == 0)) | length' <$json)
     if [ ${synth} = yes ]; then
       test $zero = 1 || ret=1
     else
@@ -801,9 +798,9 @@ for ns in 2 4 5 6; do
 
     echo_i "check JSON for 'CacheNSECNodes' with (synth-from-dnssec ${description};) ($n)"
     ret=0
-    count=$(grep '"CacheNSECNodes":' $json | wc -l)
+    count=$("${JQ}" '.views | map(select(.resolver.cachestats | has("CacheNSECNodes"))) | length' <$json)
     test $count = 2 || ret=1
-    zero=$(grep '"CacheNSECNodes":0' $json | wc -l)
+    zero=$("${JQ}" '.views | map(select(.resolver.cachestats.CacheNSECNodes == 0)) | length' <$json)
     if [ ${ad} = yes ]; then
       test $zero = 1 || ret=1
     else
@@ -823,9 +820,9 @@ for ns in 2 4 5 6; do
       echo_i "check JSON for '$synthesized}' with (synth-from-dnssec ${description};) ($n)"
       ret=0
       if [ ${synth} = yes ]; then
-        grep '"'$synthesized'":'$count'' $json >/dev/null || ret=1
+        test $("${JQ}" ".nsstats.${synthesized}" <$json) -eq $count || ret=1
       else
-        grep '"'$synthesized'":' $json >/dev/null && ret=1
+        "${JQ}" -e '.nsstats | has("'"${synthesized}"'")' <$json >/dev/null && ret=1
       fi
       n=$((n + 1))
       if [ $ret != 0 ]; then echo_i "failed"; fi

configure.ac

@@ -327,8 +327,10 @@ AC_SUBST([PYTEST])
 AM_CONDITIONAL([HAVE_PYTEST], [test -n "$PYTEST"])
 
 #
-# xsltproc is optional, it is used only by system test scripts.
+# Optional utilities, only used by system tests.
 #
+AC_PATH_PROG([JQ], [jq])
+AC_PATH_PROG([XMLLINT], [xmllint])
 AC_PATH_PROG([XSLTPROC], [xsltproc])
 
 #
@@ -1643,11 +1645,13 @@ AC_CONFIG_FILES([bin/tests/Makefile
 		 bin/tests/system/isctest/vars/.ac_vars/TOP_BUILDDIR
 		 bin/tests/system/isctest/vars/.ac_vars/TOP_SRCDIR
 		 bin/tests/system/isctest/vars/.ac_vars/FSTRM_CAPTURE
+		 bin/tests/system/isctest/vars/.ac_vars/JQ
 		 bin/tests/system/isctest/vars/.ac_vars/SHELL
 		 bin/tests/system/isctest/vars/.ac_vars/PYTHON
 		 bin/tests/system/isctest/vars/.ac_vars/PERL
 		 bin/tests/system/isctest/vars/.ac_vars/CURL
 		 bin/tests/system/isctest/vars/.ac_vars/NC
+		 bin/tests/system/isctest/vars/.ac_vars/XMLLINT
 		 bin/tests/system/isctest/vars/.ac_vars/XSLTPROC
 		 bin/tests/system/isctest/vars/.ac_vars/PYTEST
 		 bin/tests/system/dyndb/driver/Makefile

util/xmllint-html.sh

@@ -1,22 +0,0 @@
-#!/bin/sh -f
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-files=$(git ls-files '*.html')
-if test -n "$files"; then
-  xmllint --noout --nonet --html $files 2>&1 \
-    | awk 'BEGIN { status = 0; }
-	     # suppress HTML 5 <section> tag errors
-	     /HTML parser error : Tag section invalid/ { getline; getline; next; }
-	     { print; status = 1; }
-	     END { exit status }'
-fi