Merge branch '3099-reimplement-the-gnutls-cli-check-in-python' into 'main'

Reimplement the gnutls-cli check in Python

Closes #3099

See merge request isc-projects/bind9!5732

bin/tests/system/conf.sh.in

@@ -114,9 +114,6 @@ SHELL=@SHELL@
 # CURL will be empty if no program was found by configure
 CURL=@CURL@
 
-# GNUTLS_CLI will be empty if no program was found by configure
-GNUTLS_CLI=@GNUTLS_CLI@
-
 # NC will be empty if no program was found by configure
 NC=@NC@
 

bin/tests/system/doth/.gitignore

@@ -0,0 +1,4 @@
+gnutls-cli.*
+headers.*
+ns*/example.db
+ns*/named.conf

bin/tests/system/doth/clean.sh

@@ -20,6 +20,6 @@ rm -f ./*/named.memstats
 rm -f ./*/named.run
 rm -f ./*/named.run.prev
 rm -f ./dig.out.*
-rm -f ./example-soa-*.test*
+rm -f ./gnutls-cli.*
 rm -f ./*/example*.db
 rm -rf ./headers.*

bin/tests/system/doth/conftest.py

@@ -0,0 +1,43 @@
+#!/usr/bin/python3
+
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+import os
+import shutil
+import subprocess
+
+import pytest
+
+
+@pytest.fixture
+def gnutls_cli_executable():
+    # Ensure gnutls-cli is available.
+    executable = shutil.which('gnutls-cli')
+    if not executable:
+        pytest.skip('gnutls-cli not found in PATH')
+
+    # Ensure gnutls-cli supports the --logfile command-line option.
+    args = [executable, '--logfile=/dev/null']
+    try:
+        with subprocess.check_output(args, stderr=subprocess.STDOUT) as _:
+            pass
+    except subprocess.CalledProcessError as exc:
+        stderr = exc.output
+    if b'illegal option' in stderr:
+        pytest.skip('gnutls-cli does not support the --logfile option')
+
+    return executable
+
+
+@pytest.fixture
+def named_tlsport():
+    return int(os.environ.get('TLSPORT', '853'))

bin/tests/system/doth/tests.sh

@@ -582,29 +582,5 @@ if [ -n "$testcurl" ]; then
 	status=$((status + ret))
 fi
 
-# check whether we can use gnutls-cli for sending test queries.
-if [ -x "${GNUTLS_CLI}" ] ; then
-	GNUTLS_CLI_CHECK="$(${GNUTLS_CLI} --logfile=/dev/null 2>&1 | grep -i 'illegal option')"
-
-	if [ -n "$GNUTLS_CLI_CHECK" ]; then
-		echo_i "The available version of gnutls-cli does not support the required features"
-	else
-		testgnutls=1
-	fi
-fi
-
-if [ -n "${testgnutls}" ] ; then
-	n=$((n + 1))
-	echo_i "checking sending a DoT query using gnutls-cli ($n)"
-	ret=0
-	# use gnutls-cli to query for 'example/SOA',
-	# use a timeout with a second empty `cat` because EOF in `stdin`
-	# causes gnutls-cli to disconnect without waiting for the answer
-	( cat example-soa-request.saved && timeout 10 cat ) | "${GNUTLS_CLI}" --no-ca-verification --no-ocsp --alpn=dot --logfile=/dev/null --port=${TLSPORT} 10.53.0.1 > example-soa-answer.test$n 2>&1
-	diff example-soa-answer.good example-soa-answer.test$n > /dev/null 2>&1 || ret=1
-	if [ $ret != 0 ]; then echo_i "failed"; fi
-	status=$((status + ret))
-fi
-
 echo_i "exit status: $status"
 [ $status -eq 0 ] || exit 1

bin/tests/system/doth/tests_gnutls.py

@@ -0,0 +1,93 @@
+#!/usr/bin/python3
+
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+import selectors
+import struct
+import subprocess
+import time
+
+import pytest
+
+pytest.importorskip('dns')
+import dns.exception
+import dns.message
+import dns.rdatatype
+
+
+def test_gnutls_cli_query(gnutls_cli_executable, named_tlsport):
+    # Prepare the example/SOA query which will be sent over TLS.
+    query = dns.message.make_query('example.', dns.rdatatype.SOA)
+    query_wire = query.to_wire()
+    query_with_length = struct.pack('>H', len(query_wire)) + query_wire
+
+    # Run gnutls-cli.
+    gnutls_cli_args = [gnutls_cli_executable, '--no-ca-verification', '-V',
+                       '--no-ocsp', '--alpn=dot', '--logfile=gnutls-cli.log',
+                       '--port=%d' % named_tlsport, '10.53.0.1']
+    with open('gnutls-cli.err', 'wb') as gnutls_cli_stderr, \
+         subprocess.Popen(gnutls_cli_args, stdin=subprocess.PIPE,
+                          stdout=subprocess.PIPE, stderr=gnutls_cli_stderr,
+                          bufsize=0) as gnutls_cli:
+        # Send the example/SOA query to the standard input of gnutls-cli.  Do
+        # not close standard input yet because that causes gnutls-cli to close
+        # the TLS connection immediately, preventing the response from being
+        # read.
+        gnutls_cli.stdin.write(query_with_length)
+        gnutls_cli.stdin.flush()
+
+        # Keep reading data from the standard output of gnutls-cli until a full
+        # DNS message is received or a timeout is exceeded or gnutls-cli exits.
+        # Popen.communicate() cannot be used here because: a) it closes
+        # standard input after sending data to the process (see above why this
+        # is a problem), b) gnutls-cli is not DNS-aware, so it does not exit
+        # upon receiving a DNS response.
+        selector = selectors.DefaultSelector()
+        selector.register(gnutls_cli.stdout, selectors.EVENT_READ)
+        deadline = time.time() + 10
+        gnutls_cli_output = b''
+        response = b''
+        while not response and not gnutls_cli.poll():
+            if not selector.select(timeout=deadline - time.time()):
+                break
+            gnutls_cli_output += gnutls_cli.stdout.read(512)
+            try:
+                # Ignore TCP length, just try to parse a DNS message from
+                # the rest of the data received.
+                response = dns.message.from_wire(gnutls_cli_output[2:])
+            except dns.exception.FormError:
+                continue
+
+        # At this point either a DNS response was received or a timeout fired
+        # or gnutls-cli exited prematurely.  Close the standard input of
+        # gnutls-cli.  Terminate it if that does not cause it to shut down
+        # gracefully.
+        gnutls_cli.stdin.close()
+        try:
+            gnutls_cli.wait(5)
+        except subprocess.TimeoutExpired:
+            gnutls_cli.kill()
+
+    # Store the response received for diagnostic purposes.
+    with open('gnutls-cli.out.bin', 'wb') as response_bin:
+        response_bin.write(gnutls_cli_output)
+    if response:
+        with open('gnutls-cli.out.txt', 'w', encoding='utf-8') as response_txt:
+            response_txt.write(response.to_text())
+
+    # Check whether a response was received and whether it is sane.
+    assert response
+    assert query.id == response.id
+    assert len(response.answer) == 1
+    assert response.answer[0].match(dns.name.from_text('example.'),
+                                    dns.rdataclass.IN, dns.rdatatype.SOA,
+                                    dns.rdatatype.NONE)

configure.ac

@@ -1270,13 +1270,6 @@ AC_CONFIG_FILES([doc/doxygen/doxygen-input-filter],
 AC_PATH_PROG(CURL, curl, curl)
 AC_SUBST(CURL)
 
-#
-# Look for gnutls-cli
-#
-
-AC_PATH_PROG([GNUTLS_CLI], [gnutls-cli], [])
-AC_SUBST(GNUTLS_CLI)
-
 #
 # Look for nc
 #