Merge branch '2779-wildcard_test_property_based' into 'main'

Add property based test for wildcard expansion

See merge request isc-projects/bind9!5203

.gitlab-ci.yml

@@ -481,7 +481,9 @@ flake8:
       artifacts: true
   script:
     - *configure
-    - flake8 --max-line-length=80 $(git ls-files '*.py' | grep -vE '(ans\.py|dangerfile\.py)')
+    - flake8 --max-line-length=80 $(git ls-files '*.py' | grep -vE '(ans\.py|dangerfile\.py|^bin/tests/system/)')
+      # Ignore Flake8 E402 error (module level import not at top of file) in system test to enable use of pytest.importorskip
+    - flake8 --max-line-length=80 --extend-ignore=E402 $(git ls-files 'bin/tests/system/*.py' | grep -vE 'ans\.py')
 
 pylint:
   <<: *default_triggering_rules
@@ -493,7 +495,9 @@ pylint:
   script:
     - *configure
     - PYTHONPATH="$PYTHONPATH:$CI_PROJECT_DIR/bin/python"
-    - pylint --rcfile $CI_PROJECT_DIR/.pylintrc $(git ls-files '*.py' | grep -vE '(ans\.py|dangerfile\.py)')
+    - pylint --rcfile $CI_PROJECT_DIR/.pylintrc $(git ls-files '*.py' | grep -vE '(ans\.py|dangerfile\.py|^bin/tests/system/)')
+      # Ignore Pylint wrong-import-position error in system test to enable use of pytest.importorskip
+    - pylint --rcfile $CI_PROJECT_DIR/.pylintrc --disable=wrong-import-position $(git ls-files 'bin/tests/system/*.py' | grep -vE 'ans\.py')
 
 tarball-create:
   stage: precheck

bin/tests/system/.gitignore

@@ -1,4 +1,5 @@
 .cache
+.hypothesis
 __pycache__
 dig.out*
 rndc.out*

bin/tests/system/checkds/clean.sh

@@ -21,5 +21,4 @@ rm -f ns*/keygen.out.* ns*/settime.out.* ns*/signer.out.*
 rm -f ns*/managed-keys.bind*
 rm -f ns*/*.mkeys
 rm -f ns*/zones
-rm -f tests-checkds.py.status
 rm -f *.checkds.out

bin/tests/system/rpzextra/clean.sh

@@ -14,4 +14,3 @@ rm -f ns*/named.memstats
 rm -f ns*/named.run
 rm -f ns*/rpz*.txt
 rm -rf __pycache__
-rm -f *.status

bin/tests/system/run.sh.in

@@ -201,23 +201,30 @@ fi
 
 if [ $status -eq 0 ]; then
     if [ -n "$PYTEST" ]; then
-        run=$((run+1))
         for test in $(cd "${systest}" && find . -name "tests*.py"); do
+            rm -f "$systest/$test.status"
             if start_servers; then
-                rm -f "$systest/$test.status"
+                run=$((run+1))
                 test_status=0
                 (cd "$systest" && "$PYTEST" -v "$test" "$@" || echo "$?" > "$test.status") | SYSTESTDIR="$systest" cat_d
                 if [ -f "$systest/$test.status" ]; then
-                    echo_i "FAILED"
-                    test_status=$(cat "$systest/$test.status")
+                    if [ "$(cat "$systest/$test.status")" = "5" ]; then
+                        echowarn "R:$systest:SKIPPED"
+                    else
+                        echo_i "FAILED"
+                        test_status=$(cat "$systest/$test.status")
+                    fi
                 fi
                 status=$((status+test_status))
                 stop_servers || status=1
             else
                 status=1
+            fi
+            if [ $status -ne 0 ]; then
                 break
             fi
         done
+        rm -f "$systest/$test.status"
     else
         echoinfo "I:$systest:pytest not installed, skipping python tests"
     fi

bin/tests/system/shutdown/clean.sh

@@ -14,4 +14,3 @@ rm -f ns*/rpz*.txt
 rm -f */named.conf
 rm -f */named.run
 rm -rf __pycache__
-rm -f *.status

bin/tests/system/wildcard/conftest.py

@@ -0,0 +1,18 @@
+############################################################################
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+############################################################################
+
+import os
+import pytest
+
+
+@pytest.fixture(scope='module')
+def named_port():
+    return int(os.environ.get("PORT", default=5300))

bin/tests/system/wildcard/ns1/allwild.db.in

@@ -0,0 +1,4 @@
+$ORIGIN allwild.test.
+allwild.test.                 3600 IN  SOA   . . 0 0 0 0 0
+allwild.test.                 3600     NS    ns.example.test.
+*.allwild.test.           3600     A     192.0.2.1

bin/tests/system/wildcard/ns1/named.conf.in

@@ -27,6 +27,7 @@ zone "." { type primary; file "root.db.signed"; };
 /*
  * RFC 4592 example zone.
  */
+zone "allwild.test" { type primary; file "allwild.db"; };
 zone "example" { type primary; file "example.db"; };
 zone "nsec" { type primary; file "nsec.db.signed"; };
 zone "private.nsec" { type primary; file "private.nsec.db.signed"; };

bin/tests/system/wildcard/ns1/sign.sh

@@ -16,6 +16,7 @@ SYSTESTDIR=wildcard
 dssets=
 
 # RFC 4592 example zone.
+cp allwild.db.in allwild.db
 cp example.db.in example.db
 
 zone=nsec

bin/tests/system/wildcard/tests-wildcard.py

@@ -0,0 +1,103 @@
+#!/usr/bin/python3
+############################################################################
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+############################################################################
+
+"""
+Example property-based test for wildcard synthesis.
+Verifies that otherwise-empty zone with single wildcard record * A 192.0.2.1
+produces synthesized answers for <random_label>.test. A, and returns NODATA for
+<random_label>.test. when rdtype is not A.
+
+Limitations - untested properties:
+    - expansion works with multiple labels
+    - asterisk in qname does not cause expansion
+    - empty non-terminals prevent expansion
+    - or more generally any existing node prevents expansion
+    - DNSSEC record inclusion
+    - possibly others, see RFC 4592 and company
+    - content of authority & additional sections
+    - flags beyond RCODE
+    - special behavior of rdtypes like CNAME
+"""
+import pytest
+
+pytest.importorskip("dns")
+import dns.message
+import dns.name
+import dns.query
+import dns.rcode
+import dns.rdatatype
+
+pytest.importorskip("hypothesis")
+from hypothesis import given
+from hypothesis.strategies import binary, integers
+
+
+# labels of a zone with * A 192.0.2.1 wildcard
+WILDCARD_ZONE = ('allwild', 'test', '')
+WILDCARD_RDTYPE = dns.rdatatype.A
+WILDCARD_RDATA = '192.0.2.1'
+IPADDR = '10.53.0.1'
+TIMEOUT = 5  # seconds, just a sanity check
+
+
+# Helpers
+def is_nonexpanding_rdtype(rdtype):
+    """skip meta types to avoid weird rcodes caused by AXFR etc.; RFC 6895"""
+    return not(rdtype == WILDCARD_RDTYPE
+               or dns.rdatatype.is_metatype(rdtype)  # known metatypes: OPT ...
+               or 128 <= rdtype <= 255)  # unknown meta types
+
+
+def tcp_query(where, port, qname, qtype):
+    querymsg = dns.message.make_query(qname, qtype)
+    assert len(querymsg.question) == 1
+    return querymsg, dns.query.tcp(querymsg, where, port=port, timeout=TIMEOUT)
+
+
+def query(where, port, label, rdtype):
+    labels = (label, ) + WILDCARD_ZONE
+    qname = dns.name.Name(labels)
+    return tcp_query(where, port, qname, rdtype)
+
+
+# Tests
+@given(label=binary(min_size=1, max_size=63),
+       rdtype=integers(min_value=0, max_value=65535).filter(
+           is_nonexpanding_rdtype))
+def test_wildcard_rdtype_mismatch(label, rdtype, named_port):
+    """any label non-matching rdtype must result in to NODATA"""
+    check_answer_nodata(*query(IPADDR, named_port, label, rdtype))
+
+
+def check_answer_nodata(querymsg, answer):
+    assert querymsg.is_response(answer), str(answer)
+    assert answer.rcode() == dns.rcode.NOERROR, str(answer)
+    assert answer.answer == [], str(answer)
+
+
+@given(label=binary(min_size=1, max_size=63))
+def test_wildcard_match(label, named_port):
+    """any label with maching rdtype must result in wildcard data in answer"""
+    check_answer_noerror(*query(IPADDR, named_port, label, WILDCARD_RDTYPE))
+
+
+def check_answer_noerror(querymsg, answer):
+    assert querymsg.is_response(answer), str(answer)
+    assert answer.rcode() == dns.rcode.NOERROR, str(answer)
+    assert len(querymsg.question) == 1, str(answer)
+    expected_answer = [dns.rrset.from_text(
+                            querymsg.question[0].name,
+                            300,  # TTL, ignored by dnspython comparison
+                            dns.rdataclass.IN,
+                            WILDCARD_RDTYPE,
+                            WILDCARD_RDATA)]
+    assert answer.answer == expected_answer, str(answer)

util/copyrights

@@ -868,8 +868,10 @@
 ./bin/tests/system/views/setup.sh		SH	2000,2001,2004,2007,2012,2014,2016,2017,2018,2019,2020,2021
 ./bin/tests/system/views/tests.sh		SH	2000,2001,2004,2007,2012,2013,2014,2016,2018,2019,2020,2021
 ./bin/tests/system/wildcard/clean.sh		SH	2012,2013,2014,2016,2018,2019,2020,2021
+./bin/tests/system/wildcard/conftest.py		PYTHON	2021
 ./bin/tests/system/wildcard/ns1/sign.sh		SH	2012,2013,2014,2016,2018,2019,2020,2021
 ./bin/tests/system/wildcard/setup.sh		SH	2012,2014,2016,2017,2018,2019,2020,2021
+./bin/tests/system/wildcard/tests-wildcard.py	PYTHON-BIN	2021
 ./bin/tests/system/wildcard/tests.sh		SH	2012,2013,2016,2018,2019,2020,2021
 ./bin/tests/system/xfer/ans5/badkeydata		X	2011,2018,2019,2020,2021
 ./bin/tests/system/xfer/ans5/badmessageid	X	2020,2021