From 543bab699d4822ecc6c72cd298db99acfb140561 Mon Sep 17 00:00:00 2001
From: Paul Frieden <pfrieden@verizonmedia.com>
Date: Wed, 30 Oct 2019 17:06:06 -0500
Subject: [PATCH 1/7] Export zone timers via stats channels

---
 bin/named/statschannel.c | 71 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)

diff --git a/bin/named/statschannel.c b/bin/named/statschannel.c
index b702915ada..59fabdea89 100644
--- a/bin/named/statschannel.c
+++ b/bin/named/statschannel.c
@@ -1809,6 +1809,43 @@ zone_xmlrender(dns_zone_t *zone, void *arg) {
 	}
 	TRY0(xmlTextWriterEndElement(writer)); /* serial */
 
+	/*
+	 * Export zone timers to the statistics channel in XML format.  For
+	 * master zones, only include the loaded time.  For slave zones, also
+	 * include the expires and refresh times.
+	 */
+	isc_time_t timestamp;
+
+	result = dns_zone_getloadtime(zone, &timestamp);
+	if (result != ISC_R_SUCCESS) {
+		goto error;
+	}
+
+	isc_time_formatISO8601(&timestamp, buf, 64);
+	TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "loaded"));
+	TRY0(xmlTextWriterWriteString(writer, ISC_XMLCHAR buf));
+	TRY0(xmlTextWriterEndElement(writer));
+
+	if (dns_zone_gettype(zone) == dns_zone_slave) {
+		result = dns_zone_getexpiretime(zone, &timestamp);
+		if (result != ISC_R_SUCCESS) {
+			goto error;
+		}
+		isc_time_formatISO8601(&timestamp, buf, 64);
+		TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "expires"));
+		TRY0(xmlTextWriterWriteString(writer, ISC_XMLCHAR buf));
+		TRY0(xmlTextWriterEndElement(writer));
+
+		result = dns_zone_getrefreshtime(zone, &timestamp);
+		if (result != ISC_R_SUCCESS) {
+			goto error;
+		}
+		isc_time_formatISO8601(&timestamp, buf, 64);
+		TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "refresh"));
+		TRY0(xmlTextWriterWriteString(writer, ISC_XMLCHAR buf));
+		TRY0(xmlTextWriterEndElement(writer));
+	}
+
 	if (statlevel == dns_zonestat_full) {
 		isc_stats_t *zonestats;
 		isc_stats_t *gluecachestats;
@@ -2619,6 +2656,40 @@ zone_jsonrender(dns_zone_t *zone, void *arg) {
 		return (ISC_R_NOMEMORY);
 	}
 
+	/*
+	 * Export zone timers to the statistics channel in JSON format.  For
+	 * master zones, only include the loaded time.  For slave zones, also
+	 * include the expires and refresh times.
+	 */
+
+	isc_time_t timestamp;
+
+	result = dns_zone_getloadtime(zone, &timestamp);
+	if (result != ISC_R_SUCCESS) {
+		goto error;
+	}
+
+	isc_time_formatISO8601(&timestamp, buf, 64);
+	json_object_object_add(zoneobj, "loaded", json_object_new_string(buf));
+
+	if (dns_zone_gettype(zone) == dns_zone_slave) {
+		result = dns_zone_getexpiretime(zone, &timestamp);
+		if (result != ISC_R_SUCCESS) {
+			goto error;
+		}
+		isc_time_formatISO8601(&timestamp, buf, 64);
+		json_object_object_add(zoneobj, "expires",
+				       json_object_new_string(buf));
+
+		result = dns_zone_getrefreshtime(zone, &timestamp);
+		if (result != ISC_R_SUCCESS) {
+			goto error;
+		}
+		isc_time_formatISO8601(&timestamp, buf, 64);
+		json_object_object_add(zoneobj, "refresh",
+				       json_object_new_string(buf));
+	}
+
 	if (statlevel == dns_zonestat_full) {
 		isc_stats_t *zonestats;
 		isc_stats_t *gluecachestats;

From 72ffa194e2a8647583aa7927d47557f5fe88b2e7 Mon Sep 17 00:00:00 2001
From: Paul Frieden <pfrieden@verizonmedia.com>
Date: Mon, 4 Nov 2019 16:49:55 -0600
Subject: [PATCH 2/7] Add the zone timers to the XSL

---
 bin/named/bind9.xsl | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/bin/named/bind9.xsl b/bin/named/bind9.xsl
index f36927794f..ba0ad27487 100644
--- a/bin/named/bind9.xsl
+++ b/bin/named/bind9.xsl
@@ -775,7 +775,7 @@
           <xsl:for-each select="views/view">
             <h3>Zones for View <xsl:value-of select="@name"/></h3>
             <table class="zones">
-              <thead><tr><th>Name</th><th>Class</th><th>Type</th><th>Serial</th></tr></thead>
+              <thead><tr><th>Name</th><th>Class</th><th>Type</th><th>Serial</th><th>Loaded</th><th>Expires</th><th>Refresh</th></tr></thead>
               <tbody>
                 <xsl:for-each select="zones/zone">
                   <xsl:variable name="css-class15">
@@ -788,7 +788,10 @@
                       <td><xsl:value-of select="@name"/></td>
                       <td><xsl:value-of select="@rdataclass"/></td>
                       <td><xsl:value-of select="type"/></td>
-                      <td><xsl:value-of select="serial"/></td></tr>
+                      <td><xsl:value-of select="serial"/></td>
+                      <td><xsl:value-of select="loaded"/></td>
+                      <td><xsl:value-of select="expires"/></td>
+                      <td><xsl:value-of select="refresh"/></td></tr>
                 </xsl:for-each>
               </tbody>
             </table>

From 6aa6d7be58e66a9e2669af6bbf7e9c709dc36548 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= <ondrej@sury.org>
Date: Fri, 27 Mar 2020 10:13:31 +0100
Subject: [PATCH 3/7] Add tests for zone timers using the pytest testing
 framework

---
 bin/tests/system/statschannel/.gitignore      |  2 +
 bin/tests/system/statschannel/clean.sh        | 10 ++-
 bin/tests/system/statschannel/conftest.py     | 74 ++++++++++++++++
 bin/tests/system/statschannel/helper.py       | 58 +++++++++++++
 bin/tests/system/statschannel/ns1/example.db  | 47 +++++++++++
 .../system/statschannel/ns1/named.conf.in     | 41 +++++++++
 .../system/statschannel/ns2/named.conf.in     |  2 +-
 .../system/statschannel/ns3/named.conf.in     | 41 +++++++++
 bin/tests/system/statschannel/setup.sh        |  9 +-
 bin/tests/system/statschannel/tests-json.py   | 75 +++++++++++++++++
 bin/tests/system/statschannel/tests-xml.py    | 84 +++++++++++++++++++
 bin/tests/system/statschannel/tests.sh        |  0
 util/copyrights                               |  4 +
 13 files changed, 437 insertions(+), 10 deletions(-)
 create mode 100644 bin/tests/system/statschannel/.gitignore
 create mode 100644 bin/tests/system/statschannel/conftest.py
 create mode 100644 bin/tests/system/statschannel/helper.py
 create mode 100644 bin/tests/system/statschannel/ns1/example.db
 create mode 100644 bin/tests/system/statschannel/ns1/named.conf.in
 create mode 100644 bin/tests/system/statschannel/ns3/named.conf.in
 create mode 100755 bin/tests/system/statschannel/tests-json.py
 create mode 100755 bin/tests/system/statschannel/tests-xml.py
 mode change 100644 => 100755 bin/tests/system/statschannel/tests.sh

diff --git a/bin/tests/system/statschannel/.gitignore b/bin/tests/system/statschannel/.gitignore
new file mode 100644
index 0000000000..44fb46c3c0
--- /dev/null
+++ b/bin/tests/system/statschannel/.gitignore
@@ -0,0 +1,2 @@
+/.cache/
+/__pycache__/
diff --git a/bin/tests/system/statschannel/clean.sh b/bin/tests/system/statschannel/clean.sh
index c9edbde9ee..243a90612c 100644
--- a/bin/tests/system/statschannel/clean.sh
+++ b/bin/tests/system/statschannel/clean.sh
@@ -12,9 +12,9 @@
 rm -f traffic traffic.out.* traffic.json.* traffic.xml.*
 rm -f zones zones.out.* zones.json.* zones.xml.* zones.expect.*
 rm -f dig.out*
-rm -f */named.memstats
-rm -f */named.conf
-rm -f */named.run*
+rm -f ns*/named.memstats
+rm -f ns*/named.conf
+rm -f ns*/named.run*
 rm -f ns*/named.lock
 rm -f ns*/named.stats
 rm -f xml.*stats json.*stats
@@ -24,4 +24,6 @@ rm -f ns*/managed-keys.bind*
 rm -f ns2/Kdnssec* ns2/dnssec.*.id
 rm -f ns2/Kmanykeys* ns2/manykeys.*.id
 rm -f ns2/*.db.signed* ns2/dsset-*. ns2/*.jbk
-rm -f ns2/core
+rm -f ns2/dnssec.db.signed* ns2/dsset-dnssec.
+rm -f ns3/*.db
+rm -rf /.cache /__pycache__
diff --git a/bin/tests/system/statschannel/conftest.py b/bin/tests/system/statschannel/conftest.py
new file mode 100644
index 0000000000..d62b6af29b
--- /dev/null
+++ b/bin/tests/system/statschannel/conftest.py
@@ -0,0 +1,74 @@
+############################################################################
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+############################################################################
+
+import pytest
+import os
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "requests: mark tests that need requests to function"
+    )
+    config.addinivalue_line(
+        "markers", "json: mark tests that need json to function"
+    )
+    config.addinivalue_line(
+        "markers", "xml: mark tests that need xml.etree to function"
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    # Test for requests module
+    skip_requests = pytest.mark.skip(reason="need requests module to run")
+    try:
+        import requests  # noqa: F401
+    except ModuleNotFoundError:
+        for item in items:
+            if "requests" in item.keywords:
+                item.add_marker(skip_requests)
+    # Test for json module
+    skip_json = pytest.mark.skip(reason="need json module to run")
+    try:
+        import json  # noqa: F401
+    except ModuleNotFoundError:
+        for item in items:
+            if "json" in item.keywords:
+                item.add_marker(skip_json)
+    # Test for xml module
+    skip_xml = pytest.mark.skip(reason="need xml module to run")
+    try:
+        import xml  # noqa: F401
+    except ModuleNotFoundError:
+        for item in items:
+            if "xml" in item.keywords:
+                item.add_marker(skip_xml)
+    # Test if JSON statistics channel was enabled
+    no_jsonstats = pytest.mark.skip(reason="need JSON statistics to be enabled")
+    if os.getenv("HAVEJSONSTATS") is None:
+        for item in items:
+            if "json" in item.keywords:
+                item.add_marker(no_jsonstats)
+    # Test if XML statistics channel was enabled
+    no_xmlstats = pytest.mark.skip(reason="need XML statistics to be enabled")
+    if os.getenv("HAVEXMLSTATS") is None:
+        for item in items:
+            if "xml" in item.keywords:
+                item.add_marker(no_xmlstats)
+
+
+@pytest.fixture
+def statsport(request):
+    port = os.getenv("EXTRAPORT1")
+    if port is None:
+        port = 5301
+    else:
+        port = int(port)
+
+    return port
diff --git a/bin/tests/system/statschannel/helper.py b/bin/tests/system/statschannel/helper.py
new file mode 100644
index 0000000000..0865b3031b
--- /dev/null
+++ b/bin/tests/system/statschannel/helper.py
@@ -0,0 +1,58 @@
+############################################################################
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+############################################################################
+
+from datetime import datetime, timedelta
+
+# ISO datetime format without msec
+fmt = '%Y-%m-%dT%H:%M:%SZ'
+
+# The constants were taken from BIND 9 source code (lib/dns/zone.c)
+max_refresh = timedelta(seconds=2419200)  # 4 weeks
+max_expires = timedelta(seconds=14515200)  # 24 weeks
+now = datetime.utcnow().replace(microsecond=0)
+dayzero = datetime.utcfromtimestamp(0).replace(microsecond=0)
+
+
+# Generic helper functions
+def check_expires(expires, min, max):
+    assert expires >= min
+    assert expires <= max
+
+
+def check_refresh(refresh, min, max):
+    assert refresh >= min
+    assert refresh <= max
+
+
+def check_loaded(loaded, expected):
+    # Sanity check the zone timers values
+    assert loaded == expected
+    assert loaded < now
+
+
+def check_zone_timers(loaded, expires, refresh, loaded_exp):
+    # Sanity checks the zone timers values
+    if expires is not None:
+        check_expires(expires, now, now + max_expires)
+    if refresh is not None:
+        check_refresh(refresh, now, now + max_refresh)
+    check_loaded(loaded, loaded_exp)
+
+
+def zone_mtime(zonedir, name):
+    import os
+    import os.path
+    from datetime import datetime
+
+    si = os.stat(os.path.join(zonedir, "{}.db".format(name)))
+    mtime = datetime.utcfromtimestamp(si.st_mtime).replace(microsecond=0)
+
+    return mtime
diff --git a/bin/tests/system/statschannel/ns1/example.db b/bin/tests/system/statschannel/ns1/example.db
new file mode 100644
index 0000000000..b65651aa2d
--- /dev/null
+++ b/bin/tests/system/statschannel/ns1/example.db
@@ -0,0 +1,47 @@
+; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+;
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0. If a copy of the MPL was not distributed with this
+; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;
+; See the COPYRIGHT file distributed with this work for additional
+; information regarding copyright ownership.
+
+$ORIGIN .
+$TTL 300	; 5 minutes
+example			IN SOA	mname1. . (
+				1          ; serial
+				20         ; refresh (20 seconds)
+				20         ; retry (20 seconds)
+				1814400    ; expire (3 weeks)
+				3600       ; minimum (1 hour)
+				)
+example.		NS	ns2.example.
+ns2.example.		A	10.53.0.2
+
+$ORIGIN example.
+a			A	10.0.0.1
+			MX	10 mail.example.
+short                   TXT     "short text"
+long                    TXT     (
+                                  "longlonglonglonglonglonglonglonglonglong"
+                                  "longlonglonglonglonglonglonglonglonglong"
+                                  "longlonglonglonglonglonglonglonglonglong"
+                                  "longlonglonglonglonglonglonglonglonglong"
+                                  "longlonglonglonglonglonglonglonglonglong"
+                                  "longlonglonglonglonglonglonglonglonglong"
+                                  "longlonglonglonglonglonglonglonglonglong"
+                                  "longlonglonglonglonglonglonglonglonglong"
+                                  "longlonglonglonglonglonglonglonglonglong"
+                                  "longlonglonglonglonglonglonglonglonglong"
+                                  "longlonglonglonglonglonglonglonglonglong"
+                                  "longlonglonglonglonglonglonglonglonglong"
+                                  "longlonglonglonglonglonglonglonglonglong"
+                                  "longlonglonglonglonglonglonglonglonglong"
+                                  "longlonglonglonglonglonglonglonglonglong"
+                                  "longlonglonglonglonglonglonglonglonglong"
+                                  "longlonglonglonglonglonglonglonglonglong"
+                                  "longlonglonglonglonglonglonglonglonglong"
+                                )
+
+mail			A	10.0.0.2
diff --git a/bin/tests/system/statschannel/ns1/named.conf.in b/bin/tests/system/statschannel/ns1/named.conf.in
new file mode 100644
index 0000000000..fa1cd57a6c
--- /dev/null
+++ b/bin/tests/system/statschannel/ns1/named.conf.in
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+options {
+	query-source address 10.53.0.1;
+	notify-source 10.53.0.1;
+	transfer-source 10.53.0.1;
+	port @PORT@;
+	pid-file "named.pid";
+	listen-on { 10.53.0.1; };
+	listen-on-v6 { none; };
+	recursion no;
+	notify explicit;
+	minimal-responses no;
+	version none;  // make statistics independent of the version number
+};
+
+statistics-channels { inet 10.53.0.1 port @EXTRAPORT1@ allow { localhost; }; };
+
+key rndc_key {
+	secret "1234abcd8765";
+	algorithm hmac-sha256;
+};
+
+controls {
+	inet 10.53.0.1 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+zone "example" {
+	type master;
+	file "example.db";
+	allow-transfer { any; };
+};
diff --git a/bin/tests/system/statschannel/ns2/named.conf.in b/bin/tests/system/statschannel/ns2/named.conf.in
index 70aadf8b28..9eebe560d3 100644
--- a/bin/tests/system/statschannel/ns2/named.conf.in
+++ b/bin/tests/system/statschannel/ns2/named.conf.in
@@ -18,7 +18,7 @@ options {
 	listen-on { 10.53.0.2; };
 	listen-on-v6 { none; };
 	recursion no;
-	notify yes;
+	notify no;
 	minimal-responses no;
 	version none;  // make statistics independent of the version number
 };
diff --git a/bin/tests/system/statschannel/ns3/named.conf.in b/bin/tests/system/statschannel/ns3/named.conf.in
new file mode 100644
index 0000000000..e78cff9a7a
--- /dev/null
+++ b/bin/tests/system/statschannel/ns3/named.conf.in
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+options {
+	query-source address 10.53.0.3;
+	notify-source 10.53.0.3;
+	transfer-source 10.53.0.3;
+	port @PORT@;
+	pid-file "named.pid";
+	listen-on { 10.53.0.3; };
+	listen-on-v6 { none; };
+	recursion no;
+	notify no;
+	minimal-responses no;
+	version none;  // make statistics independent of the version number
+};
+
+statistics-channels { inet 10.53.0.3 port @EXTRAPORT1@ allow { localhost; }; };
+
+key rndc_key {
+	secret "1234abcd8765";
+	algorithm hmac-sha256;
+};
+
+controls {
+	inet 10.53.0.3 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+zone "example" {
+	type secondary;
+	file "example.db";
+	masters { 10.53.0.1; };
+};
diff --git a/bin/tests/system/statschannel/setup.sh b/bin/tests/system/statschannel/setup.sh
index 3a8577eb72..2ce6fde81d 100644
--- a/bin/tests/system/statschannel/setup.sh
+++ b/bin/tests/system/statschannel/setup.sh
@@ -12,9 +12,8 @@
 # shellcheck source=conf.sh
 . "$SYSTEMTESTTOP/conf.sh"
 
-copy_setports ns2/named.conf.in ns2/named.conf
+for conf in ns*/named.conf.in; do
+    copy_setports "$conf" "$(dirname "$conf")/$(basename "$conf" .in)"
+done
 
-(
-    cd ns2
-    $SHELL sign.sh
-)
+(cd ns2 && $SHELL sign.sh)
diff --git a/bin/tests/system/statschannel/tests-json.py b/bin/tests/system/statschannel/tests-json.py
new file mode 100755
index 0000000000..f9bd5ec78f
--- /dev/null
+++ b/bin/tests/system/statschannel/tests-json.py
@@ -0,0 +1,75 @@
+#!/usr/bin/python3
+############################################################################
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+############################################################################
+
+import pytest
+from datetime import datetime
+from helper import fmt, zone_mtime, check_zone_timers, dayzero
+
+
+# JSON helper functions
+def fetch_json(statsip, statsport):
+    import requests
+
+    r = requests.get("http://{}:{}/json/v1/zones".format(statsip, statsport))
+    assert r.status_code == 200
+
+    data = r.json()
+
+    return data["views"]["_default"]["zones"]
+
+
+def load_timers_from_json(zone, primary=True):
+    name = zone['name']
+
+    # Check if the primary zone timer exists
+    assert 'loaded' in zone
+    loaded = datetime.strptime(zone['loaded'], fmt)
+
+    if primary:
+        # Check if the secondary zone timers does not exist
+        assert 'expires' not in zone
+        assert 'refresh' not in zone
+        expires = None
+        refresh = None
+    else:
+        assert 'expires' in zone
+        assert 'refresh' in zone
+        expires = datetime.strptime(zone['expires'], fmt)
+        refresh = datetime.strptime(zone['refresh'], fmt)
+
+    return (name, loaded, expires, refresh)
+
+
+@pytest.mark.json
+@pytest.mark.requests
+def test_zone_timers_primary_json(statsport):
+    statsip = "10.53.0.1"
+    zonedir = "ns1"
+
+    zones = fetch_json(statsip, statsport)
+
+    for zone in zones:
+        (name, loaded, expires, refresh) = load_timers_from_json(zone, True)
+        mtime = zone_mtime(zonedir, name)
+        check_zone_timers(loaded, expires, refresh, mtime)
+
+
+@pytest.mark.json
+@pytest.mark.requests
+def test_zone_timers_secondary_json(statsport):
+    statsip = "10.53.0.3"
+
+    zones = fetch_json(statsip, statsport)
+
+    for zone in zones:
+        (name, loaded, expires, refresh) = load_timers_from_json(zone, False)
+        check_zone_timers(loaded, expires, refresh, dayzero)
diff --git a/bin/tests/system/statschannel/tests-xml.py b/bin/tests/system/statschannel/tests-xml.py
new file mode 100755
index 0000000000..dcd2d76598
--- /dev/null
+++ b/bin/tests/system/statschannel/tests-xml.py
@@ -0,0 +1,84 @@
+#!/usr/bin/python3
+############################################################################
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+############################################################################
+
+import pytest
+from datetime import datetime
+from helper import fmt, zone_mtime, check_zone_timers, dayzero
+
+
+# XML helper functions
+def fetch_xml(statsip, statsport):
+    import xml.etree.ElementTree as ET
+    import requests
+
+    r = requests.get("http://{}:{}/xml/v3/zones".format(statsip, statsport))
+    assert r.status_code == 200
+
+    root = ET.fromstring(r.text)
+
+    default_view = None
+    for view in root.find('views').iter('view'):
+        if view.attrib['name'] == "_default":
+            default_view = view
+            break
+    assert default_view is not None
+
+    return default_view.find('zones').findall('zone')
+
+
+def load_timers_from_xml(zone, primary=True):
+    name = zone.attrib['name']
+
+    loaded_el = zone.find('loaded')
+    assert loaded_el is not None
+    loaded = datetime.strptime(loaded_el.text, fmt)
+
+    expires_el = zone.find('expires')
+    refresh_el = zone.find('refresh')
+    if primary:
+        assert expires_el is None
+        assert refresh_el is None
+        expires = None
+        refresh = None
+    else:
+        assert expires_el is not None
+        assert refresh_el is not None
+        expires = datetime.strptime(expires_el.text, fmt)
+        refresh = datetime.strptime(refresh_el.text, fmt)
+
+    return (name, loaded, expires, refresh)
+
+
+@pytest.mark.xml
+@pytest.mark.requests
+def test_zone_timers_primary_xml(statsport):
+    statsip = "10.53.0.1"
+    zonedir = "ns1"
+
+    zones = fetch_xml(statsip, statsport)
+
+    for zone in zones:
+        (name, loaded, expires, refresh) = load_timers_from_xml(zone, True)
+        mtime = zone_mtime(zonedir, name)
+        check_zone_timers(loaded, expires, refresh, mtime)
+
+
+@pytest.mark.xml
+@pytest.mark.requests
+def test_zone_timers_secondary_xml(statsport):
+    statsip = "10.53.0.3"
+
+    zones = fetch_xml(statsip, statsport)
+
+    for zone in zones:
+        (name, loaded, expires, refresh) = load_timers_from_xml(zone, False)
+        check_zone_timers(loaded, expires, refresh, dayzero)
diff --git a/bin/tests/system/statschannel/tests.sh b/bin/tests/system/statschannel/tests.sh
old mode 100644
new mode 100755
diff --git a/util/copyrights b/util/copyrights
index 91a31b2ea5..40db450438 100644
--- a/util/copyrights
+++ b/util/copyrights
@@ -808,12 +808,16 @@
 ./bin/tests/system/statistics/setup.sh		SH	2018,2019,2020
 ./bin/tests/system/statistics/tests.sh		SH	2012,2015,2016,2017,2018,2019,2020
 ./bin/tests/system/statschannel/clean.sh	SH	2015,2016,2017,2018,2019,2020
+./bin/tests/system/statschannel/conftest.py	PYTHON	2020
 ./bin/tests/system/statschannel/fetch.pl	PERL	2015,2016,2018,2019,2020
+./bin/tests/system/statschannel/helper.py	PYTHON	2020
 ./bin/tests/system/statschannel/mem-xml.pl	PERL	2017,2018,2019,2020
 ./bin/tests/system/statschannel/ns2/sign.sh	SH	2019,2020
 ./bin/tests/system/statschannel/server-json.pl	PERL	2015,2016,2017,2018,2019,2020
 ./bin/tests/system/statschannel/server-xml.pl	PERL	2015,2016,2017,2018,2019,2020
 ./bin/tests/system/statschannel/setup.sh	SH	2018,2019,2020
+./bin/tests/system/statschannel/tests-json.py	PYTHON-BIN	2020
+./bin/tests/system/statschannel/tests-xml.py	PYTHON-BIN	2020
 ./bin/tests/system/statschannel/tests.sh	SH	2015,2016,2017,2018,2019,2020
 ./bin/tests/system/statschannel/traffic-json.pl	PERL	2015,2016,2017,2018,2019,2020
 ./bin/tests/system/statschannel/traffic-xml.pl	PERL	2015,2016,2017,2018,2019,2020

From 1202fd912a1baa9c299f17caf4494bc21234da85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= <ondrej@isc.org>
Date: Tue, 14 Apr 2020 17:02:21 +0200
Subject: [PATCH 4/7] Rewrite the statschannel traffic tests to pytest

---
 bin/tests/system/statschannel/clean.sh      |   2 +-
 bin/tests/system/statschannel/conftest.py   |  55 +++++++--
 bin/tests/system/statschannel/generic.py    |  95 ++++++++++++++++
 bin/tests/system/statschannel/helper.py     | 117 ++++++++++++++++++--
 bin/tests/system/statschannel/tests-json.py |  65 +++++++----
 bin/tests/system/statschannel/tests-xml.py  |  86 ++++++++++----
 bin/tests/system/statschannel/tests.sh      | 109 ------------------
 doc/man/named.conf.5in                      |  20 ++--
 doc/misc/options                            |  36 +++---
 doc/misc/options.active                     |  36 +++---
 doc/misc/options.grammar.rst                |  15 +--
 util/copyrights                             |   1 +
 12 files changed, 404 insertions(+), 233 deletions(-)
 create mode 100644 bin/tests/system/statschannel/generic.py
 mode change 100755 => 100644 bin/tests/system/statschannel/tests.sh

diff --git a/bin/tests/system/statschannel/clean.sh b/bin/tests/system/statschannel/clean.sh
index 243a90612c..4904d91e43 100644
--- a/bin/tests/system/statschannel/clean.sh
+++ b/bin/tests/system/statschannel/clean.sh
@@ -26,4 +26,4 @@ rm -f ns2/Kmanykeys* ns2/manykeys.*.id
 rm -f ns2/*.db.signed* ns2/dsset-*. ns2/*.jbk
 rm -f ns2/dnssec.db.signed* ns2/dsset-dnssec.
 rm -f ns3/*.db
-rm -rf /.cache /__pycache__
+rm -rf ./.cache ./__pycache__
diff --git a/bin/tests/system/statschannel/conftest.py b/bin/tests/system/statschannel/conftest.py
index d62b6af29b..ebb4d1e6f5 100644
--- a/bin/tests/system/statschannel/conftest.py
+++ b/bin/tests/system/statschannel/conftest.py
@@ -9,8 +9,9 @@
 # information regarding copyright ownership.
 ############################################################################
 
-import pytest
 import os
+import pytest
+
 
 def pytest_configure(config):
     config.addinivalue_line(
@@ -22,11 +23,17 @@ def pytest_configure(config):
     config.addinivalue_line(
         "markers", "xml: mark tests that need xml.etree to function"
     )
+    config.addinivalue_line(
+        "markers", "dnspython: mark tests that need dnspython to function"
+    )
 
 
 def pytest_collection_modifyitems(config, items):
+    # pylint: disable=unused-argument,unused-import,too-many-branches
+    # pylint: disable=import-outside-toplevel
     # Test for requests module
-    skip_requests = pytest.mark.skip(reason="need requests module to run")
+    skip_requests = pytest.mark.skip(
+        reason="need requests module to run")
     try:
         import requests  # noqa: F401
     except ModuleNotFoundError:
@@ -34,7 +41,8 @@ def pytest_collection_modifyitems(config, items):
             if "requests" in item.keywords:
                 item.add_marker(skip_requests)
     # Test for json module
-    skip_json = pytest.mark.skip(reason="need json module to run")
+    skip_json = pytest.mark.skip(
+        reason="need json module to run")
     try:
         import json  # noqa: F401
     except ModuleNotFoundError:
@@ -42,33 +50,58 @@ def pytest_collection_modifyitems(config, items):
             if "json" in item.keywords:
                 item.add_marker(skip_json)
     # Test for xml module
-    skip_xml = pytest.mark.skip(reason="need xml module to run")
+    skip_xml = pytest.mark.skip(
+        reason="need xml module to run")
     try:
-        import xml  # noqa: F401
+        import xml.etree.ElementTree  # noqa: F401
     except ModuleNotFoundError:
         for item in items:
             if "xml" in item.keywords:
                 item.add_marker(skip_xml)
     # Test if JSON statistics channel was enabled
-    no_jsonstats = pytest.mark.skip(reason="need JSON statistics to be enabled")
+    no_jsonstats = pytest.mark.skip(
+        reason="need JSON statistics to be enabled")
     if os.getenv("HAVEJSONSTATS") is None:
         for item in items:
             if "json" in item.keywords:
                 item.add_marker(no_jsonstats)
     # Test if XML statistics channel was enabled
-    no_xmlstats = pytest.mark.skip(reason="need XML statistics to be enabled")
+    no_xmlstats = pytest.mark.skip(
+        reason="need XML statistics to be enabled")
     if os.getenv("HAVEXMLSTATS") is None:
         for item in items:
             if "xml" in item.keywords:
                 item.add_marker(no_xmlstats)
+    # Test for dnspython module
+    skip_dnspython = pytest.mark.skip(
+        reason="need dnspython module to run")
+    try:
+        import dns.query  # noqa: F401
+    except ModuleNotFoundError:
+        for item in items:
+            if "dnspython" in item.keywords:
+                item.add_marker(skip_dnspython)
 
 
 @pytest.fixture
 def statsport(request):
-    port = os.getenv("EXTRAPORT1")
+    # pylint: disable=unused-argument
+    env_port = os.getenv("EXTRAPORT1")
     if port is None:
-        port = 5301
+        env_port = 5301
     else:
-        port = int(port)
+        env_port = int(env_port)
 
-    return port
+    return env_port
+
+
+@pytest.fixture
+def port(request):
+    # pylint: disable=unused-argument
+    env_port = os.getenv("PORT")
+    if port is None:
+        env_port = 5300
+    else:
+        env_port = int(env_port)
+
+    return env_port
diff --git a/bin/tests/system/statschannel/generic.py b/bin/tests/system/statschannel/generic.py
new file mode 100644
index 0000000000..d927793cc2
--- /dev/null
+++ b/bin/tests/system/statschannel/generic.py
@@ -0,0 +1,95 @@
+############################################################################
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+############################################################################
+
+import helper
+
+
+def test_zone_timers_primary(fetch_zones, load_timers, **kwargs):
+
+    statsip = kwargs['statsip']
+    statsport = kwargs['statsport']
+    zonedir = kwargs['zonedir']
+
+    zones = fetch_zones(statsip, statsport)
+
+    for zone in zones:
+        (name, loaded, expires, refresh) = load_timers(zone, True)
+        mtime = helper.zone_mtime(zonedir, name)
+        helper.check_zone_timers(loaded, expires, refresh, mtime)
+
+
+def test_zone_timers_secondary(fetch_zones, load_timers, **kwargs):
+
+    statsip = kwargs['statsip']
+    statsport = kwargs['statsport']
+    zonedir = kwargs['zonedir']
+
+    zones = fetch_zones(statsip, statsport)
+
+    for zone in zones:
+        (name, loaded, expires, refresh) = load_timers(zone, False)
+        mtime = helper.zone_mtime(zonedir, name)
+        helper.check_zone_timers(loaded, expires, refresh, mtime)
+
+
+def test_zone_with_many_keys(fetch_zones, load_zone, **kwargs):
+
+    statsip = kwargs['statsip']
+    statsport = kwargs['statsport']
+
+    zones = fetch_zones(statsip, statsport)
+
+    for zone in zones:
+        name = load_zone(zone)
+        if name == 'manykeys':
+            helper.check_manykeys(name)
+
+
+def test_traffic(fetch_traffic, **kwargs):
+
+    statsip = kwargs['statsip']
+    statsport = kwargs['statsport']
+    port = kwargs['port']
+
+    data = fetch_traffic(statsip, statsport)
+    exp = helper.create_expected(data)
+
+    msg = helper.create_msg("short.example.", "TXT")
+    helper.update_expected(exp, "dns-udp-requests-sizes-received-ipv4", msg)
+    ans = helper.udp_query(statsip, port, msg)
+    helper.update_expected(exp, "dns-udp-responses-sizes-sent-ipv4", ans)
+    data = fetch_traffic(statsip, statsport)
+
+    helper.check_traffic(data, exp)
+
+    msg = helper.create_msg("long.example.", "TXT")
+    helper.update_expected(exp, "dns-udp-requests-sizes-received-ipv4", msg)
+    ans = helper.udp_query(statsip, port, msg)
+    helper.update_expected(exp, "dns-udp-responses-sizes-sent-ipv4", ans)
+    data = fetch_traffic(statsip, statsport)
+
+    helper.check_traffic(data, exp)
+
+    msg = helper.create_msg("short.example.", "TXT")
+    helper.update_expected(exp, "dns-tcp-requests-sizes-received-ipv4", msg)
+    ans = helper.tcp_query(statsip, port, msg)
+    helper.update_expected(exp, "dns-tcp-responses-sizes-sent-ipv4", ans)
+    data = fetch_traffic(statsip, statsport)
+
+    helper.check_traffic(data, exp)
+
+    msg = helper.create_msg("long.example.", "TXT")
+    helper.update_expected(exp, "dns-tcp-requests-sizes-received-ipv4", msg)
+    ans = helper.tcp_query(statsip, port, msg)
+    helper.update_expected(exp, "dns-tcp-responses-sizes-sent-ipv4", ans)
+    data = fetch_traffic(statsip, statsport)
+
+    helper.check_traffic(data, exp)
diff --git a/bin/tests/system/statschannel/helper.py b/bin/tests/system/statschannel/helper.py
index 0865b3031b..7b53e9e9fe 100644
--- a/bin/tests/system/statschannel/helper.py
+++ b/bin/tests/system/statschannel/helper.py
@@ -9,8 +9,16 @@
 # information regarding copyright ownership.
 ############################################################################
 
+import os
+import os.path
+
+from collections import defaultdict
 from datetime import datetime, timedelta
 
+import dns.message
+import dns.query
+import dns.rcode
+
 # ISO datetime format without msec
 fmt = '%Y-%m-%dT%H:%M:%SZ'
 
@@ -21,15 +29,18 @@ now = datetime.utcnow().replace(microsecond=0)
 dayzero = datetime.utcfromtimestamp(0).replace(microsecond=0)
 
 
+TIMEOUT = 10
+
+
 # Generic helper functions
-def check_expires(expires, min, max):
-    assert expires >= min
-    assert expires <= max
+def check_expires(expires, min_time, max_time):
+    assert expires >= min_time
+    assert expires <= max_time
 
 
-def check_refresh(refresh, min, max):
-    assert refresh >= min
-    assert refresh <= max
+def check_refresh(refresh, min_time, max_time):
+    assert refresh >= min_time
+    assert refresh <= max_time
 
 
 def check_loaded(loaded, expected):
@@ -47,12 +58,96 @@ def check_zone_timers(loaded, expires, refresh, loaded_exp):
     check_loaded(loaded, loaded_exp)
 
 
-def zone_mtime(zonedir, name):
-    import os
-    import os.path
-    from datetime import datetime
+#
+# The output is gibberish, but at least make sure it does not crash.
+#
+def check_manykeys(name, zone=None):
+    # pylint: disable=unused-argument
+    assert name == "manykeys"
+
+
+def zone_mtime(zonedir, name):
+
+    try:
+        si = os.stat(os.path.join(zonedir, "{}.db".format(name)))
+    except FileNotFoundError:
+        return dayzero
 
-    si = os.stat(os.path.join(zonedir, "{}.db".format(name)))
     mtime = datetime.utcfromtimestamp(si.st_mtime).replace(microsecond=0)
 
     return mtime
+
+
+def zone_keyid(nameserver, zone, key):
+    with open(f'{nameserver}/{zone}.{key}.id') as f:
+        keyid = f.read().strip()
+        print(f'{zone}-{key} ID: {keyid}')
+    return keyid
+
+
+def create_msg(qname, qtype):
+    msg = dns.message.make_query(qname, qtype, want_dnssec=True,
+                                 use_edns=0, payload=4096)
+
+    return msg
+
+
+def udp_query(ip, port, msg):
+
+    ans = dns.query.udp(msg, ip, TIMEOUT, port=port)
+    assert ans.rcode() == dns.rcode.NOERROR
+
+    return ans
+
+
+def tcp_query(ip, port, msg):
+
+    ans = dns.query.tcp(msg, ip, TIMEOUT, port=port)
+    assert ans.rcode() == dns.rcode.NOERROR
+
+    return ans
+
+
+def create_expected(data):
+    expected = {"dns-tcp-requests-sizes-received-ipv4": defaultdict(int),
+                "dns-tcp-responses-sizes-sent-ipv4": defaultdict(int),
+                "dns-tcp-requests-sizes-received-ipv6": defaultdict(int),
+                "dns-tcp-responses-sizes-sent-ipv6": defaultdict(int),
+                "dns-udp-requests-sizes-received-ipv4": defaultdict(int),
+                "dns-udp-requests-sizes-received-ipv6": defaultdict(int),
+                "dns-udp-responses-sizes-sent-ipv4": defaultdict(int),
+                "dns-udp-responses-sizes-sent-ipv6": defaultdict(int),
+                }
+
+    for k, v in data.items():
+        for kk, vv in v.items():
+            expected[k][kk] += vv
+
+    return expected
+
+
+def update_expected(expected, key, msg):
+    msg_len = len(msg.to_wire())
+    bucket_num = (msg_len // 16) * 16
+    bucket = "{}-{}".format(bucket_num, bucket_num + 15)
+
+    expected[key][bucket] += 1
+
+
+def check_traffic(data, expected):
+    def ordered(obj):
+        if isinstance(obj, dict):
+            return sorted((k, ordered(v)) for k, v in obj.items())
+        if isinstance(obj, list):
+            return sorted(ordered(x) for x in obj)
+        return obj
+
+    ordered_data = ordered(data)
+    ordered_expected = ordered(expected)
+
+    assert len(ordered_data) == 8
+    assert len(ordered_expected) == 8
+    assert len(data) == len(ordered_data)
+    assert len(expected) == len(ordered_expected)
+
+    assert ordered_data == ordered_expected
diff --git a/bin/tests/system/statschannel/tests-json.py b/bin/tests/system/statschannel/tests-json.py
index f9bd5ec78f..32a06c2a39 100755
--- a/bin/tests/system/statschannel/tests-json.py
+++ b/bin/tests/system/statschannel/tests-json.py
@@ -10,24 +10,37 @@
 # information regarding copyright ownership.
 ############################################################################
 
-import pytest
 from datetime import datetime
-from helper import fmt, zone_mtime, check_zone_timers, dayzero
+
+import pytest
+import requests
+
+import generic
+from helper import fmt
 
 
 # JSON helper functions
-def fetch_json(statsip, statsport):
-    import requests
+def fetch_zones_json(statsip, statsport):
 
     r = requests.get("http://{}:{}/json/v1/zones".format(statsip, statsport))
     assert r.status_code == 200
 
     data = r.json()
-
     return data["views"]["_default"]["zones"]
 
 
-def load_timers_from_json(zone, primary=True):
+def fetch_traffic_json(statsip, statsport):
+
+    r = requests.get("http://{}:{}/json/v1/traffic".format(statsip, statsport))
+    assert r.status_code == 200
+
+    data = r.json()
+
+    return data["traffic"]
+
+
+def load_timers_json(zone, primary=True):
+
     name = zone['name']
 
     # Check if the primary zone timer exists
@@ -49,27 +62,39 @@ def load_timers_from_json(zone, primary=True):
     return (name, loaded, expires, refresh)
 
 
+def load_zone_json(zone):
+    name = zone['name']
+
+    return name
+
+
 @pytest.mark.json
 @pytest.mark.requests
 def test_zone_timers_primary_json(statsport):
-    statsip = "10.53.0.1"
-    zonedir = "ns1"
-
-    zones = fetch_json(statsip, statsport)
-
-    for zone in zones:
-        (name, loaded, expires, refresh) = load_timers_from_json(zone, True)
-        mtime = zone_mtime(zonedir, name)
-        check_zone_timers(loaded, expires, refresh, mtime)
+    generic.test_zone_timers_primary(fetch_zones_json, load_timers_json,
+                                     statsip="10.53.0.1", statsport=statsport,
+                                     zonedir="ns1")
 
 
 @pytest.mark.json
 @pytest.mark.requests
 def test_zone_timers_secondary_json(statsport):
-    statsip = "10.53.0.3"
+    generic.test_zone_timers_secondary(fetch_zones_json, load_timers_json,
+                                       statsip="10.53.0.3", statsport=statsport,
+                                       zonedir="ns3")
 
-    zones = fetch_json(statsip, statsport)
 
-    for zone in zones:
-        (name, loaded, expires, refresh) = load_timers_from_json(zone, False)
-        check_zone_timers(loaded, expires, refresh, dayzero)
+@pytest.mark.json
+@pytest.mark.requests
+def test_zone_with_many_keys_json(statsport):
+    generic.test_zone_with_many_keys(fetch_zones_json, load_zone_json,
+                                     statsip="10.53.0.2", statsport=statsport)
+
+
+@pytest.mark.json
+@pytest.mark.requests
+@pytest.mark.dnspython
+def test_traffic_json(port, statsport):
+    generic.test_traffic(fetch_traffic_json,
+                         statsip="10.53.0.2", statsport=statsport,
+                         port=port)
diff --git a/bin/tests/system/statschannel/tests-xml.py b/bin/tests/system/statschannel/tests-xml.py
index dcd2d76598..f7f0608309 100755
--- a/bin/tests/system/statschannel/tests-xml.py
+++ b/bin/tests/system/statschannel/tests-xml.py
@@ -10,15 +10,18 @@
 # information regarding copyright ownership.
 ############################################################################
 
-import pytest
+import xml.etree.ElementTree as ET
 from datetime import datetime
-from helper import fmt, zone_mtime, check_zone_timers, dayzero
+
+import pytest
+import requests
+
+import generic
+from helper import fmt
 
 
 # XML helper functions
-def fetch_xml(statsip, statsport):
-    import xml.etree.ElementTree as ET
-    import requests
+def fetch_zones_xml(statsip, statsport):
 
     r = requests.get("http://{}:{}/xml/v3/zones".format(statsip, statsport))
     assert r.status_code == 200
@@ -35,7 +38,38 @@ def fetch_xml(statsip, statsport):
     return default_view.find('zones').findall('zone')
 
 
-def load_timers_from_xml(zone, primary=True):
+def fetch_traffic_xml(statsip, statsport):
+
+    def load_counters(data):
+        out = {}
+        for counter in data.findall("counter"):
+            out[counter.attrib['name']] = int(counter.text)
+
+        return out
+
+    r = requests.get("http://{}:{}/xml/v3/traffic".format(statsip, statsport))
+    assert r.status_code == 200
+
+    root = ET.fromstring(r.text)
+
+    traffic = {}
+    for ip in ["ipv4", "ipv6"]:
+        for proto in ["udp", "tcp"]:
+            proto_root = root.find("traffic").find(ip).find(proto)
+            for counters in proto_root.findall("counters"):
+                if counters.attrib['type'] == "request-size":
+                    key = "dns-{}-requests-sizes-received-{}".format(proto, ip)
+                else:
+                    key = "dns-{}-responses-sizes-sent-{}".format(proto, ip)
+
+                values = load_counters(counters)
+                traffic[key] = values
+
+    return traffic
+
+
+def load_timers_xml(zone, primary=True):
+
     name = zone.attrib['name']
 
     loaded_el = zone.find('loaded')
@@ -58,27 +92,39 @@ def load_timers_from_xml(zone, primary=True):
     return (name, loaded, expires, refresh)
 
 
+def load_zone_xml(zone):
+    name = zone.attrib['name']
+
+    return name
+
+
 @pytest.mark.xml
 @pytest.mark.requests
 def test_zone_timers_primary_xml(statsport):
-    statsip = "10.53.0.1"
-    zonedir = "ns1"
-
-    zones = fetch_xml(statsip, statsport)
-
-    for zone in zones:
-        (name, loaded, expires, refresh) = load_timers_from_xml(zone, True)
-        mtime = zone_mtime(zonedir, name)
-        check_zone_timers(loaded, expires, refresh, mtime)
+    generic.test_zone_timers_primary(fetch_zones_xml, load_timers_xml,
+                                     statsip="10.53.0.1", statsport=statsport,
+                                     zonedir="ns1")
 
 
 @pytest.mark.xml
 @pytest.mark.requests
 def test_zone_timers_secondary_xml(statsport):
-    statsip = "10.53.0.3"
+    generic.test_zone_timers_secondary(fetch_zones_xml, load_timers_xml,
+                                       statsip="10.53.0.3", statsport=statsport,
+                                       zonedir="ns3")
 
-    zones = fetch_xml(statsip, statsport)
 
-    for zone in zones:
-        (name, loaded, expires, refresh) = load_timers_from_xml(zone, False)
-        check_zone_timers(loaded, expires, refresh, dayzero)
+@pytest.mark.xml
+@pytest.mark.requests
+def test_zone_with_many_keys_xml(statsport):
+    generic.test_zone_with_many_keys(fetch_zones_xml, load_zone_xml,
+                                     statsip="10.53.0.2", statsport=statsport)
+
+
+@pytest.mark.xml
+@pytest.mark.requests
+@pytest.mark.dnspython
+def test_traffic_xml(port, statsport):
+    generic.test_traffic(fetch_traffic_xml,
+                         statsip="10.53.0.2", statsport=statsport,
+                         port=port)
diff --git a/bin/tests/system/statschannel/tests.sh b/bin/tests/system/statschannel/tests.sh
old mode 100755
new mode 100644
index aaa221b5d7..8bac41fa86
--- a/bin/tests/system/statschannel/tests.sh
+++ b/bin/tests/system/statschannel/tests.sh
@@ -45,21 +45,6 @@ if [ ! "$PERL_JSON" -a ! "$PERL_XML" ]; then
 fi
 
 
-gettraffic() {
-    sleep 1
-    echo_i "... using $1"
-    case $1 in
-        xml) path='xml/v3/traffic' ;;
-        json) path='json/v1/traffic' ;;
-        *) return 1 ;;
-    esac
-    file=`$PERL fetch.pl -p ${EXTRAPORT1} $path`
-    cp $file $file.$1.$2
-    $PERL traffic-${1}.pl $file 2>/dev/null | sort > traffic.out.$2
-    result=$?
-    return $result
-}
-
 getzones() {
     sleep 1
     echo_i "... using $1"
@@ -86,81 +71,6 @@ loadkeys_on() {
 
 status=0
 n=1
-ret=0
-echo_i "fetching traffic size data ($n)"
-if [ $PERL_XML ]; then
-    gettraffic xml x$n || ret=1
-    cmp traffic.out.x$n traffic.expect.$n || ret=1
-fi
-if [ $PERL_JSON ]; then
-    gettraffic json j$n || ret=1
-    cmp traffic.out.j$n traffic.expect.$n || ret=1
-fi
-if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
-n=`expr $n + 1`
-
-ret=0
-echo_i "fetching traffic size data after small UDP query ($n)"
-$DIGCMD short.example txt > dig.out.$n || ret=1
-if [ $PERL_XML ]; then
-    gettraffic xml x$n || ret=1
-    cmp traffic.out.x$n traffic.expect.$n || ret=1
-fi
-if [ $PERL_JSON ]; then
-    gettraffic json j$n || ret=1
-    cmp traffic.out.j$n traffic.expect.$n || ret=1
-fi
-if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
-n=`expr $n + 1`
-
-ret=0
-n=`expr $n + 1`
-echo_i "fetching traffic size data after large UDP query ($n)"
-$DIGCMD long.example txt > dig.out.$n || ret=1
-if [ $PERL_XML ]; then
-    gettraffic xml x$n || ret=1
-    cmp traffic.out.x$n traffic.expect.$n || ret=1
-fi
-if [ $PERL_JSON ]; then
-    gettraffic json j$n || ret=1
-    cmp traffic.out.j$n traffic.expect.$n || ret=1
-fi
-if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
-n=`expr $n + 1`
-
-ret=0
-echo_i "fetching traffic size data after small TCP query ($n)"
-$DIGCMD +tcp short.example txt > dig.out.$n || ret=1
-if [ $PERL_XML ]; then
-    gettraffic xml x$n || ret=1
-    cmp traffic.out.x$n traffic.expect.$n || ret=1
-fi
-if [ $PERL_JSON ]; then
-    gettraffic json j$n || ret=1
-    cmp traffic.out.j$n traffic.expect.$n || ret=1
-fi
-if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
-n=`expr $n + 1`
-
-ret=0
-echo_i "fetching traffic size data after large TCP query ($n)"
-$DIGCMD +tcp long.example txt > dig.out.$n || ret=1
-if [ $PERL_XML ]; then
-    gettraffic xml x$n || ret=1
-    cmp traffic.out.x$n traffic.expect.$n || ret=1
-fi
-if [ $PERL_JSON ]; then
-    gettraffic json j$n || ret=1
-    cmp traffic.out.j$n traffic.expect.$n || ret=1
-fi
-if [ $ret != 0 ]; then echo_i "failed"; fi
-status=`expr $status + $ret`
-n=`expr $n + 1`
-
 ret=0
 echo_i "checking consistency between named.stats and xml/json ($n)"
 rm -f ns2/named.stats
@@ -359,25 +269,6 @@ if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 n=`expr $n + 1`
 
-# 4. Test a zone with more than four keys.
-zone="manykeys"
-ksk8_id=`cat ns2/$zone.ksk8.id`
-zsk8_id=`cat ns2/$zone.zsk8.id`
-ksk13_id=`cat ns2/$zone.ksk13.id`
-zsk13_id=`cat ns2/$zone.zsk13.id`
-ksk14_id=`cat ns2/$zone.ksk14.id`
-zsk14_id=`cat ns2/$zone.zsk14.id`
-
-ret=0
-echo_i "fetch zone stats data for a zone with many keys ($n)"
-# Fetch and check the dnssec sign statistics.
-if [ $PERL_XML ]; then
-    getzones xml $zone x$n || ret=1
-fi
-if [ $PERL_JSON ]; then
-    getzones json $zone j$n || ret=1
-fi
-# The output is gibberish, but at least make sure it does not crash.
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 n=`expr $n + 1`
diff --git a/doc/man/named.conf.5in b/doc/man/named.conf.5in
index 512d2fc59a..98266cd82c 100644
--- a/doc/man/named.conf.5in
+++ b/doc/man/named.conf.5in
@@ -288,15 +288,12 @@ options {
       dnssec\-secure\-to\-insecure boolean;
       dnssec\-update\-mode ( maintain | no\-resign );
       dnssec\-validation ( yes | no | auto );
-      dnstap { ( all | auth | client | forwarder |
-          resolver | update ) [ ( query | response ) ];
-          ... };
-      dnstap\-identity ( quoted_string | none |
-          hostname );
-      dnstap\-output ( file | unix ) quoted_string [
-          size ( unlimited | size ) ] [ versions (
-          unlimited | integer ) ] [ suffix ( increment
-          | timestamp ) ];
+      dnstap { ( all | auth | client | forwarder | resolver | update ) [
+          ( query | response ) ]; ... };
+      dnstap\-identity ( quoted_string | none | hostname );
+      dnstap\-output ( file | unix ) quoted_string [ size ( unlimited |
+          size ) ] [ versions ( unlimited | integer ) ] [ suffix (
+          increment | timestamp ) ];
       dnstap\-version ( quoted_string | none );
       dscp integer;
       dual\-stack\-servers [ port integer ] { ( quoted_string [ port
@@ -686,9 +683,8 @@ view string [ class ] {
       dnssec\-secure\-to\-insecure boolean;
       dnssec\-update\-mode ( maintain | no\-resign );
       dnssec\-validation ( yes | no | auto );
-      dnstap { ( all | auth | client | forwarder |
-          resolver | update ) [ ( query | response ) ];
-          ... };
+      dnstap { ( all | auth | client | forwarder | resolver | update ) [
+          ( query | response ) ]; ... };
       dual\-stack\-servers [ port integer ] { ( quoted_string [ port
           integer ] [ dscp integer ] | ipv4_address [ port
           integer ] [ dscp integer ] | ipv6_address [ port
diff --git a/doc/misc/options b/doc/misc/options
index 5161474545..ea9242ef67 100644
--- a/doc/misc/options
+++ b/doc/misc/options
@@ -166,16 +166,13 @@ options {
         dnssec-secure-to-insecure <boolean>;
         dnssec-update-mode ( maintain | no-resign );
         dnssec-validation ( yes | no | auto );
-        dnstap { ( all | auth | client | forwarder |
-            resolver | update ) [ ( query | response ) ];
-            ... }; // not configured
-        dnstap-identity ( <quoted_string> | none |
-            hostname ); // not configured
-        dnstap-output ( file | unix ) <quoted_string> [
-            size ( unlimited | <size> ) ] [ versions (
-            unlimited | <integer> ) ] [ suffix ( increment
-            | timestamp ) ]; // not configured
-        dnstap-version ( <quoted_string> | none ); // not configured
+        dnstap { ( all | auth | client | forwarder | resolver | update ) [
+            ( query | response ) ]; ... };
+        dnstap-identity ( <quoted_string> | none | hostname );
+        dnstap-output ( file | unix ) <quoted_string> [ size ( unlimited |
+            <size> ) ] [ versions ( unlimited | <integer> ) ] [ suffix (
+            increment | timestamp ) ];
+        dnstap-version ( <quoted_string> | none );
         dscp <integer>;
         dual-stack-servers [ port <integer> ] { ( <quoted_string> [ port
             <integer> ] [ dscp <integer> ] | <ipv4_address> [ port
@@ -199,13 +196,13 @@ options {
         forward ( first | only );
         forwarders [ port <integer> ] [ dscp <integer> ] { ( <ipv4_address>
             | <ipv6_address> ) [ port <integer> ] [ dscp <integer> ]; ... };
-        fstrm-set-buffer-hint <integer>; // not configured
-        fstrm-set-flush-timeout <integer>; // not configured
-        fstrm-set-input-queue-size <integer>; // not configured
-        fstrm-set-output-notify-threshold <integer>; // not configured
-        fstrm-set-output-queue-model ( mpsc | spsc ); // not configured
-        fstrm-set-output-queue-size <integer>; // not configured
-        fstrm-set-reopen-interval <duration>; // not configured
+        fstrm-set-buffer-hint <integer>;
+        fstrm-set-flush-timeout <integer>;
+        fstrm-set-input-queue-size <integer>;
+        fstrm-set-output-notify-threshold <integer>;
+        fstrm-set-output-queue-model ( mpsc | spsc );
+        fstrm-set-output-queue-size <integer>;
+        fstrm-set-reopen-interval <duration>;
         geoip-directory ( <quoted_string> | none );
         geoip-use-ecs <boolean>; // obsolete
         glue-cache <boolean>;
@@ -550,9 +547,8 @@ view <string> [ <class> ] {
         dnssec-secure-to-insecure <boolean>;
         dnssec-update-mode ( maintain | no-resign );
         dnssec-validation ( yes | no | auto );
-        dnstap { ( all | auth | client | forwarder |
-            resolver | update ) [ ( query | response ) ];
-            ... }; // not configured
+        dnstap { ( all | auth | client | forwarder | resolver | update ) [
+            ( query | response ) ]; ... };
         dual-stack-servers [ port <integer> ] { ( <quoted_string> [ port
             <integer> ] [ dscp <integer> ] | <ipv4_address> [ port
             <integer> ] [ dscp <integer> ] | <ipv6_address> [ port
diff --git a/doc/misc/options.active b/doc/misc/options.active
index 3ce6b6cb61..8abd4e1f5d 100644
--- a/doc/misc/options.active
+++ b/doc/misc/options.active
@@ -153,16 +153,13 @@ options {
         dnssec-secure-to-insecure <boolean>;
         dnssec-update-mode ( maintain | no-resign );
         dnssec-validation ( yes | no | auto );
-        dnstap { ( all | auth | client | forwarder |
-            resolver | update ) [ ( query | response ) ];
-            ... }; // not configured
-        dnstap-identity ( <quoted_string> | none |
-            hostname ); // not configured
-        dnstap-output ( file | unix ) <quoted_string> [
-            size ( unlimited | <size> ) ] [ versions (
-            unlimited | <integer> ) ] [ suffix ( increment
-            | timestamp ) ]; // not configured
-        dnstap-version ( <quoted_string> | none ); // not configured
+        dnstap { ( all | auth | client | forwarder | resolver | update ) [
+            ( query | response ) ]; ... };
+        dnstap-identity ( <quoted_string> | none | hostname );
+        dnstap-output ( file | unix ) <quoted_string> [ size ( unlimited |
+            <size> ) ] [ versions ( unlimited | <integer> ) ] [ suffix (
+            increment | timestamp ) ];
+        dnstap-version ( <quoted_string> | none );
         dscp <integer>;
         dual-stack-servers [ port <integer> ] { ( <quoted_string> [ port
             <integer> ] [ dscp <integer> ] | <ipv4_address> [ port
@@ -181,13 +178,13 @@ options {
         forward ( first | only );
         forwarders [ port <integer> ] [ dscp <integer> ] { ( <ipv4_address>
             | <ipv6_address> ) [ port <integer> ] [ dscp <integer> ]; ... };
-        fstrm-set-buffer-hint <integer>; // not configured
-        fstrm-set-flush-timeout <integer>; // not configured
-        fstrm-set-input-queue-size <integer>; // not configured
-        fstrm-set-output-notify-threshold <integer>; // not configured
-        fstrm-set-output-queue-model ( mpsc | spsc ); // not configured
-        fstrm-set-output-queue-size <integer>; // not configured
-        fstrm-set-reopen-interval <duration>; // not configured
+        fstrm-set-buffer-hint <integer>;
+        fstrm-set-flush-timeout <integer>;
+        fstrm-set-input-queue-size <integer>;
+        fstrm-set-output-notify-threshold <integer>;
+        fstrm-set-output-queue-model ( mpsc | spsc );
+        fstrm-set-output-queue-size <integer>;
+        fstrm-set-reopen-interval <duration>;
         geoip-directory ( <quoted_string> | none );
         glue-cache <boolean>;
         heartbeat-interval <integer>;
@@ -495,9 +492,8 @@ view <string> [ <class> ] {
         dnssec-secure-to-insecure <boolean>;
         dnssec-update-mode ( maintain | no-resign );
         dnssec-validation ( yes | no | auto );
-        dnstap { ( all | auth | client | forwarder |
-            resolver | update ) [ ( query | response ) ];
-            ... }; // not configured
+        dnstap { ( all | auth | client | forwarder | resolver | update ) [
+            ( query | response ) ]; ... };
         dual-stack-servers [ port <integer> ] { ( <quoted_string> [ port
             <integer> ] [ dscp <integer> ] | <ipv4_address> [ port
             <integer> ] [ dscp <integer> ] | <ipv6_address> [ port
diff --git a/doc/misc/options.grammar.rst b/doc/misc/options.grammar.rst
index d99a89fbcc..f7d9657b75 100644
--- a/doc/misc/options.grammar.rst
+++ b/doc/misc/options.grammar.rst
@@ -82,15 +82,12 @@
   	dnssec-secure-to-insecure <boolean>;
   	dnssec-update-mode ( maintain | no-resign );
   	dnssec-validation ( yes | no | auto );
-  	dnstap { ( all | auth | client | forwarder |
-  	    resolver | update ) [ ( query | response ) ];
-  	    ... };
-  	dnstap-identity ( <quoted_string> | none |
-  	    hostname );
-  	dnstap-output ( file | unix ) <quoted_string> [
-  	    size ( unlimited | <size> ) ] [ versions (
-  	    unlimited | <integer> ) ] [ suffix ( increment
-  	    | timestamp ) ];
+  	dnstap { ( all | auth | client | forwarder | resolver | update ) [
+  	    ( query | response ) ]; ... };
+  	dnstap-identity ( <quoted_string> | none | hostname );
+  	dnstap-output ( file | unix ) <quoted_string> [ size ( unlimited |
+  	    <size> ) ] [ versions ( unlimited | <integer> ) ] [ suffix (
+  	    increment | timestamp ) ];
   	dnstap-version ( <quoted_string> | none );
   	dscp <integer>;
   	dual-stack-servers [ port <integer> ] { ( <quoted_string> [ port
diff --git a/util/copyrights b/util/copyrights
index 40db450438..4aecf882c5 100644
--- a/util/copyrights
+++ b/util/copyrights
@@ -810,6 +810,7 @@
 ./bin/tests/system/statschannel/clean.sh	SH	2015,2016,2017,2018,2019,2020
 ./bin/tests/system/statschannel/conftest.py	PYTHON	2020
 ./bin/tests/system/statschannel/fetch.pl	PERL	2015,2016,2018,2019,2020
+./bin/tests/system/statschannel/generic.py	PYTHON	2020
 ./bin/tests/system/statschannel/helper.py	PYTHON	2020
 ./bin/tests/system/statschannel/mem-xml.pl	PERL	2017,2018,2019,2020
 ./bin/tests/system/statschannel/ns2/sign.sh	SH	2019,2020

From 0fe57533155ee6cbc1befc751636d5a9d7b7f7f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= <ondrej@isc.org>
Date: Mon, 11 May 2020 17:43:12 +0200
Subject: [PATCH 5/7] Use the result of clean.sh and setup.sh command directly

---
 bin/tests/system/run.sh.in | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/bin/tests/system/run.sh.in b/bin/tests/system/run.sh.in
index 4585cd039f..6217160357 100644
--- a/bin/tests/system/run.sh.in
+++ b/bin/tests/system/run.sh.in
@@ -185,10 +185,8 @@ fi
 # Clean up files left from any potential previous runs
 if test -f "$systest/clean.sh"
 then
-    ( cd "${systest}" && $SHELL clean.sh "$@" )
-    ret=$?
-    if [ $ret -ne 0 ]; then
-	echowarn "I:$systest:clean.sh script failed with $ret"
+    if ! ( cd "${systest}" && $SHELL clean.sh "$@" ); then
+	echowarn "I:$systest:clean.sh script failed"
     fi
 
 fi
@@ -196,10 +194,8 @@ fi
 # Set up any dynamically generated test data
 if test -f "$systest/setup.sh"
 then
-    ( cd "${systest}" && $SHELL setup.sh "$@" )
-    ret=$?
-    if [ $ret -ne 0 ]; then
-	echowarn "I:$systest:clean.sh script failed with $ret"
+    if ! ( cd "${systest}" && $SHELL setup.sh "$@" ); then
+	echowarn "I:$systest:clean.sh script failed"
     fi
 fi
 

From e0aa62bc4c698a7c9b2cf9689d618e767070931b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= <ondrej@sury.org>
Date: Thu, 31 Oct 2019 11:04:12 -0500
Subject: [PATCH 6/7] Add release notes

---
 doc/arm/notes-9.17.2.xml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/arm/notes-9.17.2.xml b/doc/arm/notes-9.17.2.xml
index 173c7ad946..193c13c2c1 100644
--- a/doc/arm/notes-9.17.2.xml
+++ b/doc/arm/notes-9.17.2.xml
@@ -88,6 +88,14 @@
           actions to be logged into a separate channel. [GL #54]
         </para>
       </listitem>
+      <listitem>
+        <para>
+          The zone timers are now exported to the statistics channel.  For the
+          primary zones, only the loaded time is exported.  For the secondary
+          zones, the exported timers also include expire and refresh times.
+          Contributed by Paul Frieden, Verizon Media. [GL #1232]
+        </para>
+      </listitem>
     </itemizedlist>
   </section>
 

From ce6cb62862ec948abbee10e5ed3efde6897ca86a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= <ondrej@sury.org>
Date: Thu, 31 Oct 2019 11:05:12 -0500
Subject: [PATCH 7/7] Add CHANGES

---
 CHANGES | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGES b/CHANGES
index 1d188bfff5..44b2e55848 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,7 @@
+5407.	[func]		The zone timers are now exported to the statistics
+			channel.  Thanks to Paul Frieden, Verizon Media.
+			[GL #1232]
+
 5406.	[func]		Added a new logging category "rpz-passthru". It allows
 			RPZ passthru actions to be logged into a separate
 			channel. [GL #54]