diff --git a/CHANGES b/CHANGES index 1d188bfff5..44b2e55848 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +5407. [func] The zone timers are now exported to the statistics + channel. Thanks to Paul Frieden, Verizon Media. + [GL #1232] + 5406. [func] Added a new logging category "rpz-passthru". It allows RPZ passthru actions to be logged into a separate channel. [GL #54] diff --git a/bin/named/bind9.xsl b/bin/named/bind9.xsl index f36927794f..ba0ad27487 100644 --- a/bin/named/bind9.xsl +++ b/bin/named/bind9.xsl @@ -775,7 +775,7 @@

Zones for View

- + @@ -788,7 +788,10 @@ - + + + +
NameClassTypeSerial
NameClassTypeSerialLoadedExpiresRefresh
diff --git a/bin/named/statschannel.c b/bin/named/statschannel.c index b702915ada..59fabdea89 100644 --- a/bin/named/statschannel.c +++ b/bin/named/statschannel.c @@ -1809,6 +1809,43 @@ zone_xmlrender(dns_zone_t *zone, void *arg) { } TRY0(xmlTextWriterEndElement(writer)); /* serial */ + /* + * Export zone timers to the statistics channel in XML format. For + * master zones, only include the loaded time. For slave zones, also + * include the expires and refresh times. + */ + isc_time_t timestamp; + + result = dns_zone_getloadtime(zone, ×tamp); + if (result != ISC_R_SUCCESS) { + goto error; + } + + isc_time_formatISO8601(×tamp, buf, 64); + TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "loaded")); + TRY0(xmlTextWriterWriteString(writer, ISC_XMLCHAR buf)); + TRY0(xmlTextWriterEndElement(writer)); + + if (dns_zone_gettype(zone) == dns_zone_slave) { + result = dns_zone_getexpiretime(zone, ×tamp); + if (result != ISC_R_SUCCESS) { + goto error; + } + isc_time_formatISO8601(×tamp, buf, 64); + TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "expires")); + TRY0(xmlTextWriterWriteString(writer, ISC_XMLCHAR buf)); + TRY0(xmlTextWriterEndElement(writer)); + + result = dns_zone_getrefreshtime(zone, ×tamp); + if (result != ISC_R_SUCCESS) { + goto error; + } + isc_time_formatISO8601(×tamp, buf, 64); + TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "refresh")); + TRY0(xmlTextWriterWriteString(writer, ISC_XMLCHAR buf)); + TRY0(xmlTextWriterEndElement(writer)); + } + if (statlevel == dns_zonestat_full) { isc_stats_t *zonestats; isc_stats_t *gluecachestats; @@ -2619,6 +2656,40 @@ zone_jsonrender(dns_zone_t *zone, void *arg) { return (ISC_R_NOMEMORY); } + /* + * Export zone timers to the statistics channel in JSON format. For + * master zones, only include the loaded time. For slave zones, also + * include the expires and refresh times. + */ + + isc_time_t timestamp; + + result = dns_zone_getloadtime(zone, ×tamp); + if (result != ISC_R_SUCCESS) { + goto error; + } + + isc_time_formatISO8601(×tamp, buf, 64); + json_object_object_add(zoneobj, "loaded", json_object_new_string(buf)); + + if (dns_zone_gettype(zone) == dns_zone_slave) { + result = dns_zone_getexpiretime(zone, ×tamp); + if (result != ISC_R_SUCCESS) { + goto error; + } + isc_time_formatISO8601(×tamp, buf, 64); + json_object_object_add(zoneobj, "expires", + json_object_new_string(buf)); + + result = dns_zone_getrefreshtime(zone, ×tamp); + if (result != ISC_R_SUCCESS) { + goto error; + } + isc_time_formatISO8601(×tamp, buf, 64); + json_object_object_add(zoneobj, "refresh", + json_object_new_string(buf)); + } + if (statlevel == dns_zonestat_full) { isc_stats_t *zonestats; isc_stats_t *gluecachestats; diff --git a/bin/tests/system/run.sh.in b/bin/tests/system/run.sh.in index 4585cd039f..6217160357 100644 --- a/bin/tests/system/run.sh.in +++ b/bin/tests/system/run.sh.in @@ -185,10 +185,8 @@ fi # Clean up files left from any potential previous runs if test -f "$systest/clean.sh" then - ( cd "${systest}" && $SHELL clean.sh "$@" ) - ret=$? - if [ $ret -ne 0 ]; then - echowarn "I:$systest:clean.sh script failed with $ret" + if ! ( cd "${systest}" && $SHELL clean.sh "$@" ); then + echowarn "I:$systest:clean.sh script failed" fi fi @@ -196,10 +194,8 @@ fi # Set up any dynamically generated test data if test -f "$systest/setup.sh" then - ( cd "${systest}" && $SHELL setup.sh "$@" ) - ret=$? - if [ $ret -ne 0 ]; then - echowarn "I:$systest:clean.sh script failed with $ret" + if ! ( cd "${systest}" && $SHELL setup.sh "$@" ); then + echowarn "I:$systest:clean.sh script failed" fi fi diff --git a/bin/tests/system/statschannel/.gitignore b/bin/tests/system/statschannel/.gitignore new file mode 100644 index 0000000000..44fb46c3c0 --- /dev/null +++ b/bin/tests/system/statschannel/.gitignore @@ -0,0 +1,2 @@ +/.cache/ +/__pycache__/ diff --git a/bin/tests/system/statschannel/clean.sh b/bin/tests/system/statschannel/clean.sh index c9edbde9ee..4904d91e43 100644 --- a/bin/tests/system/statschannel/clean.sh +++ b/bin/tests/system/statschannel/clean.sh @@ -12,9 +12,9 @@ rm -f traffic traffic.out.* traffic.json.* traffic.xml.* rm -f zones zones.out.* zones.json.* zones.xml.* zones.expect.* rm -f dig.out* -rm -f */named.memstats -rm -f */named.conf -rm -f */named.run* +rm -f ns*/named.memstats +rm -f ns*/named.conf +rm -f ns*/named.run* rm -f ns*/named.lock rm -f ns*/named.stats rm -f xml.*stats json.*stats @@ -24,4 +24,6 @@ rm -f ns*/managed-keys.bind* rm -f ns2/Kdnssec* ns2/dnssec.*.id rm -f ns2/Kmanykeys* ns2/manykeys.*.id rm -f ns2/*.db.signed* ns2/dsset-*. ns2/*.jbk -rm -f ns2/core +rm -f ns2/dnssec.db.signed* ns2/dsset-dnssec. +rm -f ns3/*.db +rm -rf ./.cache ./__pycache__ diff --git a/bin/tests/system/statschannel/conftest.py b/bin/tests/system/statschannel/conftest.py new file mode 100644 index 0000000000..ebb4d1e6f5 --- /dev/null +++ b/bin/tests/system/statschannel/conftest.py @@ -0,0 +1,107 @@ +############################################################################ +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. +############################################################################ + +import os +import pytest + + +def pytest_configure(config): + config.addinivalue_line( + "markers", "requests: mark tests that need requests to function" + ) + config.addinivalue_line( + "markers", "json: mark tests that need json to function" + ) + config.addinivalue_line( + "markers", "xml: mark tests that need xml.etree to function" + ) + config.addinivalue_line( + "markers", "dnspython: mark tests that need dnspython to function" + ) + + +def pytest_collection_modifyitems(config, items): + # pylint: disable=unused-argument,unused-import,too-many-branches + # pylint: disable=import-outside-toplevel + # Test for requests module + skip_requests = pytest.mark.skip( + reason="need requests module to run") + try: + import requests # noqa: F401 + except ModuleNotFoundError: + for item in items: + if "requests" in item.keywords: + item.add_marker(skip_requests) + # Test for json module + skip_json = pytest.mark.skip( + reason="need json module to run") + try: + import json # noqa: F401 + except ModuleNotFoundError: + for item in items: + if "json" in item.keywords: + item.add_marker(skip_json) + # Test for xml module + skip_xml = pytest.mark.skip( + reason="need xml module to run") + try: + import xml.etree.ElementTree # noqa: F401 + except ModuleNotFoundError: + for item in items: + if "xml" in item.keywords: + item.add_marker(skip_xml) + # Test if JSON statistics channel was enabled + no_jsonstats = pytest.mark.skip( + reason="need JSON statistics to be enabled") + if os.getenv("HAVEJSONSTATS") is None: + for item in items: + if "json" in item.keywords: + item.add_marker(no_jsonstats) + # Test if XML statistics channel was enabled + no_xmlstats = pytest.mark.skip( + reason="need XML statistics to be enabled") + if os.getenv("HAVEXMLSTATS") is None: + for item in items: + if "xml" in item.keywords: + item.add_marker(no_xmlstats) + # Test for dnspython module + skip_dnspython = pytest.mark.skip( + reason="need dnspython module to run") + try: + import dns.query # noqa: F401 + except ModuleNotFoundError: + for item in items: + if "dnspython" in item.keywords: + item.add_marker(skip_dnspython) + + +@pytest.fixture +def statsport(request): + # pylint: disable=unused-argument + env_port = os.getenv("EXTRAPORT1") + if port is None: + env_port = 5301 + else: + env_port = int(env_port) + + return env_port + + +@pytest.fixture +def port(request): + # pylint: disable=unused-argument + env_port = os.getenv("PORT") + if port is None: + env_port = 5300 + else: + env_port = int(env_port) + + return env_port diff --git a/bin/tests/system/statschannel/generic.py b/bin/tests/system/statschannel/generic.py new file mode 100644 index 0000000000..d927793cc2 --- /dev/null +++ b/bin/tests/system/statschannel/generic.py @@ -0,0 +1,95 @@ +############################################################################ +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. +############################################################################ + +import helper + + +def test_zone_timers_primary(fetch_zones, load_timers, **kwargs): + + statsip = kwargs['statsip'] + statsport = kwargs['statsport'] + zonedir = kwargs['zonedir'] + + zones = fetch_zones(statsip, statsport) + + for zone in zones: + (name, loaded, expires, refresh) = load_timers(zone, True) + mtime = helper.zone_mtime(zonedir, name) + helper.check_zone_timers(loaded, expires, refresh, mtime) + + +def test_zone_timers_secondary(fetch_zones, load_timers, **kwargs): + + statsip = kwargs['statsip'] + statsport = kwargs['statsport'] + zonedir = kwargs['zonedir'] + + zones = fetch_zones(statsip, statsport) + + for zone in zones: + (name, loaded, expires, refresh) = load_timers(zone, False) + mtime = helper.zone_mtime(zonedir, name) + helper.check_zone_timers(loaded, expires, refresh, mtime) + + +def test_zone_with_many_keys(fetch_zones, load_zone, **kwargs): + + statsip = kwargs['statsip'] + statsport = kwargs['statsport'] + + zones = fetch_zones(statsip, statsport) + + for zone in zones: + name = load_zone(zone) + if name == 'manykeys': + helper.check_manykeys(name) + + +def test_traffic(fetch_traffic, **kwargs): + + statsip = kwargs['statsip'] + statsport = kwargs['statsport'] + port = kwargs['port'] + + data = fetch_traffic(statsip, statsport) + exp = helper.create_expected(data) + + msg = helper.create_msg("short.example.", "TXT") + helper.update_expected(exp, "dns-udp-requests-sizes-received-ipv4", msg) + ans = helper.udp_query(statsip, port, msg) + helper.update_expected(exp, "dns-udp-responses-sizes-sent-ipv4", ans) + data = fetch_traffic(statsip, statsport) + + helper.check_traffic(data, exp) + + msg = helper.create_msg("long.example.", "TXT") + helper.update_expected(exp, "dns-udp-requests-sizes-received-ipv4", msg) + ans = helper.udp_query(statsip, port, msg) + helper.update_expected(exp, "dns-udp-responses-sizes-sent-ipv4", ans) + data = fetch_traffic(statsip, statsport) + + helper.check_traffic(data, exp) + + msg = helper.create_msg("short.example.", "TXT") + helper.update_expected(exp, "dns-tcp-requests-sizes-received-ipv4", msg) + ans = helper.tcp_query(statsip, port, msg) + helper.update_expected(exp, "dns-tcp-responses-sizes-sent-ipv4", ans) + data = fetch_traffic(statsip, statsport) + + helper.check_traffic(data, exp) + + msg = helper.create_msg("long.example.", "TXT") + helper.update_expected(exp, "dns-tcp-requests-sizes-received-ipv4", msg) + ans = helper.tcp_query(statsip, port, msg) + helper.update_expected(exp, "dns-tcp-responses-sizes-sent-ipv4", ans) + data = fetch_traffic(statsip, statsport) + + helper.check_traffic(data, exp) diff --git a/bin/tests/system/statschannel/helper.py b/bin/tests/system/statschannel/helper.py new file mode 100644 index 0000000000..7b53e9e9fe --- /dev/null +++ b/bin/tests/system/statschannel/helper.py @@ -0,0 +1,153 @@ +############################################################################ +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. +############################################################################ + +import os +import os.path + +from collections import defaultdict +from datetime import datetime, timedelta + +import dns.message +import dns.query +import dns.rcode + +# ISO datetime format without msec +fmt = '%Y-%m-%dT%H:%M:%SZ' + +# The constants were taken from BIND 9 source code (lib/dns/zone.c) +max_refresh = timedelta(seconds=2419200) # 4 weeks +max_expires = timedelta(seconds=14515200) # 24 weeks +now = datetime.utcnow().replace(microsecond=0) +dayzero = datetime.utcfromtimestamp(0).replace(microsecond=0) + + +TIMEOUT = 10 + + +# Generic helper functions +def check_expires(expires, min_time, max_time): + assert expires >= min_time + assert expires <= max_time + + +def check_refresh(refresh, min_time, max_time): + assert refresh >= min_time + assert refresh <= max_time + + +def check_loaded(loaded, expected): + # Sanity check the zone timers values + assert loaded == expected + assert loaded < now + + +def check_zone_timers(loaded, expires, refresh, loaded_exp): + # Sanity checks the zone timers values + if expires is not None: + check_expires(expires, now, now + max_expires) + if refresh is not None: + check_refresh(refresh, now, now + max_refresh) + check_loaded(loaded, loaded_exp) + + +# +# The output is gibberish, but at least make sure it does not crash. +# +def check_manykeys(name, zone=None): + # pylint: disable=unused-argument + assert name == "manykeys" + + +def zone_mtime(zonedir, name): + + try: + si = os.stat(os.path.join(zonedir, "{}.db".format(name))) + except FileNotFoundError: + return dayzero + + mtime = datetime.utcfromtimestamp(si.st_mtime).replace(microsecond=0) + + return mtime + + +def zone_keyid(nameserver, zone, key): + with open(f'{nameserver}/{zone}.{key}.id') as f: + keyid = f.read().strip() + print(f'{zone}-{key} ID: {keyid}') + return keyid + + +def create_msg(qname, qtype): + msg = dns.message.make_query(qname, qtype, want_dnssec=True, + use_edns=0, payload=4096) + + return msg + + +def udp_query(ip, port, msg): + + ans = dns.query.udp(msg, ip, TIMEOUT, port=port) + assert ans.rcode() == dns.rcode.NOERROR + + return ans + + +def tcp_query(ip, port, msg): + + ans = dns.query.tcp(msg, ip, TIMEOUT, port=port) + assert ans.rcode() == dns.rcode.NOERROR + + return ans + + +def create_expected(data): + expected = {"dns-tcp-requests-sizes-received-ipv4": defaultdict(int), + "dns-tcp-responses-sizes-sent-ipv4": defaultdict(int), + "dns-tcp-requests-sizes-received-ipv6": defaultdict(int), + "dns-tcp-responses-sizes-sent-ipv6": defaultdict(int), + "dns-udp-requests-sizes-received-ipv4": defaultdict(int), + "dns-udp-requests-sizes-received-ipv6": defaultdict(int), + "dns-udp-responses-sizes-sent-ipv4": defaultdict(int), + "dns-udp-responses-sizes-sent-ipv6": defaultdict(int), + } + + for k, v in data.items(): + for kk, vv in v.items(): + expected[k][kk] += vv + + return expected + + +def update_expected(expected, key, msg): + msg_len = len(msg.to_wire()) + bucket_num = (msg_len // 16) * 16 + bucket = "{}-{}".format(bucket_num, bucket_num + 15) + + expected[key][bucket] += 1 + + +def check_traffic(data, expected): + def ordered(obj): + if isinstance(obj, dict): + return sorted((k, ordered(v)) for k, v in obj.items()) + if isinstance(obj, list): + return sorted(ordered(x) for x in obj) + return obj + + ordered_data = ordered(data) + ordered_expected = ordered(expected) + + assert len(ordered_data) == 8 + assert len(ordered_expected) == 8 + assert len(data) == len(ordered_data) + assert len(expected) == len(ordered_expected) + + assert ordered_data == ordered_expected diff --git a/bin/tests/system/statschannel/ns1/example.db b/bin/tests/system/statschannel/ns1/example.db new file mode 100644 index 0000000000..b65651aa2d --- /dev/null +++ b/bin/tests/system/statschannel/ns1/example.db @@ -0,0 +1,47 @@ +; Copyright (C) Internet Systems Consortium, Inc. ("ISC") +; +; This Source Code Form is subject to the terms of the Mozilla Public +; License, v. 2.0. If a copy of the MPL was not distributed with this +; file, You can obtain one at http://mozilla.org/MPL/2.0/. +; +; See the COPYRIGHT file distributed with this work for additional +; information regarding copyright ownership. + +$ORIGIN . +$TTL 300 ; 5 minutes +example IN SOA mname1. . ( + 1 ; serial + 20 ; refresh (20 seconds) + 20 ; retry (20 seconds) + 1814400 ; expire (3 weeks) + 3600 ; minimum (1 hour) + ) +example. NS ns2.example. +ns2.example. A 10.53.0.2 + +$ORIGIN example. +a A 10.0.0.1 + MX 10 mail.example. +short TXT "short text" +long TXT ( + "longlonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglong" + "longlonglonglonglonglonglonglonglonglong" + ) + +mail A 10.0.0.2 diff --git a/bin/tests/system/statschannel/ns1/named.conf.in b/bin/tests/system/statschannel/ns1/named.conf.in new file mode 100644 index 0000000000..fa1cd57a6c --- /dev/null +++ b/bin/tests/system/statschannel/ns1/named.conf.in @@ -0,0 +1,41 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +options { + query-source address 10.53.0.1; + notify-source 10.53.0.1; + transfer-source 10.53.0.1; + port @PORT@; + pid-file "named.pid"; + listen-on { 10.53.0.1; }; + listen-on-v6 { none; }; + recursion no; + notify explicit; + minimal-responses no; + version none; // make statistics independent of the version number +}; + +statistics-channels { inet 10.53.0.1 port @EXTRAPORT1@ allow { localhost; }; }; + +key rndc_key { + secret "1234abcd8765"; + algorithm hmac-sha256; +}; + +controls { + inet 10.53.0.1 port @CONTROLPORT@ allow { any; } keys { rndc_key; }; +}; + +zone "example" { + type master; + file "example.db"; + allow-transfer { any; }; +}; diff --git a/bin/tests/system/statschannel/ns2/named.conf.in b/bin/tests/system/statschannel/ns2/named.conf.in index 70aadf8b28..9eebe560d3 100644 --- a/bin/tests/system/statschannel/ns2/named.conf.in +++ b/bin/tests/system/statschannel/ns2/named.conf.in @@ -18,7 +18,7 @@ options { listen-on { 10.53.0.2; }; listen-on-v6 { none; }; recursion no; - notify yes; + notify no; minimal-responses no; version none; // make statistics independent of the version number }; diff --git a/bin/tests/system/statschannel/ns3/named.conf.in b/bin/tests/system/statschannel/ns3/named.conf.in new file mode 100644 index 0000000000..e78cff9a7a --- /dev/null +++ b/bin/tests/system/statschannel/ns3/named.conf.in @@ -0,0 +1,41 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +options { + query-source address 10.53.0.3; + notify-source 10.53.0.3; + transfer-source 10.53.0.3; + port @PORT@; + pid-file "named.pid"; + listen-on { 10.53.0.3; }; + listen-on-v6 { none; }; + recursion no; + notify no; + minimal-responses no; + version none; // make statistics independent of the version number +}; + +statistics-channels { inet 10.53.0.3 port @EXTRAPORT1@ allow { localhost; }; }; + +key rndc_key { + secret "1234abcd8765"; + algorithm hmac-sha256; +}; + +controls { + inet 10.53.0.3 port @CONTROLPORT@ allow { any; } keys { rndc_key; }; +}; + +zone "example" { + type secondary; + file "example.db"; + masters { 10.53.0.1; }; +}; diff --git a/bin/tests/system/statschannel/setup.sh b/bin/tests/system/statschannel/setup.sh index 3a8577eb72..2ce6fde81d 100644 --- a/bin/tests/system/statschannel/setup.sh +++ b/bin/tests/system/statschannel/setup.sh @@ -12,9 +12,8 @@ # shellcheck source=conf.sh . "$SYSTEMTESTTOP/conf.sh" -copy_setports ns2/named.conf.in ns2/named.conf +for conf in ns*/named.conf.in; do + copy_setports "$conf" "$(dirname "$conf")/$(basename "$conf" .in)" +done -( - cd ns2 - $SHELL sign.sh -) +(cd ns2 && $SHELL sign.sh) diff --git a/bin/tests/system/statschannel/tests-json.py b/bin/tests/system/statschannel/tests-json.py new file mode 100755 index 0000000000..32a06c2a39 --- /dev/null +++ b/bin/tests/system/statschannel/tests-json.py @@ -0,0 +1,100 @@ +#!/usr/bin/python3 +############################################################################ +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. +############################################################################ + +from datetime import datetime + +import pytest +import requests + +import generic +from helper import fmt + + +# JSON helper functions +def fetch_zones_json(statsip, statsport): + + r = requests.get("http://{}:{}/json/v1/zones".format(statsip, statsport)) + assert r.status_code == 200 + + data = r.json() + return data["views"]["_default"]["zones"] + + +def fetch_traffic_json(statsip, statsport): + + r = requests.get("http://{}:{}/json/v1/traffic".format(statsip, statsport)) + assert r.status_code == 200 + + data = r.json() + + return data["traffic"] + + +def load_timers_json(zone, primary=True): + + name = zone['name'] + + # Check if the primary zone timer exists + assert 'loaded' in zone + loaded = datetime.strptime(zone['loaded'], fmt) + + if primary: + # Check if the secondary zone timers does not exist + assert 'expires' not in zone + assert 'refresh' not in zone + expires = None + refresh = None + else: + assert 'expires' in zone + assert 'refresh' in zone + expires = datetime.strptime(zone['expires'], fmt) + refresh = datetime.strptime(zone['refresh'], fmt) + + return (name, loaded, expires, refresh) + + +def load_zone_json(zone): + name = zone['name'] + + return name + + +@pytest.mark.json +@pytest.mark.requests +def test_zone_timers_primary_json(statsport): + generic.test_zone_timers_primary(fetch_zones_json, load_timers_json, + statsip="10.53.0.1", statsport=statsport, + zonedir="ns1") + + +@pytest.mark.json +@pytest.mark.requests +def test_zone_timers_secondary_json(statsport): + generic.test_zone_timers_secondary(fetch_zones_json, load_timers_json, + statsip="10.53.0.3", statsport=statsport, + zonedir="ns3") + + +@pytest.mark.json +@pytest.mark.requests +def test_zone_with_many_keys_json(statsport): + generic.test_zone_with_many_keys(fetch_zones_json, load_zone_json, + statsip="10.53.0.2", statsport=statsport) + + +@pytest.mark.json +@pytest.mark.requests +@pytest.mark.dnspython +def test_traffic_json(port, statsport): + generic.test_traffic(fetch_traffic_json, + statsip="10.53.0.2", statsport=statsport, + port=port) diff --git a/bin/tests/system/statschannel/tests-xml.py b/bin/tests/system/statschannel/tests-xml.py new file mode 100755 index 0000000000..f7f0608309 --- /dev/null +++ b/bin/tests/system/statschannel/tests-xml.py @@ -0,0 +1,130 @@ +#!/usr/bin/python3 +############################################################################ +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. +############################################################################ + +import xml.etree.ElementTree as ET +from datetime import datetime + +import pytest +import requests + +import generic +from helper import fmt + + +# XML helper functions +def fetch_zones_xml(statsip, statsport): + + r = requests.get("http://{}:{}/xml/v3/zones".format(statsip, statsport)) + assert r.status_code == 200 + + root = ET.fromstring(r.text) + + default_view = None + for view in root.find('views').iter('view'): + if view.attrib['name'] == "_default": + default_view = view + break + assert default_view is not None + + return default_view.find('zones').findall('zone') + + +def fetch_traffic_xml(statsip, statsport): + + def load_counters(data): + out = {} + for counter in data.findall("counter"): + out[counter.attrib['name']] = int(counter.text) + + return out + + r = requests.get("http://{}:{}/xml/v3/traffic".format(statsip, statsport)) + assert r.status_code == 200 + + root = ET.fromstring(r.text) + + traffic = {} + for ip in ["ipv4", "ipv6"]: + for proto in ["udp", "tcp"]: + proto_root = root.find("traffic").find(ip).find(proto) + for counters in proto_root.findall("counters"): + if counters.attrib['type'] == "request-size": + key = "dns-{}-requests-sizes-received-{}".format(proto, ip) + else: + key = "dns-{}-responses-sizes-sent-{}".format(proto, ip) + + values = load_counters(counters) + traffic[key] = values + + return traffic + + +def load_timers_xml(zone, primary=True): + + name = zone.attrib['name'] + + loaded_el = zone.find('loaded') + assert loaded_el is not None + loaded = datetime.strptime(loaded_el.text, fmt) + + expires_el = zone.find('expires') + refresh_el = zone.find('refresh') + if primary: + assert expires_el is None + assert refresh_el is None + expires = None + refresh = None + else: + assert expires_el is not None + assert refresh_el is not None + expires = datetime.strptime(expires_el.text, fmt) + refresh = datetime.strptime(refresh_el.text, fmt) + + return (name, loaded, expires, refresh) + + +def load_zone_xml(zone): + name = zone.attrib['name'] + + return name + + +@pytest.mark.xml +@pytest.mark.requests +def test_zone_timers_primary_xml(statsport): + generic.test_zone_timers_primary(fetch_zones_xml, load_timers_xml, + statsip="10.53.0.1", statsport=statsport, + zonedir="ns1") + + +@pytest.mark.xml +@pytest.mark.requests +def test_zone_timers_secondary_xml(statsport): + generic.test_zone_timers_secondary(fetch_zones_xml, load_timers_xml, + statsip="10.53.0.3", statsport=statsport, + zonedir="ns3") + + +@pytest.mark.xml +@pytest.mark.requests +def test_zone_with_many_keys_xml(statsport): + generic.test_zone_with_many_keys(fetch_zones_xml, load_zone_xml, + statsip="10.53.0.2", statsport=statsport) + + +@pytest.mark.xml +@pytest.mark.requests +@pytest.mark.dnspython +def test_traffic_xml(port, statsport): + generic.test_traffic(fetch_traffic_xml, + statsip="10.53.0.2", statsport=statsport, + port=port) diff --git a/bin/tests/system/statschannel/tests.sh b/bin/tests/system/statschannel/tests.sh index aaa221b5d7..8bac41fa86 100644 --- a/bin/tests/system/statschannel/tests.sh +++ b/bin/tests/system/statschannel/tests.sh @@ -45,21 +45,6 @@ if [ ! "$PERL_JSON" -a ! "$PERL_XML" ]; then fi -gettraffic() { - sleep 1 - echo_i "... using $1" - case $1 in - xml) path='xml/v3/traffic' ;; - json) path='json/v1/traffic' ;; - *) return 1 ;; - esac - file=`$PERL fetch.pl -p ${EXTRAPORT1} $path` - cp $file $file.$1.$2 - $PERL traffic-${1}.pl $file 2>/dev/null | sort > traffic.out.$2 - result=$? - return $result -} - getzones() { sleep 1 echo_i "... using $1" @@ -86,81 +71,6 @@ loadkeys_on() { status=0 n=1 -ret=0 -echo_i "fetching traffic size data ($n)" -if [ $PERL_XML ]; then - gettraffic xml x$n || ret=1 - cmp traffic.out.x$n traffic.expect.$n || ret=1 -fi -if [ $PERL_JSON ]; then - gettraffic json j$n || ret=1 - cmp traffic.out.j$n traffic.expect.$n || ret=1 -fi -if [ $ret != 0 ]; then echo_i "failed"; fi -status=`expr $status + $ret` -n=`expr $n + 1` - -ret=0 -echo_i "fetching traffic size data after small UDP query ($n)" -$DIGCMD short.example txt > dig.out.$n || ret=1 -if [ $PERL_XML ]; then - gettraffic xml x$n || ret=1 - cmp traffic.out.x$n traffic.expect.$n || ret=1 -fi -if [ $PERL_JSON ]; then - gettraffic json j$n || ret=1 - cmp traffic.out.j$n traffic.expect.$n || ret=1 -fi -if [ $ret != 0 ]; then echo_i "failed"; fi -status=`expr $status + $ret` -n=`expr $n + 1` - -ret=0 -n=`expr $n + 1` -echo_i "fetching traffic size data after large UDP query ($n)" -$DIGCMD long.example txt > dig.out.$n || ret=1 -if [ $PERL_XML ]; then - gettraffic xml x$n || ret=1 - cmp traffic.out.x$n traffic.expect.$n || ret=1 -fi -if [ $PERL_JSON ]; then - gettraffic json j$n || ret=1 - cmp traffic.out.j$n traffic.expect.$n || ret=1 -fi -if [ $ret != 0 ]; then echo_i "failed"; fi -status=`expr $status + $ret` -n=`expr $n + 1` - -ret=0 -echo_i "fetching traffic size data after small TCP query ($n)" -$DIGCMD +tcp short.example txt > dig.out.$n || ret=1 -if [ $PERL_XML ]; then - gettraffic xml x$n || ret=1 - cmp traffic.out.x$n traffic.expect.$n || ret=1 -fi -if [ $PERL_JSON ]; then - gettraffic json j$n || ret=1 - cmp traffic.out.j$n traffic.expect.$n || ret=1 -fi -if [ $ret != 0 ]; then echo_i "failed"; fi -status=`expr $status + $ret` -n=`expr $n + 1` - -ret=0 -echo_i "fetching traffic size data after large TCP query ($n)" -$DIGCMD +tcp long.example txt > dig.out.$n || ret=1 -if [ $PERL_XML ]; then - gettraffic xml x$n || ret=1 - cmp traffic.out.x$n traffic.expect.$n || ret=1 -fi -if [ $PERL_JSON ]; then - gettraffic json j$n || ret=1 - cmp traffic.out.j$n traffic.expect.$n || ret=1 -fi -if [ $ret != 0 ]; then echo_i "failed"; fi -status=`expr $status + $ret` -n=`expr $n + 1` - ret=0 echo_i "checking consistency between named.stats and xml/json ($n)" rm -f ns2/named.stats @@ -359,25 +269,6 @@ if [ $ret != 0 ]; then echo_i "failed"; fi status=`expr $status + $ret` n=`expr $n + 1` -# 4. Test a zone with more than four keys. -zone="manykeys" -ksk8_id=`cat ns2/$zone.ksk8.id` -zsk8_id=`cat ns2/$zone.zsk8.id` -ksk13_id=`cat ns2/$zone.ksk13.id` -zsk13_id=`cat ns2/$zone.zsk13.id` -ksk14_id=`cat ns2/$zone.ksk14.id` -zsk14_id=`cat ns2/$zone.zsk14.id` - -ret=0 -echo_i "fetch zone stats data for a zone with many keys ($n)" -# Fetch and check the dnssec sign statistics. -if [ $PERL_XML ]; then - getzones xml $zone x$n || ret=1 -fi -if [ $PERL_JSON ]; then - getzones json $zone j$n || ret=1 -fi -# The output is gibberish, but at least make sure it does not crash. if [ $ret != 0 ]; then echo_i "failed"; fi status=`expr $status + $ret` n=`expr $n + 1` diff --git a/doc/arm/notes-9.17.2.xml b/doc/arm/notes-9.17.2.xml index 173c7ad946..193c13c2c1 100644 --- a/doc/arm/notes-9.17.2.xml +++ b/doc/arm/notes-9.17.2.xml @@ -88,6 +88,14 @@ actions to be logged into a separate channel. [GL #54] + + + The zone timers are now exported to the statistics channel. For the + primary zones, only the loaded time is exported. For the secondary + zones, the exported timers also include expire and refresh times. + Contributed by Paul Frieden, Verizon Media. [GL #1232] + + diff --git a/doc/man/named.conf.5in b/doc/man/named.conf.5in index 512d2fc59a..98266cd82c 100644 --- a/doc/man/named.conf.5in +++ b/doc/man/named.conf.5in @@ -288,15 +288,12 @@ options { dnssec\-secure\-to\-insecure boolean; dnssec\-update\-mode ( maintain | no\-resign ); dnssec\-validation ( yes | no | auto ); - dnstap { ( all | auth | client | forwarder | - resolver | update ) [ ( query | response ) ]; - ... }; - dnstap\-identity ( quoted_string | none | - hostname ); - dnstap\-output ( file | unix ) quoted_string [ - size ( unlimited | size ) ] [ versions ( - unlimited | integer ) ] [ suffix ( increment - | timestamp ) ]; + dnstap { ( all | auth | client | forwarder | resolver | update ) [ + ( query | response ) ]; ... }; + dnstap\-identity ( quoted_string | none | hostname ); + dnstap\-output ( file | unix ) quoted_string [ size ( unlimited | + size ) ] [ versions ( unlimited | integer ) ] [ suffix ( + increment | timestamp ) ]; dnstap\-version ( quoted_string | none ); dscp integer; dual\-stack\-servers [ port integer ] { ( quoted_string [ port @@ -686,9 +683,8 @@ view string [ class ] { dnssec\-secure\-to\-insecure boolean; dnssec\-update\-mode ( maintain | no\-resign ); dnssec\-validation ( yes | no | auto ); - dnstap { ( all | auth | client | forwarder | - resolver | update ) [ ( query | response ) ]; - ... }; + dnstap { ( all | auth | client | forwarder | resolver | update ) [ + ( query | response ) ]; ... }; dual\-stack\-servers [ port integer ] { ( quoted_string [ port integer ] [ dscp integer ] | ipv4_address [ port integer ] [ dscp integer ] | ipv6_address [ port diff --git a/doc/misc/options b/doc/misc/options index 5161474545..ea9242ef67 100644 --- a/doc/misc/options +++ b/doc/misc/options @@ -166,16 +166,13 @@ options { dnssec-secure-to-insecure ; dnssec-update-mode ( maintain | no-resign ); dnssec-validation ( yes | no | auto ); - dnstap { ( all | auth | client | forwarder | - resolver | update ) [ ( query | response ) ]; - ... }; // not configured - dnstap-identity ( | none | - hostname ); // not configured - dnstap-output ( file | unix ) [ - size ( unlimited | ) ] [ versions ( - unlimited | ) ] [ suffix ( increment - | timestamp ) ]; // not configured - dnstap-version ( | none ); // not configured + dnstap { ( all | auth | client | forwarder | resolver | update ) [ + ( query | response ) ]; ... }; + dnstap-identity ( | none | hostname ); + dnstap-output ( file | unix ) [ size ( unlimited | + ) ] [ versions ( unlimited | ) ] [ suffix ( + increment | timestamp ) ]; + dnstap-version ( | none ); dscp ; dual-stack-servers [ port ] { ( [ port ] [ dscp ] | [ port @@ -199,13 +196,13 @@ options { forward ( first | only ); forwarders [ port ] [ dscp ] { ( | ) [ port ] [ dscp ]; ... }; - fstrm-set-buffer-hint ; // not configured - fstrm-set-flush-timeout ; // not configured - fstrm-set-input-queue-size ; // not configured - fstrm-set-output-notify-threshold ; // not configured - fstrm-set-output-queue-model ( mpsc | spsc ); // not configured - fstrm-set-output-queue-size ; // not configured - fstrm-set-reopen-interval ; // not configured + fstrm-set-buffer-hint ; + fstrm-set-flush-timeout ; + fstrm-set-input-queue-size ; + fstrm-set-output-notify-threshold ; + fstrm-set-output-queue-model ( mpsc | spsc ); + fstrm-set-output-queue-size ; + fstrm-set-reopen-interval ; geoip-directory ( | none ); geoip-use-ecs ; // obsolete glue-cache ; @@ -550,9 +547,8 @@ view [ ] { dnssec-secure-to-insecure ; dnssec-update-mode ( maintain | no-resign ); dnssec-validation ( yes | no | auto ); - dnstap { ( all | auth | client | forwarder | - resolver | update ) [ ( query | response ) ]; - ... }; // not configured + dnstap { ( all | auth | client | forwarder | resolver | update ) [ + ( query | response ) ]; ... }; dual-stack-servers [ port ] { ( [ port ] [ dscp ] | [ port ] [ dscp ] | [ port diff --git a/doc/misc/options.active b/doc/misc/options.active index 3ce6b6cb61..8abd4e1f5d 100644 --- a/doc/misc/options.active +++ b/doc/misc/options.active @@ -153,16 +153,13 @@ options { dnssec-secure-to-insecure ; dnssec-update-mode ( maintain | no-resign ); dnssec-validation ( yes | no | auto ); - dnstap { ( all | auth | client | forwarder | - resolver | update ) [ ( query | response ) ]; - ... }; // not configured - dnstap-identity ( | none | - hostname ); // not configured - dnstap-output ( file | unix ) [ - size ( unlimited | ) ] [ versions ( - unlimited | ) ] [ suffix ( increment - | timestamp ) ]; // not configured - dnstap-version ( | none ); // not configured + dnstap { ( all | auth | client | forwarder | resolver | update ) [ + ( query | response ) ]; ... }; + dnstap-identity ( | none | hostname ); + dnstap-output ( file | unix ) [ size ( unlimited | + ) ] [ versions ( unlimited | ) ] [ suffix ( + increment | timestamp ) ]; + dnstap-version ( | none ); dscp ; dual-stack-servers [ port ] { ( [ port ] [ dscp ] | [ port @@ -181,13 +178,13 @@ options { forward ( first | only ); forwarders [ port ] [ dscp ] { ( | ) [ port ] [ dscp ]; ... }; - fstrm-set-buffer-hint ; // not configured - fstrm-set-flush-timeout ; // not configured - fstrm-set-input-queue-size ; // not configured - fstrm-set-output-notify-threshold ; // not configured - fstrm-set-output-queue-model ( mpsc | spsc ); // not configured - fstrm-set-output-queue-size ; // not configured - fstrm-set-reopen-interval ; // not configured + fstrm-set-buffer-hint ; + fstrm-set-flush-timeout ; + fstrm-set-input-queue-size ; + fstrm-set-output-notify-threshold ; + fstrm-set-output-queue-model ( mpsc | spsc ); + fstrm-set-output-queue-size ; + fstrm-set-reopen-interval ; geoip-directory ( | none ); glue-cache ; heartbeat-interval ; @@ -495,9 +492,8 @@ view [ ] { dnssec-secure-to-insecure ; dnssec-update-mode ( maintain | no-resign ); dnssec-validation ( yes | no | auto ); - dnstap { ( all | auth | client | forwarder | - resolver | update ) [ ( query | response ) ]; - ... }; // not configured + dnstap { ( all | auth | client | forwarder | resolver | update ) [ + ( query | response ) ]; ... }; dual-stack-servers [ port ] { ( [ port ] [ dscp ] | [ port ] [ dscp ] | [ port diff --git a/doc/misc/options.grammar.rst b/doc/misc/options.grammar.rst index d99a89fbcc..f7d9657b75 100644 --- a/doc/misc/options.grammar.rst +++ b/doc/misc/options.grammar.rst @@ -82,15 +82,12 @@ dnssec-secure-to-insecure ; dnssec-update-mode ( maintain | no-resign ); dnssec-validation ( yes | no | auto ); - dnstap { ( all | auth | client | forwarder | - resolver | update ) [ ( query | response ) ]; - ... }; - dnstap-identity ( | none | - hostname ); - dnstap-output ( file | unix ) [ - size ( unlimited | ) ] [ versions ( - unlimited | ) ] [ suffix ( increment - | timestamp ) ]; + dnstap { ( all | auth | client | forwarder | resolver | update ) [ + ( query | response ) ]; ... }; + dnstap-identity ( | none | hostname ); + dnstap-output ( file | unix ) [ size ( unlimited | + ) ] [ versions ( unlimited | ) ] [ suffix ( + increment | timestamp ) ]; dnstap-version ( | none ); dscp ; dual-stack-servers [ port ] { ( [ port diff --git a/util/copyrights b/util/copyrights index 91a31b2ea5..4aecf882c5 100644 --- a/util/copyrights +++ b/util/copyrights @@ -808,12 +808,17 @@ ./bin/tests/system/statistics/setup.sh SH 2018,2019,2020 ./bin/tests/system/statistics/tests.sh SH 2012,2015,2016,2017,2018,2019,2020 ./bin/tests/system/statschannel/clean.sh SH 2015,2016,2017,2018,2019,2020 +./bin/tests/system/statschannel/conftest.py PYTHON 2020 ./bin/tests/system/statschannel/fetch.pl PERL 2015,2016,2018,2019,2020 +./bin/tests/system/statschannel/generic.py PYTHON 2020 +./bin/tests/system/statschannel/helper.py PYTHON 2020 ./bin/tests/system/statschannel/mem-xml.pl PERL 2017,2018,2019,2020 ./bin/tests/system/statschannel/ns2/sign.sh SH 2019,2020 ./bin/tests/system/statschannel/server-json.pl PERL 2015,2016,2017,2018,2019,2020 ./bin/tests/system/statschannel/server-xml.pl PERL 2015,2016,2017,2018,2019,2020 ./bin/tests/system/statschannel/setup.sh SH 2018,2019,2020 +./bin/tests/system/statschannel/tests-json.py PYTHON-BIN 2020 +./bin/tests/system/statschannel/tests-xml.py PYTHON-BIN 2020 ./bin/tests/system/statschannel/tests.sh SH 2015,2016,2017,2018,2019,2020 ./bin/tests/system/statschannel/traffic-json.pl PERL 2015,2016,2017,2018,2019,2020 ./bin/tests/system/statschannel/traffic-xml.pl PERL 2015,2016,2017,2018,2019,2020