From 01b293b055263eb1cc76b64968efd554fb4e0b42 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Tue, 25 Oct 2022 14:02:30 +0200 Subject: [PATCH 1/6] Unify indentation level in testcrypto.sh --- bin/tests/system/testcrypto.sh | 78 +++++++++++++++++----------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/bin/tests/system/testcrypto.sh b/bin/tests/system/testcrypto.sh index 5957dc09e5..96cc39651e 100644 --- a/bin/tests/system/testcrypto.sh +++ b/bin/tests/system/testcrypto.sh @@ -21,45 +21,45 @@ quiet=0 msg="cryptography" while test "$#" -gt 0; do - case $1 in - -q) - args="$args -q" - quiet=1 - ;; - rsa|RSA|rsasha1|RSASHA1) - alg="-a RSASHA1" - msg="RSA cryptography" - ;; - rsasha256|RSASHA256) - alg="-a RSASHA256" - msg="RSA cryptography" - ;; - rsasha512|RSASHA512) - alg="-a RSASHA512" - msg="RSA cryptography" - ;; - ecdsa|ECDSA|ecdsap256sha256|ECDSAP256SHA256) - alg="-a ECDSAP256SHA256" - msg="ECDSA cryptography" - ;; - ecdsap384sha384|ECDSAP384SHA384) - alg="-a ECDSAP384SHA384" - msg="ECDSA cryptography" - ;; - eddsa|EDDSA|ed25519|ED25519) - alg="-a ED25519" - msg="EDDSA cryptography" - ;; - ed448|ED448) - alg="-a ED448" - msg="EDDSA cryptography" - ;; - *) - echo "${prog}: unknown argument" - exit 1 - ;; - esac - shift + case $1 in + -q) + args="$args -q" + quiet=1 + ;; + rsa|RSA|rsasha1|RSASHA1) + alg="-a RSASHA1" + msg="RSA cryptography" + ;; + rsasha256|RSASHA256) + alg="-a RSASHA256" + msg="RSA cryptography" + ;; + rsasha512|RSASHA512) + alg="-a RSASHA512" + msg="RSA cryptography" + ;; + ecdsa|ECDSA|ecdsap256sha256|ECDSAP256SHA256) + alg="-a ECDSAP256SHA256" + msg="ECDSA cryptography" + ;; + ecdsap384sha384|ECDSAP384SHA384) + alg="-a ECDSAP384SHA384" + msg="ECDSA cryptography" + ;; + eddsa|EDDSA|ed25519|ED25519) + alg="-a ED25519" + msg="EDDSA cryptography" + ;; + ed448|ED448) + alg="-a ED448" + msg="EDDSA cryptography" + ;; + *) + echo "${prog}: unknown argument" + exit 1 + ;; + esac + shift done if $KEYGEN $args $alg foo > /dev/null 2>&1 From bb1c6bbdc701cdbadfe7c796acf8b5ecd719f1b9 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Tue, 25 Oct 2022 14:05:07 +0200 Subject: [PATCH 2/6] Support testcrypto.sh usage without including conf.sh The only variable really needed for the script to work is the path to the $KEYGEN binary. Allow setting this via an environment variable to avoid loading conf.sh (and causing a chicken-egg problem). Also make testcrypto.sh executable to allow its use from conf.sh. --- bin/tests/system/testcrypto.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) mode change 100644 => 100755 bin/tests/system/testcrypto.sh diff --git a/bin/tests/system/testcrypto.sh b/bin/tests/system/testcrypto.sh old mode 100644 new mode 100755 index 96cc39651e..72237777d9 --- a/bin/tests/system/testcrypto.sh +++ b/bin/tests/system/testcrypto.sh @@ -11,14 +11,16 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. -. ../conf.sh +if test -z "$KEYGEN"; then + . ../conf.sh + alg="-a $DEFAULT_ALGORITHM -b $DEFAULT_BITS" +else + alg="" +fi prog=$0 - args="" -alg="-a $DEFAULT_ALGORITHM -b $DEFAULT_BITS" quiet=0 - msg="cryptography" while test "$#" -gt 0; do case $1 in @@ -62,6 +64,11 @@ while test "$#" -gt 0; do shift done +if test -z "$alg"; then + echo "${prog}: no algorithm selected" + exit 1 +fi + if $KEYGEN $args $alg foo > /dev/null 2>&1 then rm -f Kfoo* From 37d14c69c050a6c2bc0ebbbdc80a788bb6795b7e Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Tue, 25 Oct 2022 18:00:27 +0200 Subject: [PATCH 3/6] Export env variables in system tests Certain variables have to be exported in order for the system tests to work. It makes little sense to export the variables in one place/script while they're defined in another place. Since it makes no harm, export all the variables to make the behaviour more predictable and consistent. Previously, some variables were exported as environment variables, while others were just shell variables which could be used once the configuration was sourced from another script. However, they wouldn't be exposed to spawned processes. For simplicity sake (and for the upcoming effort to run system tests with pytest), export all variables that are used. TESTS, PARALLEL_UNIX and SUBDIRS variables are automake-specific, aren't used anywhere else and thus not exported. --- bin/tests/system/conf.sh.common | 91 +++++++----------------- bin/tests/system/conf.sh.in | 122 ++++++++++++++++---------------- 2 files changed, 85 insertions(+), 128 deletions(-) diff --git a/bin/tests/system/conf.sh.common b/bin/tests/system/conf.sh.common index 3f6381703c..165b2b1152 100644 --- a/bin/tests/system/conf.sh.common +++ b/bin/tests/system/conf.sh.common @@ -131,25 +131,25 @@ zonechecks" # Set up color-coded test output # if [ ${SYSTEMTEST_FORCE_COLOR:-0} -eq 1 ] || test -t 1 && type tput > /dev/null 2>&1 && tput setaf 7 > /dev/null 2>&1 ; then - COLOR_END=`tput setaf 4` # blue - COLOR_FAIL=`tput setaf 1` # red - COLOR_INFO=`tput bold` # bold - COLOR_NONE=`tput sgr0` - COLOR_PASS=`tput setaf 2` # green - COLOR_START=`tput setaf 4` # blue - COLOR_WARN=`tput setaf 3` # yellow + export COLOR_END=`tput setaf 4` # blue + export COLOR_FAIL=`tput setaf 1` # red + export COLOR_INFO=`tput bold` # bold + export COLOR_NONE=`tput sgr0` + export COLOR_PASS=`tput setaf 2` # green + export COLOR_START=`tput setaf 4` # blue + export COLOR_WARN=`tput setaf 3` # yellow else # set to empty strings so printf succeeds - COLOR_END='' - COLOR_FAIL='' - COLOR_INFO='' - COLOR_NONE='' - COLOR_PASS='' - COLOR_START='' - COLOR_WARN='' + export COLOR_END='' + export COLOR_FAIL='' + export COLOR_INFO='' + export COLOR_NONE='' + export COLOR_PASS='' + export COLOR_START='' + export COLOR_WARN='' fi -SYSTESTDIR="`basename $PWD`" +export SYSTESTDIR="`basename $PWD`" if type printf > /dev/null 2>&1 then @@ -263,27 +263,27 @@ send() { # # Default algorithm for testing. -DEFAULT_ALGORITHM=ECDSAP256SHA256 -DEFAULT_ALGORITHM_NUMBER=13 -DEFAULT_BITS=256 +export DEFAULT_ALGORITHM=ECDSAP256SHA256 +export DEFAULT_ALGORITHM_NUMBER=13 +export DEFAULT_BITS=256 # This is an alternative algorithm for test cases that require more than # one algorithm (for example algorithm rollover). Must be different from # DEFAULT_ALGORITHM. -ALTERNATIVE_ALGORITHM=RSASHA256 -ALTERNATIVE_ALGORITHM_NUMBER=8 -ALTERNATIVE_BITS=1280 +export ALTERNATIVE_ALGORITHM=RSASHA256 +export ALTERNATIVE_ALGORITHM_NUMBER=8 +export ALTERNATIVE_BITS=1280 # This is an algorithm that is used for tests against the # "disable-algorithms" configuration option. Must be different from above # algorithms. -DISABLED_ALGORITHM=ECDSAP384SHA384 -DISABLED_ALGORITHM_NUMBER=14 -DISABLED_BITS=384 +export DISABLED_ALGORITHM=ECDSAP384SHA384 +export DISABLED_ALGORITHM_NUMBER=14 +export DISABLED_BITS=384 # Default HMAC algorithm. # also update common/rndc.conf and common/rndc.key when updating DEFAULT_HMAC -DEFAULT_HMAC=hmac-sha256 +export DEFAULT_HMAC=hmac-sha256 # # Useful functions in test scripts @@ -719,44 +719,3 @@ copy_setports() { -e "s/@DISABLED_BITS@/${DISABLED_BITS}/g" \ $1 > $2 } - -# -# Export command paths -# -export ARPANAME -export BIGKEY -export CDS -export CHECKZONE -export DESCRIPTION -export DIG -export DNSTAPREAD -export FEATURETEST -export FSTRM_CAPTURE -export GENCHECK -export JOURNALPRINT -export KEYCREATE -export KEYDELETE -export KEYFRLAB -export KEYGEN -export KEYSETTOOL -export KEYSIGNER -export KRB5_CONFIG -export KRB5_KTNAME -export MAKEJOURNAL -export MDIG -export NAMED -export NSEC3HASH -export NSLOOKUP -export NSUPDATE -export NZD2NZF -export PERL -export PIPEQUERIES -export PYTHON -export RNDC -export RRCHECKER -export SIGNER -export SUBDIRS -export TMPDIR -export TSIGKEYGEN -export VERIFY -export WIRETEST diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in index 295c657479..70edf5bc3e 100644 --- a/bin/tests/system/conf.sh.in +++ b/bin/tests/system/conf.sh.in @@ -17,57 +17,57 @@ # # Find the top of the BIND9 tree. -TOP_BUILDDIR=@abs_top_builddir@ -TOP_SRCDIR=@abs_top_srcdir@ +export TOP_BUILDDIR=@abs_top_builddir@ +export TOP_SRCDIR=@abs_top_srcdir@ # Provide TMPDIR variable for tests that need it. -TMPDIR=${TMPDIR:-/tmp} +export TMPDIR=${TMPDIR:-/tmp} # Load common values . $TOP_SRCDIR/bin/tests/system/conf.sh.common -ARPANAME=$TOP_BUILDDIR/bin/tools/arpaname -CDS=$TOP_BUILDDIR/bin/dnssec/dnssec-cds -CHECKCONF=$TOP_BUILDDIR/bin/check/named-checkconf -CHECKZONE=$TOP_BUILDDIR/bin/check/named-checkzone -DELV=$TOP_BUILDDIR/bin/delv/delv -DIG=$TOP_BUILDDIR/bin/dig/dig -DNSTAPREAD=$TOP_BUILDDIR/bin/tools/dnstap-read -DSFROMKEY=$TOP_BUILDDIR/bin/dnssec/dnssec-dsfromkey -FEATURETEST=$TOP_BUILDDIR/bin/tests/system/feature-test -FSTRM_CAPTURE=@FSTRM_CAPTURE@ -HOST=$TOP_BUILDDIR/bin/dig/host -IMPORTKEY=$TOP_BUILDDIR/bin/dnssec/dnssec-importkey -JOURNALPRINT=$TOP_BUILDDIR/bin/tools/named-journalprint -KEYFRLAB=$TOP_BUILDDIR/bin/dnssec/dnssec-keyfromlabel -KEYGEN=$TOP_BUILDDIR/bin/dnssec/dnssec-keygen -MDIG=$TOP_BUILDDIR/bin/tools/mdig -NAMED=$TOP_BUILDDIR/bin/named/named -NSEC3HASH=$TOP_BUILDDIR/bin/tools/nsec3hash -NSLOOKUP=$TOP_BUILDDIR/bin/dig/nslookup -NSUPDATE=$TOP_BUILDDIR/bin/nsupdate/nsupdate -NZD2NZF=$TOP_BUILDDIR/bin/tools/named-nzd2nzf -REVOKE=$TOP_BUILDDIR/bin/dnssec/dnssec-revoke -RNDC=$TOP_BUILDDIR/bin/rndc/rndc -RNDCCONFGEN=$TOP_BUILDDIR/bin/confgen/rndc-confgen -RRCHECKER=$TOP_BUILDDIR/bin/tools/named-rrchecker -SETTIME=$TOP_BUILDDIR/bin/dnssec/dnssec-settime -SIGNER=$TOP_BUILDDIR/bin/dnssec/dnssec-signzone -TSIGKEYGEN=$TOP_BUILDDIR/bin/confgen/tsig-keygen -VERIFY=$TOP_BUILDDIR/bin/dnssec/dnssec-verify -WIRETEST=$TOP_BUILDDIR/bin/tests/wire_test +export ARPANAME=$TOP_BUILDDIR/bin/tools/arpaname +export CDS=$TOP_BUILDDIR/bin/dnssec/dnssec-cds +export CHECKCONF=$TOP_BUILDDIR/bin/check/named-checkconf +export CHECKZONE=$TOP_BUILDDIR/bin/check/named-checkzone +export DELV=$TOP_BUILDDIR/bin/delv/delv +export DIG=$TOP_BUILDDIR/bin/dig/dig +export DNSTAPREAD=$TOP_BUILDDIR/bin/tools/dnstap-read +export DSFROMKEY=$TOP_BUILDDIR/bin/dnssec/dnssec-dsfromkey +export FEATURETEST=$TOP_BUILDDIR/bin/tests/system/feature-test +export FSTRM_CAPTURE=@FSTRM_CAPTURE@ +export HOST=$TOP_BUILDDIR/bin/dig/host +export IMPORTKEY=$TOP_BUILDDIR/bin/dnssec/dnssec-importkey +export JOURNALPRINT=$TOP_BUILDDIR/bin/tools/named-journalprint +export KEYFRLAB=$TOP_BUILDDIR/bin/dnssec/dnssec-keyfromlabel +export KEYGEN=$TOP_BUILDDIR/bin/dnssec/dnssec-keygen +export MDIG=$TOP_BUILDDIR/bin/tools/mdig +export NAMED=$TOP_BUILDDIR/bin/named/named +export NSEC3HASH=$TOP_BUILDDIR/bin/tools/nsec3hash +export NSLOOKUP=$TOP_BUILDDIR/bin/dig/nslookup +export NSUPDATE=$TOP_BUILDDIR/bin/nsupdate/nsupdate +export NZD2NZF=$TOP_BUILDDIR/bin/tools/named-nzd2nzf +export REVOKE=$TOP_BUILDDIR/bin/dnssec/dnssec-revoke +export RNDC=$TOP_BUILDDIR/bin/rndc/rndc +export RNDCCONFGEN=$TOP_BUILDDIR/bin/confgen/rndc-confgen +export RRCHECKER=$TOP_BUILDDIR/bin/tools/named-rrchecker +export SETTIME=$TOP_BUILDDIR/bin/dnssec/dnssec-settime +export SIGNER=$TOP_BUILDDIR/bin/dnssec/dnssec-signzone +export TSIGKEYGEN=$TOP_BUILDDIR/bin/confgen/tsig-keygen +export VERIFY=$TOP_BUILDDIR/bin/dnssec/dnssec-verify +export WIRETEST=$TOP_BUILDDIR/bin/tests/wire_test -BIGKEY=$TOP_BUILDDIR/bin/tests/system/rsabigexponent/bigkey -GENCHECK=$TOP_BUILDDIR/bin/tests/system/rndc/gencheck -KEYCREATE=$TOP_BUILDDIR/bin/tests/system/tkey/keycreate -KEYDELETE=$TOP_BUILDDIR/bin/tests/system/tkey/keydelete -MAKEJOURNAL=$TOP_BUILDDIR/bin/tests/system/makejournal -PIPEQUERIES=$TOP_BUILDDIR/bin/tests/system/pipelined/pipequeries +export BIGKEY=$TOP_BUILDDIR/bin/tests/system/rsabigexponent/bigkey +export GENCHECK=$TOP_BUILDDIR/bin/tests/system/rndc/gencheck +export KEYCREATE=$TOP_BUILDDIR/bin/tests/system/tkey/keycreate +export KEYDELETE=$TOP_BUILDDIR/bin/tests/system/tkey/keydelete +export MAKEJOURNAL=$TOP_BUILDDIR/bin/tests/system/makejournal +export PIPEQUERIES=$TOP_BUILDDIR/bin/tests/system/pipelined/pipequeries # we don't want a KRB5_CONFIG setting breaking the tests -KRB5_CONFIG=/dev/null +export KRB5_CONFIG=/dev/null # use local keytab instead of default /etc/krb5.keytab -KRB5_KTNAME=dns.keytab +export KRB5_KTNAME=dns.keytab # # Construct the lists of tests to run @@ -94,39 +94,37 @@ tcp" SUBDIRS="$PARALLEL_COMMON $PARALLEL_UNIX" # Use the CONFIG_SHELL detected by configure for tests -SHELL=@SHELL@ +export SHELL=@SHELL@ # CURL will be empty if no program was found by configure -CURL=@CURL@ +export CURL=@CURL@ # NC will be empty if no program was found by configure -NC=@NC@ +export NC=@NC@ # XMLLINT will be empty if no program was found by configure -XMLLINT=@XMLLINT@ +export XMLLINT=@XMLLINT@ # XSLTPROC will be empty if no program was found by configure -XSLTPROC=@XSLTPROC@ +export XSLTPROC=@XSLTPROC@ # PERL will be an empty string if no perl interpreter was found. -PERL=$(command -v "@PERL@") +export PERL=$(command -v "@PERL@") -PYTHON=$(command -v "@PYTHON@" || true) -PYTEST=@PYTEST@ +export PYTHON=$(command -v "@PYTHON@" || true) +export PYTEST=@PYTEST@ # # Determine if we support various optional features. # -LIBXML2_LIBS="@LIBXML2_LIBS@" -HAVEXMLSTATS=${LIBXML2_LIBS:+1} -JSON_C_LIBS="@JSON_C_LIBS@" -HAVEJSONSTATS=${JSON_C_LIBS:+1} -MAXMINDDB_LIBS="@MAXMINDDB_LIBS@" -HAVEGEOIP2=${MAXMINDDB_LIBS:+1} -ZLIB_LIBS="@ZLIB_LIBS@" -HAVEZLIB=${ZLIB_LIBS:+1} -LMDB_LIBS="@LMDB_LIBS@" -NZD=${LMDB_LIBS:+1} -CRYPTO=@CRYPTO@ - -export HAVEXMLSTATS HAVEJSONSTATS +export LIBXML2_LIBS="@LIBXML2_LIBS@" +export HAVEXMLSTATS=${LIBXML2_LIBS:+1} +export JSON_C_LIBS="@JSON_C_LIBS@" +export HAVEJSONSTATS=${JSON_C_LIBS:+1} +export MAXMINDDB_LIBS="@MAXMINDDB_LIBS@" +export HAVEGEOIP2=${MAXMINDDB_LIBS:+1} +export ZLIB_LIBS="@ZLIB_LIBS@" +export HAVEZLIB=${ZLIB_LIBS:+1} +export LMDB_LIBS="@LMDB_LIBS@" +export NZD=${LMDB_LIBS:+1} +export CRYPTO=@CRYPTO@ From 5f480c8485261f32b0e9b15630cebed0e8d80eaa Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Tue, 25 Oct 2022 17:45:16 +0200 Subject: [PATCH 4/6] Script for random algorithm selection in system tests Multiple algorithm sets can be defined in this script. These can be selected via the ALGORITHM_SET environment variable. For compatibility reasons, "stable" set contains the currently used algorithms, since our system tests need some changes before being compatible with randomly selected algorithms. The script operation is similar to the get_ports.py - environment variables are created and then printed out as `export NAME=VALUE` commands, to be interpreted by shell. Once we support pytest runner for system tests, this should be a fixture instead. --- bin/tests/system/get_algorithms.py | 239 +++++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100755 bin/tests/system/get_algorithms.py diff --git a/bin/tests/system/get_algorithms.py b/bin/tests/system/get_algorithms.py new file mode 100755 index 0000000000..5429649758 --- /dev/null +++ b/bin/tests/system/get_algorithms.py @@ -0,0 +1,239 @@ +#!/usr/bin/python3 + +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# SPDX-License-Identifier: MPL-2.0 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +# This script is a 'port' broker. It keeps track of ports given to the +# individual system subtests, so every test is given a unique port range. + +import logging +import os +from pathlib import Path +import platform +import random +import subprocess +import time +from typing import Dict, List, NamedTuple, Union + +# Uncomment to enable DEBUG logging +# logging.basicConfig( +# format="get_algorithms.py %(levelname)s %(message)s", level=logging.DEBUG +# ) + +STABLE_PERIOD = 3600 * 3 +"""number of secs during which algorithm selection remains stable""" + + +class Algorithm(NamedTuple): + name: str + number: int + bits: int + + +class AlgorithmSet(NamedTuple): + """Collection of DEFAULT, ALTERNATIVE and DISABLED algorithms""" + + default: Union[Algorithm, List[Algorithm]] + """DEFAULT is the algorithm for testing.""" + + alternative: Union[Algorithm, List[Algorithm]] + """ALTERNATIVE is an alternative algorithm for test cases that require more + than one algorithm (for example algorithm rollover).""" + + disabled: Union[Algorithm, List[Algorithm]] + """DISABLED is an algorithm that is used for tests against the + "disable-algorithms" configuration option.""" + + +RSASHA1 = Algorithm("RSASHA1", 5, 1280) +RSASHA256 = Algorithm("RSASHA256", 8, 1280) +RSASHA512 = Algorithm("RSASHA512", 10, 1280) +ECDSAP256SHA256 = Algorithm("ECDSAP256SHA256", 13, 256) +ECDSAP384SHA384 = Algorithm("ECDSAP384SHA384", 14, 384) +ED25519 = Algorithm("ED25519", 15, 256) +ED448 = Algorithm("ED448", 16, 456) + +ALL_ALGORITHMS = [ + RSASHA1, + RSASHA256, + RSASHA512, + ECDSAP256SHA256, + ECDSAP384SHA384, + ED25519, + ED448, +] + +ALGORITHM_SETS = { + "stable": AlgorithmSet( + default=ECDSAP256SHA256, alternative=RSASHA256, disabled=ECDSAP384SHA384 + ), + "ecc_default": AlgorithmSet( + default=[ + ECDSAP256SHA256, + ECDSAP384SHA384, + ED25519, + ED448, + ], + alternative=RSASHA256, + disabled=RSASHA512, + ), + # FUTURE The system tests needs more work before they're ready for this. + # "random": AlgorithmSet( + # default=ALL_ALGORITHMS, + # alternative=ALL_ALGORITHMS, + # disabled=ALL_ALGORITHMS, + # ), +} + +TESTCRYPTO = Path(__file__).resolve().parent / "testcrypto.sh" + +KEYGEN = os.getenv("KEYGEN", "") +if not KEYGEN: + raise RuntimeError("KEYGEN environment variable has to be set") + +ALGORITHM_SET = os.getenv("ALGORITHM_SET", "stable") +assert ALGORITHM_SET in ALGORITHM_SETS, f'ALGORITHM_SET "{ALGORITHM_SET}" unknown' +logging.debug('choosing from ALGORITHM_SET "%s"', ALGORITHM_SET) + + +def is_supported(alg: Algorithm) -> bool: + """Test whether a given algorithm is supported on the current platform.""" + try: + subprocess.run( + f"{TESTCRYPTO} -q {alg.name}", + shell=True, + check=True, + env={"KEYGEN": KEYGEN}, + stdout=subprocess.DEVNULL, + ) + except subprocess.CalledProcessError as exc: + logging.debug(exc) + logging.info("algorithm %s not supported", alg.name) + return False + return True + + +def filter_supported(algs: AlgorithmSet) -> AlgorithmSet: + """Select supported algorithms from the set.""" + filtered = {} + for alg_type in algs._fields: + candidates = getattr(algs, alg_type) + if isinstance(candidates, Algorithm): + candidates = [candidates] + supported = list(filter(is_supported, candidates)) + if len(supported) == 1: + supported = supported.pop() + elif not supported: + raise RuntimeError( + f'no {alg_type.upper()} algorithm from "{ALGORITHM_SET}" set ' + "supported on this platform" + ) + filtered[alg_type] = supported + return AlgorithmSet(**filtered) + + +def select_random(algs: AlgorithmSet, stable_period=STABLE_PERIOD) -> AlgorithmSet: + """Select random DEFAULT, ALTERNATIVE and DISABLED algorithms from the set. + + The algorithm selection is deterministic for a given time period and + platform. This should make potential issues more reproducible. + + To increase the likelyhood of detecting an issue with a given algorithm in + CI, the current platform is used as a randomness source. When testing on + multiple platforms at the same time, this ensures more algorithm variance + while keeping reproducibility for a single platform. + + The function also ensures that DEFAULT, ALTERNATIVE and DISABLED algorithms + are all different. + """ + # FUTURE Random selection of ALTERNATIVE and DISABLED algorithms needs to + # be implemented. + alternative = algs.alternative + disabled = algs.disabled + assert isinstance( + alternative, Algorithm + ), "ALTERNATIVE algorithm randomization not supported yet" + assert isinstance( + disabled, Algorithm + ), "DISABLED algorithm randomization not supported yet" + + # initialize randomness + now = time.time() + time_seed = int(now - now % stable_period) + seed = f"{platform.platform()}_{time_seed}" + random.seed(seed) + + # DEFAULT selection + if isinstance(algs.default, Algorithm): + default = algs.default + else: + candidates = algs.default + for taken in [alternative, disabled]: + try: + candidates.remove(taken) + except ValueError: + pass + assert len(candidates), "no possible choice for DEFAULT algorithm" + random.shuffle(candidates) + default = candidates[0] + + # Ensure only single algorithm is present for each option + assert isinstance(default, Algorithm) + assert isinstance(alternative, Algorithm) + assert isinstance(disabled, Algorithm) + + assert default != alternative, "DEFAULT and ALTERNATIVE algorithms are the same" + assert default != disabled, "DEFAULT and DISABLED algorithms are the same" + assert alternative != disabled, "ALTERNATIVE and DISABLED algorithms are the same" + + return AlgorithmSet(default, alternative, disabled) + + +def algorithms_env(algs: AlgorithmSet) -> Dict[str, str]: + """Return environment variables with selected algorithms as a dict.""" + algs_env: Dict[str, str] = {} + + def set_alg_env(alg: Algorithm, prefix): + algs_env[f"{prefix}_ALGORITHM"] = alg.name + algs_env[f"{prefix}_ALGORITHM_NUMBER"] = str(alg.number) + algs_env[f"{prefix}_BITS"] = str(alg.bits) + + assert isinstance(algs.default, Algorithm) + assert isinstance(algs.alternative, Algorithm) + assert isinstance(algs.disabled, Algorithm) + + set_alg_env(algs.default, "DEFAULT") + set_alg_env(algs.alternative, "ALTERNATIVE") + set_alg_env(algs.disabled, "DISABLED") + + logging.info("selected algorithms: %s", algs_env) + return algs_env + + +def main(): + try: + algs = ALGORITHM_SETS[ALGORITHM_SET] + algs = filter_supported(algs) + algs = select_random(algs) + algs_env = algorithms_env(algs) + except Exception: + # if anything goes wrong, the conf.sh ignores error codes, so make sure + # we set an environment variable to an error value that can be checked + # later by run.sh + print("export ALGORITHM_SET=error") + raise + else: + for name, value in algs_env.items(): + print(f"export {name}={value}") + + +if __name__ == "__main__": + main() From 69b608ee9f90af0a351c176167825bfc335b982d Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Wed, 26 Oct 2022 17:38:32 +0200 Subject: [PATCH 5/6] Set algorithms for system tests at runtime Use the get_algorithms.py script to detect supported algorithms and select random algorithms to use for the tests. Make sure to load common.conf.sh after KEYGEN env var is exported. --- bin/tests/system/conf.sh.common | 41 ++++++++++++++++++++------------- bin/tests/system/conf.sh.in | 6 ++--- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/bin/tests/system/conf.sh.common b/bin/tests/system/conf.sh.common index 165b2b1152..b10cfeff07 100644 --- a/bin/tests/system/conf.sh.common +++ b/bin/tests/system/conf.sh.common @@ -262,24 +262,33 @@ send() { # Useful variables in test scripts # +# The following script sets the following algorithm-related variables. These +# are selected randomly at runtime from a list of supported algorithms. The +# randomization is deterministic and remains stable for a period of time for a +# given platform. +# # Default algorithm for testing. -export DEFAULT_ALGORITHM=ECDSAP256SHA256 -export DEFAULT_ALGORITHM_NUMBER=13 -export DEFAULT_BITS=256 - -# This is an alternative algorithm for test cases that require more than -# one algorithm (for example algorithm rollover). Must be different from +# DEFAULT_ALGORITHM +# DEFAULT_ALGORITHM_NUMBER +# DEFAULT_BITS +# +# This is an alternative algorithm for test cases that require more than one +# algorithm (for example algorithm rollover). Must be different from # DEFAULT_ALGORITHM. -export ALTERNATIVE_ALGORITHM=RSASHA256 -export ALTERNATIVE_ALGORITHM_NUMBER=8 -export ALTERNATIVE_BITS=1280 - -# This is an algorithm that is used for tests against the -# "disable-algorithms" configuration option. Must be different from above -# algorithms. -export DISABLED_ALGORITHM=ECDSAP384SHA384 -export DISABLED_ALGORITHM_NUMBER=14 -export DISABLED_BITS=384 +# ALTERNATIVE_ALGORITHM +# ALTERNATIVE_ALGORITHM_NUMBER +# ALTERNATIVE_BITS +# +# This is an algorithm that is used for tests against the "disable-algorithms" +# configuration option. Must be different from above algorithms. +# DISABLED_ALGORITHM +# DISABLED_ALGORITHM_NUMBER +# DISABLED_BITS +# +# There are multiple algoritms sets to choose from (see get_algorithms.py). To +# override the default choice, set the ALGORITHM_SET env var (see mkeys system +# test for example). +eval "$($PYTHON "$TOP_SRCDIR/bin/tests/system/get_algorithms.py")" # Default HMAC algorithm. # also update common/rndc.conf and common/rndc.key when updating DEFAULT_HMAC diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in index 70edf5bc3e..a41ac10b63 100644 --- a/bin/tests/system/conf.sh.in +++ b/bin/tests/system/conf.sh.in @@ -23,9 +23,6 @@ export TOP_SRCDIR=@abs_top_srcdir@ # Provide TMPDIR variable for tests that need it. export TMPDIR=${TMPDIR:-/tmp} -# Load common values -. $TOP_SRCDIR/bin/tests/system/conf.sh.common - export ARPANAME=$TOP_BUILDDIR/bin/tools/arpaname export CDS=$TOP_BUILDDIR/bin/dnssec/dnssec-cds export CHECKCONF=$TOP_BUILDDIR/bin/check/named-checkconf @@ -69,6 +66,9 @@ export KRB5_CONFIG=/dev/null # use local keytab instead of default /etc/krb5.keytab export KRB5_KTNAME=dns.keytab +# Load common values +. $TOP_SRCDIR/bin/tests/system/conf.sh.common + # # Construct the lists of tests to run # From f65f276f986fe1e0498698f7058722a0b7a9aec1 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Wed, 26 Oct 2022 16:20:57 +0200 Subject: [PATCH 6/6] Randomize algorithm selection for mkeys test Use the ALGORITHM_SET option to use randomly selected default algorithm in this test. Make sure the test works by using variables instead of hard-coding values. --- bin/tests/system/mkeys/setup.sh | 7 +++++++ bin/tests/system/mkeys/tests.sh | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/bin/tests/system/mkeys/setup.sh b/bin/tests/system/mkeys/setup.sh index 1cba2b5c19..3d4337071b 100644 --- a/bin/tests/system/mkeys/setup.sh +++ b/bin/tests/system/mkeys/setup.sh @@ -11,8 +11,15 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. +export ALGORITHM_SET="ecc_default" . ../conf.sh +# Ensure the selected algorithm set is okay. +if [ "$ALGORITHM_SET" = "error" ]; then + echofail "Algorithm selection failed." >&2 + exit 1 +fi + copy_setports ns1/named1.conf.in ns1/named.conf copy_setports ns2/named.conf.in ns2/named.conf copy_setports ns3/named.conf.in ns3/named.conf diff --git a/bin/tests/system/mkeys/tests.sh b/bin/tests/system/mkeys/tests.sh index 1fdc9eee49..30740226c2 100644 --- a/bin/tests/system/mkeys/tests.sh +++ b/bin/tests/system/mkeys/tests.sh @@ -13,6 +13,7 @@ set -e +export ALGORITHM_SET="ecc_default" #shellcheck source=conf.sh . ../conf.sh @@ -690,7 +691,7 @@ ret=0 # compare against the known key. tathex=$(grep "query '_ta-[0-9a-f][0-9a-f]*/NULL/IN' approved" ns1/named.run | awk '{print $6; exit 0}' | sed -e 's/(_ta-\([0-9a-f][0-9a-f]*\)):/\1/') || true tatkey=$($PERL -e 'printf("%d\n", hex(@ARGV[0]));' "$tathex") -realkey=$(rndccmd 10.53.0.2 secroots - | sed -n 's#.*SHA256/\([0-9][0-9]*\) ; .*managed.*#\1#p') +realkey=$(rndccmd 10.53.0.2 secroots - | sed -n "s#.*${DEFAULT_ALGORITHM}/\([0-9][0-9]*\) ; .*managed.*#\1#p") [ "$tatkey" -eq "$realkey" ] || ret=1 if [ $ret != 0 ]; then echo_i "failed"; fi status=$((status+ret))