From 420986bf187ed2a33321d5af362732184426f439 Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Wed, 1 Jul 2020 10:29:36 +0200 Subject: [PATCH] Add pairwise testing Pairwise testing is a test case generation technique based on the observation that most faults are caused by interactions of at most two factors. For BIND, its configure options can be thought of as such factors. Process BIND configure options into a model that is subsequently processed by the PICT tool in order to find an effective test vector. That test vector is then used for configuring and building BIND using various combinations of configure options. --- .gitlab-ci.yml | 19 +++++++++++++ configure.ac | 60 ++++++++++++++++++++++++++++++++++++++-- util/copyrights | 1 + util/pairwise-testing.sh | 32 +++++++++++++++++++++ 4 files changed, 110 insertions(+), 2 deletions(-) create mode 100755 util/pairwise-testing.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index be17b22631..ca3b373d69 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1376,3 +1376,22 @@ gcov: only: - main@isc-projects/bind9 - /^v9_[1-9][0-9]$/@isc-projects/bind9 + +# Pairwise testing of ./configure options + +pairwise: + <<: *base_image + stage: build + needs: + - job: autoreconf + artifacts: true + script: + - util/pairwise-testing.sh + artifacts: + paths: + - pairwise-commands.txt + - pairwise-model.txt + when: on_failure + only: + variables: + - $PAIRWISE_TESTING diff --git a/configure.ac b/configure.ac index 1787043f9f..be261f8646 100644 --- a/configure.ac +++ b/configure.ac @@ -154,6 +154,7 @@ AX_CHECK_COMPILE_FLAG([-fdiagnostics-show-option], # Change defaults for developers if not explicity set. # Needs to be before the option is tested. # +# [pairwise: --enable-developer, --disable-developer] AC_ARG_ENABLE([developer], [AS_HELP_STRING([--enable-developer], [enable developer build settings])]) @@ -173,6 +174,7 @@ AC_SUBST([DEVELOPER_MODE]) AC_SUBST([STD_CFLAGS]) AC_SUBST([STD_CPPFLAGS]) +# [pairwise: --enable-warn-error, --disable-warn-error] AC_ARG_ENABLE([warn_error], [AS_HELP_STRING([--enable-warn-error], [turn on -Werror when compiling])], @@ -188,6 +190,7 @@ PKG_PROG_PKG_CONFIG AS_IF([test -z "$PKG_CONFIG"], [AC_MSG_ERROR([The pkg-config script could not be found or is too old.])]) +# [pairwise: --enable-buffer-useinline, --disable-buffer-useinline] AC_ARG_ENABLE(buffer_useinline, AS_HELP_STRING([--enable-buffer-useinline], [define ISC_BUFFER_USEINLINE when compiling @@ -199,6 +202,10 @@ AC_ARG_ENABLE(buffer_useinline, fi, AC_DEFINE([ISC_BUFFER_USEINLINE], [1])) +# Fuzzing is not included in pairwise testing as fuzzing tools are +# not present in the relevant Docker image. +# +# [pairwise: skip] AC_ARG_ENABLE([fuzzing], [AS_HELP_STRING([--enable-fuzzing=], [Enable fuzzing using American Fuzzy Lop or libFuzzer (default=no)])], @@ -239,6 +246,7 @@ AS_IF([test "$enable_fuzzing" = "afl"], [AC_MSG_ERROR([set CC=afl- when --enable-fuzzing=afl is used])]) ]) +# [pairwise: --enable-mutex-atomics, --disable-mutex-atomics] AC_ARG_ENABLE(mutex_atomics, AS_HELP_STRING([--enable-mutex-atomics], [emulate atomics by mutex-locked variables, useful for debugging @@ -419,6 +427,7 @@ AX_GCC_FUNC_ATTRIBUTE([noreturn]) # # check if we have kqueue # +# [pairwise: --enable-kqueue, --disable-kqueue] AC_ARG_ENABLE([kqueue], [AS_HELP_STRING([--enable-kqueue], [use BSD kqueue when available [default=yes]])], @@ -431,6 +440,7 @@ AS_IF([test "$enable_kqueue" = "yes"], # check if we have epoll. Linux kernel 2.4 has epoll_create() which fails, # so we need to try running the code, not just test its existence. # +# [pairwise: --enable-epoll, --disable-epoll] AC_ARG_ENABLE([epoll], [AS_HELP_STRING([--enable-epoll], [use Linux epoll when available [default=auto]])], @@ -442,6 +452,7 @@ AS_IF([test "$enable_epoll" = "yes"], # # check if we support /dev/poll # +# [pairwise: --enable-devpoll, --disable-devpoll] AC_ARG_ENABLE([devpoll], [AS_HELP_STRING([--enable-devpoll], [use /dev/poll when available [default=yes]])], @@ -454,11 +465,13 @@ AS_IF([test "$enable_devpoll" = "yes"], # # Should be on by default if libmaxminddb exists. # +# [pairwise: --enable-geoip --with-maxminddb=auto, --enable-geoip --with-maxminddb=yes, --disable-geoip] AC_ARG_ENABLE([geoip], [AS_HELP_STRING([--disable-geoip], [support GeoIP2 geolocation ACLs if available [default=yes]])], [], [enable_geoip="yes"]) +# [pairwise: skip] AC_ARG_WITH([maxminddb], [AS_HELP_STRING([--with-maxminddb=PATH], [Build with MaxMind GeoIP2 support (auto|yes|no|path) [default=auto]])], @@ -506,6 +519,7 @@ CC="$PTHREAD_CC" AC_CHECK_FUNCS([pthread_attr_getstacksize pthread_attr_setstacksize]) +# [pairwise: --with-locktype=adaptive, --with-locktype=standard] AC_ARG_WITH([locktype], AS_HELP_STRING([--with-locktype=ARG], [Specify mutex lock type @@ -576,6 +590,7 @@ AC_CHECK_FUNCS([sysconf]) # # Do we want to use pthread rwlock? # +# [pairwise: --enable-pthread-rwlock, --disable-pthread-rwlock] AC_ARG_ENABLE([pthread_rwlock], [AS_HELP_STRING([--enable-pthread-rwlock], [use pthread rwlock instead of internal rwlock implementation])], @@ -686,6 +701,10 @@ AC_CHECK_FUNCS([DH_get0_key ECDSA_SIG_get0 RSA_set0_key]) # # Check whether FIPS mode is available and whether we should enable it # +# FIPS is not included in pairwise testing as the relevant Docker image +# does not support FIPS mode. +# +# [pairwise: skip] AC_ARG_ENABLE([fips-mode], [AS_HELP_STRING([--enable-fips-mode], [enable FIPS mode in OpenSSL library [default=no]])], @@ -706,6 +725,7 @@ AC_SUBST([OPENSSL_LIBS]) # # was --enable-native-pkcs11 specified? # +# [pairwise: --enable-native-pkcs11, --disable-native-pkcs11] AC_ARG_ENABLE([native-pkcs11], AS_HELP_STRING([--enable-native-pkcs11], [use native PKCS11 for public-key crypto [default=no]]), @@ -728,6 +748,7 @@ AS_CASE([$CRYPTO], # # was --with-pkcs11 specified? # +# [pairwise: skip] AC_ARG_WITH([pkcs11], [AS_HELP_STRING([--with-pkcs11[=PATH]], [Build with PKCS11 support [no|path] (PATH is for the PKCS11 provider)])], @@ -740,6 +761,7 @@ AC_DEFINE_UNQUOTED([PK11_LIB_LOCATION], ["$with_pkcs11"], [define the default PK AC_CHECK_FUNCS([clock_gettime]) +# [pairwise: --with-gssapi=yes, --with-gssapi=auto, --without-gssapi] AC_ARG_WITH([gssapi], [AS_HELP_STRING([--with-gssapi=[PATH|[/path/]krb5-config]], [Specify path for system-supplied GSSAPI @@ -806,6 +828,7 @@ AC_SUBST([KRB5_LIBS]) # was --with-lmdb specified? # +# [pairwise: --with-lmdb=auto, --with-lmdb=yes, --without-lmdb] AC_ARG_WITH([lmdb], [AS_HELP_STRING([--with-lmdb=@<:@PATH@:>@], [use LMDB library @<:@default=auto@:>@, optionally specify the prefix for lmdb library])], @@ -843,6 +866,7 @@ AM_CONDITIONAL([HAVE_LMDB], [test -n "$LMDB_LIBS"]) # # was --with-libxml2 specified? # +# [pairwise: --with-libxml2=auto, --with-libxml2=yes, --without-libxml2] AC_ARG_WITH([libxml2], [AS_HELP_STRING([--with-libxml2], [build with libxml2 library [yes|no|auto] (default is auto)])], @@ -862,6 +886,7 @@ AM_CONDITIONAL([HAVE_LIBXML2], [test -n "$LIBXML2_LIBS"]) # # was --with-json-c specified? # +# [pairwise: --with-json-c=detect, --with-json-c=yes, --without-json-c] AC_ARG_WITH([json-c], [AS_HELP_STRING([--with-json-c], [build with json-c library [yes|no|detect] (default is detect)])], @@ -885,6 +910,7 @@ AC_SUBST([JSON_C_LIBS]) # # was --with-zlib specified? # +# [pairwise: --with-zlib=auto, --with-zlib=yes, --without-zlib] AC_ARG_WITH([zlib], [AS_HELP_STRING([--with-zlib], [build with zlib for HTTP compression @@ -941,7 +967,7 @@ AC_COMPILE_IFELSE( # confusing results on some systems (e.g. FreeBSD; see set_tcp_fastopen() # comment in lib/isc/unix/socket.c). # - +# [pairwise: --enable-tcp-fastopen, --disable-tcp-fastopen] AC_ARG_ENABLE([tcp_fastopen], [AS_HELP_STRING([--disable-tcp-fastopen], [disable TCP Fast Open support [default=yes]])], @@ -959,6 +985,13 @@ AC_CHECK_FUNCS([strlcpy strlcat]) # Check for readline support # +# editline is not included in pairwise testing as the library +# is not present in the relevant Docker image. +# +# GNU Readline is not included in pairwise testing as the library +# in the relevant Docker image does not support pkg-config. +# +# [pairwise: --with-readline=auto, --with-readline=yes, --with-readline=libedit, --without-readline] AC_ARG_WITH([readline], [AS_HELP_STRING([--with-readline=yes|no|libedit|readline], [specify readline library [default auto]])], @@ -994,6 +1027,8 @@ AC_SUBST([READLINE_LIBS]) # # Note it is very recommended to *not* disable chroot(), # this is only because chroot() was made obsolete by Posix. +# +# [pairwise: --enable-chroot, --disable-chroot] AC_ARG_ENABLE(chroot, AS_HELP_STRING([--disable-chroot], [disable chroot])) case "$enable_chroot" in yes|'') @@ -1005,6 +1040,8 @@ esac LIBCAP_LIBS="" AC_MSG_CHECKING([whether to enable Linux capabilities]) + +# [pairwise: --enable-linux-caps, --disable-linux-caps] AC_ARG_ENABLE([linux-caps], [AS_HELP_STRING([--disable-linux-caps], [disable Linux capabilities])], @@ -1190,6 +1227,7 @@ fi # # Activate "rrset-order fixed" or not? # +# [pairwise: --enable-fixed-rrset, --disable-fixed-rrset] AC_ARG_ENABLE([fixed-rrset], [AS_HELP_STRING([--enable-fixed-rrset], [enable fixed rrset ordering [default=no]])], @@ -1201,6 +1239,7 @@ AS_IF([test "$enable_fixed_rrset" = "yes"], # # Activate dnstap? # +# [pairwise: --enable-dnstap, --disable-dnstap] AC_ARG_ENABLE([dnstap], [AS_HELP_STRING([--enable-dnstap], [enable dnstap support @@ -1266,6 +1305,8 @@ AC_SUBST(CURL) LIBIDN2_CFLAGS= LIBIDN2_LDFLAGS= LIBIDN2_LIBS= + +# [pairwise: --with-libidn2=yes, --without-libidn2] AC_ARG_WITH([libidn2], [AS_HELP_STRING([--with-libidn2[=PATH]], [enable IDN support using GNU libidn2 [yes|no(default)|path]])], [with_libidn2="$withval"], [with_libidn2="no"]) @@ -1294,7 +1335,7 @@ AC_SUBST([LIBIDN2_LIBS]) # # Check whether to build with cmocka unit testing framework # - +# [pairwise: --with-cmocka=detect, --with-cmocka=yes, --without-cmocka] AC_ARG_WITH([cmocka], [AS_HELP_STRING([--with-cmocka=detect],[enable CMocka based tests (default is detect)])], [],[with_cmocka=detect]) @@ -1360,6 +1401,7 @@ AC_CHECK_FUNCS(setlocale) # # was --with-tuning specified? # +# [pairwise: --with-tuning=small, --without-tuning] AC_ARG_WITH([tuning], AS_HELP_STRING([--with-tuning=ARG], [Specify server tuning (default or small)]), @@ -1373,6 +1415,7 @@ AS_CASE([$with_tuning], # # was --enable-querytrace or --enable-singletrace specified? # +# [pairwise: --enable-singletrace --enable-querytrace, --disable-singletrace --enable-querytrace, --disable-singletrace --disable-querytrace] AC_ARG_ENABLE([singletrace], AS_HELP_STRING([--enable-singletrace], [enable single-query trace logging @@ -1394,6 +1437,7 @@ no) ;; esac +# [pairwise: skip] AC_ARG_ENABLE(querytrace, AS_HELP_STRING([--enable-querytrace], [enable very verbose query trace logging @@ -1420,7 +1464,10 @@ esac # # Was --disable-auto-validation specified? # + validation_default=auto + +# [pairwise: --enable-auto-validation, --disable-auto-validation] AC_ARG_ENABLE(auto-validation, AS_HELP_STRING([--enable-auto-validation], [turn on DNSSEC validation by default, using the IANA root key [default=yes]]), @@ -1474,6 +1521,10 @@ AC_COMPILE_IFELSE( AC_MSG_RESULT([no]) ]) +# DNSRPS is not included in pairwise testing as the librpz library is not +# present in the relevant Docker image. +# +# [pairwise: skip] AC_ARG_ENABLE([dnsrps-dl], [AS_HELP_STRING([--enable-dnsrps-dl], [DNS Response Policy Service delayed link @@ -1483,10 +1534,13 @@ AC_ARG_ENABLE([dnsrps-dl], AS_IF([test "$enable_librpz_dl" = "yes" -a "$with_dlopen" = "no"], [AC_MSG_ERROR([DNS Response Policy Service delayed link requires dlopen to be enabled])]) +# [pairwise: skip] AC_ARG_WITH([dnsrps-libname], [AS_HELP_STRING([--with-dnsrps-libname], [DNSRPS provider library name (librpz.so)])], [librpz_name="$withval"], [librpz_name="librpz.so"]) + +# [pairwise: skip] AC_ARG_WITH([dnsrps-dir], [AS_HELP_STRING([--with-dnsrps-dir], [path to DNSRPS provider library])], @@ -1506,6 +1560,7 @@ AS_IF([test "$enable_librpz_dl" = "yes"], AC_DEFINE_UNQUOTED([DNSRPS_LIB_OPEN], [$dnsrps_lib_open], [0=no DNSRPS 1=static link 2=dlopen()]) +# [pairwise: skip] AC_ARG_ENABLE([dnsrps], AS_HELP_STRING([--enable-dnsrps], [enable DNS Response Policy Service API]), @@ -1646,6 +1701,7 @@ AC_CONFIG_FILES([util/check-make-install]) AC_OUTPUT +# [pairwise: --enable-full-report, --disable-full-report] AC_ARG_ENABLE(full-report, AS_HELP_STRING([--enable-full-report], [report values of all configure options])) diff --git a/util/copyrights b/util/copyrights index 4763526922..95314a5679 100644 --- a/util/copyrights +++ b/util/copyrights @@ -2190,6 +2190,7 @@ ./util/nanny.pl PERL 2000,2001,2004,2007,2012,2016,2018,2019,2020 ./util/new-func PERL 2005,2007,2012,2016,2018,2019,2020 ./util/nt-kit SH 1999,2000,2001,2004,2007,2012,2016,2018,2019,2020 +./util/pairwise-testing.sh SH 2020 ./util/parse_tsan.py PYTHON-BIN 2019,2020 ./util/run-clang-tidy X 2020 ./util/spacewhack.pl PERL 2000,2001,2004,2007,2012,2016,2018,2019,2020 diff --git a/util/pairwise-testing.sh b/util/pairwise-testing.sh new file mode 100755 index 0000000000..e67145b8fe --- /dev/null +++ b/util/pairwise-testing.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +set -e +set -o pipefail + +grep -v -F "pairwise: skip" configure.ac | sed -n -E "s|.*# \[pairwise: (.*)\]|\1|p" | \ + while read -r SWITCH; do + echo "${RANDOM}: ${SWITCH}" +done > pairwise-model.txt + +pict pairwise-model.txt 2>/dev/null | tr "\t" " " | sed "1d" > pairwise-commands.txt + +while read -r -a configure_switches; do + runid=${RANDOM} + mkdir "pairwise-${runid}" + cd "pairwise-${runid}" + echo "${configure_switches[@]}" > "../pairwise-output.${runid}.txt" + ../configure "${configure_switches[@]}" 2>&1 | tee -a "../pairwise-output.${runid}.txt" + grep -F "WARNING: unrecognized options:" "../pairwise-output.${runid}.txt" && exit 1 + make "-j${BUILD_PARALLEL_JOBS:-1}" all 2>&1 | tee -a "../pairwise-output.${runid}.txt" + cd .. + rm -rf "pairwise-${runid}" "pairwise-output.${runid}.txt" +done < pairwise-commands.txt