From fbdd464038d9890c56ebdd848d6473e156bd17a7 Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Wed, 30 Dec 2020 13:23:18 +0100 Subject: [PATCH 1/5] Record skipped test as skipped in testsuite summary When system test execution was ported to Automake, SKIPPED and UNTESTED system test result were not made to match Automake expectations, therefore a skipped test is recorded by Automake as "PASS": $ make check TESTS=cpu V=1 I:cpu:cpu test only runs on Linux, skipping test I:cpu:Prerequisites missing, skipping test. R:cpu:SKIPPED E:cpu:2020-12-16T11:36:58+0000 PASS: cpu ==================================================================== Testsuite summary for BIND 9.17.7 ==================================================================== # TOTAL: 1 # PASS: 1 For a test to be recorded by Automake as skipped, the test, or it's test driver, needs to exit with code 77: $ make check TESTS=cpu V=1 I:cpu:cpu test only runs on Linux, skipping test I:cpu:Prerequisites missing, skipping test. R:cpu:SKIPPED E:cpu:2020-12-16T11:39:10+0000 SKIP: cpu ==================================================================== Testsuite summary for BIND 9.17.7 ==================================================================== # TOTAL: 1 # PASS: 0 # SKIP: 1 --- bin/tests/system/run.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/tests/system/run.sh.in b/bin/tests/system/run.sh.in index 7027ab2192..3a71afef08 100644 --- a/bin/tests/system/run.sh.in +++ b/bin/tests/system/run.sh.in @@ -172,7 +172,7 @@ else echowarn "R:$systest:UNTESTED" fi echoend "E:$systest:$(date_with_args)" - exit 0 + exit 77 fi # Check for PKCS#11 support From 29d7c6e449f74653fd911e856b8b06a7f9994891 Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Wed, 30 Dec 2020 13:22:46 +0100 Subject: [PATCH 2/5] Merge UNTESTED and SKIPPED system test results Descriptions of UNTESTED and SKIPPED system test results are very similar to one another and it may be confusing when to pick one and when the other. Merging these two system test results removes the confusion and also makes system test more aligned with Automake, which does not know about UNTESTED test result. --- bin/tests/system/README | 9 +++------ bin/tests/system/run.sh.in | 6 +----- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/bin/tests/system/README b/bin/tests/system/README index a6d9558101..3bc6210a9a 100644 --- a/bin/tests/system/README +++ b/bin/tests/system/README @@ -214,9 +214,6 @@ This indicates the type of message. This is one of: FAIL The test failed SKIPPED The test was not run, usually because some prerequisites required to run the test are missing. - UNTESTED The test was not run for some other reason, e.g. a - prerequisite is available but is not compatible with - the platform on which the test is run. This is the name of the test from which the message emanated, which is also the @@ -270,9 +267,9 @@ As noted above, each test is in a separate directory. To interact with the test framework, the directories contain the following standard files: prereq.sh Run at the beginning to determine whether the test can be run at - all; if not, we see a result of R:SKIPPED or R:UNTESTED. This file - is optional: if not present, the test is assumed to have all its - prerequisites met. + all; if not, we see a R:SKIPPED result. This file is optional: + if not present, the test is assumed to have all its prerequisites + met. setup.sh Run after prereq.sh, this sets up the preconditions for the tests. Although optional, virtually all tests will require such a file to diff --git a/bin/tests/system/run.sh.in b/bin/tests/system/run.sh.in index 3a71afef08..99c93da1ca 100644 --- a/bin/tests/system/run.sh.in +++ b/bin/tests/system/run.sh.in @@ -166,11 +166,7 @@ if [ $result -eq 0 ]; then : prereqs ok else echowarn "I:$systest:Prerequisites missing, skipping test." - if [ $result -eq 255 ]; then - echowarn "R:$systest:SKIPPED"; - else - echowarn "R:$systest:UNTESTED" - fi + echowarn "R:$systest:SKIPPED"; echoend "E:$systest:$(date_with_args)" exit 77 fi From fa505bfb0e7623d7cfc94ae15a0246ae71000904 Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Mon, 18 Jan 2021 19:15:44 +0100 Subject: [PATCH 3/5] Record skipped unit test as skipped in Automake framework --- Makefile.tests | 5 +++-- lib/dns/tests/acl_test.c | 2 +- lib/dns/tests/db_test.c | 2 +- lib/dns/tests/dbdiff_test.c | 2 +- lib/dns/tests/dbiterator_test.c | 2 +- lib/dns/tests/dbversion_test.c | 2 +- lib/dns/tests/dh_test.c | 3 ++- lib/dns/tests/dispatch_test.c | 2 +- lib/dns/tests/dns64_test.c | 2 +- lib/dns/tests/dnstap_test.c | 2 +- lib/dns/tests/dst_test.c | 2 +- lib/dns/tests/geoip_test.c | 2 +- lib/dns/tests/keytable_test.c | 2 +- lib/dns/tests/master_test.c | 2 +- lib/dns/tests/name_test.c | 2 +- lib/dns/tests/nsec3_test.c | 2 +- lib/dns/tests/nsec3param_test.c | 2 +- lib/dns/tests/peer_test.c | 2 +- lib/dns/tests/private_test.c | 2 +- lib/dns/tests/rbt_serialize_test.c | 2 +- lib/dns/tests/rbt_test.c | 2 +- lib/dns/tests/rdata_test.c | 2 +- lib/dns/tests/rdataset_test.c | 2 +- lib/dns/tests/rdatasetstats_test.c | 2 +- lib/dns/tests/resolver_test.c | 2 +- lib/dns/tests/result_test.c | 2 +- lib/dns/tests/rsa_test.c | 2 +- lib/dns/tests/sigs_test.c | 2 +- lib/dns/tests/time_test.c | 2 +- lib/dns/tests/tsig_test.c | 2 +- lib/dns/tests/update_test.c | 2 +- lib/dns/tests/zonemgr_test.c | 2 +- lib/dns/tests/zt_test.c | 2 +- lib/irs/tests/resconf_test.c | 2 +- lib/isc/tests/aes_test.c | 2 +- lib/isc/tests/buffer_test.c | 2 +- lib/isc/tests/counter_test.c | 2 +- lib/isc/tests/crc64_test.c | 2 +- lib/isc/tests/errno_test.c | 2 +- lib/isc/tests/file_test.c | 2 +- lib/isc/tests/hash_test.c | 2 +- lib/isc/tests/heap_test.c | 2 +- lib/isc/tests/hmac_test.c | 2 +- lib/isc/tests/ht_test.c | 2 +- lib/isc/tests/lex_test.c | 2 +- lib/isc/tests/md_test.c | 2 +- lib/isc/tests/mem_test.c | 2 +- lib/isc/tests/netaddr_test.c | 2 +- lib/isc/tests/parse_test.c | 2 +- lib/isc/tests/pool_test.c | 2 +- lib/isc/tests/quota_test.c | 2 +- lib/isc/tests/radix_test.c | 2 +- lib/isc/tests/random_test.c | 2 +- lib/isc/tests/regex_test.c | 2 +- lib/isc/tests/result_test.c | 2 +- lib/isc/tests/safe_test.c | 2 +- lib/isc/tests/siphash_test.c | 2 +- lib/isc/tests/sockaddr_test.c | 2 +- lib/isc/tests/socket_test.c | 2 +- lib/isc/tests/symtab_test.c | 2 +- lib/isc/tests/task_test.c | 2 +- lib/isc/tests/taskpool_test.c | 2 +- lib/isc/tests/tcp_quota_test.c | 2 +- lib/isc/tests/tcp_test.c | 2 +- lib/isc/tests/tcpdns_test.c | 2 +- lib/isc/tests/time_test.c | 2 +- lib/isc/tests/timer_test.c | 2 +- lib/isc/tests/udp_test.c | 2 +- lib/isccc/tests/result_test.c | 2 +- lib/isccfg/tests/duration_test.c | 2 +- lib/isccfg/tests/parser_test.c | 2 +- lib/ns/tests/listenlist_test.c | 4 ++-- lib/ns/tests/notify_test.c | 4 ++-- lib/ns/tests/plugin_test.c | 2 +- lib/ns/tests/query_test.c | 4 ++-- 75 files changed, 81 insertions(+), 79 deletions(-) diff --git a/Makefile.tests b/Makefile.tests index 2d39eb8f25..3b5e07d90c 100644 --- a/Makefile.tests +++ b/Makefile.tests @@ -3,8 +3,9 @@ AM_CPPFLAGS += \ $(CMOCKA_CFLAGS) \ - -DTESTS_DIR=\"$(abs_srcdir)\" \ - -DNAMED_PLUGINDIR=\"$(libdir)/named\" + -DNAMED_PLUGINDIR=\"$(libdir)/named\" \ + -DSKIPPED_TEST_EXIT_CODE=77 \ + -DTESTS_DIR=\"$(abs_srcdir)\" LDADD = \ $(CMOCKA_LIBS) diff --git a/lib/dns/tests/acl_test.c b/lib/dns/tests/acl_test.c index 93758ad252..f04b8a8e87 100644 --- a/lib/dns/tests/acl_test.c +++ b/lib/dns/tests/acl_test.c @@ -150,7 +150,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/db_test.c b/lib/dns/tests/db_test.c index 0d348999bf..c12e90ee1c 100644 --- a/lib/dns/tests/db_test.c +++ b/lib/dns/tests/db_test.c @@ -417,7 +417,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/dbdiff_test.c b/lib/dns/tests/dbdiff_test.c index 3246d60814..a62496f6eb 100644 --- a/lib/dns/tests/dbdiff_test.c +++ b/lib/dns/tests/dbdiff_test.c @@ -177,7 +177,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/dbiterator_test.c b/lib/dns/tests/dbiterator_test.c index e6ae81ea36..13a4976d45 100644 --- a/lib/dns/tests/dbiterator_test.c +++ b/lib/dns/tests/dbiterator_test.c @@ -386,7 +386,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/dbversion_test.c b/lib/dns/tests/dbversion_test.c index 0891bf19cd..d90316504f 100644 --- a/lib/dns/tests/dbversion_test.c +++ b/lib/dns/tests/dbversion_test.c @@ -491,7 +491,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/dh_test.c b/lib/dns/tests/dh_test.c index 176f938763..db7cac9b6a 100644 --- a/lib/dns/tests/dh_test.c +++ b/lib/dns/tests/dh_test.c @@ -100,6 +100,7 @@ main(void) { return (cmocka_run_group_tests(tests, NULL, NULL)); #else /* if USE_OPENSSL */ print_message("1..0 # Skipped: dh test broken with PKCS11"); + return (SKIPPED_TEST_EXIT_CODE); #endif /* if USE_OPENSSL */ } @@ -110,7 +111,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/dispatch_test.c b/lib/dns/tests/dispatch_test.c index b8cc954c88..7bd0712d56 100644 --- a/lib/dns/tests/dispatch_test.c +++ b/lib/dns/tests/dispatch_test.c @@ -352,7 +352,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/dns64_test.c b/lib/dns/tests/dns64_test.c index 450b09ac67..c95504cfa8 100644 --- a/lib/dns/tests/dns64_test.c +++ b/lib/dns/tests/dns64_test.c @@ -244,7 +244,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif diff --git a/lib/dns/tests/dnstap_test.c b/lib/dns/tests/dnstap_test.c index 0a64de7544..b41cd14fd4 100644 --- a/lib/dns/tests/dnstap_test.c +++ b/lib/dns/tests/dnstap_test.c @@ -391,7 +391,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* HAVE_CMOCKA */ diff --git a/lib/dns/tests/dst_test.c b/lib/dns/tests/dst_test.c index 4cffaae0a8..288ac5b7b7 100644 --- a/lib/dns/tests/dst_test.c +++ b/lib/dns/tests/dst_test.c @@ -302,7 +302,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/geoip_test.c b/lib/dns/tests/geoip_test.c index 2f96692d51..b7020ccc1d 100644 --- a/lib/dns/tests/geoip_test.c +++ b/lib/dns/tests/geoip_test.c @@ -418,7 +418,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* HAVE_CMOCKA */ diff --git a/lib/dns/tests/keytable_test.c b/lib/dns/tests/keytable_test.c index e6a96350bd..9d482d7bb6 100644 --- a/lib/dns/tests/keytable_test.c +++ b/lib/dns/tests/keytable_test.c @@ -712,7 +712,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/master_test.c b/lib/dns/tests/master_test.c index 55c415a84d..a4389f296c 100644 --- a/lib/dns/tests/master_test.c +++ b/lib/dns/tests/master_test.c @@ -644,7 +644,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/name_test.c b/lib/dns/tests/name_test.c index 161d2c4504..5cb94f505e 100644 --- a/lib/dns/tests/name_test.c +++ b/lib/dns/tests/name_test.c @@ -788,7 +788,7 @@ main(int argc, char **argv) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/nsec3_test.c b/lib/dns/tests/nsec3_test.c index dadb6688f6..a71b3c908b 100644 --- a/lib/dns/tests/nsec3_test.c +++ b/lib/dns/tests/nsec3_test.c @@ -189,7 +189,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/nsec3param_test.c b/lib/dns/tests/nsec3param_test.c index 3ae5da4168..65206bed7a 100644 --- a/lib/dns/tests/nsec3param_test.c +++ b/lib/dns/tests/nsec3param_test.c @@ -296,7 +296,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/peer_test.c b/lib/dns/tests/peer_test.c index 9342ba0bf2..8f0759c555 100644 --- a/lib/dns/tests/peer_test.c +++ b/lib/dns/tests/peer_test.c @@ -167,7 +167,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/private_test.c b/lib/dns/tests/private_test.c index 4da68a29da..47d7dd261c 100644 --- a/lib/dns/tests/private_test.c +++ b/lib/dns/tests/private_test.c @@ -232,7 +232,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/rbt_serialize_test.c b/lib/dns/tests/rbt_serialize_test.c index e04422c6eb..71cc2903a6 100644 --- a/lib/dns/tests/rbt_serialize_test.c +++ b/lib/dns/tests/rbt_serialize_test.c @@ -481,7 +481,7 @@ main(int argc, char **argv) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/rbt_test.c b/lib/dns/tests/rbt_test.c index c17b503f8f..08276827b7 100644 --- a/lib/dns/tests/rbt_test.c +++ b/lib/dns/tests/rbt_test.c @@ -1382,7 +1382,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/rdata_test.c b/lib/dns/tests/rdata_test.c index e44b2f1cab..9cf32c2849 100644 --- a/lib/dns/tests/rdata_test.c +++ b/lib/dns/tests/rdata_test.c @@ -2704,7 +2704,7 @@ main(int argc, char **argv) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/rdataset_test.c b/lib/dns/tests/rdataset_test.c index 87e773a468..72b6fd3946 100644 --- a/lib/dns/tests/rdataset_test.c +++ b/lib/dns/tests/rdataset_test.c @@ -138,7 +138,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/rdatasetstats_test.c b/lib/dns/tests/rdatasetstats_test.c index 3cf15f1b82..a6df467e8e 100644 --- a/lib/dns/tests/rdatasetstats_test.c +++ b/lib/dns/tests/rdatasetstats_test.c @@ -304,7 +304,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/resolver_test.c b/lib/dns/tests/resolver_test.c index 81457284a3..66f7c96b83 100644 --- a/lib/dns/tests/resolver_test.c +++ b/lib/dns/tests/resolver_test.c @@ -220,7 +220,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/result_test.c b/lib/dns/tests/result_test.c index 44605edad2..b21828cbe5 100644 --- a/lib/dns/tests/result_test.c +++ b/lib/dns/tests/result_test.c @@ -125,7 +125,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/rsa_test.c b/lib/dns/tests/rsa_test.c index 851a4220bf..74b7bf5961 100644 --- a/lib/dns/tests/rsa_test.c +++ b/lib/dns/tests/rsa_test.c @@ -231,7 +231,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* HAVE_CMOCKA */ diff --git a/lib/dns/tests/sigs_test.c b/lib/dns/tests/sigs_test.c index cff7cd908c..9c59afa770 100644 --- a/lib/dns/tests/sigs_test.c +++ b/lib/dns/tests/sigs_test.c @@ -454,7 +454,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/time_test.c b/lib/dns/tests/time_test.c index d6edf74de3..3dc636b63b 100644 --- a/lib/dns/tests/time_test.c +++ b/lib/dns/tests/time_test.c @@ -210,7 +210,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/tsig_test.c b/lib/dns/tests/tsig_test.c index 897f0f741f..6c8e6a3c05 100644 --- a/lib/dns/tests/tsig_test.c +++ b/lib/dns/tests/tsig_test.c @@ -597,7 +597,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/update_test.c b/lib/dns/tests/update_test.c index f15d015360..8036229a94 100644 --- a/lib/dns/tests/update_test.c +++ b/lib/dns/tests/update_test.c @@ -321,7 +321,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/zonemgr_test.c b/lib/dns/tests/zonemgr_test.c index e508a2a1d1..ca7559e181 100644 --- a/lib/dns/tests/zonemgr_test.c +++ b/lib/dns/tests/zonemgr_test.c @@ -257,7 +257,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/dns/tests/zt_test.c b/lib/dns/tests/zt_test.c index bd1a325ea9..7945a0bd9e 100644 --- a/lib/dns/tests/zt_test.c +++ b/lib/dns/tests/zt_test.c @@ -368,7 +368,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/irs/tests/resconf_test.c b/lib/irs/tests/resconf_test.c index 99876e8a76..8ca9762bff 100644 --- a/lib/irs/tests/resconf_test.c +++ b/lib/irs/tests/resconf_test.c @@ -133,7 +133,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/aes_test.c b/lib/isc/tests/aes_test.c index b808cd4531..919c572786 100644 --- a/lib/isc/tests/aes_test.c +++ b/lib/isc/tests/aes_test.c @@ -241,7 +241,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/buffer_test.c b/lib/isc/tests/buffer_test.c index 228b089318..a50431ce81 100644 --- a/lib/isc/tests/buffer_test.c +++ b/lib/isc/tests/buffer_test.c @@ -343,7 +343,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/counter_test.c b/lib/isc/tests/counter_test.c index 9642059ef6..52ee95bd69 100644 --- a/lib/isc/tests/counter_test.c +++ b/lib/isc/tests/counter_test.c @@ -97,7 +97,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/crc64_test.c b/lib/isc/tests/crc64_test.c index 8fd22aca15..0ee45441b2 100644 --- a/lib/isc/tests/crc64_test.c +++ b/lib/isc/tests/crc64_test.c @@ -100,7 +100,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/errno_test.c b/lib/isc/tests/errno_test.c index 4d66d9efd8..47fe24b9b7 100644 --- a/lib/isc/tests/errno_test.c +++ b/lib/isc/tests/errno_test.c @@ -115,7 +115,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/file_test.c b/lib/isc/tests/file_test.c index 74166e61d2..d1d6a956e7 100644 --- a/lib/isc/tests/file_test.c +++ b/lib/isc/tests/file_test.c @@ -146,7 +146,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/hash_test.c b/lib/isc/tests/hash_test.c index 3444f9f0ca..6d4918b218 100644 --- a/lib/isc/tests/hash_test.c +++ b/lib/isc/tests/hash_test.c @@ -106,7 +106,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/heap_test.c b/lib/isc/tests/heap_test.c index 2ac3a546c2..fb61f3fd99 100644 --- a/lib/isc/tests/heap_test.c +++ b/lib/isc/tests/heap_test.c @@ -110,7 +110,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/hmac_test.c b/lib/isc/tests/hmac_test.c index 1cf4002695..cfd3ae3c9b 100644 --- a/lib/isc/tests/hmac_test.c +++ b/lib/isc/tests/hmac_test.c @@ -941,7 +941,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/ht_test.c b/lib/isc/tests/ht_test.c index cc8724546b..c139ca8de4 100644 --- a/lib/isc/tests/ht_test.c +++ b/lib/isc/tests/ht_test.c @@ -353,7 +353,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/lex_test.c b/lib/isc/tests/lex_test.c index 34ae9ba9bd..059d323bf0 100644 --- a/lib/isc/tests/lex_test.c +++ b/lib/isc/tests/lex_test.c @@ -136,7 +136,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/md_test.c b/lib/isc/tests/md_test.c index a41471a05a..260fda861e 100644 --- a/lib/isc/tests/md_test.c +++ b/lib/isc/tests/md_test.c @@ -579,7 +579,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/mem_test.c b/lib/isc/tests/mem_test.c index e096c54c67..3b7c3e9ee1 100644 --- a/lib/isc/tests/mem_test.c +++ b/lib/isc/tests/mem_test.c @@ -533,7 +533,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/netaddr_test.c b/lib/isc/tests/netaddr_test.c index 6a01904163..9ec2cb894d 100644 --- a/lib/isc/tests/netaddr_test.c +++ b/lib/isc/tests/netaddr_test.c @@ -151,7 +151,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/parse_test.c b/lib/isc/tests/parse_test.c index 5fe9e013a7..68f08d97ef 100644 --- a/lib/isc/tests/parse_test.c +++ b/lib/isc/tests/parse_test.c @@ -88,7 +88,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/pool_test.c b/lib/isc/tests/pool_test.c index 45c3b9c1d4..9ead94a592 100644 --- a/lib/isc/tests/pool_test.c +++ b/lib/isc/tests/pool_test.c @@ -186,7 +186,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/quota_test.c b/lib/isc/tests/quota_test.c index 99ba08f75b..1def0c7456 100644 --- a/lib/isc/tests/quota_test.c +++ b/lib/isc/tests/quota_test.c @@ -348,7 +348,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/radix_test.c b/lib/isc/tests/radix_test.c index 2149ae1715..d444804802 100644 --- a/lib/isc/tests/radix_test.c +++ b/lib/isc/tests/radix_test.c @@ -116,7 +116,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/random_test.c b/lib/isc/tests/random_test.c index 4b6b643967..7161cd96a9 100644 --- a/lib/isc/tests/random_test.c +++ b/lib/isc/tests/random_test.c @@ -885,7 +885,7 @@ main(int argc, char **argv) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/regex_test.c b/lib/isc/tests/regex_test.c index 7fe4bcb2de..5d7ae5e5b4 100644 --- a/lib/isc/tests/regex_test.c +++ b/lib/isc/tests/regex_test.c @@ -2366,7 +2366,7 @@ main(int argc, char **argv) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/result_test.c b/lib/isc/tests/result_test.c index f1242dc668..248dea6f8c 100644 --- a/lib/isc/tests/result_test.c +++ b/lib/isc/tests/result_test.c @@ -124,7 +124,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/safe_test.c b/lib/isc/tests/safe_test.c index fbe3190e17..b4de3be002 100644 --- a/lib/isc/tests/safe_test.c +++ b/lib/isc/tests/safe_test.c @@ -99,7 +99,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/siphash_test.c b/lib/isc/tests/siphash_test.c index 1657fa5b0d..3cc85fac90 100644 --- a/lib/isc/tests/siphash_test.c +++ b/lib/isc/tests/siphash_test.c @@ -175,7 +175,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/sockaddr_test.c b/lib/isc/tests/sockaddr_test.c index 59ed4c6353..c944b9bf1c 100644 --- a/lib/isc/tests/sockaddr_test.c +++ b/lib/isc/tests/sockaddr_test.c @@ -180,7 +180,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/socket_test.c b/lib/isc/tests/socket_test.c index f894393ca0..47e98a7b46 100644 --- a/lib/isc/tests/socket_test.c +++ b/lib/isc/tests/socket_test.c @@ -824,7 +824,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/symtab_test.c b/lib/isc/tests/symtab_test.c index 85b0dabaf9..6ac0204c78 100644 --- a/lib/isc/tests/symtab_test.c +++ b/lib/isc/tests/symtab_test.c @@ -163,7 +163,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/task_test.c b/lib/isc/tests/task_test.c index f4a44a12c5..0aad028fb1 100644 --- a/lib/isc/tests/task_test.c +++ b/lib/isc/tests/task_test.c @@ -1598,7 +1598,7 @@ main(int argc, char **argv) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/taskpool_test.c b/lib/isc/tests/taskpool_test.c index 5b97a0d765..195ac4b44f 100644 --- a/lib/isc/tests/taskpool_test.c +++ b/lib/isc/tests/taskpool_test.c @@ -201,7 +201,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/tcp_quota_test.c b/lib/isc/tests/tcp_quota_test.c index 849213b677..17bec56f24 100644 --- a/lib/isc/tests/tcp_quota_test.c +++ b/lib/isc/tests/tcp_quota_test.c @@ -731,7 +731,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/tcp_test.c b/lib/isc/tests/tcp_test.c index a27ab80ebb..7e911fe948 100644 --- a/lib/isc/tests/tcp_test.c +++ b/lib/isc/tests/tcp_test.c @@ -1042,7 +1042,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/tcpdns_test.c b/lib/isc/tests/tcpdns_test.c index 2145964592..f576fe8bd0 100644 --- a/lib/isc/tests/tcpdns_test.c +++ b/lib/isc/tests/tcpdns_test.c @@ -873,7 +873,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/time_test.c b/lib/isc/tests/time_test.c index 1bc0391744..e16fd3e4ed 100644 --- a/lib/isc/tests/time_test.c +++ b/lib/isc/tests/time_test.c @@ -315,7 +315,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/timer_test.c b/lib/isc/tests/timer_test.c index d29241fe27..4d02ff3b74 100644 --- a/lib/isc/tests/timer_test.c +++ b/lib/isc/tests/timer_test.c @@ -647,7 +647,7 @@ main(int argc, char **argv) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isc/tests/udp_test.c b/lib/isc/tests/udp_test.c index a1a99314e2..00210a18e9 100644 --- a/lib/isc/tests/udp_test.c +++ b/lib/isc/tests/udp_test.c @@ -886,7 +886,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isccc/tests/result_test.c b/lib/isccc/tests/result_test.c index fd90020118..341088a04b 100644 --- a/lib/isccc/tests/result_test.c +++ b/lib/isccc/tests/result_test.c @@ -76,7 +76,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isccfg/tests/duration_test.c b/lib/isccfg/tests/duration_test.c index 67777aa14b..3c5d45092c 100644 --- a/lib/isccfg/tests/duration_test.c +++ b/lib/isccfg/tests/duration_test.c @@ -195,7 +195,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/isccfg/tests/parser_test.c b/lib/isccfg/tests/parser_test.c index bd44120497..79e9f50541 100644 --- a/lib/isccfg/tests/parser_test.c +++ b/lib/isccfg/tests/parser_test.c @@ -282,7 +282,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/ns/tests/listenlist_test.c b/lib/ns/tests/listenlist_test.c index 7e48e4c6fd..1a7f46451a 100644 --- a/lib/ns/tests/listenlist_test.c +++ b/lib/ns/tests/listenlist_test.c @@ -130,9 +130,9 @@ main(void) { int main(void) { - printf("1..0 # Skip cmocka not available\n"); + printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* HAVE_CMOCKA */ diff --git a/lib/ns/tests/notify_test.c b/lib/ns/tests/notify_test.c index 61d4a251eb..5fbea8966e 100644 --- a/lib/ns/tests/notify_test.c +++ b/lib/ns/tests/notify_test.c @@ -156,9 +156,9 @@ main(void) { int main(void) { - printf("1..0 # Skip cmocka not available\n"); + printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* HAVE_CMOCKA */ diff --git a/lib/ns/tests/plugin_test.c b/lib/ns/tests/plugin_test.c index 289ea2e2bd..ada102a99e 100644 --- a/lib/ns/tests/plugin_test.c +++ b/lib/ns/tests/plugin_test.c @@ -202,7 +202,7 @@ main(void) { int main(void) { printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* if HAVE_CMOCKA */ diff --git a/lib/ns/tests/query_test.c b/lib/ns/tests/query_test.c index cb7c4b82f7..6630ee3369 100644 --- a/lib/ns/tests/query_test.c +++ b/lib/ns/tests/query_test.c @@ -1522,9 +1522,9 @@ main(void) { int main(void) { - printf("1..0 # Skip cmocka not available\n"); + printf("1..0 # Skipped: cmocka not available\n"); - return (0); + return (SKIPPED_TEST_EXIT_CODE); } #endif /* HAVE_CMOCKA */ From cfd0f7d2ebb5ddb84f79ef65d120a40331485810 Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Fri, 12 Feb 2021 18:19:01 +0100 Subject: [PATCH 4/5] Do not build geoip_test when GeoIP is not available --- lib/dns/tests/Makefile.am | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/dns/tests/Makefile.am b/lib/dns/tests/Makefile.am index d28608bb5f..d57d446483 100644 --- a/lib/dns/tests/Makefile.am +++ b/lib/dns/tests/Makefile.am @@ -26,7 +26,6 @@ check_PROGRAMS = \ dispatch_test \ dns64_test \ dst_test \ - geoip_test \ keytable_test \ name_test \ nsec3_test \ From 3cc3f1712b8a99dcfedd73dacc1b1b749e0603fb Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Wed, 30 Dec 2020 14:15:23 +0100 Subject: [PATCH 5/5] Add CHANGES note for [GL !4517] --- CHANGES | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES b/CHANGES index fafaf01b3d..6f335a8187 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,8 @@ +5580. [test] The system test framework no longer differentiates + between SKIPPED and UNTESTED system test results. Any + system test which is not run is now marked as SKIPPED. + [GL !4517] + 5579. [bug] If an invalid key name (e.g. "a..b") is specified in an primaries list in named.conf the wrong size is passed to isc_mem_put