diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8ffe8ae77c..dfb583ce5c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -385,6 +385,9 @@ stages:
$EXTRA_CONFIGURE
build
+.git-clone-bind9-qa: &git_clone_bind9-qa
+ - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
+
# change directory to the workspace before including this
.find_python: &find_python
- PYTHON="$(cat build/bin/tests/system/isctest/vars/.build_vars/PYTHON)"
@@ -429,21 +432,6 @@ stages:
fi
fi
-.check_for_junit_xml: &check_for_junit_xml
- # test if junit.xml file exists and is longer 40 bytes
- # (i.e., contains more than ``)
- - if [ -f "$CI_PROJECT_DIR"/junit.xml ]; then
- if [ $(wc -c < "$CI_PROJECT_DIR"/junit.xml) -gt 40 ]; then
- echo "junit.xml file exists and is longer than 40 bytes.";
- else
- echo "junit.xml file exists but is too short.";
- exit 1;
- fi
- else
- echo "junit.xml file does not exist.";
- exit 1;
- fi
-
.build: &build_job
<<: *default_triggering_rules
stage: build
@@ -555,7 +543,9 @@ stages:
- if pytest --version | grep -F "pytest 9.0" >/dev/null; then echo "filterwarnings = ignore::pytest.PytestRemovedIn9Warning" >> pytest.ini; fi
- >
("$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt) || RET=1
- - *check_for_junit_xml
+ - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
+ - >
+ "$PYTHON" bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit.xml --output "$CI_PROJECT_DIR"/junit.xml
- (exit $RET)
- '( ! grep -F "grep: warning:" pytest.out.txt )'
- test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ( cd ../../.. && ninja -C build clean >/dev/null 2>&1 )
@@ -579,18 +569,33 @@ stages:
<<: *default_triggering_rules
stage: unit
# This script needs to: 1) fail if the unit tests fail, 2) fail if the
- # junit.xml file is broken, 3) produce the junit.xml file even if the
+ # junit.xml file is broken, 3) produce the JUnit reports even if the
# unit tests fail. Therefore, $RET is used to "cache" the result of
# running "meson test" as interrupting the script immediately when
# unit tests fail would make checking the contents of the junit.xml
# file impossible (GitLab Runner uses "set -o pipefail").
+
+ # Additionally, both flaky and CMocka test need special handling:
+ # - flaky tests are retried a number of times (default 2) before being
+ # considered failed
+ # - for CMocka tests, we use the CMocka's XML report to get more detailed
+ # information (subtest results and timings). Meson also produces
+ # a JUnit report, so we need to not pass it to GitLab to avoid duplication.
script:
- *fips_feature_test
+ - *find_python
+ - *git_clone_bind9-qa
+ # Set CMocka JUnit XML output.
+ - export CMOCKA_MESSAGE_OUTPUT="xml"
+ - export CMOCKA_XML_FILE="$(pwd)/build/meson-logs/include-cmocka-%g.junit.xml"
- RET=0
- - meson test -C build --no-rebuild --no-suite flaky || RET=1
- - cp build/meson-logs/testlog.junit.xml $CI_PROJECT_DIR/junit.xml
- - meson test -C build --no-rebuild --suite flaky --logbase testlog-flaky || meson test -C build --no-rebuild --suite flaky --logbase testlog-flaky || RET=1
- - *check_for_junit_xml
+ - MESON_WRAPPER="$(pwd)/bind9-qa/ci/meson_retry_if_flaky.sh 2"
+ # CMocka tests: Mark Meson-generated XMLs to exclude them later with --logbase, CMocka generates better ones.
+ - meson test -C build --suite cmocka --wrapper "$MESON_WRAPPER" --no-rebuild --logbase "exclude-" || RET=1
+ # Non-CMocka test: Use Meson-generated XMLs.
+ - meson test -C build --no-suite cmocka --wrapper "$MESON_WRAPPER" --no-rebuild --logbase "include-" || RET=1
+ - >
+ "$PYTHON" bind9-qa/ci/postprocess_junit_files.py build/meson-logs/include-*.junit.xml --output "$CI_PROJECT_DIR/junit.xml"
- (exit $RET)
- test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ninja -C build clean >/dev/null 2>&1
artifacts:
@@ -599,7 +604,6 @@ stages:
reports:
junit:
- junit.xml
- - build/meson-logs/testlog-flaky.junit.xml
.unit_test_tsan: &unit_test_tsan_job
<<: *unit_test_job
@@ -624,7 +628,7 @@ stages:
- *configure
- meson compile -C build
- *setup_interfaces
- - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
+ - *git_clone_bind9-qa
- cd bind9-qa/respdiff
needs: []
artifacts:
@@ -692,7 +696,7 @@ ci-variables:
ci-orphaned-anchors:
<<: *precheck_job
script:
- - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
+ - *git_clone_bind9-qa
- bind9-qa/ci-orphaned-anchors/check-orphaned-anchors-ci.py .gitlab-ci.yml
needs: []
rules:
@@ -920,6 +924,7 @@ cross-version-config-tests:
- *setup_interfaces
- meson compile -C build system-test-init system-test-dependencies
- meson compile -C build
+ - *find_python
- *find_pytest
- git clone --branch "${BIND_BASELINE_VERSION}" --depth 1 https://gitlab.isc.org/isc-projects/bind9.git "bind-${BIND_BASELINE_VERSION}"
- cd "bind-${BIND_BASELINE_VERSION}"
@@ -936,8 +941,19 @@ cross-version-config-tests:
# should not be run.
- rm -r dlzexternal
- rm -r dyndb
+ # This script needs to: 1) fail if the tests fail, 2) fail if
+ # the junit.xml file is broken, 3) produce the junit.xml file even if
+ # the tests fail. Therefore, $RET is used to "cache" the
+ # result of running pytest as interrupting the script immediately when
+ # system tests fail would make checking the contents of the junit.xml
+ # file impossible (GitLab Runner uses "set -o pipefail").
+ - RET=0
- >
- "$PYTEST" --setup-only --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "${TEST_PARALLEL_JOBS:-1}"
+ "$PYTEST" --setup-only --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "${TEST_PARALLEL_JOBS:-1}" || RET=1
+ - *git_clone_bind9-qa
+ - >
+ "$PYTHON" bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit.xml --output "$CI_PROJECT_DIR"/junit.xml
+ - (exit $RET)
needs:
- job: ci-variables
artifacts: true
@@ -1784,7 +1800,7 @@ publish:
<<: *manual_release_job
<<: *base_image
before_script:
- - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
+ - *git_clone_bind9-qa
needs:
- job: staging
artifacts: false
@@ -1902,7 +1918,7 @@ customer-git:branch:
BRANCH: '$CI_COMMIT_BRANCH'
before_script:
- test -n "$CUSTOMER"
- - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
+ - *git_clone_bind9-qa
script:
- git checkout -b "$BRANCH" # ensure refs/heads/$BRANCH exists; GitLab clones with detached HEAD
- bind9-qa/releng/push_to_customer_repository.py --branch "$BRANCH" --customer "$CUSTOMER" --force
@@ -1915,7 +1931,7 @@ customer-git:tag:
rules:
- *rule_tag
before_script:
- - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
+ - *git_clone_bind9-qa
- git clone --depth 1 "https://token:${ISC_CUSTOMERS_WRITE_TOKEN}@gitlab.isc.org/isc-customers/isc-customer-settings.git"
script:
- bind9-qa/releng/push_to_customer_repository.py --tag "$CI_COMMIT_TAG" --entitlements isc-customer-settings/entitlements.yaml --force
@@ -2089,7 +2105,7 @@ generate-stress-test-configs:
<<: *default_triggering_rules
stage: precheck
script:
- - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
+ - *git_clone_bind9-qa
- bind9-qa/stress/generate-stress-test-configs.py > stress-test-configs.yml
artifacts:
paths:
@@ -2219,7 +2235,7 @@ pairwise:
- >
: stop if this is not a merge request in the current project\'s namespace
- test -n "$MERGE_REQUEST_ID" || exit 0
- - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
+ - *git_clone_bind9-qa
backports:
<<: *post_merge
diff --git a/tests/dns/meson.build b/tests/dns/meson.build
index dcbf853c72..73c811f10e 100644
--- a/tests/dns/meson.build
+++ b/tests/dns/meson.build
@@ -94,7 +94,7 @@ foreach unit : dns_tests
unit,
test_bin,
depends: master_data,
- suite: 'dns',
+ suite: ['dns', 'cmocka'],
timeout: 300,
workdir: meson.current_source_dir(),
)
diff --git a/tests/include/tests/isc.h b/tests/include/tests/isc.h
index 78e7e10614..3e77f7f1b5 100644
--- a/tests/include/tests/isc.h
+++ b/tests/include/tests/isc.h
@@ -20,6 +20,7 @@
#include
#include
+#include
#include
#include
#include
@@ -57,6 +58,16 @@ setup_managers(void **state);
int
teardown_managers(void **state);
+isc_result_t
+file_path_to_groupname(const char *path, char *out, size_t outlen);
+/*%<
+ * Example:
+ * "/.../tests/dns/dbdiff" -> "dns_dbdiff"
+ *
+ * Returns ISC_R_SUCCESS on success, ISC_R_NOSPACE if outlen is too small,
+ * or other error codes returned by isc_file_splitpath.
+ */
+
#ifndef TESTS_DIR
#define TESTS_DIR "./"
#endif
@@ -240,10 +251,18 @@ teardown_managers(void **state);
} \
} \
\
+ char group_name[1024]; \
+ isc_result_t res = file_path_to_groupname(argv[0], group_name, \
+ sizeof(group_name)); \
+ if (res != ISC_R_SUCCESS) { \
+ strncpy(group_name, "tests", sizeof(group_name)); \
+ } \
if (selected[0].name != NULL) { \
- r = cmocka_run_group_tests(selected, setup, teardown); \
+ r = cmocka_run_group_tests_name(group_name, selected, \
+ setup, teardown); \
} else { \
- r = cmocka_run_group_tests(tests, setup, teardown); \
+ r = cmocka_run_group_tests_name(group_name, tests, \
+ setup, teardown); \
} \
\
teardown_mctx(NULL); \
diff --git a/tests/isc/meson.build b/tests/isc/meson.build
index 0b4254c8cc..5aa7ecafee 100644
--- a/tests/isc/meson.build
+++ b/tests/isc/meson.build
@@ -89,15 +89,21 @@ foreach unit : isc_test
],
)
- suites = ['isc']
+ suites = ['isc', 'cmocka']
+ env = environment()
+ timeout = 300
if unit in flaky_isc_test
suites += 'flaky'
+ # Pass FLAKY and TIMEOUT to the test wrapper so it can retry appropriately
+ env.set('FLAKY', '1')
+ env.set('TIMEOUT', timeout.to_string())
endif
test(
unit,
test_bin,
suite: suites,
- timeout: 300,
+ timeout: timeout,
workdir: meson.current_source_dir(),
+ env: env,
)
endforeach
diff --git a/tests/isccfg/meson.build b/tests/isccfg/meson.build
index 419986f595..ece606e0a0 100644
--- a/tests/isccfg/meson.build
+++ b/tests/isccfg/meson.build
@@ -32,7 +32,7 @@ foreach unit : [
test(
unit,
test_bin,
- suite: 'isccfg',
+ suite: ['isccfg', 'cmocka'],
timeout: 300,
workdir: meson.current_source_dir(),
)
diff --git a/tests/libtest/isc.c b/tests/libtest/isc.c
index 5b6334a64d..433ea1487a 100644
--- a/tests/libtest/isc.c
+++ b/tests/libtest/isc.c
@@ -26,6 +26,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -126,3 +127,29 @@ teardown_managers(void **state) {
return 0;
}
+
+isc_result_t
+file_path_to_groupname(const char *path, char *out, size_t outlen) {
+ char *dir = NULL;
+ const char *base = NULL;
+ const char *parent_basename;
+ const char *file_basename;
+
+ REQUIRE(path != NULL);
+ REQUIRE(out != NULL);
+
+ RETERR(isc_file_splitpath(isc_g_mctx, path, &dir, &base));
+
+ parent_basename = isc_file_basename(dir);
+ file_basename = isc_file_basename(base);
+
+ if (strlen(parent_basename) + 1 + strlen(file_basename) + 1 > outlen) {
+ isc_mem_free(isc_g_mctx, dir);
+ return ISC_R_NOSPACE;
+ }
+
+ snprintf(out, outlen, "%s_%s", parent_basename, file_basename);
+
+ isc_mem_free(isc_g_mctx, dir);
+ return ISC_R_SUCCESS;
+}
diff --git a/tests/ns/meson.build b/tests/ns/meson.build
index 7d4e2e30fa..dc14986280 100644
--- a/tests/ns/meson.build
+++ b/tests/ns/meson.build
@@ -36,7 +36,7 @@ foreach unit : [
test(
unit,
test_bin,
- suite: 'ns',
+ suite: ['ns', 'cmocka'],
timeout: 300,
workdir: meson.current_source_dir(),
)