From c3f5db6dbc224e3c58f307fb1fdc327242241594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Thu, 11 Dec 2025 22:28:04 +0100 Subject: [PATCH 1/6] Factor the cloning of the QA repo into an anchor The unusual hyphen in the anchor name is used for symmetry with the bind9-qa repo and directory name. --- .gitlab-ci.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8ffe8ae77c..cd8fd309bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -385,6 +385,9 @@ stages: $EXTRA_CONFIGURE build +.git-clone-bind9-qa: &git_clone_bind9-qa + - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + # change directory to the workspace before including this .find_python: &find_python - PYTHON="$(cat build/bin/tests/system/isctest/vars/.build_vars/PYTHON)" @@ -624,7 +627,7 @@ stages: - *configure - meson compile -C build - *setup_interfaces - - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + - *git_clone_bind9-qa - cd bind9-qa/respdiff needs: [] artifacts: @@ -692,7 +695,7 @@ ci-variables: ci-orphaned-anchors: <<: *precheck_job script: - - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + - *git_clone_bind9-qa - bind9-qa/ci-orphaned-anchors/check-orphaned-anchors-ci.py .gitlab-ci.yml needs: [] rules: @@ -1784,7 +1787,7 @@ publish: <<: *manual_release_job <<: *base_image before_script: - - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + - *git_clone_bind9-qa needs: - job: staging artifacts: false @@ -1902,7 +1905,7 @@ customer-git:branch: BRANCH: '$CI_COMMIT_BRANCH' before_script: - test -n "$CUSTOMER" - - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + - *git_clone_bind9-qa script: - git checkout -b "$BRANCH" # ensure refs/heads/$BRANCH exists; GitLab clones with detached HEAD - bind9-qa/releng/push_to_customer_repository.py --branch "$BRANCH" --customer "$CUSTOMER" --force @@ -1915,7 +1918,7 @@ customer-git:tag: rules: - *rule_tag before_script: - - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + - *git_clone_bind9-qa - git clone --depth 1 "https://token:${ISC_CUSTOMERS_WRITE_TOKEN}@gitlab.isc.org/isc-customers/isc-customer-settings.git" script: - bind9-qa/releng/push_to_customer_repository.py --tag "$CI_COMMIT_TAG" --entitlements isc-customer-settings/entitlements.yaml --force @@ -2089,7 +2092,7 @@ generate-stress-test-configs: <<: *default_triggering_rules stage: precheck script: - - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + - *git_clone_bind9-qa - bind9-qa/stress/generate-stress-test-configs.py > stress-test-configs.yml artifacts: paths: @@ -2219,7 +2222,7 @@ pairwise: - > : stop if this is not a merge request in the current project\'s namespace - test -n "$MERGE_REQUEST_ID" || exit 0 - - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + - *git_clone_bind9-qa backports: <<: *post_merge From 5126f782b15a24022df284e25420117220b92806 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Wed, 15 Oct 2025 21:37:52 +0200 Subject: [PATCH 2/6] Set unit test group name in CMocka tests CMocka uses group names in the JUnit output. Use dirname_filename as the group name, as there duplicate testnames (e.g. time exists both in isc/ and dns/) --- tests/include/tests/isc.h | 23 +++++++++++++++++++++-- tests/libtest/isc.c | 27 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/tests/include/tests/isc.h b/tests/include/tests/isc.h index 78e7e10614..3e77f7f1b5 100644 --- a/tests/include/tests/isc.h +++ b/tests/include/tests/isc.h @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -57,6 +58,16 @@ setup_managers(void **state); int teardown_managers(void **state); +isc_result_t +file_path_to_groupname(const char *path, char *out, size_t outlen); +/*%< + * Example: + * "/.../tests/dns/dbdiff" -> "dns_dbdiff" + * + * Returns ISC_R_SUCCESS on success, ISC_R_NOSPACE if outlen is too small, + * or other error codes returned by isc_file_splitpath. + */ + #ifndef TESTS_DIR #define TESTS_DIR "./" #endif @@ -240,10 +251,18 @@ teardown_managers(void **state); } \ } \ \ + char group_name[1024]; \ + isc_result_t res = file_path_to_groupname(argv[0], group_name, \ + sizeof(group_name)); \ + if (res != ISC_R_SUCCESS) { \ + strncpy(group_name, "tests", sizeof(group_name)); \ + } \ if (selected[0].name != NULL) { \ - r = cmocka_run_group_tests(selected, setup, teardown); \ + r = cmocka_run_group_tests_name(group_name, selected, \ + setup, teardown); \ } else { \ - r = cmocka_run_group_tests(tests, setup, teardown); \ + r = cmocka_run_group_tests_name(group_name, tests, \ + setup, teardown); \ } \ \ teardown_mctx(NULL); \ diff --git a/tests/libtest/isc.c b/tests/libtest/isc.c index 5b6334a64d..433ea1487a 100644 --- a/tests/libtest/isc.c +++ b/tests/libtest/isc.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -126,3 +127,29 @@ teardown_managers(void **state) { return 0; } + +isc_result_t +file_path_to_groupname(const char *path, char *out, size_t outlen) { + char *dir = NULL; + const char *base = NULL; + const char *parent_basename; + const char *file_basename; + + REQUIRE(path != NULL); + REQUIRE(out != NULL); + + RETERR(isc_file_splitpath(isc_g_mctx, path, &dir, &base)); + + parent_basename = isc_file_basename(dir); + file_basename = isc_file_basename(base); + + if (strlen(parent_basename) + 1 + strlen(file_basename) + 1 > outlen) { + isc_mem_free(isc_g_mctx, dir); + return ISC_R_NOSPACE; + } + + snprintf(out, outlen, "%s_%s", parent_basename, file_basename); + + isc_mem_free(isc_g_mctx, dir); + return ISC_R_SUCCESS; +} From 179c101bf07fc57ccf3c63183c8fa50195ca572e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Wed, 15 Oct 2025 17:15:12 +0200 Subject: [PATCH 3/6] Put CMocka unit tests in a suite Distinguish them for JUnit report collection. --- tests/dns/meson.build | 2 +- tests/isc/meson.build | 2 +- tests/isccfg/meson.build | 2 +- tests/ns/meson.build | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/dns/meson.build b/tests/dns/meson.build index dcbf853c72..73c811f10e 100644 --- a/tests/dns/meson.build +++ b/tests/dns/meson.build @@ -94,7 +94,7 @@ foreach unit : dns_tests unit, test_bin, depends: master_data, - suite: 'dns', + suite: ['dns', 'cmocka'], timeout: 300, workdir: meson.current_source_dir(), ) diff --git a/tests/isc/meson.build b/tests/isc/meson.build index 0b4254c8cc..a73d837377 100644 --- a/tests/isc/meson.build +++ b/tests/isc/meson.build @@ -89,7 +89,7 @@ foreach unit : isc_test ], ) - suites = ['isc'] + suites = ['isc', 'cmocka'] if unit in flaky_isc_test suites += 'flaky' endif diff --git a/tests/isccfg/meson.build b/tests/isccfg/meson.build index 419986f595..ece606e0a0 100644 --- a/tests/isccfg/meson.build +++ b/tests/isccfg/meson.build @@ -32,7 +32,7 @@ foreach unit : [ test( unit, test_bin, - suite: 'isccfg', + suite: ['isccfg', 'cmocka'], timeout: 300, workdir: meson.current_source_dir(), ) diff --git a/tests/ns/meson.build b/tests/ns/meson.build index 7d4e2e30fa..dc14986280 100644 --- a/tests/ns/meson.build +++ b/tests/ns/meson.build @@ -36,7 +36,7 @@ foreach unit : [ test( unit, test_bin, - suite: 'ns', + suite: ['ns', 'cmocka'], timeout: 300, workdir: meson.current_source_dir(), ) From 5528ed7a5b1d3f3246ea43e35f4dfabe68ed7d5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Wed, 15 Oct 2025 18:33:18 +0200 Subject: [PATCH 4/6] Check the validity of cross-version-config-test's XML output This was overlooked before. --- .gitlab-ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cd8fd309bf..8687380270 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -939,8 +939,17 @@ cross-version-config-tests: # should not be run. - rm -r dlzexternal - rm -r dyndb + # This script needs to: 1) fail if the tests fail, 2) fail if + # the junit.xml file is broken, 3) produce the junit.xml file even if + # the tests fail. Therefore, $RET is used to "cache" the + # result of running pytest as interrupting the script immediately when + # system tests fail would make checking the contents of the junit.xml + # file impossible (GitLab Runner uses "set -o pipefail"). + - RET=0 - > - "$PYTEST" --setup-only --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "${TEST_PARALLEL_JOBS:-1}" + "$PYTEST" --setup-only --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "${TEST_PARALLEL_JOBS:-1}" || RET=1 + - *check_for_junit_xml + - (exit $RET) needs: - job: ci-variables artifacts: true From abecddb13afa3f8ad9611f1241a635fbe6b90818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Wed, 15 Oct 2025 18:26:48 +0200 Subject: [PATCH 5/6] Replace check_for_junit_xml anchor with a Python script This allows checking of multiple files with variable filenames rather than insisting on harcoded `junit.xml`. --- .gitlab-ci.yml | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8687380270..eb21ca579e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -432,21 +432,6 @@ stages: fi fi -.check_for_junit_xml: &check_for_junit_xml - # test if junit.xml file exists and is longer 40 bytes - # (i.e., contains more than ``) - - if [ -f "$CI_PROJECT_DIR"/junit.xml ]; then - if [ $(wc -c < "$CI_PROJECT_DIR"/junit.xml) -gt 40 ]; then - echo "junit.xml file exists and is longer than 40 bytes."; - else - echo "junit.xml file exists but is too short."; - exit 1; - fi - else - echo "junit.xml file does not exist."; - exit 1; - fi - .build: &build_job <<: *default_triggering_rules stage: build @@ -558,7 +543,9 @@ stages: - if pytest --version | grep -F "pytest 9.0" >/dev/null; then echo "filterwarnings = ignore::pytest.PytestRemovedIn9Warning" >> pytest.ini; fi - > ("$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt) || RET=1 - - *check_for_junit_xml + - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + - > + "$PYTHON" bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit.xml --output "$CI_PROJECT_DIR"/junit.xml - (exit $RET) - '( ! grep -F "grep: warning:" pytest.out.txt )' - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ( cd ../../.. && ninja -C build clean >/dev/null 2>&1 ) @@ -593,7 +580,10 @@ stages: - meson test -C build --no-rebuild --no-suite flaky || RET=1 - cp build/meson-logs/testlog.junit.xml $CI_PROJECT_DIR/junit.xml - meson test -C build --no-rebuild --suite flaky --logbase testlog-flaky || meson test -C build --no-rebuild --suite flaky --logbase testlog-flaky || RET=1 - - *check_for_junit_xml + - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + - *find_python + - > + "$PYTHON" bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit.xml --output "$CI_PROJECT_DIR"/junit.xml - (exit $RET) - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ninja -C build clean >/dev/null 2>&1 artifacts: @@ -923,6 +913,7 @@ cross-version-config-tests: - *setup_interfaces - meson compile -C build system-test-init system-test-dependencies - meson compile -C build + - *find_python - *find_pytest - git clone --branch "${BIND_BASELINE_VERSION}" --depth 1 https://gitlab.isc.org/isc-projects/bind9.git "bind-${BIND_BASELINE_VERSION}" - cd "bind-${BIND_BASELINE_VERSION}" @@ -948,7 +939,9 @@ cross-version-config-tests: - RET=0 - > "$PYTEST" --setup-only --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "${TEST_PARALLEL_JOBS:-1}" || RET=1 - - *check_for_junit_xml + - *git_clone_bind9-qa + - > + "$PYTHON" bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit.xml --output "$CI_PROJECT_DIR"/junit.xml - (exit $RET) needs: - job: ci-variables From 237489caf45b06141d7c54540e26892fac159f59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Wed, 15 Oct 2025 19:23:59 +0200 Subject: [PATCH 6/6] Use CMocka generated JUnit reports where possible Where applicable, use the more detailed CMocka generated JUnit reports which include subtest results and timings instead of the one generated by Meson. Flaky tests also require retrying, so use a wrapper and mark them with a environment variable. This is done to avoid the need to compute an intersection of suites in Meson which is not supported out-of-the-box (`meson test --suite=foo,bar` runs the union of foo and bar). --- .gitlab-ci.yml | 27 +++++++++++++++++++-------- tests/isc/meson.build | 8 +++++++- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eb21ca579e..dfb583ce5c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -569,21 +569,33 @@ stages: <<: *default_triggering_rules stage: unit # This script needs to: 1) fail if the unit tests fail, 2) fail if the - # junit.xml file is broken, 3) produce the junit.xml file even if the + # junit.xml file is broken, 3) produce the JUnit reports even if the # unit tests fail. Therefore, $RET is used to "cache" the result of # running "meson test" as interrupting the script immediately when # unit tests fail would make checking the contents of the junit.xml # file impossible (GitLab Runner uses "set -o pipefail"). + + # Additionally, both flaky and CMocka test need special handling: + # - flaky tests are retried a number of times (default 2) before being + # considered failed + # - for CMocka tests, we use the CMocka's XML report to get more detailed + # information (subtest results and timings). Meson also produces + # a JUnit report, so we need to not pass it to GitLab to avoid duplication. script: - *fips_feature_test - - RET=0 - - meson test -C build --no-rebuild --no-suite flaky || RET=1 - - cp build/meson-logs/testlog.junit.xml $CI_PROJECT_DIR/junit.xml - - meson test -C build --no-rebuild --suite flaky --logbase testlog-flaky || meson test -C build --no-rebuild --suite flaky --logbase testlog-flaky || RET=1 - - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git - *find_python + - *git_clone_bind9-qa + # Set CMocka JUnit XML output. + - export CMOCKA_MESSAGE_OUTPUT="xml" + - export CMOCKA_XML_FILE="$(pwd)/build/meson-logs/include-cmocka-%g.junit.xml" + - RET=0 + - MESON_WRAPPER="$(pwd)/bind9-qa/ci/meson_retry_if_flaky.sh 2" + # CMocka tests: Mark Meson-generated XMLs to exclude them later with --logbase, CMocka generates better ones. + - meson test -C build --suite cmocka --wrapper "$MESON_WRAPPER" --no-rebuild --logbase "exclude-" || RET=1 + # Non-CMocka test: Use Meson-generated XMLs. + - meson test -C build --no-suite cmocka --wrapper "$MESON_WRAPPER" --no-rebuild --logbase "include-" || RET=1 - > - "$PYTHON" bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit.xml --output "$CI_PROJECT_DIR"/junit.xml + "$PYTHON" bind9-qa/ci/postprocess_junit_files.py build/meson-logs/include-*.junit.xml --output "$CI_PROJECT_DIR/junit.xml" - (exit $RET) - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ninja -C build clean >/dev/null 2>&1 artifacts: @@ -592,7 +604,6 @@ stages: reports: junit: - junit.xml - - build/meson-logs/testlog-flaky.junit.xml .unit_test_tsan: &unit_test_tsan_job <<: *unit_test_job diff --git a/tests/isc/meson.build b/tests/isc/meson.build index a73d837377..5aa7ecafee 100644 --- a/tests/isc/meson.build +++ b/tests/isc/meson.build @@ -90,14 +90,20 @@ foreach unit : isc_test ) suites = ['isc', 'cmocka'] + env = environment() + timeout = 300 if unit in flaky_isc_test suites += 'flaky' + # Pass FLAKY and TIMEOUT to the test wrapper so it can retry appropriately + env.set('FLAKY', '1') + env.set('TIMEOUT', timeout.to_string()) endif test( unit, test_bin, suite: suites, - timeout: 300, + timeout: timeout, workdir: meson.current_source_dir(), + env: env, ) endforeach