chg: ci: Use CMocka generated JUnit reports where possible

Where applicable, use the more detailed CMocka generated JUnit
reports which include subtest results and timings instead of the
one generated by Meson.

Prerequisites:
- bind9-qa!137

Closes #5511

Merge branch '5511-cmocka-junit-ouput' into 'main'

See merge request isc-projects/bind9!11100

.gitlab-ci.yml

@@ -385,6 +385,9 @@ stages:
       $EXTRA_CONFIGURE
       build
 
+.git-clone-bind9-qa: &git_clone_bind9-qa
+  - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
+
 # change directory to the workspace before including this
 .find_python: &find_python
   - PYTHON="$(cat build/bin/tests/system/isctest/vars/.build_vars/PYTHON)"
@@ -429,21 +432,6 @@ stages:
         fi
       fi
 
-.check_for_junit_xml: &check_for_junit_xml
-    # test if junit.xml file exists and is longer 40 bytes
-    # (i.e., contains more than `<testsuites><testsuite /></testsuites>`)
-    - if [ -f "$CI_PROJECT_DIR"/junit.xml ]; then
-        if [ $(wc -c < "$CI_PROJECT_DIR"/junit.xml) -gt 40 ]; then
-          echo "junit.xml file exists and is longer than 40 bytes.";
-        else
-          echo "junit.xml file exists but is too short.";
-          exit 1;
-        fi
-      else
-        echo "junit.xml file does not exist.";
-        exit 1;
-      fi
-
 .build: &build_job
   <<: *default_triggering_rules
   stage: build
@@ -555,7 +543,9 @@ stages:
     - if pytest --version | grep -F "pytest 9.0" >/dev/null; then echo "filterwarnings = ignore::pytest.PytestRemovedIn9Warning" >> pytest.ini; fi
     - >
       ("$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt) || RET=1
-    - *check_for_junit_xml
+    - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
+    - >
+      "$PYTHON" bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit.xml --output "$CI_PROJECT_DIR"/junit.xml
     - (exit $RET)
     - '( ! grep -F "grep: warning:" pytest.out.txt )'
     - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ( cd ../../.. && ninja -C build clean >/dev/null 2>&1 )
@@ -579,18 +569,33 @@ stages:
   <<: *default_triggering_rules
   stage: unit
   # This script needs to: 1) fail if the unit tests fail, 2) fail if the
-  # junit.xml file is broken, 3) produce the junit.xml file even if the
+  # junit.xml file is broken, 3) produce the JUnit reports even if the
   # unit tests fail.  Therefore, $RET is used to "cache" the result of
   # running "meson test" as interrupting the script immediately when
   # unit tests fail would make checking the contents of the junit.xml
   # file impossible (GitLab Runner uses "set -o pipefail").
+
+  # Additionally, both flaky and CMocka test need special handling:
+  # - flaky tests are retried a number of times (default 2) before being
+  #   considered failed
+  # - for CMocka tests, we use the CMocka's XML report to get more detailed
+  #   information (subtest results and timings). Meson also produces
+  #   a JUnit report, so we need to not pass it to GitLab to avoid duplication.
   script:
     - *fips_feature_test
+    - *find_python
+    - *git_clone_bind9-qa
+    # Set CMocka JUnit XML output.
+    - export CMOCKA_MESSAGE_OUTPUT="xml"
+    - export CMOCKA_XML_FILE="$(pwd)/build/meson-logs/include-cmocka-%g.junit.xml"
     - RET=0
-    - meson test -C build --no-rebuild --no-suite flaky || RET=1
-    - cp build/meson-logs/testlog.junit.xml $CI_PROJECT_DIR/junit.xml
-    - meson test -C build --no-rebuild --suite flaky  --logbase testlog-flaky || meson test -C build --no-rebuild --suite flaky --logbase testlog-flaky || RET=1
-    - *check_for_junit_xml
+    - MESON_WRAPPER="$(pwd)/bind9-qa/ci/meson_retry_if_flaky.sh 2"
+    # CMocka tests: Mark Meson-generated XMLs to exclude them later with --logbase, CMocka generates better ones.
+    - meson test -C build --suite cmocka --wrapper "$MESON_WRAPPER" --no-rebuild --logbase "exclude-" || RET=1
+    # Non-CMocka test: Use Meson-generated XMLs.
+    - meson test -C build --no-suite cmocka --wrapper "$MESON_WRAPPER" --no-rebuild --logbase "include-" || RET=1
+    - >
+      "$PYTHON" bind9-qa/ci/postprocess_junit_files.py build/meson-logs/include-*.junit.xml --output "$CI_PROJECT_DIR/junit.xml"
     - (exit $RET)
     - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ninja -C build clean >/dev/null 2>&1
   artifacts:
@@ -599,7 +604,6 @@ stages:
     reports:
       junit:
         - junit.xml
-        - build/meson-logs/testlog-flaky.junit.xml
 
 .unit_test_tsan: &unit_test_tsan_job
   <<: *unit_test_job
@@ -624,7 +628,7 @@ stages:
     - *configure
     - meson compile -C build
     - *setup_interfaces
-    - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
+    - *git_clone_bind9-qa
     - cd bind9-qa/respdiff
   needs: []
   artifacts:
@@ -692,7 +696,7 @@ ci-variables:
 ci-orphaned-anchors:
   <<: *precheck_job
   script:
-    - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
+    - *git_clone_bind9-qa
     - bind9-qa/ci-orphaned-anchors/check-orphaned-anchors-ci.py .gitlab-ci.yml
   needs: []
   rules:
@@ -920,6 +924,7 @@ cross-version-config-tests:
     - *setup_interfaces
     - meson compile -C build system-test-init system-test-dependencies
     - meson compile -C build
+    - *find_python
     - *find_pytest
     - git clone --branch "${BIND_BASELINE_VERSION}" --depth 1 https://gitlab.isc.org/isc-projects/bind9.git "bind-${BIND_BASELINE_VERSION}"
     - cd "bind-${BIND_BASELINE_VERSION}"
@@ -936,8 +941,19 @@ cross-version-config-tests:
     # should not be run.
     - rm -r dlzexternal
     - rm -r dyndb
+    # This script needs to: 1) fail if the tests fail, 2) fail if
+    # the junit.xml file is broken, 3) produce the junit.xml file even if
+    # the  tests fail.  Therefore, $RET is used to "cache" the
+    # result of running pytest as interrupting the script immediately when
+    # system tests fail would make checking the contents of the junit.xml
+    # file impossible (GitLab Runner uses "set -o pipefail").
+    - RET=0
     - >
-      "$PYTEST" --setup-only --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "${TEST_PARALLEL_JOBS:-1}"
+      "$PYTEST" --setup-only --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "${TEST_PARALLEL_JOBS:-1}" || RET=1
+    - *git_clone_bind9-qa
+    - >
+      "$PYTHON" bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit.xml --output "$CI_PROJECT_DIR"/junit.xml
+    - (exit $RET)
   needs:
     - job: ci-variables
       artifacts: true
@@ -1784,7 +1800,7 @@ publish:
   <<: *manual_release_job
   <<: *base_image
   before_script:
-    - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
+    - *git_clone_bind9-qa
   needs:
     - job: staging
       artifacts: false
@@ -1902,7 +1918,7 @@ customer-git:branch:
         BRANCH: '$CI_COMMIT_BRANCH'
   before_script:
     - test -n "$CUSTOMER"
-    - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
+    - *git_clone_bind9-qa
   script:
     - git checkout -b "$BRANCH"  # ensure refs/heads/$BRANCH exists; GitLab clones with detached HEAD
     - bind9-qa/releng/push_to_customer_repository.py --branch "$BRANCH" --customer "$CUSTOMER" --force
@@ -1915,7 +1931,7 @@ customer-git:tag:
   rules:
     - *rule_tag
   before_script:
-    - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
+    - *git_clone_bind9-qa
     - git clone --depth 1 "https://token:${ISC_CUSTOMERS_WRITE_TOKEN}@gitlab.isc.org/isc-customers/isc-customer-settings.git"
   script:
     - bind9-qa/releng/push_to_customer_repository.py --tag "$CI_COMMIT_TAG" --entitlements isc-customer-settings/entitlements.yaml --force
@@ -2089,7 +2105,7 @@ generate-stress-test-configs:
   <<: *default_triggering_rules
   stage: precheck
   script:
-    - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
+    - *git_clone_bind9-qa
     - bind9-qa/stress/generate-stress-test-configs.py > stress-test-configs.yml
   artifacts:
     paths:
@@ -2219,7 +2235,7 @@ pairwise:
     - >
       : stop if this is not a merge request in the current project\'s namespace
     - test -n "$MERGE_REQUEST_ID" || exit 0
-    - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
+    - *git_clone_bind9-qa
 
 backports:
   <<: *post_merge

tests/dns/meson.build

@@ -94,7 +94,7 @@ foreach unit : dns_tests
         unit,
         test_bin,
         depends: master_data,
-        suite: 'dns',
+        suite: ['dns', 'cmocka'],
         timeout: 300,
         workdir: meson.current_source_dir(),
     )

tests/include/tests/isc.h

@@ -20,6 +20,7 @@
 
 #include <isc/buffer.h>
 #include <isc/commandline.h>
+#include <isc/file.h>
 #include <isc/hash.h>
 #include <isc/log.h>
 #include <isc/mem.h>
@@ -57,6 +58,16 @@ setup_managers(void **state);
 int
 teardown_managers(void **state);
 
+isc_result_t
+file_path_to_groupname(const char *path, char *out, size_t outlen);
+/*%<
+ * Example:
+ *   "/.../tests/dns/dbdiff" -> "dns_dbdiff"
+ *
+ * Returns ISC_R_SUCCESS on success, ISC_R_NOSPACE if outlen is too small,
+ * or other error codes returned by isc_file_splitpath.
+ */
+
 #ifndef TESTS_DIR
 #define TESTS_DIR "./"
 #endif
@@ -240,10 +251,18 @@ teardown_managers(void **state);
 			}                                                               \
 		}                                                                       \
                                                                                         \
+		char	     group_name[1024];                                          \
+		isc_result_t res = file_path_to_groupname(argv[0], group_name,          \
+							  sizeof(group_name));          \
+		if (res != ISC_R_SUCCESS) {                                             \
+			strncpy(group_name, "tests", sizeof(group_name));               \
+		}                                                                       \
 		if (selected[0].name != NULL) {                                         \
-			r = cmocka_run_group_tests(selected, setup, teardown);          \
+			r = cmocka_run_group_tests_name(group_name, selected,           \
+							setup, teardown);               \
 		} else {                                                                \
-			r = cmocka_run_group_tests(tests, setup, teardown);             \
+			r = cmocka_run_group_tests_name(group_name, tests,              \
+							setup, teardown);               \
 		}                                                                       \
                                                                                         \
 		teardown_mctx(NULL);                                                    \

tests/isc/meson.build

@@ -89,15 +89,21 @@ foreach unit : isc_test
         ],
     )
 
-    suites = ['isc']
+    suites = ['isc', 'cmocka']
+    env = environment()
+    timeout = 300
     if unit in flaky_isc_test
         suites += 'flaky'
+        # Pass FLAKY and TIMEOUT to the test wrapper so it can retry appropriately
+        env.set('FLAKY', '1')
+        env.set('TIMEOUT', timeout.to_string())
     endif
     test(
         unit,
         test_bin,
         suite: suites,
-        timeout: 300,
+        timeout: timeout,
         workdir: meson.current_source_dir(),
+        env: env,
     )
 endforeach

tests/isccfg/meson.build

@@ -32,7 +32,7 @@ foreach unit : [
     test(
         unit,
         test_bin,
-        suite: 'isccfg',
+        suite: ['isccfg', 'cmocka'],
         timeout: 300,
         workdir: meson.current_source_dir(),
     )

tests/libtest/isc.c

@@ -26,6 +26,7 @@
 #include <isc/managers.h>
 #include <isc/mem.h>
 #include <isc/os.h>
+#include <isc/result.h>
 #include <isc/string.h>
 #include <isc/timer.h>
 #include <isc/util.h>
@@ -126,3 +127,29 @@ teardown_managers(void **state) {
 
 	return 0;
 }
+
+isc_result_t
+file_path_to_groupname(const char *path, char *out, size_t outlen) {
+	char *dir = NULL;
+	const char *base = NULL;
+	const char *parent_basename;
+	const char *file_basename;
+
+	REQUIRE(path != NULL);
+	REQUIRE(out != NULL);
+
+	RETERR(isc_file_splitpath(isc_g_mctx, path, &dir, &base));
+
+	parent_basename = isc_file_basename(dir);
+	file_basename = isc_file_basename(base);
+
+	if (strlen(parent_basename) + 1 + strlen(file_basename) + 1 > outlen) {
+		isc_mem_free(isc_g_mctx, dir);
+		return ISC_R_NOSPACE;
+	}
+
+	snprintf(out, outlen, "%s_%s", parent_basename, file_basename);
+
+	isc_mem_free(isc_g_mctx, dir);
+	return ISC_R_SUCCESS;
+}

tests/ns/meson.build

@@ -36,7 +36,7 @@ foreach unit : [
     test(
         unit,
         test_bin,
-        suite: 'ns',
+        suite: ['ns', 'cmocka'],
         timeout: 300,
         workdir: meson.current_source_dir(),
     )