diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8ffe8ae77c..dfb583ce5c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -385,6 +385,9 @@ stages: $EXTRA_CONFIGURE build +.git-clone-bind9-qa: &git_clone_bind9-qa + - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + # change directory to the workspace before including this .find_python: &find_python - PYTHON="$(cat build/bin/tests/system/isctest/vars/.build_vars/PYTHON)" @@ -429,21 +432,6 @@ stages: fi fi -.check_for_junit_xml: &check_for_junit_xml - # test if junit.xml file exists and is longer 40 bytes - # (i.e., contains more than ``) - - if [ -f "$CI_PROJECT_DIR"/junit.xml ]; then - if [ $(wc -c < "$CI_PROJECT_DIR"/junit.xml) -gt 40 ]; then - echo "junit.xml file exists and is longer than 40 bytes."; - else - echo "junit.xml file exists but is too short."; - exit 1; - fi - else - echo "junit.xml file does not exist."; - exit 1; - fi - .build: &build_job <<: *default_triggering_rules stage: build @@ -555,7 +543,9 @@ stages: - if pytest --version | grep -F "pytest 9.0" >/dev/null; then echo "filterwarnings = ignore::pytest.PytestRemovedIn9Warning" >> pytest.ini; fi - > ("$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt) || RET=1 - - *check_for_junit_xml + - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + - > + "$PYTHON" bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit.xml --output "$CI_PROJECT_DIR"/junit.xml - (exit $RET) - '( ! grep -F "grep: warning:" pytest.out.txt )' - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ( cd ../../.. && ninja -C build clean >/dev/null 2>&1 ) @@ -579,18 +569,33 @@ stages: <<: *default_triggering_rules stage: unit # This script needs to: 1) fail if the unit tests fail, 2) fail if the - # junit.xml file is broken, 3) produce the junit.xml file even if the + # junit.xml file is broken, 3) produce the JUnit reports even if the # unit tests fail. Therefore, $RET is used to "cache" the result of # running "meson test" as interrupting the script immediately when # unit tests fail would make checking the contents of the junit.xml # file impossible (GitLab Runner uses "set -o pipefail"). + + # Additionally, both flaky and CMocka test need special handling: + # - flaky tests are retried a number of times (default 2) before being + # considered failed + # - for CMocka tests, we use the CMocka's XML report to get more detailed + # information (subtest results and timings). Meson also produces + # a JUnit report, so we need to not pass it to GitLab to avoid duplication. script: - *fips_feature_test + - *find_python + - *git_clone_bind9-qa + # Set CMocka JUnit XML output. + - export CMOCKA_MESSAGE_OUTPUT="xml" + - export CMOCKA_XML_FILE="$(pwd)/build/meson-logs/include-cmocka-%g.junit.xml" - RET=0 - - meson test -C build --no-rebuild --no-suite flaky || RET=1 - - cp build/meson-logs/testlog.junit.xml $CI_PROJECT_DIR/junit.xml - - meson test -C build --no-rebuild --suite flaky --logbase testlog-flaky || meson test -C build --no-rebuild --suite flaky --logbase testlog-flaky || RET=1 - - *check_for_junit_xml + - MESON_WRAPPER="$(pwd)/bind9-qa/ci/meson_retry_if_flaky.sh 2" + # CMocka tests: Mark Meson-generated XMLs to exclude them later with --logbase, CMocka generates better ones. + - meson test -C build --suite cmocka --wrapper "$MESON_WRAPPER" --no-rebuild --logbase "exclude-" || RET=1 + # Non-CMocka test: Use Meson-generated XMLs. + - meson test -C build --no-suite cmocka --wrapper "$MESON_WRAPPER" --no-rebuild --logbase "include-" || RET=1 + - > + "$PYTHON" bind9-qa/ci/postprocess_junit_files.py build/meson-logs/include-*.junit.xml --output "$CI_PROJECT_DIR/junit.xml" - (exit $RET) - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ninja -C build clean >/dev/null 2>&1 artifacts: @@ -599,7 +604,6 @@ stages: reports: junit: - junit.xml - - build/meson-logs/testlog-flaky.junit.xml .unit_test_tsan: &unit_test_tsan_job <<: *unit_test_job @@ -624,7 +628,7 @@ stages: - *configure - meson compile -C build - *setup_interfaces - - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + - *git_clone_bind9-qa - cd bind9-qa/respdiff needs: [] artifacts: @@ -692,7 +696,7 @@ ci-variables: ci-orphaned-anchors: <<: *precheck_job script: - - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + - *git_clone_bind9-qa - bind9-qa/ci-orphaned-anchors/check-orphaned-anchors-ci.py .gitlab-ci.yml needs: [] rules: @@ -920,6 +924,7 @@ cross-version-config-tests: - *setup_interfaces - meson compile -C build system-test-init system-test-dependencies - meson compile -C build + - *find_python - *find_pytest - git clone --branch "${BIND_BASELINE_VERSION}" --depth 1 https://gitlab.isc.org/isc-projects/bind9.git "bind-${BIND_BASELINE_VERSION}" - cd "bind-${BIND_BASELINE_VERSION}" @@ -936,8 +941,19 @@ cross-version-config-tests: # should not be run. - rm -r dlzexternal - rm -r dyndb + # This script needs to: 1) fail if the tests fail, 2) fail if + # the junit.xml file is broken, 3) produce the junit.xml file even if + # the tests fail. Therefore, $RET is used to "cache" the + # result of running pytest as interrupting the script immediately when + # system tests fail would make checking the contents of the junit.xml + # file impossible (GitLab Runner uses "set -o pipefail"). + - RET=0 - > - "$PYTEST" --setup-only --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "${TEST_PARALLEL_JOBS:-1}" + "$PYTEST" --setup-only --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "${TEST_PARALLEL_JOBS:-1}" || RET=1 + - *git_clone_bind9-qa + - > + "$PYTHON" bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit.xml --output "$CI_PROJECT_DIR"/junit.xml + - (exit $RET) needs: - job: ci-variables artifacts: true @@ -1784,7 +1800,7 @@ publish: <<: *manual_release_job <<: *base_image before_script: - - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + - *git_clone_bind9-qa needs: - job: staging artifacts: false @@ -1902,7 +1918,7 @@ customer-git:branch: BRANCH: '$CI_COMMIT_BRANCH' before_script: - test -n "$CUSTOMER" - - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + - *git_clone_bind9-qa script: - git checkout -b "$BRANCH" # ensure refs/heads/$BRANCH exists; GitLab clones with detached HEAD - bind9-qa/releng/push_to_customer_repository.py --branch "$BRANCH" --customer "$CUSTOMER" --force @@ -1915,7 +1931,7 @@ customer-git:tag: rules: - *rule_tag before_script: - - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + - *git_clone_bind9-qa - git clone --depth 1 "https://token:${ISC_CUSTOMERS_WRITE_TOKEN}@gitlab.isc.org/isc-customers/isc-customer-settings.git" script: - bind9-qa/releng/push_to_customer_repository.py --tag "$CI_COMMIT_TAG" --entitlements isc-customer-settings/entitlements.yaml --force @@ -2089,7 +2105,7 @@ generate-stress-test-configs: <<: *default_triggering_rules stage: precheck script: - - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + - *git_clone_bind9-qa - bind9-qa/stress/generate-stress-test-configs.py > stress-test-configs.yml artifacts: paths: @@ -2219,7 +2235,7 @@ pairwise: - > : stop if this is not a merge request in the current project\'s namespace - test -n "$MERGE_REQUEST_ID" || exit 0 - - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git + - *git_clone_bind9-qa backports: <<: *post_merge diff --git a/tests/dns/meson.build b/tests/dns/meson.build index dcbf853c72..73c811f10e 100644 --- a/tests/dns/meson.build +++ b/tests/dns/meson.build @@ -94,7 +94,7 @@ foreach unit : dns_tests unit, test_bin, depends: master_data, - suite: 'dns', + suite: ['dns', 'cmocka'], timeout: 300, workdir: meson.current_source_dir(), ) diff --git a/tests/include/tests/isc.h b/tests/include/tests/isc.h index 78e7e10614..3e77f7f1b5 100644 --- a/tests/include/tests/isc.h +++ b/tests/include/tests/isc.h @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -57,6 +58,16 @@ setup_managers(void **state); int teardown_managers(void **state); +isc_result_t +file_path_to_groupname(const char *path, char *out, size_t outlen); +/*%< + * Example: + * "/.../tests/dns/dbdiff" -> "dns_dbdiff" + * + * Returns ISC_R_SUCCESS on success, ISC_R_NOSPACE if outlen is too small, + * or other error codes returned by isc_file_splitpath. + */ + #ifndef TESTS_DIR #define TESTS_DIR "./" #endif @@ -240,10 +251,18 @@ teardown_managers(void **state); } \ } \ \ + char group_name[1024]; \ + isc_result_t res = file_path_to_groupname(argv[0], group_name, \ + sizeof(group_name)); \ + if (res != ISC_R_SUCCESS) { \ + strncpy(group_name, "tests", sizeof(group_name)); \ + } \ if (selected[0].name != NULL) { \ - r = cmocka_run_group_tests(selected, setup, teardown); \ + r = cmocka_run_group_tests_name(group_name, selected, \ + setup, teardown); \ } else { \ - r = cmocka_run_group_tests(tests, setup, teardown); \ + r = cmocka_run_group_tests_name(group_name, tests, \ + setup, teardown); \ } \ \ teardown_mctx(NULL); \ diff --git a/tests/isc/meson.build b/tests/isc/meson.build index 0b4254c8cc..5aa7ecafee 100644 --- a/tests/isc/meson.build +++ b/tests/isc/meson.build @@ -89,15 +89,21 @@ foreach unit : isc_test ], ) - suites = ['isc'] + suites = ['isc', 'cmocka'] + env = environment() + timeout = 300 if unit in flaky_isc_test suites += 'flaky' + # Pass FLAKY and TIMEOUT to the test wrapper so it can retry appropriately + env.set('FLAKY', '1') + env.set('TIMEOUT', timeout.to_string()) endif test( unit, test_bin, suite: suites, - timeout: 300, + timeout: timeout, workdir: meson.current_source_dir(), + env: env, ) endforeach diff --git a/tests/isccfg/meson.build b/tests/isccfg/meson.build index 419986f595..ece606e0a0 100644 --- a/tests/isccfg/meson.build +++ b/tests/isccfg/meson.build @@ -32,7 +32,7 @@ foreach unit : [ test( unit, test_bin, - suite: 'isccfg', + suite: ['isccfg', 'cmocka'], timeout: 300, workdir: meson.current_source_dir(), ) diff --git a/tests/libtest/isc.c b/tests/libtest/isc.c index 5b6334a64d..433ea1487a 100644 --- a/tests/libtest/isc.c +++ b/tests/libtest/isc.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -126,3 +127,29 @@ teardown_managers(void **state) { return 0; } + +isc_result_t +file_path_to_groupname(const char *path, char *out, size_t outlen) { + char *dir = NULL; + const char *base = NULL; + const char *parent_basename; + const char *file_basename; + + REQUIRE(path != NULL); + REQUIRE(out != NULL); + + RETERR(isc_file_splitpath(isc_g_mctx, path, &dir, &base)); + + parent_basename = isc_file_basename(dir); + file_basename = isc_file_basename(base); + + if (strlen(parent_basename) + 1 + strlen(file_basename) + 1 > outlen) { + isc_mem_free(isc_g_mctx, dir); + return ISC_R_NOSPACE; + } + + snprintf(out, outlen, "%s_%s", parent_basename, file_basename); + + isc_mem_free(isc_g_mctx, dir); + return ISC_R_SUCCESS; +} diff --git a/tests/ns/meson.build b/tests/ns/meson.build index 7d4e2e30fa..dc14986280 100644 --- a/tests/ns/meson.build +++ b/tests/ns/meson.build @@ -36,7 +36,7 @@ foreach unit : [ test( unit, test_bin, - suite: 'ns', + suite: ['ns', 'cmocka'], timeout: 300, workdir: meson.current_source_dir(), )