diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eb21ca579e..dfb583ce5c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -569,21 +569,33 @@ stages: <<: *default_triggering_rules stage: unit # This script needs to: 1) fail if the unit tests fail, 2) fail if the - # junit.xml file is broken, 3) produce the junit.xml file even if the + # junit.xml file is broken, 3) produce the JUnit reports even if the # unit tests fail. Therefore, $RET is used to "cache" the result of # running "meson test" as interrupting the script immediately when # unit tests fail would make checking the contents of the junit.xml # file impossible (GitLab Runner uses "set -o pipefail"). + + # Additionally, both flaky and CMocka test need special handling: + # - flaky tests are retried a number of times (default 2) before being + # considered failed + # - for CMocka tests, we use the CMocka's XML report to get more detailed + # information (subtest results and timings). Meson also produces + # a JUnit report, so we need to not pass it to GitLab to avoid duplication. script: - *fips_feature_test - - RET=0 - - meson test -C build --no-rebuild --no-suite flaky || RET=1 - - cp build/meson-logs/testlog.junit.xml $CI_PROJECT_DIR/junit.xml - - meson test -C build --no-rebuild --suite flaky --logbase testlog-flaky || meson test -C build --no-rebuild --suite flaky --logbase testlog-flaky || RET=1 - - git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git - *find_python + - *git_clone_bind9-qa + # Set CMocka JUnit XML output. + - export CMOCKA_MESSAGE_OUTPUT="xml" + - export CMOCKA_XML_FILE="$(pwd)/build/meson-logs/include-cmocka-%g.junit.xml" + - RET=0 + - MESON_WRAPPER="$(pwd)/bind9-qa/ci/meson_retry_if_flaky.sh 2" + # CMocka tests: Mark Meson-generated XMLs to exclude them later with --logbase, CMocka generates better ones. + - meson test -C build --suite cmocka --wrapper "$MESON_WRAPPER" --no-rebuild --logbase "exclude-" || RET=1 + # Non-CMocka test: Use Meson-generated XMLs. + - meson test -C build --no-suite cmocka --wrapper "$MESON_WRAPPER" --no-rebuild --logbase "include-" || RET=1 - > - "$PYTHON" bind9-qa/ci/postprocess_junit_files.py "$CI_PROJECT_DIR"/junit.xml --output "$CI_PROJECT_DIR"/junit.xml + "$PYTHON" bind9-qa/ci/postprocess_junit_files.py build/meson-logs/include-*.junit.xml --output "$CI_PROJECT_DIR/junit.xml" - (exit $RET) - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ninja -C build clean >/dev/null 2>&1 artifacts: @@ -592,7 +604,6 @@ stages: reports: junit: - junit.xml - - build/meson-logs/testlog-flaky.junit.xml .unit_test_tsan: &unit_test_tsan_job <<: *unit_test_job diff --git a/tests/isc/meson.build b/tests/isc/meson.build index a73d837377..5aa7ecafee 100644 --- a/tests/isc/meson.build +++ b/tests/isc/meson.build @@ -90,14 +90,20 @@ foreach unit : isc_test ) suites = ['isc', 'cmocka'] + env = environment() + timeout = 300 if unit in flaky_isc_test suites += 'flaky' + # Pass FLAKY and TIMEOUT to the test wrapper so it can retry appropriately + env.set('FLAKY', '1') + env.set('TIMEOUT', timeout.to_string()) endif test( unit, test_bin, suite: suites, - timeout: 300, + timeout: timeout, workdir: meson.current_source_dir(), + env: env, ) endforeach