From 57b343c6247e6943787a2455c2c18bd8ca92bfbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Wed, 4 Jun 2025 14:41:35 +0200 Subject: [PATCH 1/6] Add a check for existence and sanity of the junit.xml file And use if after system tests are run using pytest. (cherry picked from commit dfd6097f7bb573dfa9e713149d33e6d9b92e3e99) --- .gitlab-ci.yml | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e553988d2c..6bc8630a19 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -334,6 +334,21 @@ stages: .fips-feature-test: &fips_feature_test - if bin/tests/system/feature-test --have-fips-mode; then fips-mode-setup --check; fips-mode-setup --is-enabled; fi +.check_for_junit_xml: &check_for_junit_xml + # test if junit.xml file exists and is longer 40 bytes + # (i.e., contains more than ``) + - if [ -f "$CI_PROJECT_DIR"/junit.xml ]; then + if [ $(wc -c < "$CI_PROJECT_DIR"/junit.xml) -gt 40 ]; then + echo "junit.xml file exists and is longer than 40 bytes."; + else + echo "junit.xml file exists but is too short."; + exit 1; + fi + else + echo "junit.xml file does not exist."; + exit 1; + fi + .build: &build_job <<: *default_triggering_rules stage: build @@ -421,14 +436,23 @@ stages: before_script: - test -n "${OUT_OF_TREE_WORKSPACE}" && cp -r bin/tests/system/* "${OUT_OF_TREE_WORKSPACE}/bin/tests/system/" && cd "${OUT_OF_TREE_WORKSPACE}" - *setup_interfaces + # This script needs to: 1) fail if the system tests fail, 2) fail if + # the junit.xml file is broken, 3) produce the junit.xml file even if + # the system tests fail. Therefore, $RET is used to "cache" the + # result of running pytest as interrupting the script immediately when + # system tests fail would make checking the contents of the junit.xml + # file impossible (GitLab Runner uses "set -o pipefail"). script: - *fips_feature_test - *find_pytest - *find_python - ( if [ "${CI_DISPOSABLE_ENVIRONMENT}" = "true" ]; then sleep 3000; "$PYTHON" "${CI_PROJECT_DIR}/util/get-running-system-tests.py"; fi ) & - cd bin/tests/system + - RET=0 - > - "$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt + ("$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt) || RET=1 + - *check_for_junit_xml + - (exit $RET) - '( ! grep -F "grep: warning:" pytest.out.txt )' - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ( cd ../../.. && make clean >/dev/null 2>&1 ) after_script: From 85df35c6a3ede5416409d32a0c701600d9c57bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Wed, 4 Jun 2025 15:06:09 +0200 Subject: [PATCH 2/6] Hoist the artifact handling to the `&system_test_common` anchor In the past artifacts of different types of system test jobs were treated differently but this is no longer the case. (cherry picked from commit c61ff639b3a5aa7d4513efdc893aadff95a56c60) --- .gitlab-ci.yml | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6bc8630a19..68f2fd5375 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -458,9 +458,6 @@ stages: after_script: - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}" - *display_pytest_failures - -.system_test: &system_test_job - <<: *system_test_common artifacts: untracked: true exclude: @@ -469,21 +466,32 @@ stages: reports: junit: junit.xml +.system_test: &system_test_job + <<: *system_test_common + .system_test_make_check: &system_test_make_check_job <<: *system_test_common + # This script needs to: 1) fail if the system tests fail, 2) fail if + # the junit.xml file is broken, 3) produce the junit.xml file even if + # the system tests fail. Therefore, $RET is used to "cache" the + # result of "make check" as interrupting the script immediately when + # system tests fail would prevent the junit.xml file from being + # produced. script: - cd bin/tests/system - - make -j${TEST_PARALLEL_JOBS:-1} check + - RET=0 + - make -j${TEST_PARALLEL_JOBS:-1} check || RET=$? + - cd "$CI_PROJECT_DIR" + - *find_python + - > + "$PYTHON" bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml + - *check_for_junit_xml + - (exit $RET) after_script: - cat bin/tests/system/test-suite.log || true .system_test_gcov: &system_test_gcov_job <<: *system_test_common - artifacts: - untracked: true - exclude: - - "**/__pycache__/**/*" - when: always .system_test_tsan: &system_test_tsan_job <<: *system_test_common @@ -492,15 +500,6 @@ stages: - find bin/tests/system -name "*dig.*" | xargs grep "error" || true - *find_python - *parse_tsan - - > - "$PYTHON" bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml - artifacts: - untracked: true - exclude: - - "**/__pycache__/**/*" - when: always - reports: - junit: junit.xml .unit_test_common: &unit_test_common <<: *default_triggering_rules From aa10df51110fba744b74d6cad0f6e2d37926d187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Wed, 4 Jun 2025 15:13:23 +0200 Subject: [PATCH 3/6] Clean up the definitions and usages of `&system_test_*` anchors Remove redundant indirections and overwrites. (cherry picked from commit cbe9972d3e3c682621c8b6ddf338ffd2f8a872ca) --- .gitlab-ci.yml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 68f2fd5375..68374614ac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -430,7 +430,7 @@ stages: artifacts: true timeout: 2h -.system_test_common: &system_test_common +.system_test_common: &system_test_job <<: *default_triggering_rules stage: system before_script: @@ -466,11 +466,8 @@ stages: reports: junit: junit.xml -.system_test: &system_test_job - <<: *system_test_common - .system_test_make_check: &system_test_make_check_job - <<: *system_test_common + <<: *system_test_job # This script needs to: 1) fail if the system tests fail, 2) fail if # the junit.xml file is broken, 3) produce the junit.xml file even if # the system tests fail. Therefore, $RET is used to "cache" the @@ -490,11 +487,8 @@ stages: after_script: - cat bin/tests/system/test-suite.log || true -.system_test_gcov: &system_test_gcov_job - <<: *system_test_common - .system_test_tsan: &system_test_tsan_job - <<: *system_test_common + <<: *system_test_job after_script: - *display_pytest_failures - find bin/tests/system -name "*dig.*" | xargs grep "error" || true @@ -1014,7 +1008,7 @@ gcc:bookworm:amd64: system:gcc:bookworm:amd64: <<: *debian_bookworm_amd64_image - <<: *system_test_gcov_job + <<: *system_test_job variables: CI_ENABLE_ALL_TESTS: 1 CLEAN_BUILD_ARTIFACTS_ON_SUCCESS: 0 @@ -1247,7 +1241,6 @@ gcc:tumbleweed:amd64: system:gcc:tumbleweed:amd64: <<: *tumbleweed_latest_amd64_image - <<: *system_test_job <<: *system_test_make_check_job needs: - job: gcc:tumbleweed:amd64 From c052007d05a4e73cc54000e61ac9489db85b4fc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Wed, 4 Jun 2025 15:18:29 +0200 Subject: [PATCH 4/6] Hoist the junit and artifact handling to the `&unit_test_common` anchor In some cases the report wasn't generated, sometimes it wasn't kept properly. This unifies the way artifacts are generated and kept. (cherry picked from commit 4ec1a37ca090df41f046e20c6a6c788e7a4a0afe) --- .gitlab-ci.yml | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 68374614ac..08d0e8dbd7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -500,44 +500,38 @@ stages: stage: unit before_script: - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}" + # This script needs to: 1) fail if the unit tests fail, 2) fail if the + # junit.xml file is broken, 3) produce the junit.xml file even if the + # unit tests fail. Therefore, $RET is used to "cache" the result of + # "make unit" as interrupting the script immediately when unit tests + # fail would prevent the junit.xml file from being produced. script: - *fips_feature_test - - make -j${TEST_PARALLEL_JOBS:-1} -k unit V=1 - - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || make clean >/dev/null 2>&1 - after_script: - - test -d bind-* && cd bind-* - - REALSOURCEDIR="$PWD" - - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}" + - RET=0 + - make -j${TEST_PARALLEL_JOBS:-1} -k unit V=1 || RET=$? - *find_python - > - "$PYTHON" "$REALSOURCEDIR"/bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml - -.unit_test: &unit_test_job - <<: *unit_test_common + "$PYTHON" "$CI_PROJECT_DIR"/bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml + - *check_for_junit_xml + - (exit $RET) + - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || make clean >/dev/null 2>&1 artifacts: untracked: true when: always reports: junit: junit.xml +.unit_test: &unit_test_job + <<: *unit_test_common + .unit_test_gcov: &unit_test_gcov_job <<: *unit_test_common - artifacts: - untracked: true - when: always .unit_test_tsan: &unit_test_tsan_job <<: *unit_test_common after_script: - *find_python - *parse_tsan - - > - "$PYTHON" bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml - artifacts: - untracked: true - when: always - reports: - junit: junit.xml .docs: &docs_job variables: From af4b44525775ce855daf2f447fe9b43a49f618cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Wed, 4 Jun 2025 15:22:12 +0200 Subject: [PATCH 5/6] Clean up the definitions and usages of `&unit_test_*` anchors Remove redundant indirections and overwrites. (cherry picked from commit 4303e0691968824ae8b8938e1ab121386f32f83a) --- .gitlab-ci.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 08d0e8dbd7..a1b9b113af 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -495,7 +495,7 @@ stages: - *find_python - *parse_tsan -.unit_test_common: &unit_test_common +.unit_test_common: &unit_test_job <<: *default_triggering_rules stage: unit before_script: @@ -521,14 +521,8 @@ stages: reports: junit: junit.xml -.unit_test: &unit_test_job - <<: *unit_test_common - -.unit_test_gcov: &unit_test_gcov_job - <<: *unit_test_common - .unit_test_tsan: &unit_test_tsan_job - <<: *unit_test_common + <<: *unit_test_job after_script: - *find_python - *parse_tsan @@ -1013,7 +1007,7 @@ system:gcc:bookworm:amd64: unit:gcc:bookworm:amd64: <<: *debian_bookworm_amd64_image - <<: *unit_test_gcov_job + <<: *unit_test_job variables: CI_ENABLE_ALL_TESTS: 1 CLEAN_BUILD_ARTIFACTS_ON_SUCCESS: 0 From d8e6609aaaa5fb752d4105a1b558f200a08aa4c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Fri, 6 Jun 2025 23:58:46 +0200 Subject: [PATCH 6/6] Use :global-test-result: when converting trs file to JUnit XML There might be more than one :test-result: and they are collated into the :global-test-result: field. This only happens when system tests are run with `make check`. --- bin/tests/convert-trs-to-junit.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/tests/convert-trs-to-junit.py b/bin/tests/convert-trs-to-junit.py index 6d7b4aaa1b..fdfd8cf3ce 100755 --- a/bin/tests/convert-trs-to-junit.py +++ b/bin/tests/convert-trs-to-junit.py @@ -28,14 +28,14 @@ def read_trs_result(filename): items = line.split() if len(items) < 2: raise ValueError("unsupported line in trs file", filename, line) - if items[0] != (":test-result:"): + if items[0] != (":global-test-result:"): continue if result is not None: - raise NotImplementedError("double :test-result:", filename) + raise NotImplementedError("double :global-test-result:", filename) result = items[1].upper() if result is None: - raise ValueError(":test-result: not found", filename) + raise ValueError(":global-test-result: not found", filename) return result