From 8bfe95a6a93297c15e4ffa1c0f28145c3f48afb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Wed, 4 Jun 2025 14:41:35 +0200 Subject: [PATCH 1/6] Add a check for existence and sanity of the junit.xml file And use if after system tests are run using pytest. (cherry picked from commit dfd6097f7bb573dfa9e713149d33e6d9b92e3e99) --- .gitlab-ci.yml | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c22e56e740..6476f9f2d5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -314,6 +314,21 @@ stages: - rm -f bind-*.tar.xz - cd bind-* +.check_for_junit_xml: &check_for_junit_xml + # test if junit.xml file exists and is longer 40 bytes + # (i.e., contains more than ``) + - if [ -f "$CI_PROJECT_DIR"/junit.xml ]; then + if [ $(wc -c < "$CI_PROJECT_DIR"/junit.xml) -gt 40 ]; then + echo "junit.xml file exists and is longer than 40 bytes."; + else + echo "junit.xml file exists but is too short."; + exit 1; + fi + else + echo "junit.xml file does not exist."; + exit 1; + fi + .build: &build_job <<: *default_triggering_rules stage: build @@ -400,13 +415,22 @@ stages: before_script: - test -n "${OUT_OF_TREE_WORKSPACE}" && cp -r bin/tests/system/* "${OUT_OF_TREE_WORKSPACE}/bin/tests/system/" && cd "${OUT_OF_TREE_WORKSPACE}" - *setup_interfaces + # This script needs to: 1) fail if the system tests fail, 2) fail if + # the junit.xml file is broken, 3) produce the junit.xml file even if + # the system tests fail. Therefore, $RET is used to "cache" the + # result of running pytest as interrupting the script immediately when + # system tests fail would make checking the contents of the junit.xml + # file impossible (GitLab Runner uses "set -o pipefail"). script: - *find_pytest - *find_python - ( if [ "${CI_DISPOSABLE_ENVIRONMENT}" = "true" ]; then sleep 3000; "$PYTHON" "${CI_PROJECT_DIR}/util/get-running-system-tests.py"; fi ) & - cd bin/tests/system + - RET=0 - > - "$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt + ("$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt) || RET=1 + - *check_for_junit_xml + - (exit $RET) - '( ! grep -F "grep: warning:" pytest.out.txt )' - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ( cd ../../.. && make clean >/dev/null 2>&1 ) after_script: From 1866f47284f1e9f13c75d03f91fce937fa996ff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Wed, 4 Jun 2025 15:06:09 +0200 Subject: [PATCH 2/6] Hoist the artifact handling to the `&system_test_common` anchor In the past artifacts of different types of system test jobs were treated differently but this is no longer the case. (cherry picked from commit c61ff639b3a5aa7d4513efdc893aadff95a56c60) --- .gitlab-ci.yml | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6476f9f2d5..3b49f629c4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -436,6 +436,13 @@ stages: after_script: - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}" - *display_pytest_failures + artifacts: + untracked: true + exclude: + - "**/__pycache__/**/*" + when: always + reports: + junit: junit.xml .system_test_legacy: &system_test_legacy script: @@ -454,29 +461,30 @@ stages: .system_test: &system_test_job <<: *system_test_common - artifacts: - untracked: true - exclude: - - "**/__pycache__/**/*" - when: always - reports: - junit: junit.xml .system_test_make_check: &system_test_make_check_job <<: *system_test_common + # This script needs to: 1) fail if the system tests fail, 2) fail if + # the junit.xml file is broken, 3) produce the junit.xml file even if + # the system tests fail. Therefore, $RET is used to "cache" the + # result of "make check" as interrupting the script immediately when + # system tests fail would prevent the junit.xml file from being + # produced. script: - cd bin/tests/system - - make -j${TEST_PARALLEL_JOBS:-1} check + - RET=0 + - make -j${TEST_PARALLEL_JOBS:-1} check || RET=$? + - cd "$CI_PROJECT_DIR" + - *find_python + - > + "$PYTHON" bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml + - *check_for_junit_xml + - (exit $RET) after_script: - cat bin/tests/system/test-suite.log || true .system_test_gcov: &system_test_gcov_job <<: *system_test_common - artifacts: - untracked: true - exclude: - - "**/__pycache__/**/*" - when: always .system_test_tsan: &system_test_tsan_job <<: *system_test_common @@ -485,15 +493,6 @@ stages: - find bin/tests/system -name "*dig.*" | xargs grep "error" || true - *find_python - *parse_tsan - - > - "$PYTHON" bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml - artifacts: - untracked: true - exclude: - - "**/__pycache__/**/*" - when: always - reports: - junit: junit.xml .unit_test_common: &unit_test_common <<: *default_triggering_rules From a692f57d6fed33929a83801726b87b406af82e74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Wed, 4 Jun 2025 15:13:23 +0200 Subject: [PATCH 3/6] Clean up the definitions and usages of `&system_test_*` anchors Remove redundant indirections and overwrites. (cherry picked from commit cbe9972d3e3c682621c8b6ddf338ffd2f8a872ca) --- .gitlab-ci.yml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3b49f629c4..7a2e5f7be8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -409,7 +409,7 @@ stages: artifacts: true timeout: 2h -.system_test_common: &system_test_common +.system_test_common: &system_test_job <<: *default_triggering_rules stage: system before_script: @@ -459,11 +459,8 @@ stages: - > "$PYTHON" "$REALSOURCEDIR"/bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml -.system_test: &system_test_job - <<: *system_test_common - .system_test_make_check: &system_test_make_check_job - <<: *system_test_common + <<: *system_test_job # This script needs to: 1) fail if the system tests fail, 2) fail if # the junit.xml file is broken, 3) produce the junit.xml file even if # the system tests fail. Therefore, $RET is used to "cache" the @@ -483,11 +480,8 @@ stages: after_script: - cat bin/tests/system/test-suite.log || true -.system_test_gcov: &system_test_gcov_job - <<: *system_test_common - .system_test_tsan: &system_test_tsan_job - <<: *system_test_common + <<: *system_test_job after_script: - *display_pytest_failures - find bin/tests/system -name "*dig.*" | xargs grep "error" || true @@ -950,7 +944,7 @@ gcc:bookworm:amd64: system:gcc:bookworm:amd64: <<: *debian_bookworm_amd64_image - <<: *system_test_gcov_job + <<: *system_test_job variables: CI_ENABLE_ALL_TESTS: 1 CLEAN_BUILD_ARTIFACTS_ON_SUCCESS: 0 @@ -1127,7 +1121,6 @@ gcc:tumbleweed:amd64: system:gcc:tumbleweed:amd64: <<: *tumbleweed_latest_amd64_image - <<: *system_test_job <<: *system_test_make_check_job needs: - job: gcc:tumbleweed:amd64 From 0de9d07f5edc1d0355763ad9533727e92d2204df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Wed, 4 Jun 2025 15:18:29 +0200 Subject: [PATCH 4/6] Hoist the junit and artifact handling to the `&unit_test_common` anchor In some cases the report wasn't generated, sometimes it wasn't kept properly. This unifies the way artifacts are generated and kept. (cherry picked from commit 4ec1a37ca090df41f046e20c6a6c788e7a4a0afe) --- .gitlab-ci.yml | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a2e5f7be8..6dc2553c63 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -493,43 +493,37 @@ stages: stage: unit before_script: - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}" + # This script needs to: 1) fail if the unit tests fail, 2) fail if the + # junit.xml file is broken, 3) produce the junit.xml file even if the + # unit tests fail. Therefore, $RET is used to "cache" the result of + # "make unit" as interrupting the script immediately when unit tests + # fail would prevent the junit.xml file from being produced. script: - - make -j${TEST_PARALLEL_JOBS:-1} -k unit V=1 - - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || make clean >/dev/null 2>&1 - after_script: - - test -d bind-* && cd bind-* - - REALSOURCEDIR="$PWD" - - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}" + - RET=0 + - make -j${TEST_PARALLEL_JOBS:-1} -k unit V=1 || RET=$? - *find_python - > - "$PYTHON" "$REALSOURCEDIR"/bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml - -.unit_test: &unit_test_job - <<: *unit_test_common + "$PYTHON" "$CI_PROJECT_DIR"/bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml + - *check_for_junit_xml + - (exit $RET) + - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || make clean >/dev/null 2>&1 artifacts: untracked: true when: always reports: junit: junit.xml +.unit_test: &unit_test_job + <<: *unit_test_common + .unit_test_gcov: &unit_test_gcov_job <<: *unit_test_common - artifacts: - untracked: true - when: always .unit_test_tsan: &unit_test_tsan_job <<: *unit_test_common after_script: - *find_python - *parse_tsan - - > - "$PYTHON" bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml - artifacts: - untracked: true - when: always - reports: - junit: junit.xml .docs: &docs_job variables: From 277ab295066f4d7a39e9917aba79612561ab2038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Wed, 4 Jun 2025 15:22:12 +0200 Subject: [PATCH 5/6] Clean up the definitions and usages of `&unit_test_*` anchors Remove redundant indirections and overwrites. (cherry picked from commit 4303e0691968824ae8b8938e1ab121386f32f83a) --- .gitlab-ci.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6dc2553c63..1e036c32df 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -488,7 +488,7 @@ stages: - *find_python - *parse_tsan -.unit_test_common: &unit_test_common +.unit_test_common: &unit_test_job <<: *default_triggering_rules stage: unit before_script: @@ -513,14 +513,8 @@ stages: reports: junit: junit.xml -.unit_test: &unit_test_job - <<: *unit_test_common - -.unit_test_gcov: &unit_test_gcov_job - <<: *unit_test_common - .unit_test_tsan: &unit_test_tsan_job - <<: *unit_test_common + <<: *unit_test_job after_script: - *find_python - *parse_tsan @@ -948,7 +942,7 @@ system:gcc:bookworm:amd64: unit:gcc:bookworm:amd64: <<: *debian_bookworm_amd64_image - <<: *unit_test_gcov_job + <<: *unit_test_job variables: CI_ENABLE_ALL_TESTS: 1 CLEAN_BUILD_ARTIFACTS_ON_SUCCESS: 0 From b49e785e6f5eddecf809b8de3900daca26528025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Fri, 6 Jun 2025 23:58:46 +0200 Subject: [PATCH 6/6] Use :global-test-result: when converting trs file to JUnit XML There might be more than one :test-result: and they are collated into the :global-test-result: field. This only happens when system tests are run with `make check`. (cherry picked from commit d8e6609aaaa5fb752d4105a1b558f200a08aa4c1) --- bin/tests/convert-trs-to-junit.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/tests/convert-trs-to-junit.py b/bin/tests/convert-trs-to-junit.py index 6d7b4aaa1b..fdfd8cf3ce 100755 --- a/bin/tests/convert-trs-to-junit.py +++ b/bin/tests/convert-trs-to-junit.py @@ -28,14 +28,14 @@ def read_trs_result(filename): items = line.split() if len(items) < 2: raise ValueError("unsupported line in trs file", filename, line) - if items[0] != (":test-result:"): + if items[0] != (":global-test-result:"): continue if result is not None: - raise NotImplementedError("double :test-result:", filename) + raise NotImplementedError("double :global-test-result:", filename) result = items[1].upper() if result is None: - raise ValueError(":test-result: not found", filename) + raise ValueError(":global-test-result: not found", filename) return result