From e3d2f5ad948f977df67a349601bd4743f6777539 Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Wed, 25 Mar 2026 13:31:49 +0100 Subject: [PATCH 1/3] Set RUNNER_SCRIPT_TIMEOUTs Sometimes jobs can get stuck and be terminated by GitLab, leaving us without artefacts that could contain useful information about why the job got stuck. Assisted-by: Claude:claude-opus-4-7 --- .gitlab-ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 91a0945f00..f41f49b4b5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,7 @@ variables: + # The default timeout is 60 minutes. Set RUNNER_SCRIPT_TIMEOUT to 55m to + # leave room for the after_script timeout that defaults to 5m. + RUNNER_SCRIPT_TIMEOUT: 55m # Not normally needed, but may be if some script uses `apt-get install`. DEBIAN_FRONTEND: noninteractive # Locale settings do not affect the build, but might affect tests. @@ -503,6 +506,9 @@ stages: SHOTGUN_ROUNDS: 5 when: manual allow_failure: true + timeout: 2h + variables: + RUNNER_SCRIPT_TIMEOUT: 1h55m script: - if [ -n "$WEEKDAY" ] && [ "$(date +%w)" != "$WEEKDAY" ]; then exit 0; fi # if WEEKDAY is set, only run on those days - if [ -z "$BASELINE" ]; then export BASELINE=$BIND_BASELINE_VERSION; fi # this dotenv variable can't be set in the rules section, because rules are evaluated before any jobs run @@ -531,7 +537,6 @@ stages: needs: - job: ci-variables artifacts: true - timeout: 2h .system_test_common: &system_test_job <<: *default_triggering_rules From 4f410ee1e6b7f5a8cceb3331aa5be3dc74680aee Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Mon, 11 May 2026 19:19:53 +0200 Subject: [PATCH 2/3] Pass -r option to respdiff.sh Tell respdiff.sh where to find the respdiff Python tools (msgdiff.py, diffsum.py, ...) so the in-tree copy from bind9-qa is used. Assisted-by: Claude:claude-opus-4-7 --- .gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f41f49b4b5..b0b9450e1d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2376,7 +2376,7 @@ respdiff: EXTRA_CONFIGURE: "-Doptimization=g" MAX_DISAGREEMENTS_PERCENTAGE: "0.3" script: - - bash respdiff.sh -m /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named" + - bash respdiff.sh -m /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" -r "${PWD}/respdiff" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named" - cd ../.. && ninja -C build clean >/dev/null 2>&1 respdiff:asan: @@ -2388,7 +2388,7 @@ respdiff:asan: EXTRA_CONFIGURE: "-Doptimization=g -Db_sanitize=address,undefined -Djemalloc=disabled" MAX_DISAGREEMENTS_PERCENTAGE: "0.3" script: - - bash respdiff.sh -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named" + - bash respdiff.sh -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" -r "${PWD}/respdiff" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named" - cd ../.. && ninja -C build clean >/dev/null 2>&1 respdiff:tsan: @@ -2402,7 +2402,7 @@ respdiff:tsan: MAX_DISAGREEMENTS_PERCENTAGE: "0.3" TSAN_SYMBOLIZER_PATH: "${TSAN_SYMBOLIZER_PATH_DEBIAN}" script: - - bash respdiff.sh -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named" + - bash respdiff.sh -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" -r "${PWD}/respdiff" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named" - cd ../.. && ninja -C build clean >/dev/null 2>&1 after_script: - *parse_tsan @@ -2416,7 +2416,7 @@ respdiff-third-party: EXTRA_CONFIGURE: "-Doptimization=g" MAX_DISAGREEMENTS_PERCENTAGE: "0.4" script: - - bash respdiff.sh -s third_party -q "${PWD}/100k_mixed.txt" -c 1 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" + - bash respdiff.sh -s third_party -q "${PWD}/100k_mixed.txt" -c 1 -w "${PWD}/rspworkdir" -r "${PWD}/respdiff" "${CI_PROJECT_DIR}" - cd ../.. && ninja -C build clean >/dev/null 2>&1 .respdiff-recent-named: &respdiff_recent_named @@ -2434,7 +2434,7 @@ respdiff-third-party: - *configure - meson compile -C build - cd "$CI_PROJECT_DIR"/bind9-qa/respdiff - - bash respdiff.sh -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "${CI_PROJECT_DIR}/baseline-build/build/named" + - bash respdiff.sh -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" -r "${PWD}/respdiff" "${CI_PROJECT_DIR}" "${CI_PROJECT_DIR}/baseline-build/build/named" respdiff:recent-named: <<: *respdiff_recent_named From 7928127d8b143b30001083cd106cf3519cd9626d Mon Sep 17 00:00:00 2001 From: Michal Nowak Date: Mon, 11 May 2026 19:20:23 +0200 Subject: [PATCH 3/3] Get some useful data out of respdiff even in case of a failure Assisted-by: Claude:claude-opus-4-7 --- .gitlab-ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b0b9450e1d..6dc73e5516 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -645,6 +645,11 @@ stages: - *setup_interfaces - *git_clone_bind9-qa - cd "$CI_PROJECT_DIR"/bind9-qa/respdiff + after_script: + - *parse_tsan + - cd "$CI_PROJECT_DIR"/bind9-qa/respdiff + - if [ ! -e rspworkdir/report.json ]; then respdiff/msgdiff.py --config rspworkdir/respdiff.cfg rspworkdir; fi + - if [ ! -e rspworkdir/report.txt ]; then respdiff/diffsum.py --config rspworkdir/respdiff.cfg --limit 0 rspworkdir | tee rspworkdir/report.txt; fi needs: [] artifacts: paths: @@ -2404,8 +2409,6 @@ respdiff:tsan: script: - bash respdiff.sh -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" -r "${PWD}/respdiff" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named" - cd ../.. && ninja -C build clean >/dev/null 2>&1 - after_script: - - *parse_tsan respdiff-third-party: <<: *respdiff_job