From 1c1a7aa2e19f471d74fd5416c454ad279f4ac41d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicki=20K=C5=99=C3=AD=C5=BEek?= Date: Mon, 13 Jan 2025 14:29:24 +0100 Subject: [PATCH] Set stricter limits for respdiff testing Adjust the limit of maximum disagreements in respdiff results based on recent pipeline results. The respdiff and respdiff:asan seem to have almost identical results, typically around 0.07 % of differences with ocassional spikes up to around 0.11 %. Similar results are for respdiff:tsan, perhaps with more common spikes with values up to around 0.12 %. Set the limit to 0.15 % to allow for some tolerance due to network conditions, time of day etc. The respdiff:third-party has a slightly higher disagreements average, with typical values being around 0.12 %. Set the limit to 0.2 %. Exceeding either of those values should be quite clear indication that some resolution behaviour has changed, since the values appear to be very stable within the newly configured limits. (cherry picked from commit 0584d3f65f4fea05eeb8c3f70c5460467b6637ef) --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 10eede6212..a11e466771 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1514,7 +1514,7 @@ respdiff: variables: CC: gcc CFLAGS: "${CFLAGS_COMMON} -Og" - MAX_DISAGREEMENTS_PERCENTAGE: "0.5" + MAX_DISAGREEMENTS_PERCENTAGE: "0.15" script: - bash respdiff.sh -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named" @@ -1527,7 +1527,7 @@ respdiff:asan: CFLAGS: "${CFLAGS_COMMON} -Og -fsanitize=address,undefined" LDFLAGS: "-fsanitize=address,undefined" EXTRA_CONFIGURE: "--disable-dnsrps --without-jemalloc" - MAX_DISAGREEMENTS_PERCENTAGE: "0.5" + MAX_DISAGREEMENTS_PERCENTAGE: "0.15" script: - bash respdiff.sh -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named" @@ -1542,7 +1542,7 @@ respdiff:tsan: OPENSSL_CFLAGS: -I/opt/tsan/include OPENSSL_LIBS: -L/opt/tsan/lib -lssl -lcrypto EXTRA_CONFIGURE: "--disable-dnsrps --enable-pthread-rwlock --without-jemalloc PKG_CONFIG_PATH=/opt/tsan/lib/pkgconfig" - MAX_DISAGREEMENTS_PERCENTAGE: "0.5" + MAX_DISAGREEMENTS_PERCENTAGE: "0.15" TSAN_OPTIONS: "${TSAN_OPTIONS_DEBIAN}" script: - bash respdiff.sh -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named" @@ -1557,7 +1557,7 @@ respdiff-third-party: variables: CC: gcc CFLAGS: "${CFLAGS_COMMON} -Og" - MAX_DISAGREEMENTS_PERCENTAGE: "0.5" + MAX_DISAGREEMENTS_PERCENTAGE: "0.2" script: - bash respdiff.sh -s third_party -q "${PWD}/100k_mixed.txt" -c 1 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}"