From a7e89a0712fe85d52d437038de278cd867d225ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= <michal@isc.org>
Date: Mon, 18 Jul 2022 14:39:02 +0200
Subject: [PATCH 1/3] Use a pre-built executable as the reference named

The "respdiff" GitLab CI job compares DNS responses produced by the
current version of named with those produced by a reference version.
The latter is built from source in each "respdiff" job, despite the fact
that the reference version changes very rarely.  Use a pre-built named
executable as the reference version instead, assuming it is available in
the OS image used for "respdiff" tests.
---
 .gitlab-ci.yml | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6b7b81b55f..f24e4d69f7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1230,7 +1230,6 @@ respdiff:
   variables:
     CC: gcc
     CFLAGS: "${CFLAGS_COMMON} -Og -DISC_TRACK_PTHREADS_OBJECTS"
-    BIND_BASELINE_VERSION: v9_11_24
     MAX_DISAGREEMENTS_PERCENTAGE: "0.5"
   script:
     - autoreconf -fi
@@ -1238,16 +1237,11 @@ respdiff:
     - make -j${BUILD_PARALLEL_JOBS:-1} V=1
     - *setup_interfaces
     - git clone --depth 1 https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.isc.org/isc-private/bind-qa.git
-    - git clone --branch "${BIND_BASELINE_VERSION}" --depth 1 https://gitlab.isc.org/isc-projects/bind9.git refserver
-    - cd refserver/
-    - ./configure --with-randomdev=/dev/urandom
-    - make -j${BUILD_PARALLEL_JOBS:-1} V=1
-    - cd ../bind-qa/bind9/respdiff
-    - bash respdiff.sh -m /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "${CI_PROJECT_DIR}/refserver"
+    - cd bind-qa/bind9/respdiff
+    - bash respdiff.sh -m /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named"
   needs: []
   artifacts:
     paths:
-      - refserver
       - bind-qa/bind9/respdiff
     exclude:
       - bind-qa/bind9/respdiff/rspworkdir/data.mdb # Exclude a 10 GB file.

From 004a7bb37684adaac8ee2aa686411f42bceb732b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= <michal@isc.org>
Date: Mon, 18 Jul 2022 14:39:02 +0200
Subject: [PATCH 2/3] Extract respdiff job definition to a YAML anchor

Ensure the common parts of all jobs using respdiff are available in the
form of a reusable YAML anchor, to reduce code duplication and to
simplify adding more respdiff-based jobs to GitLab CI.
---
 .gitlab-ci.yml | 56 ++++++++++++++++++++------------------------------
 1 file changed, 22 insertions(+), 34 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f24e4d69f7..a8b650c645 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -390,6 +390,26 @@ stages:
     - qpdf --check doc/arm/_build/latex/Bv9ARM.pdf
     - find doc/man/ -maxdepth 1 -name "*.[0-9]" -exec mandoc -T lint "{}" \; | ( ! grep -v -e "skipping paragraph macro. sp after" -e "unknown font, skipping request. ft C" )
 
+.respdiff: &respdiff_job
+  <<: *base_image
+  stage: system
+  before_script:
+    - autoreconf -fi
+    - *configure
+    - make -j${BUILD_PARALLEL_JOBS:-1} V=1
+    - *setup_interfaces
+    - git clone --depth 1 https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.isc.org/isc-private/bind-qa.git
+    - cd bind-qa/bind9/respdiff
+  needs: []
+  artifacts:
+    paths:
+      - bind-qa/bind9/respdiff
+    exclude:
+      - bind-qa/bind9/respdiff/rspworkdir/data.mdb # Exclude a 10 GB file.
+    untracked: true
+    expire_in: "1 day"
+    when: always
+
 ### Job Definitions
 
 # Jobs in the precheck stage
@@ -1224,56 +1244,24 @@ coverity:
 # Respdiff tests
 
 respdiff:
-  <<: *base_image
+  <<: *respdiff_job
   <<: *api_schedules_tags_triggers_web_triggering_rules
-  stage: system
   variables:
     CC: gcc
     CFLAGS: "${CFLAGS_COMMON} -Og -DISC_TRACK_PTHREADS_OBJECTS"
     MAX_DISAGREEMENTS_PERCENTAGE: "0.5"
   script:
-    - autoreconf -fi
-    - *configure
-    - make -j${BUILD_PARALLEL_JOBS:-1} V=1
-    - *setup_interfaces
-    - git clone --depth 1 https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.isc.org/isc-private/bind-qa.git
-    - cd bind-qa/bind9/respdiff
     - bash respdiff.sh -m /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named"
-  needs: []
-  artifacts:
-    paths:
-      - bind-qa/bind9/respdiff
-    exclude:
-      - bind-qa/bind9/respdiff/rspworkdir/data.mdb # Exclude a 10 GB file.
-    untracked: true
-    expire_in: "1 day"
-    when: always
 
 respdiff-third-party:
-  <<: *base_image
+  <<: *respdiff_job
   <<: *api_schedules_tags_triggers_web_triggering_rules
-  stage: system
   variables:
     CC: gcc
     CFLAGS: "${CFLAGS_COMMON} -Og"
     MAX_DISAGREEMENTS_PERCENTAGE: "0.5"
   script:
-    - autoreconf -fi
-    - *configure
-    - make -j${BUILD_PARALLEL_JOBS:-1} V=1
-    - *setup_interfaces
-    - git clone --depth 1 https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.isc.org/isc-private/bind-qa.git
-    - cd bind-qa/bind9/respdiff
     - bash respdiff.sh -s third_party -q "${PWD}/100k_mixed.txt" -c 1 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}"
-  needs: []
-  artifacts:
-    paths:
-      - bind-qa/bind9/respdiff
-    exclude:
-      - bind-qa/bind9/respdiff/rspworkdir/data.mdb # Exclude a 10 GB file.
-    untracked: true
-    expire_in: "1 day"
-    when: always
 
 # "Stress" tests
 

From ea8a0bde8ea306893b9a523206b41754e444f9fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= <michal@isc.org>
Date: Mon, 18 Jul 2022 14:39:02 +0200
Subject: [PATCH 3/3] Run a short respdiff test for all merge requests

Now that the respdiff tests can detect memory leaks, it is worth running
them for every merge request.  However, the existing respdiff-based
tests take a while to complete (about half an hour with our current CI
infrastructure), which does not make them a good fit for this purpose.
Add a new GitLab CI job, "respdiff-short", which uses a smaller query
set that gets processed within a couple of minutes on our current CI
infrastructure.  Rename the existing respdiff-based jobs to make
distinguishing them easier.
---
 .gitlab-ci.yml | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a8b650c645..9400189ca0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1243,7 +1243,17 @@ coverity:
 
 # Respdiff tests
 
-respdiff:
+respdiff-short:
+  <<: *respdiff_job
+  <<: *default_triggering_rules
+  variables:
+    CC: gcc
+    CFLAGS: "${CFLAGS_COMMON} -Og -DISC_TRACK_PTHREADS_OBJECTS"
+    MAX_DISAGREEMENTS_PERCENTAGE: "0.5"
+  script:
+    - bash respdiff.sh -m /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 -s named -q "${PWD}/10k_a.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named"
+
+respdiff-long:
   <<: *respdiff_job
   <<: *api_schedules_tags_triggers_web_triggering_rules
   variables:
@@ -1253,7 +1263,7 @@ respdiff:
   script:
     - bash respdiff.sh -m /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named"
 
-respdiff-third-party:
+respdiff-long-third-party:
   <<: *respdiff_job
   <<: *api_schedules_tags_triggers_web_triggering_rules
   variables: