From 4c8ce37e26223c1ec35844bfad28f0a999cbb206 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= <michal@isc.org>
Date: Thu, 26 Mar 2020 11:03:52 +0100
Subject: [PATCH 1/8] Define TSAN options in a global variable

All assignments for the TSAN_OPTIONS variable are identical across the
entire .gitlab-ci.yml file.  Define a global TSAN_OPTIONS_COMMON
variable and use it in job definitions to reduce code duplication.

(cherry picked from commit 6325c0993a4f73b7c79dea4835a98c894820f189)
---
 .gitlab-ci.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5ed62125b7..937ecb8163 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,6 +26,7 @@ variables:
 
   # Pass run-time flags to AddressSanitizer to get core dumps on error.
   ASAN_OPTIONS_COMMON: abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1
+  TSAN_OPTIONS_COMMON: "second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan external_symbolizer_path=$SYMBOLIZER"
 
   TARBALL_COMPRESSOR: gzip
   TARBALL_EXTENSION: gz
@@ -915,7 +916,7 @@ gcc:tsan:
 
 system:gcc:tsan:
   variables:
-    TSAN_OPTIONS: "second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan external_symbolizer_path=$SYMBOLIZER exitcode=0"
+    TSAN_OPTIONS: ${TSAN_OPTIONS_COMMON}
   before_script:
     - *setup_interfaces
     - echo $TSAN_OPTIONS
@@ -937,7 +938,7 @@ system:gcc:tsan:
 
 unit:gcc:tsan:
   variables:
-    TSAN_OPTIONS: "second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan external_symbolizer_path=$SYMBOLIZER"
+    TSAN_OPTIONS: ${TSAN_OPTIONS_COMMON}
   before_script:
     - echo $TSAN_OPTIONS
     - lib/isc/tests/result_test
@@ -970,7 +971,7 @@ clang:tsan:
 
 system:clang:tsan:
   variables:
-    TSAN_OPTIONS: "second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan external_symbolizer_path=$SYMBOLIZER exitcode=0"
+    TSAN_OPTIONS: ${TSAN_OPTIONS_COMMON}
   before_script:
     - *setup_interfaces
     - echo $TSAN_OPTIONS
@@ -992,7 +993,7 @@ system:clang:tsan:
 
 unit:clang:tsan:
   variables:
-    TSAN_OPTIONS: "second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan external_symbolizer_path=$SYMBOLIZER"
+    TSAN_OPTIONS: ${TSAN_OPTIONS_COMMON}
   before_script:
     - echo $TSAN_OPTIONS
     - lib/isc/tests/result_test

From 5ba8fac3513aab5107e098b806c6a27be8efcf04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= <michal@isc.org>
Date: Thu, 26 Mar 2020 11:03:52 +0100
Subject: [PATCH 2/8] Drop "before_script" key from TSAN job definitions

Both "system_test_job" and "unit_test_job" YAML anchors contain a
"before_script" key.  TSAN job definitions first specify their own value
of the "before_script" key and then include the aforementioned YAML
anchors, which results in the value of the "before_script" key being
overridden with the value specified by the included anchor.  Given this,
remove "before_script" definitions specific to TSAN jobs as they serve
no practical purpose.

(cherry picked from commit 8ef01c7b50d9700f81f4836657a62ccf80b91ac7)
---
 .gitlab-ci.yml | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 937ecb8163..8c581dd79f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -917,9 +917,6 @@ gcc:tsan:
 system:gcc:tsan:
   variables:
     TSAN_OPTIONS: ${TSAN_OPTIONS_COMMON}
-  before_script:
-    - *setup_interfaces
-    - echo $TSAN_OPTIONS
   <<: *base_image
   <<: *system_test_job
   needs:
@@ -939,9 +936,6 @@ system:gcc:tsan:
 unit:gcc:tsan:
   variables:
     TSAN_OPTIONS: ${TSAN_OPTIONS_COMMON}
-  before_script:
-    - echo $TSAN_OPTIONS
-    - lib/isc/tests/result_test
   <<: *base_image
   <<: *unit_test_job
   needs:
@@ -972,9 +966,6 @@ clang:tsan:
 system:clang:tsan:
   variables:
     TSAN_OPTIONS: ${TSAN_OPTIONS_COMMON}
-  before_script:
-    - *setup_interfaces
-    - echo $TSAN_OPTIONS
   <<: *base_image
   <<: *system_test_job
   needs:
@@ -994,9 +985,6 @@ system:clang:tsan:
 unit:clang:tsan:
   variables:
     TSAN_OPTIONS: ${TSAN_OPTIONS_COMMON}
-  before_script:
-    - echo $TSAN_OPTIONS
-    - lib/isc/tests/result_test
   <<: *base_image
   <<: *unit_test_job
   needs:

From a46c97f36d08b8031bcc223a3480030423cc0255 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= <michal@isc.org>
Date: Thu, 26 Mar 2020 11:03:52 +0100
Subject: [PATCH 3/8] Refactor TSAN system test job definitions

Multiple YAML keys have identical values for both TSAN system test job
definitions.  Extract these common keys to a YAML anchor and use it in
TSAN system test job definitions to reduce code duplication.

(cherry picked from commit a9aa295f1f3a5af1861d5be150f66522358da8aa)
---
 .gitlab-ci.yml | 42 +++++++++++++++++++-----------------------
 1 file changed, 19 insertions(+), 23 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8c581dd79f..584a0272bd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -259,7 +259,7 @@ stages:
 .setup_softhsm: &setup_softhsm |
     sh -x bin/tests/prepare-softhsm2.sh
 
-.system_test: &system_test_job
+.system_test_common: &system_test_common
   <<: *default_triggering_rules
   stage: system
   before_script:
@@ -268,11 +268,27 @@ stages:
   script:
     - ( cd bin/tests/system && make -j${TEST_PARALLEL_JOBS:-1} -k test V=1 )
     - test -s bin/tests/system/systests.output
+
+.system_test: &system_test_job
+  <<: *system_test_common
   artifacts:
     untracked: true
     expire_in: "1 day"
     when: on_failure
 
+.system_test_tsan: &system_test_tsan_job
+  <<: *system_test_common
+  allow_failure: true
+  after_script:
+    - find bin -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
+  artifacts:
+    expire_in: "1 day"
+    paths:
+      - bin/tests/system/*/tsan.*
+      - bin/tests/system/*/*/tsan.*
+      - tsan/
+    when: on_failure
+
 .kyua_report: &kyua_report_html |
   kyua --logfile /dev/null report-html \
        --force \
@@ -918,20 +934,10 @@ system:gcc:tsan:
   variables:
     TSAN_OPTIONS: ${TSAN_OPTIONS_COMMON}
   <<: *base_image
-  <<: *system_test_job
+  <<: *system_test_tsan_job
   needs:
     - job: gcc:tsan
       artifacts: true
-  allow_failure: true
-  after_script:
-    - find bin -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
-  artifacts:
-    expire_in: "1 day"
-    paths:
-      - bin/tests/system/*/tsan.*
-      - bin/tests/system/*/*/tsan.*
-      - tsan/
-    when: on_failure
 
 unit:gcc:tsan:
   variables:
@@ -967,20 +973,10 @@ system:clang:tsan:
   variables:
     TSAN_OPTIONS: ${TSAN_OPTIONS_COMMON}
   <<: *base_image
-  <<: *system_test_job
+  <<: *system_test_tsan_job
   needs:
     - job: clang:tsan
       artifacts: true
-  allow_failure: true
-  after_script:
-    - find bin -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
-  artifacts:
-    expire_in: "1 day"
-    paths:
-      - bin/tests/system/*/tsan.*
-      - bin/tests/system/*/*/tsan.*
-      - tsan/
-    when: on_failure
 
 unit:clang:tsan:
   variables:

From 5e2428a545b791fd1f48c0faa43ce96a93352416 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= <michal@isc.org>
Date: Thu, 26 Mar 2020 11:03:52 +0100
Subject: [PATCH 4/8] Run "kyua report-html" for TSAN unit test jobs

Definitions of jobs running unit tests under TSAN contain an
"after_script" YAML key.  Since the "unit_test_job" anchor is included
in those job definitions before "after_script" is defined, the
job-specific value of that key overrides the one defined in the included
anchor.  This prevents "kyua report-html" from being run for TSAN unit
test jobs.  Moving the invocation of "kyua report-html" to the "script"
key in the "unit_test_job" anchor is not acceptable as it would cause
the exit code of that command to determine the result of all unit test
jobs and we need that to be the exit code of "make unit".  Instead, add
"kyua report-html" invocations to the "after_script" key of TSAN unit
test job definitions to address the problem without affecting other job
definitions.

(cherry picked from commit 6ebce9425e21615bfbe2aafe3bbc0b001885e882)
---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 584a0272bd..036933bc7d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -949,6 +949,7 @@ unit:gcc:tsan:
       artifacts: true
   allow_failure: true
   after_script:
+    - *kyua_report_html
     - find lib -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
   artifacts:
     expire_in: "1 day"
@@ -988,6 +989,7 @@ unit:clang:tsan:
       artifacts: true
   allow_failure: true
   after_script:
+    - *kyua_report_html
     - find lib -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
   artifacts:
     expire_in: "1 day"

From 91276d9a8cd91d856193268b453ec3934eb67cf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= <michal@isc.org>
Date: Thu, 26 Mar 2020 11:03:52 +0100
Subject: [PATCH 5/8] Refactor TSAN unit test job definitions

Multiple YAML keys have identical values for both TSAN unit test job
definitions.  Extract these common keys to a YAML anchor and use it in
TSAN unit test job definitions to reduce code duplication.

(cherry picked from commit 84463f33bfe7777b5884696820cdf484ea9a1073)
---
 .gitlab-ci.yml | 51 ++++++++++++++++++++++----------------------------
 1 file changed, 22 insertions(+), 29 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 036933bc7d..883045987f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -320,7 +320,7 @@ stages:
     - tags
     - web
 
-.unit_test: &unit_test_job
+.unit_test_common: &unit_test_common
   <<: *default_triggering_rules
   stage: unit
   before_script:
@@ -329,6 +329,9 @@ stages:
     - make unit
   after_script:
     - *kyua_report_html
+
+.unit_test: &unit_test_job
+  <<: *unit_test_common
   artifacts:
     paths:
       - kyua.log
@@ -337,6 +340,22 @@ stages:
     expire_in: "1 day"
     when: on_failure
 
+.unit_test_tsan: &unit_test_tsan_job
+  <<: *unit_test_common
+  allow_failure: true
+  after_script:
+    - *kyua_report_html
+    - find lib -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
+  artifacts:
+    expire_in: "1 day"
+    paths:
+      - lib/*/tests/tsan.*
+      - tsan/
+      - kyua.log
+      - kyua.results
+      - kyua_html/
+    when: on_failure
+
 .cppcheck_args: &run_cppcheck |
   cppcheck --enable=warning,performance,portability,information,missingInclude --include=config.h --std=c11 --language=c --project=compile_commands.json --error-exitcode=2 -j ${TEST_PARALLEL_JOBS:-1} --xml --output-file=cppcheck.results --relative-paths="$CI_PROJECT_DIR" --inline-suppr --suppressions-list=util/suppressions.txt
 
@@ -943,23 +962,10 @@ unit:gcc:tsan:
   variables:
     TSAN_OPTIONS: ${TSAN_OPTIONS_COMMON}
   <<: *base_image
-  <<: *unit_test_job
+  <<: *unit_test_tsan_job
   needs:
     - job: gcc:tsan
       artifacts: true
-  allow_failure: true
-  after_script:
-    - *kyua_report_html
-    - find lib -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
-  artifacts:
-    expire_in: "1 day"
-    paths:
-      - lib/*/tests/tsan.*
-      - tsan/
-      - kyua.log
-      - kyua.results
-      - kyua_html/
-    when: on_failure
 
 clang:tsan:
   <<: *base_image
@@ -983,23 +989,10 @@ unit:clang:tsan:
   variables:
     TSAN_OPTIONS: ${TSAN_OPTIONS_COMMON}
   <<: *base_image
-  <<: *unit_test_job
+  <<: *unit_test_tsan_job
   needs:
     - job: clang:tsan
       artifacts: true
-  allow_failure: true
-  after_script:
-    - *kyua_report_html
-    - find lib -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
-  artifacts:
-    expire_in: "1 day"
-    paths:
-      - lib/*/tests/tsan.*
-      - tsan/
-      - kyua.log
-      - kyua.results
-      - kyua_html/
-    when: on_failure
 
 # Jobs for mutex-based atomics on Debian SID (amd64)
 mutexatomics:

From 824204ae19af994c1f13c27522efafa9f416cc5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= <michal@isc.org>
Date: Thu, 26 Mar 2020 11:03:52 +0100
Subject: [PATCH 6/8] Include compiler name in all build/test job names

Most build/test job names already contain a "clang", "gcc", or "msvc"
prefix which indicates the compiler used for a given job.  Apply that
naming convention to all build/test job names.

(cherry picked from commit 0c898084cd0f3abf82a40ad82d384952dcb3def5)
---
 .gitlab-ci.yml | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 883045987f..693ef09af9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -685,7 +685,7 @@ cppcheck:
 
 # Job for out-of-tree GCC build on Debian Sid (amd64)
 
-out-of-tree:
+gcc:out-of-tree:
   variables:
     CC: gcc
     CFLAGS: "${CFLAGS_COMMON} -Og"
@@ -698,7 +698,7 @@ out-of-tree:
 
 # Jobs for tarball GCC builds on Debian Sid (amd64)
 
-tarball:
+gcc:tarball:
   variables:
     CC: gcc
     EXTRA_CONFIGURE: "--enable-dnstap --with-libidn2"
@@ -715,25 +715,25 @@ tarball:
   only:
     - tags
 
-system:tarball:
+system:gcc:tarball:
   <<: *base_image
   <<: *system_test_job
   before_script:
     - cd bind-*
     - *setup_interfaces
   needs:
-    - job: tarball
+    - job: gcc:tarball
       artifacts: true
   only:
     - tags
 
-unit:tarball:
+unit:gcc:tarball:
   <<: *base_image
   <<: *unit_test_job
   before_script:
     - cd bind-*
   needs:
-    - job: tarball
+    - job: gcc:tarball
       artifacts: true
   only:
     - tags
@@ -995,7 +995,7 @@ unit:clang:tsan:
       artifacts: true
 
 # Jobs for mutex-based atomics on Debian SID (amd64)
-mutexatomics:
+gcc:mutexatomics:
   variables:
     CC: gcc
     CFLAGS: "${CFLAGS_COMMON} -DISC_MEM_USE_INTERNAL_MALLOC=0"
@@ -1003,18 +1003,18 @@ mutexatomics:
   <<: *base_image
   <<: *build_job
 
-system:mutexatomics:
+system:gcc:mutexatomics:
   <<: *base_image
   <<: *system_test_job
   needs:
-    - job: mutexatomics
+    - job: gcc:mutexatomics
       artifacts: true
 
-unit:mutexatomics:
+unit:gcc:mutexatomics:
   <<: *base_image
   <<: *unit_test_job
   needs:
-    - job: mutexatomics
+    - job: gcc:mutexatomics
       artifacts: true
 
 # Jobs for Clang builds on Debian Buster (amd64)
@@ -1043,7 +1043,7 @@ unit:clang:buster:amd64:
 
 # Jobs for PKCS#11-enabled GCC builds on Debian Sid (amd64)
 
-pkcs11:
+gcc:pkcs11:
   variables:
     CC: gcc
     CFLAGS: "${CFLAGS_COMMON}"
@@ -1051,18 +1051,18 @@ pkcs11:
   <<: *base_image
   <<: *build_job
 
-system:pkcs11:
+system:gcc:pkcs11:
   <<: *base_image
   <<: *system_test_job
   needs:
-    - job: pkcs11
+    - job: gcc:pkcs11
       artifacts: true
 
-unit:pkcs11:
+unit:gcc:pkcs11:
   <<: *base_image
   <<: *unit_test_job
   needs:
-    - job: pkcs11
+    - job: gcc:pkcs11
       artifacts: true
 
 # Jobs for Clang builds on FreeBSD 11.3 (amd64)

From 7be0cc640f752f8b11a14736f7cabec1fcc52cfd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= <michal@isc.org>
Date: Thu, 26 Mar 2020 11:03:52 +0100
Subject: [PATCH 7/8] Do not install compiledb in cppcheck job

compiledb is already included in the Docker image used by the cppcheck
job.  Do not attempt installing it again.

(cherry picked from commit 3d264dbe819cd045cde72acac4222cbadf71ee15)
---
 .gitlab-ci.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 693ef09af9..4dd564313a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -365,8 +365,6 @@ stages:
 .cppcheck: &cppcheck_job
   <<: *default_triggering_rules
   stage: postcheck
-  before_script:
-    - pip3 install compiledb
   script:
     - *configure
     - (make -nwk all || true) | compiledb

From 03f764f7087dc0b4890fb2baa4825b3f72f9f9f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= <michal@isc.org>
Date: Thu, 26 Mar 2020 11:03:52 +0100
Subject: [PATCH 8/8] Remove unused YAML anchors

Some YAML anchors defined in .gitlab-ci.yml are not subsequently used.
Remove them to prevent confusion.

(cherry picked from commit 3d121ede6cf9876890fa80c1050d569c5c653d05)
---
 .gitlab-ci.yml | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4dd564313a..4d1c2f7e82 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -98,10 +98,6 @@ stages:
   image: "$CI_REGISTRY_IMAGE:debian-stretch-amd64"
   <<: *linux_amd64
 
-.debian-stretch-i386: &debian_stretch_i386_image
-  image: "$CI_REGISTRY_IMAGE:debian-stretch-i386"
-  <<: *linux_i386
-
 .debian-buster-amd64: &debian_buster_amd64_image
   image: "$CI_REGISTRY_IMAGE:debian-buster-amd64"
   <<: *linux_amd64
@@ -136,18 +132,10 @@ stages:
   image: "$CI_REGISTRY_IMAGE:ubuntu-xenial-amd64"
   <<: *linux_amd64
 
-.ubuntu-xenial-i386: &ubuntu_xenial_i386_image
-  image: "$CI_REGISTRY_IMAGE:ubuntu-xenial-i386"
-  <<: *linux_i386
-
 .ubuntu-bionic-amd64: &ubuntu_bionic_amd64_image
   image: "$CI_REGISTRY_IMAGE:ubuntu-bionic-amd64"
   <<: *linux_amd64
 
-.ubuntu-bionic-i386: &ubuntu_bionic_i386_image
-  image: "$CI_REGISTRY_IMAGE:ubuntu-bionic-i386"
-  <<: *linux_i386
-
 # Base image
 # This is a meta image that is used as a base for non-specific jobs