From 4bc2b3be48e23152cdced9517f7b1d2839fde8f6 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Tue, 22 Nov 2022 10:47:15 +0100
Subject: [PATCH] Use pytest system test runner in CI

Replace the legacy system test runner by the pytest system test runner.
Since EL7 and OpenBSD have only ancient versions of pytest / xdist, keep
using the legacy test runner there for now.

Out of tree tests aren't supported by the pytest runner yet. Use the
legacy test runner for that purpose as well.

Use awk to display failures and errors at the end of the log for
convenience, since pytest displays them first, which makes them
difficult to find.
---
 .gitlab-ci.yml | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 62f811d0e0..5f388c9f6f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -253,6 +253,10 @@ stages:
   - PYTHON="$(source bin/tests/system/conf.sh; echo $PYTHON)"
   - test -x "$PYTHON"
 
+.find_pytest: &find_pytest
+  - PYTEST="$(source bin/tests/system/conf.sh; echo $PYTEST)"
+  - test -x "$PYTEST"
+
 .parse_tsan: &parse_tsan
     - find -name 'tsan.*' -exec "$PYTHON" util/parse_tsan.py {} \;
 
@@ -317,6 +321,17 @@ stages:
   before_script:
     - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}"
     - *setup_interfaces
+  script:
+    - *find_pytest
+    - cd bin/tests/system
+    - >
+      "$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" --dist loadscope | tee pytest.out.txt
+    - '( ! grep -F "grep: warning:" pytest.out.txt )'
+  after_script:
+    - awk '/^=+ FAILURES =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
+    - awk '/^=+ ERRORS =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
+
+.system_test_legacy: &system_test_legacy
   script:
     - cd bin/tests/system
     - make -j${TEST_PARALLEL_JOBS:-1} -k check V=1
@@ -336,6 +351,8 @@ stages:
   <<: *system_test_common
   artifacts:
     untracked: true
+    exclude:
+      - "**/__pycache__/**/*"
     when: always
     reports:
       junit: junit.xml
@@ -344,12 +361,15 @@ stages:
   <<: *system_test_common
   artifacts:
     untracked: true
+    exclude:
+      - "**/__pycache__/**/*"
     when: always
 
 .system_test_tsan: &system_test_tsan_job
   <<: *system_test_common
   after_script:
-    - cat bin/tests/system/test-suite.log
+    - awk '/^=+ FAILURES =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
+    - awk '/^=+ ERRORS =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
     - find bin/tests/system -name "*dig.*" | xargs grep "error" || true
     - *find_python
     - *parse_tsan
@@ -357,6 +377,8 @@ stages:
       "$PYTHON" bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml
   artifacts:
     untracked: true
+    exclude:
+      - "**/__pycache__/**/*"
     when: always
     reports:
       junit: junit.xml
@@ -859,6 +881,7 @@ system:gcc:out-of-tree:
       artifacts: true
   <<: *base_image
   <<: *system_test_job
+  <<: *system_test_legacy
   <<: *api_schedules_tags_triggers_web_triggering_rules
 
 unit:gcc:out-of-tree:
@@ -1193,6 +1216,7 @@ clang:openbsd:amd64:
 system:clang:openbsd:amd64:
   <<: *openbsd_amd64_image
   <<: *system_test_job
+  <<: *system_test_legacy
   <<: *api_schedules_triggers_web_triggering_rules
   variables:
     USER: gitlab-runner