From ca392c6f64312cf79b08434e2e0ec4e8e0db4071 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Tue, 5 Sep 2023 10:29:13 +0200
Subject: [PATCH] Modify custom-test-driver to interpret JUnit results

Pytest provides JUnit output and uses different exit codes from
Automake. Use the conversion script to interpret the JUnit test results
from python rather than relying on the status code.

(cherry picked from commit 295890a16b24f740b8bc8b7c75a4ded9802cb3ff)
---
 bin/tests/system/.gitignore         |  1 +
 bin/tests/system/custom-test-driver | 22 ++++++++++------------
 2 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore
index a54b7c9565..f946793ae7 100644
--- a/bin/tests/system/.gitignore
+++ b/bin/tests/system/.gitignore
@@ -16,6 +16,7 @@ named.run
 parallel.mk
 /*.log
 /*.trs
+/*.xml
 /resolve
 /legacy.run.sh
 /run.log
diff --git a/bin/tests/system/custom-test-driver b/bin/tests/system/custom-test-driver
index 7499aa0bc0..fed98cccc6 100755
--- a/bin/tests/system/custom-test-driver
+++ b/bin/tests/system/custom-test-driver
@@ -56,7 +56,7 @@ END
 test_name= # Used for reporting.
 log_file=  # Where to save the output of the test script.
 trs_file=  # Where to save the metadata of the test run.
-status_file= # Where to save the status of the test run.
+junit_file=  # Where to save pytest junit output.
 expect_failure=no
 color_tests=no
 enable_hard_errors=yes
@@ -67,8 +67,7 @@ while test $# -gt 0; do
   --version) echo "test-driver $scriptversion"; exit $?;;
   --test-name) test_name=$2; shift;;
   --log-file) log_file=$2; shift;;
-  --trs-file) trs_file=$2; shift;;
-  --status-file) status_file=$2; shift;;
+  --trs-file) trs_file=$2; junit_file=$(echo $trs_file | sed 's/\.trs$/\.xml/'); shift;;
   --color-tests) color_tests=$2; shift;;
   --expect-failure) expect_failure=$2; shift;;
   --enable-hard-errors) enable_hard_errors=$2; shift;;
@@ -104,22 +103,22 @@ else
   red= grn= lgn= blu= mgn= std=
 fi
 
-do_exit='rm -f $log_file $trs_file $status_file; (exit $st); exit $st'
+do_exit='rm -f $log_file $trs_file $junit_file; (exit $st); exit $st'
 trap "st=129; $do_exit" 1
 trap "st=130; $do_exit" 2
 trap "st=141; $do_exit" 13
 trap "st=143; $do_exit" 15
 
-# Set default
-test x"$status_file" = x && status_file=$(mktemp ./custom-test-runner.XXXXXX)
 # Test script is run here.
 if test $verbose = yes; then
-  ("$@" 2>&1; echo $? > "$status_file") | tee $log_file
+  "$@" --junit-xml $PWD/$junit_file 2>&1 | tee $log_file
 else
-  "$@" >$log_file 2>&1; echo $? > "$status_file"
+  "$@" --junit-xml $PWD/$junit_file >$log_file 2>&1
 fi
-read -r estatus < "$status_file"
-rm "$status_file"
+
+# Run junit to trs converter script.
+./convert-junit-to-trs.py $junit_file > $trs_file
+estatus=$?
 
 if test $enable_hard_errors = no && test $estatus -eq 99; then
   tweaked_estatus=1
@@ -145,8 +144,7 @@ echo "$res $test_name (exit status: $estatus)" >>$log_file
 # Report outcome to console.
 echo "${col}${res}${std}: $test_name"
 
-# Register the test result, and other relevant metadata.
-echo ":test-result: $res" > $trs_file
+# Register other relevant test metadata.
 echo ":global-test-result: $res" >> $trs_file
 echo ":recheck: $recheck" >> $trs_file
 echo ":copy-in-global-log: $gcopy" >> $trs_file