Merge branch 'mnowak/unit-test-debugging-enhancements-v9_11' into 'v9_11'

[v9_11] Enhance unit test debugging

See merge request isc-projects/bind9!3268

.gitlab-ci.yml

@@ -306,10 +306,7 @@ stages:
   after_script:
     - *kyua_report_html
   artifacts:
-    paths:
-      - kyua.log
-      - kyua.results
-      - kyua_html/
+    untracked: true
     expire_in: "1 day"
     when: on_failure
 
@@ -968,8 +965,6 @@ system:clang:freebsd11.3:amd64:
 unit:clang:freebsd11.3:amd64:
   <<: *freebsd_amd64
   <<: *unit_test_job
-  variables:
-    USER: gitlab-runner
   dependencies:
     - clang:freebsd11.3:amd64
   needs: ["clang:freebsd11.3:amd64"]
@@ -997,8 +992,6 @@ system:clang:freebsd12.1:amd64:
 unit:clang:freebsd12.1:amd64:
   <<: *freebsd_amd64
   <<: *unit_test_job
-  variables:
-    USER: gitlab-runner
   dependencies:
     - clang:freebsd12.1:amd64
   needs: ["clang:freebsd12.1:amd64"]

unit/unittest.sh.in

@@ -12,50 +12,64 @@ kyua_report() {
 	${KYUA} --logfile /dev/null report --results-file "${KYUA_RESULT:-LATEST}"
 }
 
+clear_kyua_work_dir() {
+	KYUA_WORK_DIR="$(grep -i -m1 "failed" "${1}" | sed -n 's|.*\(/tmp/kyua\.[A-Za-z0-9]*\).*|\1|p')"
+	if [ -n "${CI}" ] && [ -d "${KYUA_WORK_DIR}" ]; then
+		find "${KYUA_WORK_DIR}" \( -name 'core*' -o -name '*.core' \) -exec mv -v {} . \;
+		rm -rf "${KYUA_WORK_DIR}"
+	fi
+}
+
 status=0
 if [ -n "${UNITTESTS}" ] && [ -f Kyuafile ]
 then
 	echo "S:unit:$(date)"
 	echo "T:unit:1:A"
-	echo "I: unit tests (using kyua)"
+	echo "I:unit tests (using kyua)"
+
 	${KYUA} -v parallelism="${TEST_PARALLEL_JOBS:-1}" --logfile kyua.log --loglevel debug test --results-file "${KYUA_RESULT:-NEW}"
 	status=$?
 
 	kyua_report
 
-	if command -v sysctl >/dev/null; then
-		if [ "$(uname -s)" = "Linux" ] && [ "$(sysctl -n kernel.core_uses_pid)" -ne 1 ]; then
-			echo "kernel.core_uses_pid is not set on the Linux host"
-			echo "kyua may not find core file of broken tests"
-		fi
-	else
-		echo "sysctl command is not present, can't check kernel.core_uses_pid."
-		echo "kyua may not find core file of broken tests"
-	fi
+	clear_kyua_work_dir kyua.log
 
 	# Use kyua-debug(1) facility to gather additional data on failed tests.
 	# Some runs will just show verbose information from the run, some will
 	# show backtrace via gdb(1).
-	broken_tests=$(kyua_report | awk '$2 == "->" && ( $3 == "broken:" || $3 == "failed:" ) { print $1 }')
-	if [ -n "${CI}" ] && [ "$(id -u)" -eq 0 ] && [ -n "${broken_tests}" ] && [ -n "${GDB}" ]; then
+	USER_ID=$(id -u)
+	BROKEN_TESTS=$(kyua_report | awk '$2 == "->" && ( $3 == "broken:" || $3 == "failed:" ) { print $1 }')
+	# Conditions for getting kyua debug info and GDB backtrace: runs under CI
+	# (safety), GDB present, root privileges, failed tests.
+	if [ -n "${CI}" ] && [ -n "${GDB}" ] && [ "${USER_ID:-1}" -eq 0 ] && [ -n "${BROKEN_TESTS}" ]; then
+		if [ "$(uname -s)" = "Linux" ] && ! sysctl -n "kernel.core_pattern" | grep -xq "core.%p"; then
+			echo "I:*** kernel.core_pattern is not set to 'core.%p'"
+			echo "I:*** kyua may not be able to find core dumps for broken tests"
+		fi
+		if [ "$(uname -s)" = "FreeBSD" ] && ! sysctl -n "kern.corefile" | grep -xq "core.%P"; then
+			echo "I:*** kern.corefile is not set to 'core.%P'"
+			echo "I:*** kyua may not be able to find core dumps for broken tests"
+		fi
 		if grep '^#define USE_LIBTOOL 1$' "${TOP}/config.h" >/dev/null; then
-			# kyua debug command misidentifies broken binary when libtool is used
-			# to configure BIND (see https://github.com/jmmv/kyua/issues/207).
-			# Here we try "trick" kyua use our custom gdb script instead
-			# of using gdb(1) directly. That's why this part needs to be run as root
+			# kyua debug command misidentifies broken binaries when libtool
+			# is used (see https://github.com/jmmv/kyua/issues/207).
+			# Here we try to "trick" kyua to use our custom gdb script instead
+			# of using gdb(1) directly. Hence this part needs to be run as root
 			# and, for safety reasons, only in the CI.
 			mv "${GDB}" "${GDB}.orig"
 			cp "${TOP}/unit/gdb" "${GDB}"
-			for test in ${broken_tests}; do
-				echo
-				${KYUA} debug "${test}"
-			done
+		fi
+		i=1
+		for test in ${BROKEN_TESTS}; do
+			echo
+			echo "----- $test -----"
+			KYUA_DEBUG_LOG="kyua.debug.log.${i}"
+			${KYUA} debug "${test}" 2>&1 | tee "${KYUA_DEBUG_LOG}"
+			clear_kyua_work_dir "${KYUA_DEBUG_LOG}"
+			i=$((i + 1))
+		done
+		if grep '^#define USE_LIBTOOL 1$' "${TOP}/config.h" >/dev/null; then
 			mv "${GDB}.orig" "${GDB}"
-		else
-			for test in ${broken_tests}; do
-				echo
-				${KYUA} debug "${test}"
-			done
 		fi
 	fi