Enhance unit test debugging

When unit test fails, core file is created. Kyua's 'debug' command can
run GDB on it and provide backtrace. Unfortunately Kyua is picky about
location of these core files we opt to use custom Kyua fork and copy
core files from Kyua working directory to source tree and make it
available in GitLab.

(cherry picked from commit 94fa27e43f)
This commit is contained in:
Michal Nowak 2020-01-29 15:56:44 +01:00
parent c17114d132
commit 0603dc862e
No known key found for this signature in database
GPG key ID: 24A3E8463AEE5E56
2 changed files with 40 additions and 33 deletions

View file

@ -306,10 +306,7 @@ stages:
after_script:
- *kyua_report_html
artifacts:
paths:
- kyua.log
- kyua.results
- kyua_html/
untracked: true
expire_in: "1 day"
when: on_failure
@ -968,8 +965,6 @@ system:clang:freebsd11.3:amd64:
unit:clang:freebsd11.3:amd64:
<<: *freebsd_amd64
<<: *unit_test_job
variables:
USER: gitlab-runner
dependencies:
- clang:freebsd11.3:amd64
needs: ["clang:freebsd11.3:amd64"]
@ -997,8 +992,6 @@ system:clang:freebsd12.1:amd64:
unit:clang:freebsd12.1:amd64:
<<: *freebsd_amd64
<<: *unit_test_job
variables:
USER: gitlab-runner
dependencies:
- clang:freebsd12.1:amd64
needs: ["clang:freebsd12.1:amd64"]

View file

@ -12,50 +12,64 @@ kyua_report() {
${KYUA} --logfile /dev/null report --results-file "${KYUA_RESULT:-LATEST}"
}
clear_kyua_work_dir() {
KYUA_WORK_DIR="$(grep -i -m1 "failed" "${1}" | sed -n 's|.*\(/tmp/kyua\.[A-Za-z0-9]*\).*|\1|p')"
if [ -n "${CI}" ] && [ -d "${KYUA_WORK_DIR}" ]; then
find "${KYUA_WORK_DIR}" \( -name 'core*' -o -name '*.core' \) -exec mv -v {} . \;
rm -rf "${KYUA_WORK_DIR}"
fi
}
status=0
if [ -n "${UNITTESTS}" ] && [ -f Kyuafile ]
then
echo "S:unit:$(date)"
echo "T:unit:1:A"
echo "I: unit tests (using kyua)"
echo "I:unit tests (using kyua)"
${KYUA} -v parallelism="${TEST_PARALLEL_JOBS:-1}" --logfile kyua.log --loglevel debug test --results-file "${KYUA_RESULT:-NEW}"
status=$?
kyua_report
if command -v sysctl >/dev/null; then
if [ "$(uname -s)" = "Linux" ] && [ "$(sysctl -n kernel.core_uses_pid)" -ne 1 ]; then
echo "kernel.core_uses_pid is not set on the Linux host"
echo "kyua may not find core file of broken tests"
fi
else
echo "sysctl command is not present, can't check kernel.core_uses_pid."
echo "kyua may not find core file of broken tests"
fi
clear_kyua_work_dir kyua.log
# Use kyua-debug(1) facility to gather additional data on failed tests.
# Some runs will just show verbose information from the run, some will
# show backtrace via gdb(1).
broken_tests=$(kyua_report | awk '$2 == "->" && ( $3 == "broken:" || $3 == "failed:" ) { print $1 }')
if [ -n "${CI}" ] && [ "$(id -u)" -eq 0 ] && [ -n "${broken_tests}" ] && [ -n "${GDB}" ]; then
USER_ID=$(id -u)
BROKEN_TESTS=$(kyua_report | awk '$2 == "->" && ( $3 == "broken:" || $3 == "failed:" ) { print $1 }')
# Conditions for getting kyua debug info and GDB backtrace: runs under CI
# (safety), GDB present, root privileges, failed tests.
if [ -n "${CI}" ] && [ -n "${GDB}" ] && [ "${USER_ID:-1}" -eq 0 ] && [ -n "${BROKEN_TESTS}" ]; then
if [ "$(uname -s)" = "Linux" ] && ! sysctl -n "kernel.core_pattern" | grep -xq "core.%p"; then
echo "I:*** kernel.core_pattern is not set to 'core.%p'"
echo "I:*** kyua may not be able to find core dumps for broken tests"
fi
if [ "$(uname -s)" = "FreeBSD" ] && ! sysctl -n "kern.corefile" | grep -xq "core.%P"; then
echo "I:*** kern.corefile is not set to 'core.%P'"
echo "I:*** kyua may not be able to find core dumps for broken tests"
fi
if grep '^#define USE_LIBTOOL 1$' "${TOP}/config.h" >/dev/null; then
# kyua debug command misidentifies broken binary when libtool is used
# to configure BIND (see https://github.com/jmmv/kyua/issues/207).
# Here we try "trick" kyua use our custom gdb script instead
# of using gdb(1) directly. That's why this part needs to be run as root
# kyua debug command misidentifies broken binaries when libtool
# is used (see https://github.com/jmmv/kyua/issues/207).
# Here we try to "trick" kyua to use our custom gdb script instead
# of using gdb(1) directly. Hence this part needs to be run as root
# and, for safety reasons, only in the CI.
mv "${GDB}" "${GDB}.orig"
cp "${TOP}/unit/gdb" "${GDB}"
for test in ${broken_tests}; do
echo
${KYUA} debug "${test}"
done
fi
i=1
for test in ${BROKEN_TESTS}; do
echo
echo "----- $test -----"
KYUA_DEBUG_LOG="kyua.debug.log.${i}"
${KYUA} debug "${test}" 2>&1 | tee "${KYUA_DEBUG_LOG}"
clear_kyua_work_dir "${KYUA_DEBUG_LOG}"
i=$((i + 1))
done
if grep '^#define USE_LIBTOOL 1$' "${TOP}/config.h" >/dev/null; then
mv "${GDB}.orig" "${GDB}"
else
for test in ${broken_tests}; do
echo
${KYUA} debug "${test}"
done
fi
fi