From db032966830d2b448085f575cfa4b4c5c6853441 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Mon, 19 Dec 2022 17:44:35 +0100 Subject: [PATCH 1/5] testcrypto.sh: run in TMPDIR if possible Avoid creating any temporary files in the current workdir. Additional/changing files in the bin/tests/system directory are problematic for pytest/xdist collection phase, which assumes the list of files doesn't change between the collection phase of the main pytest thread and the subsequent collection phase of the xdist worker threads. Since the testcrypto.sh is also called during pytest initialization through conf.sh.common (to detect feature support), this could occasionally cause a race condition when the list of files would be different for the main pytest thread and the xdist worker. (cherry picked from commit 61330a7863c0dfe65f3add97bd363ec4998066aa) --- bin/tests/system/get_algorithms.py | 5 ++++- bin/tests/system/testcrypto.sh | 12 +++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/bin/tests/system/get_algorithms.py b/bin/tests/system/get_algorithms.py index 5429649758..9cd07d4857 100755 --- a/bin/tests/system/get_algorithms.py +++ b/bin/tests/system/get_algorithms.py @@ -111,7 +111,10 @@ def is_supported(alg: Algorithm) -> bool: f"{TESTCRYPTO} -q {alg.name}", shell=True, check=True, - env={"KEYGEN": KEYGEN}, + env={ + "KEYGEN": KEYGEN, + "TMPDIR": os.getenv("TMPDIR", "/tmp"), + }, stdout=subprocess.DEVNULL, ) except subprocess.CalledProcessError as exc: diff --git a/bin/tests/system/testcrypto.sh b/bin/tests/system/testcrypto.sh index 2be22f3652..ceaaf3726a 100755 --- a/bin/tests/system/testcrypto.sh +++ b/bin/tests/system/testcrypto.sh @@ -14,6 +14,7 @@ prog=$0 args="" quiet=0 +dir="" msg="cryptography" if test -z "$KEYGEN"; then @@ -74,9 +75,18 @@ if test -z "$alg"; then exit 1 fi +if test -n "$TMPDIR"; then + dir=$(mktemp -d "$TMPDIR/XXXXXX") + args="$args -K $dir" +fi + if $KEYGEN $args $alg foo > /dev/null 2>&1 then - rm -f Kfoo* + if test -z "$dir"; then + rm -f Kfoo* + else + rm -rf "$dir" + fi else if test $quiet -eq 0; then echo_i "This test requires support for $msg" >&2 From bcf1f5c80824f424f7aed266016a8d4bf40bdc27 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Tue, 20 Dec 2022 15:09:48 +0100 Subject: [PATCH 2/5] Factor out script to handle system test core dumps Move the core dump detection functionality for system test runs into a separate script. This enables reuse by the pytest runner. The functionality remains the same. (cherry picked from commit d9a97200d5e911d06875510ae9207e913ee42480) --- bin/tests/system/get_core_dumps.sh | 66 ++++++++++++++++++++++++++++++ bin/tests/system/run.sh.in | 48 +--------------------- 2 files changed, 68 insertions(+), 46 deletions(-) create mode 100755 bin/tests/system/get_core_dumps.sh diff --git a/bin/tests/system/get_core_dumps.sh b/bin/tests/system/get_core_dumps.sh new file mode 100755 index 0000000000..5d0255a2fd --- /dev/null +++ b/bin/tests/system/get_core_dumps.sh @@ -0,0 +1,66 @@ +#!/bin/sh + +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# SPDX-License-Identifier: MPL-2.0 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +dir=$(dirname "$0") +. "$dir/conf.sh" + +systest=$1 +status=0 + +export SYSTESTDIR="${TOP_SRCDIR}/bin/tests/system/${systest}" + +get_core_dumps() { + find "$SYSTESTDIR/" \( -name 'core' -or -name 'core.*' -or -name '*.core' \) ! -name '*.gz' ! -name '*.txt' | sort +} + +core_dumps=$(get_core_dumps | tr '\n' ' ') +assertion_failures=$(find "$SYSTESTDIR/" -name named.run -exec grep "assertion failure" {} + | wc -l) +sanitizer_summaries=$(find "$SYSTESTDIR/" -name 'tsan.*' | wc -l) +if [ -n "$core_dumps" ]; then + status=1 + echoinfo "I:$systest:Core dump(s) found: $core_dumps" + get_core_dumps | while read -r coredump; do + echoinfo "D:$systest:backtrace from $coredump:" + echoinfo "D:$systest:--------------------------------------------------------------------------------" + binary=$(gdb --batch --core="$coredump" 2>/dev/null | sed -ne "s|Core was generated by \`\([^' ]*\)[' ].*|\1|p") + if [ ! -f "${binary}" ]; then + binary=$(find "${TOP_BUILDDIR}" -path "*/.libs/${binary}" -type f) + fi + "${TOP_BUILDDIR}/libtool" --mode=execute gdb \ + -batch \ + -ex bt \ + -core="$coredump" \ + -- \ + "$binary" 2>/dev/null | sed -n '/^Core was generated by/,$p' | cat_d + echoinfo "D:$systest:--------------------------------------------------------------------------------" + coredump_backtrace="${coredump}-backtrace.txt" + echoinfo "D:$systest:full backtrace from $coredump saved in $coredump_backtrace" + "${TOP_BUILDDIR}/libtool" --mode=execute gdb \ + -batch \ + -command=run.gdb \ + -core="$coredump" \ + -- \ + "$binary" > "$coredump_backtrace" 2>&1 + echoinfo "D:$systest:core dump $coredump archived as $coredump.gz" + gzip -1 "${coredump}" + done +elif [ "$assertion_failures" -ne 0 ]; then + status=1 + echoinfo "I:$systest:$assertion_failures assertion failure(s) found" + find "$SYSTESTDIR/" -name 'tsan.*' -exec grep "SUMMARY: " {} + | sort -u | cat_d +elif [ "$sanitizer_summaries" -ne 0 ]; then + status=1 + echoinfo "I:$systest:$sanitizer_summaries sanitizer report(s) found" +fi + +exit $status diff --git a/bin/tests/system/run.sh.in b/bin/tests/system/run.sh.in index ce469cc270..550574c61f 100644 --- a/bin/tests/system/run.sh.in +++ b/bin/tests/system/run.sh.in @@ -114,7 +114,7 @@ if [ "${srcdir}" != "${builddir}" ]; then cp -a "${srcdir}/common" "${builddir}" fi # Some tests require additional files to work for out-of-tree test runs. - for file in ckdnsrps.sh conftest.py digcomp.pl ditch.pl fromhex.pl kasp.sh packet.pl pytest_custom_markers.py start.pl stop.pl testcrypto.sh; do + for file in ckdnsrps.sh conftest.py digcomp.pl ditch.pl fromhex.pl get_core_dumps.sh kasp.sh packet.pl pytest_custom_markers.py start.pl stop.pl testcrypto.sh; do if [ ! -r "${file}" ]; then cp -a "${srcdir}/${file}" "${builddir}" fi @@ -266,51 +266,7 @@ else exit $status fi -get_core_dumps() { - find "$systest/" \( -name 'core' -or -name 'core.*' -or -name '*.core' \) ! -name '*.gz' ! -name '*.txt' | sort -} - -core_dumps=$(get_core_dumps | tr '\n' ' ') -assertion_failures=$(find "$systest/" -name named.run -exec grep "assertion failure" {} + | wc -l) -sanitizer_summaries=$(find "$systest/" -name 'tsan.*' | wc -l) -if [ -n "$core_dumps" ]; then - status=1 - echoinfo "I:$systest:Core dump(s) found: $core_dumps" - get_core_dumps | while read -r coredump; do - export SYSTESTDIR="$systest" - echoinfo "D:$systest:backtrace from $coredump:" - echoinfo "D:$systest:--------------------------------------------------------------------------------" - binary=$(gdb --batch --core="$coredump" 2>/dev/null | sed -ne "s|Core was generated by \`\([^' ]*\)[' ].*|\1|p") - if [ ! -f "${binary}" ]; then - binary=$(find "${top_builddir}" -path "*/.libs/${binary}" -type f) - fi - "${top_builddir}/libtool" --mode=execute gdb \ - -batch \ - -ex bt \ - -core="$coredump" \ - -- \ - "$binary" 2>/dev/null | sed -n '/^Core was generated by/,$p' | cat_d - echoinfo "D:$systest:--------------------------------------------------------------------------------" - coredump_backtrace="${coredump}-backtrace.txt" - echoinfo "D:$systest:full backtrace from $coredump saved in $coredump_backtrace" - "${top_builddir}/libtool" --mode=execute gdb \ - -batch \ - -command=run.gdb \ - -core="$coredump" \ - -- \ - "$binary" > "$coredump_backtrace" 2>&1 - echoinfo "D:$systest:core dump $coredump archived as $coredump.gz" - gzip -1 "${coredump}" - done -elif [ "$assertion_failures" -ne 0 ]; then - status=1 - SYSTESTDIR="$systest" - echoinfo "I:$systest:$assertion_failures assertion failure(s) found" - find "$systest/" -name 'tsan.*' -exec grep "SUMMARY: " {} + | sort -u | cat_d -elif [ "$sanitizer_summaries" -ne 0 ]; then - status=1 - echoinfo "I:$systest:$sanitizer_summaries sanitizer report(s) found" -fi +$SHELL get_core_dumps.sh "$systest" || status=1 print_outstanding_files() { if test -d ${srcdir}/../../../.git; then From 8033857295c8c85668cc9deb3375b93d87b85ee9 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Mon, 2 Jan 2023 17:54:58 +0100 Subject: [PATCH 3/5] Look for ifconfig.sh.in in testsock.pl parent dir Instead of using the current working directory to find the ifconfig.sh script, look for the ifconfig.sh.in template in the directory where the testsock.pl script is located. This enables the testsock.pl script to be called from any working directory. Using the ifconfig.sh.in template is sufficient, since it contains the necessary information to be extracted: the max= value (which is hard-coded in the template). (cherry picked from commit e24d3b21d0feeb04e92b41422a6b1ac44e938e63) --- bin/tests/system/testsock.pl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/tests/system/testsock.pl b/bin/tests/system/testsock.pl index 3677e200b2..e793874d93 100755 --- a/bin/tests/system/testsock.pl +++ b/bin/tests/system/testsock.pl @@ -15,6 +15,8 @@ require 5.001; +use Cwd 'abs_path'; +use File::Basename; use Socket; use Getopt::Long; @@ -27,7 +29,8 @@ my @ids; if ($id != 0) { @ids = ($id); } else { - my $fn = "ifconfig.sh"; + my $dir = dirname(abs_path($0)); + my $fn = "$dir/ifconfig.sh.in"; open FH, "< $fn" or die "open < $fn: $!\n"; while () { @ids = (1..$1) From dd5f0d2c4c93d1d75b8503995889ee490ffab90b Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Tue, 3 Jan 2023 11:58:43 +0100 Subject: [PATCH 4/5] Tweak dupsigs test output Use a different visual separator, since "====", "----" and "____" is used by pytest to separate the log output. (cherry picked from commit 9593ff9347b0d1ddc5b679dc5e40711785b1e5e7) --- bin/tests/system/dupsigs/tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/tests/system/dupsigs/tests.sh b/bin/tests/system/dupsigs/tests.sh index 205a4ed099..85218c68ee 100644 --- a/bin/tests/system/dupsigs/tests.sh +++ b/bin/tests/system/dupsigs/tests.sh @@ -38,7 +38,7 @@ end=$((start + 140)) while [ $now -lt $end ]; do et=$((now - start)) - echo "=============== $et ============" + echo "............... $et ............" $JOURNALPRINT ns1/signing.test.db.signed.jnl | $PERL check_journal.pl $DIG axfr signing.test -p ${PORT} @10.53.0.1 > dig.out.at$et awk '$4 == "RRSIG" { print $11 }' dig.out.at$et | sort | uniq -c From f3c621704678707abe0fac4da13e78c448239d04 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Fri, 6 Jan 2023 15:08:27 +0100 Subject: [PATCH 5/5] Update the TEST_PARALLEL_JOBS value in CI The authoritative source for this value is in the project's CI/CD Variables Setting. The reason to keep it in .gitlab-ci.yaml as well is to have functional testing in forks without the need to manually specify this variable in Settings. The tests have been executed with 4 jobs for some time now. This "change" only brings .gitlab-ci.yaml file up to date, it doesn't actually change the number of jobs we currently use to test. (cherry picked from commit 03d7b45d81268708fd47bebc8d13f8bc5c49f64b) --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bd451dcf70..72c8d6068c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,8 +8,10 @@ variables: CCACHE_DIR: "/ccache" GIT_DEPTH: 1 + + # The following values may be overwritten in GitLab's CI/CD Variables Settings. BUILD_PARALLEL_JOBS: 6 - TEST_PARALLEL_JOBS: 6 + TEST_PARALLEL_JOBS: 4 CONFIGURE: ./configure CLANG_VERSION: 15