From cf5105939c1c1802e69cc63a39e9d71b6e5f67e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= <ondrej@sury.org>
Date: Fri, 27 Mar 2020 13:23:24 +0100
Subject: [PATCH] Add a pytest runner to run.sh

The system tests currently uses patchwork of shell scripts which doesn't
offer proper error handling.

This commit introduced option to write new tests in pytest framework
that also allows easier manipulation of DNS traffic (using dnspython),
native XML and JSON manipulation and proper error reporting.
---
 .gitignore                  |  1 +
 .gitlab-ci.yml              |  3 ++
 bin/tests/system/.gitignore |  1 +
 bin/tests/system/conf.sh.in |  3 ++
 bin/tests/system/run.sh.in  | 78 ++++++++++++++++++++++++++++++-------
 configure.ac                |  5 +++
 6 files changed, 78 insertions(+), 13 deletions(-)

diff --git a/.gitignore b/.gitignore
index 71607aa297..742df701d0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,6 +14,7 @@
 *_test
 *.ipch # vscode/intellisense precompiled header
 *~
+__pycache__/
 .ccache/
 .cproject
 .deps/
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b0662fc1de..65a3fc0f8f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -199,6 +199,7 @@ stages:
   before_script:
     - test -w "${CCACHE_DIR}" && export PATH="/usr/lib/ccache:${PATH}"
     - test -n "${OOT_BUILD_WORKSPACE}" && mkdir "${OOT_BUILD_WORKSPACE}" && cd "${OOT_BUILD_WORKSPACE}"
+    - pip3 install pytest requests || pip install pytest requests || true
   script:
     - *configure
     - make -j${BUILD_PARALLEL_JOBS:-1} -k all V=1
@@ -256,6 +257,7 @@ stages:
   before_script:
     - *setup_interfaces
     - *setup_softhsm
+    - pip3 install pytest requests || pip install pytest requests || true
   script:
     - cd bin/tests/system
     - make -j${TEST_PARALLEL_JOBS:-1} -k check V=1 || make -j${TEST_PARALLEL_JOBS:-1} -k recheck V=1
@@ -441,6 +443,7 @@ pylint:
   before_script:
     - pip3 install pylint
     - PYTHONPATH="$PYTHONPATH:$CI_PROJECT_DIR/bin/python"
+    - pip3 install pytest requests || pip install pytest requests
   script:
     - *configure
     - pylint --rcfile $CI_PROJECT_DIR/.pylintrc $(git ls-files '*.py' | grep -v 'ans\.py')
diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore
index 0c66e8d2cc..d7c9c7b1bc 100644
--- a/bin/tests/system/.gitignore
+++ b/bin/tests/system/.gitignore
@@ -1,3 +1,4 @@
+__pycache__
 dig.out*
 rndc.out*
 nsupdate.out*
diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in
index 85f8df69e4..4088fb4411 100644
--- a/bin/tests/system/conf.sh.in
+++ b/bin/tests/system/conf.sh.in
@@ -112,6 +112,7 @@ PERL=$(command -v "@PERL@")
 PSSUSPEND=
 
 PYTHON=$(command -v "@PYTHON@")
+PYTEST=@PYTEST@
 
 #
 # Determine if we support various optional features.
@@ -127,3 +128,5 @@ HAVEZLIB=${ZLIB_LIBS:+1}
 LMDB_LIBS="@LMDB_LIBS@"
 NZD=${LMDB_LIBS:+1}
 CRYPTO=@CRYPTO@
+
+export HAVEXMLSTATS HAVEJSONSTATS
diff --git a/bin/tests/system/run.sh.in b/bin/tests/system/run.sh.in
index 5e4095ea67..4e8f5cc6b9 100644
--- a/bin/tests/system/run.sh.in
+++ b/bin/tests/system/run.sh.in
@@ -108,6 +108,39 @@ fi
 # true, the last digit of EXTRAPORTn is "n".
 eval "$(${srcdir}/get_ports.sh -p "$baseport")"
 
+restart=false
+
+start_servers_failed() {
+    echoinfo "I:$systest:starting servers failed"
+    echofail "R:$systest:FAIL"
+    echoend  "E:$systest:$(date_with_args)"
+    exit 1
+}
+
+start_servers() {
+    echoinfo "I:$systest:starting servers"
+    if $restart; then
+        $PERL start.pl --restart --port "$PORT" "$systest" || start_fail
+    else
+        restart=true
+        $PERL start.pl --port "$PORT" "$systest" || start_fail
+    fi
+}
+
+stop_servers_failed() {
+    echoinfo "I:$systest:stopping servers failed"
+    echofail "R:$systest:FAIL"
+    echoend  "E:$systest:$(date_with_args)"
+    exit 1
+}
+
+stop_servers() {
+    if $stopservers; then
+        echoinfo "I:$systest:stopping servers"
+        $PERL stop.pl "$systest" || stop_servers_failed
+    fi
+}
+
 echostart "S:$systest:$(date_with_args)"
 echoinfo  "T:$systest:1:A"
 echoinfo  "A:$systest:System test $systest"
@@ -161,16 +194,40 @@ then
    ( cd "${systest}" && $SHELL setup.sh "$@" )
 fi
 
-# Start name servers running
-if ! $PERL ${srcdir}/start.pl --port "$PORT" "$systest"; then
-    echofail "R:$systest:FAIL"
-    echoend  "E:$systest:$(date_with_args)"
-    exit 1
+
+status=0
+run=0
+# Run the tests
+if [ -r "$systest/tests.sh" ]; then
+    start_servers
+    ( cd "$systest" && $SHELL tests.sh "$@" )
+    status=$?
+    run=$((run+1))
+    stop_servers
 fi
 
-# Run the tests
-( cd "${systest}" && $SHELL tests.sh "$@" )
-status=$?
+if [ -n "$PYTEST" ]; then
+    run=$((run+1))
+    for test in $(cd "${systest}" && find . -name "tests*.py"); do
+	start_servers
+	rm -f "$systest/$test.status"
+	test_status=0
+	(cd "$systest" && "$PYTEST" -v "$test" "$@" || echo "$?" > "$test.status") | SYSTESTDIR="$systest" cat_d
+	if [ -f "$systest/$test.status" ]; then
+	    echo_i "FAILED"
+	    test_status=$(cat "$systest/$test.status")
+	fi
+	status=$((status+test_status))
+	stop_servers
+    done
+else
+    echoinfo "I:$systest:pytest not installed, skipping python tests"
+fi
+
+if [ "$run" -eq "0" ]; then
+    echoinfo "I:$systest:No tests were found and run"
+    status=255
+fi
 
 if $stopservers
 then
@@ -179,11 +236,6 @@ else
     exit $status
 fi
 
-# Shutdown
-$PERL ${srcdir}/stop.pl "$systest"
-
-status=$((status + $?))
-
 if [ $status != 0 ]; then
     echofail "R:$systest:FAIL"
     # Do not clean up - we need the evidence.
diff --git a/configure.ac b/configure.ac
index 8ad62afb27..458bd6cda2 100644
--- a/configure.ac
+++ b/configure.ac
@@ -276,6 +276,11 @@ AM_CONDITIONAL([HAVE_PERLMOD_TIME_HIRES],
 AM_PATH_PYTHON([3.4], [], [:])
 AM_CONDITIONAL([HAVE_PYTHON], [test "$PYTHON" != ":"])
 
+AC_PATH_PROGS([PYTEST], [pytest-3 pytest pytest-pypy], [])
+AS_IF([test -z "$PYTEST"],
+      [AC_MSG_WARN([pytest not found, some system tests will be skipped])])
+AC_SUBST([PYTEST])
+
 AX_PYTHON_MODULE([dns])
 AM_CONDITIONAL([HAVE_PYMOD_DNS], [test "$HAVE_PYMOD_DNS" = "yes"])