Add a pytest runner to run.sh

The system tests currently uses patchwork of shell scripts which doesn't
offer proper error handling.

This commit introduced option to write new tests in pytest framework
that also allows easier manipulation of DNS traffic (using dnspython),
native XML and JSON manipulation and proper error reporting.

.gitignore

@@ -14,6 +14,7 @@
 *_test
 *.ipch # vscode/intellisense precompiled header
 *~
+__pycache__/
 .ccache/
 .cproject
 .deps/

.gitlab-ci.yml

@@ -199,6 +199,7 @@ stages:
   before_script:
     - test -w "${CCACHE_DIR}" && export PATH="/usr/lib/ccache:${PATH}"
     - test -n "${OOT_BUILD_WORKSPACE}" && mkdir "${OOT_BUILD_WORKSPACE}" && cd "${OOT_BUILD_WORKSPACE}"
+    - pip3 install pytest requests || pip install pytest requests || true
   script:
     - *configure
     - make -j${BUILD_PARALLEL_JOBS:-1} -k all V=1
@@ -256,6 +257,7 @@ stages:
   before_script:
     - *setup_interfaces
     - *setup_softhsm
+    - pip3 install pytest requests || pip install pytest requests || true
   script:
     - cd bin/tests/system
     - make -j${TEST_PARALLEL_JOBS:-1} -k check V=1 || make -j${TEST_PARALLEL_JOBS:-1} -k recheck V=1
@@ -441,6 +443,7 @@ pylint:
   before_script:
     - pip3 install pylint
     - PYTHONPATH="$PYTHONPATH:$CI_PROJECT_DIR/bin/python"
+    - pip3 install pytest requests || pip install pytest requests
   script:
     - *configure
     - pylint --rcfile $CI_PROJECT_DIR/.pylintrc $(git ls-files '*.py' | grep -v 'ans\.py')

bin/tests/system/.gitignore

@@ -1,3 +1,4 @@
+__pycache__
 dig.out*
 rndc.out*
 nsupdate.out*

bin/tests/system/conf.sh.in

@@ -112,6 +112,7 @@ PERL=$(command -v "@PERL@")
 PSSUSPEND=
 
 PYTHON=$(command -v "@PYTHON@")
+PYTEST=@PYTEST@
 
 #
 # Determine if we support various optional features.
@@ -127,3 +128,5 @@ HAVEZLIB=${ZLIB_LIBS:+1}
 LMDB_LIBS="@LMDB_LIBS@"
 NZD=${LMDB_LIBS:+1}
 CRYPTO=@CRYPTO@
+
+export HAVEXMLSTATS HAVEJSONSTATS

bin/tests/system/run.sh.in

@@ -108,6 +108,39 @@ fi
 # true, the last digit of EXTRAPORTn is "n".
 eval "$(${srcdir}/get_ports.sh -p "$baseport")"
 
+restart=false
+
+start_servers_failed() {
+    echoinfo "I:$systest:starting servers failed"
+    echofail "R:$systest:FAIL"
+    echoend  "E:$systest:$(date_with_args)"
+    exit 1
+}
+
+start_servers() {
+    echoinfo "I:$systest:starting servers"
+    if $restart; then
+        $PERL start.pl --restart --port "$PORT" "$systest" || start_fail
+    else
+        restart=true
+        $PERL start.pl --port "$PORT" "$systest" || start_fail
+    fi
+}
+
+stop_servers_failed() {
+    echoinfo "I:$systest:stopping servers failed"
+    echofail "R:$systest:FAIL"
+    echoend  "E:$systest:$(date_with_args)"
+    exit 1
+}
+
+stop_servers() {
+    if $stopservers; then
+        echoinfo "I:$systest:stopping servers"
+        $PERL stop.pl "$systest" || stop_servers_failed
+    fi
+}
+
 echostart "S:$systest:$(date_with_args)"
 echoinfo  "T:$systest:1:A"
 echoinfo  "A:$systest:System test $systest"
@@ -161,16 +194,40 @@ then
    ( cd "${systest}" && $SHELL setup.sh "$@" )
 fi
 
-# Start name servers running
-if ! $PERL ${srcdir}/start.pl --port "$PORT" "$systest"; then
-    echofail "R:$systest:FAIL"
-    echoend  "E:$systest:$(date_with_args)"
-    exit 1
+
+status=0
+run=0
+# Run the tests
+if [ -r "$systest/tests.sh" ]; then
+    start_servers
+    ( cd "$systest" && $SHELL tests.sh "$@" )
+    status=$?
+    run=$((run+1))
+    stop_servers
 fi
 
-# Run the tests
-( cd "${systest}" && $SHELL tests.sh "$@" )
-status=$?
+if [ -n "$PYTEST" ]; then
+    run=$((run+1))
+    for test in $(cd "${systest}" && find . -name "tests*.py"); do
+	start_servers
+	rm -f "$systest/$test.status"
+	test_status=0
+	(cd "$systest" && "$PYTEST" -v "$test" "$@" || echo "$?" > "$test.status") | SYSTESTDIR="$systest" cat_d
+	if [ -f "$systest/$test.status" ]; then
+	    echo_i "FAILED"
+	    test_status=$(cat "$systest/$test.status")
+	fi
+	status=$((status+test_status))
+	stop_servers
+    done
+else
+    echoinfo "I:$systest:pytest not installed, skipping python tests"
+fi
+
+if [ "$run" -eq "0" ]; then
+    echoinfo "I:$systest:No tests were found and run"
+    status=255
+fi
 
 if $stopservers
 then
@@ -179,11 +236,6 @@ else
     exit $status
 fi
 
-# Shutdown
-$PERL ${srcdir}/stop.pl "$systest"
-
-status=$((status + $?))
-
 if [ $status != 0 ]; then
     echofail "R:$systest:FAIL"
     # Do not clean up - we need the evidence.

configure.ac

@@ -276,6 +276,11 @@ AM_CONDITIONAL([HAVE_PERLMOD_TIME_HIRES],
 AM_PATH_PYTHON([3.4], [], [:])
 AM_CONDITIONAL([HAVE_PYTHON], [test "$PYTHON" != ":"])
 
+AC_PATH_PROGS([PYTEST], [pytest-3 pytest pytest-pypy], [])
+AS_IF([test -z "$PYTEST"],
+      [AC_MSG_WARN([pytest not found, some system tests will be skipped])])
+AC_SUBST([PYTEST])
+
 AX_PYTHON_MODULE([dns])
 AM_CONDITIONAL([HAVE_PYMOD_DNS], [test "$HAVE_PYMOD_DNS" = "yes"])