upstream/bind9
1
0
Fork 0
mirror of https://github.com/isc-projects/bind9.git synced 2026-05-28 04:34:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Merge branch 'tkrizek/system-tests-pytest-runner-9.18' into 'bind-9.18'

Browse source 
[9.18] System tests: support using pytest instead of run.sh

See merge request isc-projects/bind9!7959
This commit is contained in:
Tom Krizek 2023-05-23 16:02:15 +00:00
commit 66254cf56d
126 changed files with 2350 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -5,6 +5,7 @@
*.la
*.lo
*.log
*.log.txt
*.o
*.orig
*.plist/ # ccc-analyzer store its results in .plist directories

32
.gitlab-ci.yml
View file

@ -249,6 +249,10 @@ stages:
- PYTHON="$(source bin/tests/system/conf.sh; echo $PYTHON)"
- test -x "$PYTHON"
.find_pytest: &find_pytest
- PYTEST="$(source bin/tests/system/conf.sh; echo $PYTEST)"
- test -x "$PYTEST"
.parse_tsan: &parse_tsan
- find -name 'tsan.*' -exec "$PYTHON" util/parse_tsan.py {} \;
@ -307,12 +311,26 @@ stages:
sudo sh -x bin/tests/system/ifconfig.sh up;
fi
.display_pytest_failures: &display_pytest_failures
- awk '/^=+ FAILURES =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
- awk '/^=+ ERRORS =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
.system_test_common: &system_test_common
<<: *default_triggering_rules
stage: system
before_script:
- test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}"
- *setup_interfaces
script:
- *find_pytest
- cd bin/tests/system
- >
"$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt
- '( ! grep -F "grep: warning:" pytest.out.txt )'
after_script:
- *display_pytest_failures
.system_test_legacy: &system_test_legacy
script:
- cd bin/tests/system
- make -j${TEST_PARALLEL_JOBS:-1} -k check V=1
@ -332,6 +350,8 @@ stages:
<<: *system_test_common
artifacts:
untracked: true
exclude:
- "**/__pycache__/**/*"
when: always
reports:
junit: junit.xml
@ -340,12 +360,14 @@ stages:
<<: *system_test_common
artifacts:
untracked: true
exclude:
- "**/__pycache__/**/*"
when: always
.system_test_tsan: &system_test_tsan_job
<<: *system_test_common
after_script:
- cat bin/tests/system/test-suite.log
- *display_pytest_failures
- find bin/tests/system -name "*dig.*" | xargs grep "error" || true
- *find_python
- *parse_tsan
@ -353,6 +375,8 @@ stages:
"$PYTHON" bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml
artifacts:
untracked: true
exclude:
- "**/__pycache__/**/*"
when: always
reports:
junit: junit.xml
@ -604,6 +628,7 @@ gcc:oraclelinux7:amd64:
system:gcc:oraclelinux7:amd64:
<<: *oraclelinux_7_amd64_image
<<: *system_test_job
<<: *system_test_legacy
needs:
- job: gcc:oraclelinux7:amd64
artifacts: true
@ -830,6 +855,7 @@ system:gcc:out-of-tree:
artifacts: true
<<: *base_image
<<: *system_test_job
<<: *system_test_legacy
<<: *api_schedules_tags_triggers_web_triggering_rules
unit:gcc:out-of-tree:
@ -864,6 +890,9 @@ system:gcc:tarball:
before_script:
- cd bind-*
- *setup_interfaces
after_script:
- cd bind-*
- *display_pytest_failures
needs:
- job: gcc:tarball
artifacts: true
@ -1191,6 +1220,7 @@ clang:openbsd:amd64:
system:clang:openbsd:amd64:
<<: *openbsd_amd64_image
<<: *system_test_job
<<: *system_test_legacy
<<: *api_schedules_triggers_web_triggering_rules
variables:
USER: gitlab-runner

1
.pylintrc
View file

@ -7,3 +7,4 @@ disable=
C0209, # consider-using-f-string
C0415, # import-outside-toplevel
R0801, # duplicate-code
R0903, # too-few-public-methods

3
CHANGES
View file

@ -1,3 +1,6 @@
6176. [test] Add support for using pytest & pytest-xdist to
execute the system test suite. [GL #3978]
6174. [bug] BIND could get stuck on reconfiguration when a
'listen' statement for HTTP is removed from the
configuration. That has been fixed. [GL #4071]

3
bin/tests/system/.gitignore vendored
View file

@ -17,8 +17,9 @@ parallel.mk
/*.log
/*.trs
/resolve
/run.sh
/legacy.run.sh
/run.log
/start.sh
/stop.sh
/ifconfig.sh
/*_tmp_*

4
bin/tests/system/Makefile.am
View file

@ -247,9 +247,9 @@ LOG_DRIVER_V_1 = --verbose yes
LOG_DRIVER = $(srcdir)/custom-test-driver
AM_LOG_DRIVER_FLAGS = $(LOG_DRIVER_V)
LOG_COMPILER = $(builddir)/run.sh
LOG_COMPILER = $(builddir)/legacy.run.sh
AM_LOG_FLAGS = -r
$(TESTS): run.sh
$(TESTS): legacy.run.sh
test-local: check

180
bin/tests/system/README
View file

@ -67,18 +67,58 @@ then run
... as root.
Running the System Tests
Running the System Tests with pytest
===
The pytest system test runner is currently in development, but it is the
recommended way to run tests. Please report issues to QA.
Running an Individual Test
---
pytest -k <test-name>
Note that in comparison to the legacy test runner, some additional tests might
be picked up when specifying just the system test directory name. To check
which tests will be executed, you can use the `--collect-only` option. You
might also be able to find a more specific test name to provide to ensure only
your desired test is executed. See help for `-k` option in `pytest --help` for
more info.
It is also possible to run a single individual pytest test case. For example,
you can use the name test_sslyze_dot to execute just the test_sslyze_dot()
function from doth/tests_sslyze.py. The entire needed setup and teardown will
be handled by the framework.
Running All the System Tests
---
Issuing plain `pytest` command without any argument will execute all tests
sequenatially. To execute them in parallel, ensure you have pytest-xdist
installed and run:
pytest -n <number-of-workers>
Running the System Tests Using the Legacy Runner
===
!!! WARNING !!!
---
The legacy way to run system tests is currently being reworked into a pytest
system test runner described in the previous section. The contents of this
section might be out of date and no longer applicable. Please try and use the
pytest runner if possible and report issues and missing features.
Running an Individual Test
---
The tests can be run individually using the following command:
sh run.sh [flags] <test-name> [<test-arguments>]
sh legacy.run.sh [flags] <test-name> [<test-arguments>]
e.g.
sh run.sh [flags] notify
sh legacy.run.sh [flags] notify
Optional flags are:
@ -120,10 +160,10 @@ To run all the system tests, enter the command:
sh runall.sh [-c] [-n] [numproc]
The optional flag "-c" forces colored output (by default system test output is
not printed in color due to run.sh being piped through "tee").
not printed in color due to legacy.run.sh being piped through "tee").
The optional flag "-n" has the same effect as it does for "run.sh" - it causes
the retention of all output files from all tests.
The optional flag "-n" has the same effect as it does for "legacy.run.sh" - it
causes the retention of all output files from all tests.
The optional "numproc" argument specifies the maximum number of tests that can
run in parallel. The default is 1, which means that all of the tests run
@ -243,9 +283,9 @@ Re-Running the Tests
---
If there is a requirement to re-run a test (or the entire test suite), the
files produced by the tests should be deleted first. Normally, these files are
deleted if the test succeeds but are retained on error. The run.sh script
automatically calls a given test's clean.sh script before invoking its setup.sh
script.
deleted if the test succeeds but are retained on error. The legacy.run.sh
script automatically calls a given test's clean.sh script before invoking its
setup.sh script.
Deletion of the files produced by the set of tests (e.g. after the execution
of "runall.sh") can be carried out using the command:
@ -285,8 +325,8 @@ tests.sh Runs the actual tests. This file is mandatory.
clean.sh Run at the end to clean up temporary files, but only if the test
was completed successfully and its running was not inhibited by the
"-n" switch being passed to "run.sh". Otherwise the temporary
files are left in place for inspection.
"-n" switch being passed to "legacy.run.sh". Otherwise the
temporary files are left in place for inspection.
ns<N> These subdirectories contain test name servers that can be queried
or can interact with each other. The value of N indicates the
@ -305,8 +345,8 @@ ans<N> Like ns[X], but these are simple mock name servers implemented in
Port Usage
---
In order for the tests to run in parallel, each test requires a unique set of
ports. These are specified by the "-p" option passed to "run.sh", which sets
environment variables that the scripts listed above can reference.
ports. These are specified by the "-p" option passed to "legacy.run.sh", which
sets environment variables that the scripts listed above can reference.
The convention used in the system tests is that the number passed is the start
of a range of 100 ports. The test is free to use the ports as required,
@ -358,10 +398,10 @@ General
directory.
2. Arguments can be only passed to the script if the test is being run as a
one-off with "run.sh". In this case, everything on the command line after the
name of the test is passed to each script. For example, the command:
one-off with "legacy.run.sh". In this case, everything on the command line
after the name of the test is passed to each script. For example, the command:
sh run.sh -p 12300 mytest -D xyz
sh legacy.run.sh -p 12300 mytest -D xyz
... will run "mytest" with a port range of 12300 to 12399. Each of the
framework scripts provided by the test will be invoked using the remaining
@ -532,8 +572,8 @@ Ideally, the directory numbers should start at 1 and work upwards.
When running a test, the servers are started using "start.sh" (which is nothing
more than a wrapper for start.pl). The options for "start.pl" are documented
in the header for that file, so will not be repeated here. In summary, when
invoked by "run.sh", start.pl looks for directories named "nsN" or "ansN" in
the test directory and starts the servers it finds there.
invoked by "legacy.run.sh", start.pl looks for directories named "nsN" or
"ansN" in the test directory and starts the servers it finds there.
"named" Command-Line Options
@ -630,8 +670,8 @@ the options available are listed in the file's header and will not be repeated
here.
In summary though, the nameservers for a given test, if left running by
specifying the "-k" flag to "run.sh" when the test is started, can be stopped
by the command:
specifying the "-k" flag to "legacy.run.sh" when the test is started, can be
stopped by the command:
sh stop.sh <test-name> [server]
@ -663,17 +703,103 @@ completed. To enable this, set the USE_VALGRIND environment variable to
"helgrind" to run the Helgrind tool, or any other value to run the Memcheck
tool. To use "helgrind" effectively, build BIND with --disable-atomic.
Developer Notes for pytest runner
===
Maintenance Notes
Test discovery and collection
---
There are two distinct types of system tests. The first is a shell script
tests.sh containing individual test cases executed sequentially and the
success/failure is determined by return code. The second type is a regular
pytest file which contains test functions.
Dealing with the regular pytest files doesn't require any special consideration
as long as the naming conventions are met. Discovering the tests.sh tests is
more complicated.
The chosen solution is to add a bit of glue for each system test. For every
tests.sh, there is an accompanying tests_sh_*.py file that contains a test
function which utilizes a custom run_tests_sh fixture to call the tests.sh
script. Other solutions were tried and eventually rejected. While this
introduces a bit of extra glue, it is the most portable, compatible and least
complex solution.
Module scope
---
Pytest fixtures can have a scope. The "module" scope is the most important for
our use. A module is a python file which contains test functions. Every system
test directory may contain multiple modules (i.e. tests_*.py files)!
The server setup/teardown is done for a module. Bundling test cases together
inside a single module may save some resources. However, test cases inside a
single module can't be executed in parallel.
It is possible to execute different modules defined within a single system test
directory in parallel. This is possible thanks to executing the tests inside a
temporary directory and proper port assignment to ensure there won't be any
conflicts.
Test logging
---
Each module has a separate log which will be saved as pytest.log.txt in the
temporary directory in which the test is executed. This log includes messages
for this module setup/teardown as well as any logging from the tests using the
`logger` fixture. Logging level DEBUG and above will be present in this log.
In general, any log messages using INFO or above will also be printed out
during pytest execution. In CI, the pytest output is also saved to
pytest.out.txt in the bin/tests/system directory.
Parallel execution
---
As mentioned in the previous section, test cases inside a single module can't
be executed in parallel. To put it differently, all tests cases inside the same
module must be performed by the same worker/thread. Otherwise, server
setup/teardown fixtures won't be shared and runtime issues due to port
collisions are likely to occur.
Pytest-xdist is used for executing pytest test cases in parallel using the `-n
N_WORKERS` option. By default, xdist will distribute any test case to any
worker, which would lead to the issue described above. Therefore, conftest.py
enforces equivalent of `--dist loadscope` option which ensures that test cases
within the same (module) scope will be handled by the same worker. Parallelism
is automatically disabled when xdist.scheduler.loadscope library is not
available.
$ pytest -n auto
Test selection
---
It is possible to run just a single pytest test case from any module. Use
standard pytest facility to select the desired test case(s), i.e. pass a
sufficiently unique identifier for `-k` parameter. You can also check which
tests will be executed by using the `--collect-only` flag to debug your `-k`
expression.
Compatibility with older pytest version
---
Keep in mind that the pytest runner must work with ancient versions of pytest.
When implementing new features, it is advisable to check feature support in
pytest and pytest-xdist in older distributions first.
As a general rule, any changes to the pytest runner need to keep working on all
platforms in CI that use the pytest runner. As of 2023-01-13, the oldest
supported version is whatever is available in EL8.
We may need to add more compat code eventually to handle breaking upstream
changes. For example, using request.fspath attribute is already deprecatred in
latest pytest.
Maintenance Notes for legacy runner
===
This section is aimed at developers maintaining BIND's system test framework.
Notes on Parallel Execution
---
Although execution of an individual test is controlled by "run.sh", which
executes the above shell scripts (and starts the relevant servers) for each
test, the running of all tests in the test suite is controlled by the Makefile.
("runall.sh" does little more than invoke "make" on the Makefile.)
Although execution of an individual test is controlled by "legacy.run.sh",
which executes the above shell scripts (and starts the relevant servers) for
each test, the running of all tests in the test suite is controlled by the
Makefile. ("runall.sh" does little more than invoke "make" on the Makefile.)
All system tests are capable of being run in parallel. For this to work, each
test needs to use a unique set of ports. To avoid the need to define which
@ -683,7 +809,7 @@ the ports are assigned when the tests are run. This is achieved by having the
when "make check" is run, and contains a target for each test of the form:
<test-name>:
@$(SHELL) run.sh -p <baseport> <test-name>
@$(SHELL) legacy.run.sh -p <baseport> <test-name>
The <baseport> is unique and the values of <baseport> for each test are
separated by at least 100 ports.
@ -707,7 +833,7 @@ If the test fails, all these files are retained. But if the test succeeds,
they are cleaned up at different times:
1. Files generated by the test itself are cleaned up by the test's own
"clean.sh", which is called from "run.sh".
"clean.sh", which is called from "legacy.run.sh".
2. Files that may not be cleaned up if named exits abnormally can be removed
using the "cleanall.sh" script.

14
bin/tests/system/acl/tests_sh_acl.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_acl(run_tests_sh):
run_tests_sh()

14
bin/tests/system/additional/tests_sh_additional.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_additional(run_tests_sh):
run_tests_sh()

14
bin/tests/system/addzone/tests_sh_addzone.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_addzone(run_tests_sh):
run_tests_sh()

14
bin/tests/system/allow-query/tests_sh_allowquery.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_allowquery(run_tests_sh):
run_tests_sh()

14
bin/tests/system/auth/tests_sh_auth.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_auth(run_tests_sh):
run_tests_sh()

14
bin/tests/system/autosign/tests_sh_autosign.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_autosign(run_tests_sh):
run_tests_sh()

14
bin/tests/system/builtin/tests_sh_builtin.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_builtin(run_tests_sh):
run_tests_sh()

14
bin/tests/system/cacheclean/tests_sh_cacheclean.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_cacheclean(run_tests_sh):
run_tests_sh()

14
bin/tests/system/case/tests_sh_case.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_case(run_tests_sh):
run_tests_sh()

14
bin/tests/system/catz/tests_sh_catz.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_catz(run_tests_sh):
run_tests_sh()

14
bin/tests/system/cds/tests_sh_cds.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_cds(run_tests_sh):
run_tests_sh()

14
bin/tests/system/chain/tests_sh_chain.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_chain(run_tests_sh):
run_tests_sh()

14
bin/tests/system/checkconf/tests_sh_checkconf.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_checkconf(run_tests_sh):
run_tests_sh()

14
bin/tests/system/checknames/tests_sh_checknames.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_checknames(run_tests_sh):
run_tests_sh()

14
bin/tests/system/checkzone/tests_sh_checkzone.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_checkzone(run_tests_sh):
run_tests_sh()

2
bin/tests/system/conf.sh.common
View file

@ -702,7 +702,7 @@ get_named_xfer_stats() {
#
# Convenience function to copy a configuration file, replacing the tokens
# QUERYPORT, CONTROLPORT and EXTRAPORT[1-8] with the values of the equivalent
# environment variables. (These values are set by "run.sh", which calls the
# environment variables. (These values are set by test runner, which calls the
# scripts invoking this function.)
#
# Usage:

598
bin/tests/system/conftest.py
View file

@ -1,5 +1,3 @@
#!/usr/bin/python3
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
@ -11,26 +9,612 @@
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
import logging
import os
import pytest
@pytest.fixture(scope="session")
# ======================= LEGACY=COMPATIBLE FIXTURES =========================
# The following fixtures are designed to work with both pytest system test
# runner and the legacy system test framework.
#
# FUTURE: Rewrite the individual port fixtures to re-use the `ports` fixture.
@pytest.fixture(scope="module")
def named_port():
return int(os.environ.get("PORT", default=5300))
@pytest.fixture(scope="session")
@pytest.fixture(scope="module")
def named_tlsport():
return int(os.environ.get("TLSPORT", default=8853))
@pytest.fixture(scope="session")
@pytest.fixture(scope="module")
def named_httpsport():
return int(os.environ.get("HTTPSPORT", default=4443))
@pytest.fixture(scope="session")
@pytest.fixture(scope="module")
def control_port():
return int(os.environ.get("CONTROLPORT", default=9953))
if os.getenv("LEGACY_TEST_RUNNER", "0") != "0":
@pytest.fixture
def logger(request):
"""Logging facility specific to a particular test."""
return logging.getLogger(request.node.name)
else:
# ======================= PYTEST SYSTEM TEST RUNNER ==========================
# From this point onward, any setting, fixtures or functions only apply to the
# new pytest runner. Ideally, these would be in a separate file. However, due
# to how pytest works and how it's used by the legacy runner, the best approach
# is to have everything in this file to avoid duplication and set the
# LEGACY_TEST_RUNNER if pytest is executed from the legacy framework.
#
# FUTURE: Once legacy runner is no longer supported, remove the env var and
# don't branch the code.
from functools import partial
from pathlib import Path
import re
import shutil
import subprocess
import tempfile
import time
from typing import Any, Dict, List, Optional
# Silence warnings caused by passing a pytest fixture to another fixture.
# pylint: disable=redefined-outer-name
# ----------------- Older pytest / xdist compatibility -------------------
# As of 2023-01-11, the minimal supported pytest / xdist versions are
# determined by what is available in EL8/EPEL8:
# - pytest 3.4.2
# - pytest-xdist 1.24.1
_pytest_ver = pytest.__version__.split(".")
_pytest_major_ver = int(_pytest_ver[0])
if _pytest_major_ver < 7:
# pytest.Stash/pytest.StashKey mechanism has been added in 7.0.0
# for older versions, use regular dictionary with string keys instead
FIXTURE_OK = "fixture_ok" # type: Any
else:
FIXTURE_OK = pytest.StashKey[bool]() # pylint: disable=no-member
# ----------------------- Globals definition -----------------------------
LOG_FORMAT = "%(asctime)s %(levelname)7s:%(name)s %(message)s"
XDIST_WORKER = os.environ.get("PYTEST_XDIST_WORKER", "")
FILE_DIR = os.path.abspath(Path(__file__).parent)
ENV_RE = re.compile(b"([^=]+)=(.*)")
PORT_MIN = 5001
PORT_MAX = 32767
PORTS_PER_TEST = 20
PRIORITY_TESTS = [
# Tests that are scheduled first. Speeds up parallel execution.
"dupsigs/",
"rpz/",
"rpzrecurse/",
"serve-stale/",
"timeouts/",
"upforwd/",
]
PRIORITY_TESTS_RE = re.compile("|".join(PRIORITY_TESTS))
CONFTEST_LOGGER = logging.getLogger("conftest")
# ---------------------- Module initialization ---------------------------
def init_pytest_conftest_logger(conftest_logger):
"""
This initializes the conftest logger which is used for pytest setup
and configuration before tests are executed -- aka any logging in this
file that is _not_ module-specific.
"""
conftest_logger.setLevel(logging.DEBUG)
file_handler = logging.FileHandler("pytest.conftest.log.txt")
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(logging.Formatter(LOG_FORMAT))
conftest_logger.addHandler(file_handler)
init_pytest_conftest_logger(CONFTEST_LOGGER)
def avoid_duplicated_logs():
"""
Remove direct root logger output to file descriptors.
This default is causing duplicates because all our messages go through
regular logging as well and are thus displayed twice.
"""
todel = []
for handler in logging.root.handlers:
if handler.__class__ == logging.StreamHandler:
# Beware: As for pytest 7.2.2, LiveLogging and LogCapture
# handlers inherit from logging.StreamHandler
todel.append(handler)
for handler in todel:
logging.root.handlers.remove(handler)
def parse_env(env_bytes):
"""Parse the POSIX env format into Python dictionary."""
out = {}
for line in env_bytes.splitlines():
match = ENV_RE.match(line)
if match:
# EL8+ workaround for https://access.redhat.com/solutions/6994985
# FUTURE: can be removed when we no longer need to parse env vars
if match.groups()[0] in [b"which_declare", b"BASH_FUNC_which%%"]:
continue
out[match.groups()[0]] = match.groups()[1]
return out
def get_env_bytes(cmd):
try:
proc = subprocess.run(
[cmd],
shell=True,
check=True,
cwd=FILE_DIR,
stdout=subprocess.PIPE,
)
except subprocess.CalledProcessError as exc:
CONFTEST_LOGGER.error("failed to get shell env: %s", exc)
raise exc
env_bytes = proc.stdout
return parse_env(env_bytes)
# Read common environment variables for running tests from conf.sh.
# FUTURE: Remove conf.sh entirely and define all variables in pytest only.
CONF_ENV = get_env_bytes(". ./conf.sh && env")
os.environb.update(CONF_ENV)
CONFTEST_LOGGER.debug(
"variables in env: %s", ", ".join([str(key) for key in CONF_ENV])
)
# --------------------------- pytest hooks -------------------------------
def pytest_addoption(parser):
parser.addoption(
"--noclean",
action="store_true",
default=False,
help="don't remove the temporary test directories with artifacts",
)
def pytest_configure(config):
# Ensure this hook only runs on the main pytest instance if xdist is
# used to spawn other workers.
if not XDIST_WORKER:
CONFTEST_LOGGER.debug("compiling required files")
env = os.environ.copy()
env["TESTS"] = "" # disable automake test framework - compile-only
try:
# FUTURE: Remove the need to run this compilation command
# before executing tests. Currently it's only here to have
# on-par functionality with the legacy test framework.
proc = subprocess.run(
"make -e check",
shell=True,
check=True,
cwd=FILE_DIR,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env,
)
except subprocess.CalledProcessError as exc:
CONFTEST_LOGGER.debug(exc.stdout)
CONFTEST_LOGGER.error("failed to compile test files: %s", exc)
raise exc
CONFTEST_LOGGER.debug(proc.stdout)
if config.pluginmanager.has_plugin("xdist") and config.option.numprocesses:
# system tests depend on module scope for setup & teardown
# enforce use "loadscope" scheduler or disable paralelism
try:
import xdist.scheduler.loadscope # pylint: disable=unused-import
except ImportError:
CONFTEST_LOGGER.debug(
"xdist is too old and does not have "
"scheduler.loadscope, disabling parallelism"
)
config.option.dist = "no"
else:
config.option.dist = "loadscope"
def pytest_ignore_collect(path):
# System tests are executed in temporary directories inside
# bin/tests/system. These temporary directories contain all files
# needed for the system tests - including tests_*.py files. Make sure to
# ignore these during test collection phase. Otherwise, test artifacts
# from previous runs could mess with the runner.
return "_tmp_" in str(path)
def pytest_collection_modifyitems(items):
"""Schedule long-running tests first to get more benefit from parallelism."""
priority = []
other = []
for item in items:
if PRIORITY_TESTS_RE.search(item.nodeid):
priority.append(item)
else:
other.append(item)
items[:] = priority + other
class NodeResult:
def __init__(self, report=None):
self.outcome = None
self.messages = []
if report is not None:
self.update(report)
def update(self, report):
if self.outcome is None or report.outcome != "passed":
self.outcome = report.outcome
if report.longreprtext:
self.messages.append(report.longreprtext)
@pytest.hookimpl(tryfirst=True, hookwrapper=True)
def pytest_runtest_makereport(item):
"""Hook that is used to expose test results to session (for use in fixtures)."""
# execute all other hooks to obtain the report object
outcome = yield
report = outcome.get_result()
# Set the test outcome in session, so we can access it from module-level
# fixture using nodeid. Note that this hook is called three times: for
# setup, call and teardown. We only care about the overall result so we
# merge the results together and preserve the information whether a test
# passed.
test_results = {}
try:
test_results = getattr(item.session, "test_results")
except AttributeError:
setattr(item.session, "test_results", test_results)
node_result = test_results.setdefault(item.nodeid, NodeResult())
node_result.update(report)
# --------------------------- Fixtures -----------------------------------
@pytest.fixture(scope="session")
def modules():
"""Sorted list of all modules. Used to determine port distribution."""
mods = []
for dirpath, _dirs, files in os.walk(os.getcwd()):
for file in files:
if file.startswith("tests_") and file.endswith(".py"):
mod = f"{dirpath}/{file}"
mods.append(mod)
return sorted(mods)
@pytest.fixture(scope="session")
def module_base_ports(modules):
"""
Dictionary containing assigned base port for every module.
Note that this is a session-wide fixture. The port numbers are
deterministically assigned before any testing starts. This fixture MUST
return the same value when called again during the same test session.
When running tests in parallel, this is exactly what happens - every
worker thread will call this fixture to determine test ports.
"""
port_min = PORT_MIN
port_max = PORT_MAX - len(modules) * PORTS_PER_TEST
if port_max < port_min:
raise RuntimeError(
"not enough ports to assign unique port set to each module"
)
# Rotate the base port value over time to detect possible test issues
# with using random ports. This introduces a very slight race condition
# risk. If this value changes between pytest invocation and spawning
# worker threads, multiple tests may have same port values assigned. If
# these tests are then executed simultaneously, the test results will
# be misleading.
base_port = int(time.time() // 3600) % (port_max - port_min)
return {mod: base_port + i * PORTS_PER_TEST for i, mod in enumerate(modules)}
@pytest.fixture(scope="module")
def base_port(request, module_base_ports):
"""Start of the port range assigned to a particular test module."""
port = module_base_ports[request.fspath]
return port
@pytest.fixture(scope="module")
def ports(base_port):
"""Dictionary containing port names and their assigned values."""
return {
"PORT": str(base_port),
"TLSPORT": str(base_port + 1),
"HTTPPORT": str(base_port + 2),
"HTTPSPORT": str(base_port + 3),
"EXTRAPORT1": str(base_port + 4),
"EXTRAPORT2": str(base_port + 5),
"EXTRAPORT3": str(base_port + 6),
"EXTRAPORT4": str(base_port + 7),
"EXTRAPORT5": str(base_port + 8),
"EXTRAPORT6": str(base_port + 9),
"EXTRAPORT7": str(base_port + 10),
"EXTRAPORT8": str(base_port + 11),
"CONTROLPORT": str(base_port + 12),
}
@pytest.fixture(scope="module")
def env(ports):
"""Dictionary containing environment variables for the test."""
env = os.environ.copy()
env.update(ports)
env["builddir"] = f"{env['TOP_BUILDDIR']}/bin/tests/system"
env["srcdir"] = f"{env['TOP_SRCDIR']}/bin/tests/system"
return env
@pytest.fixture(scope="module")
def system_test_name(request):
"""Name of the system test directory."""
path = Path(request.fspath)
return path.parent.name
@pytest.fixture(scope="module")
def mlogger(system_test_name):
"""Logging facility specific to this test module."""
avoid_duplicated_logs()
return logging.getLogger(system_test_name)
@pytest.fixture
def logger(request, system_test_name):
"""Logging facility specific to a particular test."""
return logging.getLogger(f"{system_test_name}.{request.node.name}")
@pytest.fixture(scope="module")
def system_test_dir(request, env, system_test_name, mlogger):
"""
Temporary directory for executing the test.
This fixture is responsible for creating (and potentially removing) a
copy of the system test directory which is used as a temporary
directory for the test execution.
FUTURE: This removes the need to have clean.sh scripts.
"""
def get_test_result():
"""Aggregate test results from all individual tests from this module
into a single result: failed > skipped > passed."""
try:
all_test_results = request.session.test_results
except AttributeError:
# This may happen if pytest execution is interrupted and
# pytest_runtest_makereport() is never called.
mlogger.debug("can't obtain test results, test run was interrupted")
return "error"
test_results = {
node.nodeid: all_test_results[node.nodeid]
for node in request.node.collect()
if node.nodeid in all_test_results
}
assert len(test_results)
messages = []
for node, result in test_results.items():
mlogger.debug("%s %s", result.outcome.upper(), node)
messages.extend(result.messages)
for message in messages:
mlogger.debug("\n" + message)
failed = any(res.outcome == "failed" for res in test_results.values())
skipped = any(res.outcome == "skipped" for res in test_results.values())
if failed:
return "failed"
if skipped:
return "skipped"
assert all(res.outcome == "passed" for res in test_results.values())
return "passed"
# Create a temporary directory with a copy of the original system test dir contents
system_test_root = Path(f"{env['TOP_BUILDDIR']}/bin/tests/system")
testdir = Path(
tempfile.mkdtemp(prefix=f"{system_test_name}_tmp_", dir=system_test_root)
)
shutil.rmtree(testdir)
shutil.copytree(system_test_root / system_test_name, testdir)
# Configure logger to write to a file inside the temporary test directory
mlogger.handlers.clear()
mlogger.setLevel(logging.DEBUG)
handler = logging.FileHandler(testdir / "pytest.log.txt", mode="w")
formatter = logging.Formatter(LOG_FORMAT)
handler.setFormatter(formatter)
mlogger.addHandler(handler)
# System tests are meant to be executed from their directory - switch to it.
old_cwd = os.getcwd()
os.chdir(testdir)
mlogger.info("switching to tmpdir: %s", testdir)
try:
yield testdir # other fixtures / tests will execute here
finally:
os.chdir(old_cwd)
mlogger.debug("changed workdir to: %s", old_cwd)
result = get_test_result()
# Clean temporary dir unless it should be kept
if request.config.getoption("--noclean"):
mlogger.debug("--noclean requested, keeping temporary directory")
elif result == "failed":
mlogger.debug("test failure detected, keeping temporary directory")
elif not request.node.stash[FIXTURE_OK]:
mlogger.debug(
"test setup/teardown issue detected, keeping temporary directory"
)
else:
mlogger.debug("deleting temporary directory")
handler.flush()
handler.close()
shutil.rmtree(testdir)
def _run_script( # pylint: disable=too-many-arguments
env,
mlogger,
system_test_dir: Path,
interpreter: str,
script: str,
args: Optional[List[str]] = None,
):
"""Helper function for the shell / perl script invocations (through fixtures below)."""
if args is None:
args = []
path = Path(script)
if not path.is_absolute():
# make sure relative paths are always relative to system_dir
path = system_test_dir.parent / path
script = str(path)
cwd = os.getcwd()
if not path.exists():
raise FileNotFoundError(f"script {script} not found in {cwd}")
mlogger.debug("running script: %s %s %s", interpreter, script, " ".join(args))
mlogger.debug(" workdir: %s", cwd)
returncode = 1
cmd = [interpreter, script] + args
with subprocess.Popen(
cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=1,
universal_newlines=True,
errors="backslashreplace",
) as proc:
if proc.stdout:
for line in proc.stdout:
mlogger.info(" %s", line.rstrip("\n"))
proc.communicate()
returncode = proc.returncode
if returncode:
raise subprocess.CalledProcessError(returncode, cmd)
mlogger.debug(" exited with %d", returncode)
@pytest.fixture(scope="module")
def shell(env, system_test_dir, mlogger):
"""Function to call a shell script with arguments."""
return partial(_run_script, env, mlogger, system_test_dir, env["SHELL"])
@pytest.fixture(scope="module")
def perl(env, system_test_dir, mlogger):
"""Function to call a perl script with arguments."""
return partial(_run_script, env, mlogger, system_test_dir, env["PERL"])
@pytest.fixture(scope="module")
def run_tests_sh(system_test_dir, shell):
"""Utility function to execute tests.sh as a python test."""
def run_tests():
shell(f"{system_test_dir}/tests.sh")
return run_tests
@pytest.fixture(scope="module", autouse=True)
def system_test( # pylint: disable=too-many-arguments,too-many-statements
request,
env: Dict[str, str],
mlogger,
system_test_dir,
shell,
perl,
):
"""
Driver of the test setup/teardown process. Used automatically for every test module.
This is the most important one-fixture-to-rule-them-all. Note the
autouse=True which causes this fixture to be loaded by every test
module without the need to explicitly specify it.
When this fixture is used, it utilizes other fixtures, such as
system_test_dir, which handles the creation of the temporary test
directory.
Afterwards, it checks the test environment and takes care of starting
the servers. When everything is ready, that's when the actual tests are
executed. Once that is done, this fixture stops the servers and checks
for any artifacts indicating an issue (e.g. coredumps).
Finally, when this fixture reaches an end (or encounters an exception,
which may be caused by fail/skip invocations), any fixtures which is
used by this one are finalized - e.g. system_test_dir performs final
checks and cleans up the temporary test directory.
"""
def check_net_interfaces():
try:
perl("testsock.pl", ["-p", env["PORT"]])
except subprocess.CalledProcessError as exc:
mlogger.error("testsock.pl: exited with code %d", exc.returncode)
pytest.skip("Network interface aliases not set up.")
def check_prerequisites():
try:
shell(f"{system_test_dir}/prereq.sh")
except FileNotFoundError:
pass # prereq.sh is optional
except subprocess.CalledProcessError:
pytest.skip("Prerequisites missing.")
def setup_test():
try:
shell(f"{system_test_dir}/setup.sh")
except FileNotFoundError:
pass # setup.sh is optional
except subprocess.CalledProcessError as exc:
mlogger.error("Failed to run test setup")
pytest.fail(f"setup.sh exited with {exc.returncode}")
def start_servers():
try:
perl("start.pl", ["--port", env["PORT"], system_test_dir.name])
except subprocess.CalledProcessError as exc:
mlogger.error("Failed to start servers")
pytest.fail(f"start.pl exited with {exc.returncode}")
def stop_servers():
try:
perl("stop.pl", [system_test_dir.name])
except subprocess.CalledProcessError as exc:
mlogger.error("Failed to stop servers")
pytest.fail(f"stop.pl exited with {exc.returncode}")
def get_core_dumps():
try:
shell("get_core_dumps.sh", [system_test_dir.name])
except subprocess.CalledProcessError as exc:
mlogger.error("Found core dumps")
pytest.fail(f"get_core_dumps.sh exited with {exc.returncode}")
os.environ.update(env) # Ensure pytests have the same env vars as shell tests.
mlogger.info(f"test started: {request.node.name}")
port = int(env["PORT"])
mlogger.info("using port range: <%d, %d>", port, port + PORTS_PER_TEST - 1)
if not hasattr(request.node, "stash"): # compatibility with pytest<7.0.0
request.node.stash = {} # use regular dict instead of pytest.Stash
request.node.stash[FIXTURE_OK] = True
# Perform checks which may skip this test.
check_net_interfaces()
check_prerequisites()
# Store the fact that this fixture hasn't successfully finished yet.
# This is checked before temporary directory teardown to decide whether
# it's okay to remove the directory.
request.node.stash[FIXTURE_OK] = False
setup_test()
try:
start_servers()
mlogger.debug("executing test(s)")
yield
finally:
mlogger.debug("test(s) finished")
stop_servers()
get_core_dumps()
request.node.stash[FIXTURE_OK] = True

14
bin/tests/system/cookie/tests_sh_cookie.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_cookie(run_tests_sh):
run_tests_sh()

14
bin/tests/system/database/tests_sh_database.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_database(run_tests_sh):
run_tests_sh()

14
bin/tests/system/dialup/tests_sh_dialup.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_dialup(run_tests_sh):
run_tests_sh()

14
bin/tests/system/digdelv/tests_sh_digdelv.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_digdelv(run_tests_sh):
run_tests_sh()

14
bin/tests/system/dlzexternal/tests_sh_dlzexternal.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_dlzexternal(run_tests_sh):
run_tests_sh()

14
bin/tests/system/dns64/tests_sh_dns64.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_dns64(run_tests_sh):
run_tests_sh()

14
bin/tests/system/dnssec/tests_sh_dnssec.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_dnssec(run_tests_sh):
run_tests_sh()

14
bin/tests/system/dnstap/tests_sh_dnstap.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_dnstap(run_tests_sh):
run_tests_sh()

14
bin/tests/system/doth/tests_sh_doth.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_doth(run_tests_sh):
run_tests_sh()

14
bin/tests/system/dsdigest/tests_sh_dsdigest.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_dsdigest(run_tests_sh):
run_tests_sh()

14
bin/tests/system/dupsigs/tests_sh_dupsigs.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_dupsigs(run_tests_sh):
run_tests_sh()

14
bin/tests/system/dyndb/tests_sh_dyndb.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_dyndb(run_tests_sh):
run_tests_sh()

14
bin/tests/system/ecdsa/tests_sh_ecdsa.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_ecdsa(run_tests_sh):
run_tests_sh()

14
bin/tests/system/eddsa/tests_sh_eddsa.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_eddsa(run_tests_sh):
run_tests_sh()

14
bin/tests/system/ednscompliance/tests_sh_ednscompliance.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_ednscompliance(run_tests_sh):
run_tests_sh()

14
bin/tests/system/emptyzones/tests_sh_emptyzones.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_emptyzones(run_tests_sh):
run_tests_sh()

14
bin/tests/system/enginepkcs11/tests_sh_enginepkcs11.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_enginepkcs11(run_tests_sh):
run_tests_sh()

14
bin/tests/system/fetchlimit/tests_sh_fetchlimit.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_fetchlimit(run_tests_sh):
run_tests_sh()

14
bin/tests/system/filter-aaaa/tests_sh_filter_aaaa.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_filter_aaaa(run_tests_sh):
run_tests_sh()

14
bin/tests/system/formerr/tests_sh_formerr.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_formerr(run_tests_sh):
run_tests_sh()

14
bin/tests/system/forward/tests_sh_forward.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_forward(run_tests_sh):
run_tests_sh()

14
bin/tests/system/geoip2/tests_sh_geoip2.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_geoip2(run_tests_sh):
run_tests_sh()

2
bin/tests/system/get_algorithms.py
View file

@ -230,7 +230,7 @@ def main():
except Exception:
# if anything goes wrong, the conf.sh ignores error codes, so make sure
# we set an environment variable to an error value that can be checked
# later by run.sh
# later by the test runner and/or tests themselves
print("export ALGORITHM_SET=error")
raise
for name, value in algs_env.items():

14
bin/tests/system/glue/tests_sh_glue.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_glue(run_tests_sh):
run_tests_sh()

14
bin/tests/system/host/tests_sh_host.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_host(run_tests_sh):
run_tests_sh()

14
bin/tests/system/idna/tests_sh_idna.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_idna(run_tests_sh):
run_tests_sh()

14
bin/tests/system/include-multiplecfg/tests_sh_include_multiplecfg.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_include_multiplecfg(run_tests_sh):
run_tests_sh()

14
bin/tests/system/inline/tests_sh_inline.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_inline(run_tests_sh):
run_tests_sh()

14
bin/tests/system/integrity/tests_sh_integrity.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_integrity(run_tests_sh):
run_tests_sh()

14
bin/tests/system/ixfr/tests_sh_ixfr.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_ixfr(run_tests_sh):
run_tests_sh()

14
bin/tests/system/journal/tests_sh_journal.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_journal(run_tests_sh):
run_tests_sh()

14
bin/tests/system/kasp/tests_sh_kasp.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_kasp(run_tests_sh):
run_tests_sh()

14
bin/tests/system/keepalive/tests_sh_keepalive.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_keepalive(run_tests_sh):
run_tests_sh()

14
bin/tests/system/keyfromlabel/tests_sh_keyfromlabel.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_keyfromlabel(run_tests_sh):
run_tests_sh()

14
bin/tests/system/keymgr2kasp/tests_sh_keymgr2kasp.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_keymgr2kasp(run_tests_sh):
run_tests_sh()

4
bin/tests/system/run.sh.in → bin/tests/system/legacy.run.sh.in
View file

@ -225,12 +225,12 @@ fi
if [ $status -eq 0 ]; then
if [ -n "$PYTEST" ]; then
for test in $(cd "${systest}" && find . -name "tests*.py"); do
for test in $(cd "${systest}" && find . -name "tests*.py" ! -name "tests_sh_*.py"); do
rm -f "$systest/$test.status"
if start_servers; then
run=$((run+1))
test_status=0
(cd "$systest" && "$PYTEST" -rsxX -v "$test" "$@" || echo "$?" > "$test.status") | SYSTESTDIR="$systest" cat_d
(cd "$systest" && LEGACY_TEST_RUNNER=1 "$PYTEST" -rsxX -v "$test" "$@" || echo "$?" > "$test.status") | SYSTESTDIR="$systest" cat_d
if [ -f "$systest/$test.status" ]; then
if [ "$(cat "$systest/$test.status")" = "5" ]; then
echowarn "R:$systest:SKIPPED"

14
bin/tests/system/legacy/tests_sh_legacy.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_legacy(run_tests_sh):
run_tests_sh()

14
bin/tests/system/limits/tests_sh_limits.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_limits(run_tests_sh):
run_tests_sh()

14
bin/tests/system/logfileconfig/tests_sh_logfileconfig.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_logfileconfig(run_tests_sh):
run_tests_sh()

14
bin/tests/system/masterfile/tests_sh_masterfile.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_masterfile(run_tests_sh):
run_tests_sh()

14
bin/tests/system/masterformat/tests_sh_masterformat.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_masterformat(run_tests_sh):
run_tests_sh()

14
bin/tests/system/metadata/tests_sh_metadata.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_metadata(run_tests_sh):
run_tests_sh()

14
bin/tests/system/mirror/tests_sh_mirror.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_mirror(run_tests_sh):
run_tests_sh()

14
bin/tests/system/mkeys/tests_sh_mkeys.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_mkeys(run_tests_sh):
run_tests_sh()

14
bin/tests/system/names/tests_sh_names.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_names(run_tests_sh):
run_tests_sh()

14
bin/tests/system/notify/tests_sh_notify.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_notify(run_tests_sh):
run_tests_sh()

14
bin/tests/system/nsec3/tests_sh_nsec3.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_nsec3(run_tests_sh):
run_tests_sh()

14
bin/tests/system/nslookup/tests_sh_nslookup.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_nslookup(run_tests_sh):
run_tests_sh()

14
bin/tests/system/nsupdate/tests_sh_nsupdate.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_nsupdate(run_tests_sh):
run_tests_sh()

14
bin/tests/system/nzd2nzf/tests_sh_nzd2nzf.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_nzd2nzf(run_tests_sh):
run_tests_sh()

14
bin/tests/system/padding/tests_sh_padding.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_padding(run_tests_sh):
run_tests_sh()

2
bin/tests/system/parallel.sh
View file

@ -30,5 +30,5 @@ echo "test check: $PARALLELS"
for directory in $PARALLELDIRS ; do
echo
echo "test-$(echo "$directory" | tr _ -): check_interfaces"
echo " @${SHELL} ./run.sh -r $directory 2>&1 | tee test.output.$directory"
echo " @${SHELL} ./legacy.run.sh -r $directory 2>&1 | tee test.output.$directory"
done

14
bin/tests/system/pending/tests_sh_pending.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_pending(run_tests_sh):
run_tests_sh()

14
bin/tests/system/pipelined/tests_sh_pipelined.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_pipelined(run_tests_sh):
run_tests_sh()

20
bin/tests/system/pytest.ini Normal file
View file

@ -0,0 +1,20 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
[pytest]
addopts = --tb=short -rA
log_format = %(asctime)s %(levelname)s:%(name)s %(message)s
log_date_format = %Y-%m-%d %H:%M:%S
log_cli = 1
log_level = INFO
python_files = tests_*.py
junit_logging = log
junit_log_passing_tests = 0

14
bin/tests/system/qmin/tests_sh_qmin.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_qmin(run_tests_sh):
run_tests_sh()

14
bin/tests/system/reclimit/tests_sh_reclimit.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_reclimit(run_tests_sh):
run_tests_sh()

14
bin/tests/system/redirect/tests_sh_redirect.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_redirect(run_tests_sh):
run_tests_sh()

14
bin/tests/system/resolver/tests_sh_resolver.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_resolver(run_tests_sh):
run_tests_sh()

14
bin/tests/system/rndc/tests_sh_rndc.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_rndc(run_tests_sh):
run_tests_sh()

14
bin/tests/system/rootkeysentinel/tests_sh_rootkeysentinel.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_rootkeysentinel(run_tests_sh):
run_tests_sh()

14
bin/tests/system/rpz/tests_sh_rpz.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_rpz(run_tests_sh):
run_tests_sh()

14
bin/tests/system/rpzrecurse/tests_sh_rpzrecurse.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_rpzrecurse(run_tests_sh):
run_tests_sh()

14
bin/tests/system/rrchecker/tests_sh_rrchecker.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_rrchecker(run_tests_sh):
run_tests_sh()

14
bin/tests/system/rrl/tests_sh_rrl.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_rrl(run_tests_sh):
run_tests_sh()

14
bin/tests/system/rrsetorder/tests_sh_rrsetorder.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_rrsetorder(run_tests_sh):
run_tests_sh()

14
bin/tests/system/rsabigexponent/tests_sh_rsabigexponent.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_rsabigexponent(run_tests_sh):
run_tests_sh()

27
bin/tests/system/run.sh Executable file
View file

@ -0,0 +1,27 @@
#!/bin/sh
#
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
#
# Run a single system test using the pytest runner. This is a simple wrapper
# around pytest for convenience.
#
if [ -z "$1" ] || [ ! -d "$1" ]; then
echo "Usage: $0 system_test_dir [pytest_args]"
exit 2
fi
system_test_dir="$1"
shift
(cd "$system_test_dir" || exit 2 ; /usr/bin/env python3 -m pytest "$@")

2
bin/tests/system/runall.sh
View file

@ -88,7 +88,7 @@ else
(
status=0
for testdir in $SUBDIRS; do
$SHELL run.sh -r "$testdir" || status=1
$SHELL legacy.run.sh -r "$testdir" || status=1
done
echo "$status" > systests.status
) 2>&1 | tee "systests.output"

2
bin/tests/system/runsequential.sh
View file

@ -22,5 +22,5 @@
for d in $SEQUENTIALDIRS
do
$SHELL run.sh "${@}" "$d" 2>&1 | tee "test.output.$d"
$SHELL legacy.run.sh "${@}" "$d" 2>&1 | tee "test.output.$d"
done

14
bin/tests/system/runtime/tests_sh_runtime.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_runtime(run_tests_sh):
run_tests_sh()

14
bin/tests/system/serve-stale/tests_sh_serve_stale.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_serve_stale(run_tests_sh):
run_tests_sh()

14
bin/tests/system/sfcache/tests_sh_sfcache.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_sfcache(run_tests_sh):
run_tests_sh()

14
bin/tests/system/smartsign/tests_sh_smartsign.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_smartsign(run_tests_sh):
run_tests_sh()

14
bin/tests/system/sortlist/tests_sh_sortlist.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_sortlist(run_tests_sh):
run_tests_sh()

14
bin/tests/system/spf/tests_sh_spf.py Normal file
View file

@ -0,0 +1,14 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
def test_spf(run_tests_sh):
run_tests_sh()

Some files were not shown because too many files have changed in this diff Show more