From 3fa2cc5c3d3bd93c6de221be48ad020cbabbb3dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Mon, 14 Mar 2022 08:59:32 +0100 Subject: [PATCH] Rework skipping long tests The ability to conveniently mark tests which should only be run when the CI_ENABLE_ALL_TESTS environment variable is set seems to be useful on a general level and therefore it should not be limited to the "timeouts" system test, where it is currently used. pytest documentation [1] suggests to reuse commonly used test markers by putting them all in a single Python module which then has to be imported by test files that want to use the markers defined therein. Follow that advice by creating a new bin/tests/system/pytest_custom_markers.py Python module containing the relevant marker definitions. Note that "import pytest_custom_markers" works from a test-specific subdirectory because pytest modifies sys.path so that it contains the paths to all parent directories containing a conftest.py file (and bin/tests/system/ is one). PyLint does not like that, though, so add a relevant PyLint suppression. The above changes make bin/tests/system/timeouts/conftest.py redundant, so remove it. [1] https://docs.pytest.org/en/7.0.x/how-to/skipping.html#id1 (cherry picked from commit 00392921f0983e26813f8dd0a6d2589629bcee5f) --- bin/tests/system/pytest_custom_markers.py | 20 +++++++++++++++ bin/tests/system/run.sh.in | 2 +- bin/tests/system/timeouts/conftest.py | 30 ----------------------- bin/tests/system/timeouts/tests-tcp.py | 6 +++-- 4 files changed, 25 insertions(+), 33 deletions(-) create mode 100644 bin/tests/system/pytest_custom_markers.py delete mode 100644 bin/tests/system/timeouts/conftest.py diff --git a/bin/tests/system/pytest_custom_markers.py b/bin/tests/system/pytest_custom_markers.py new file mode 100644 index 0000000000..256fb7db8d --- /dev/null +++ b/bin/tests/system/pytest_custom_markers.py @@ -0,0 +1,20 @@ +#!/usr/bin/python3 + +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# SPDX-License-Identifier: MPL-2.0 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +import os + +import pytest + + +long_test = pytest.mark.skipif(not os.environ.get('CI_ENABLE_ALL_TESTS'), + reason='CI_ENABLE_ALL_TESTS not set') diff --git a/bin/tests/system/run.sh.in b/bin/tests/system/run.sh.in index 6362f594b4..310c7ac4dc 100644 --- a/bin/tests/system/run.sh.in +++ b/bin/tests/system/run.sh.in @@ -112,7 +112,7 @@ if [ "${srcdir}" != "${builddir}" ]; then cp -a "${srcdir}/common" "${builddir}" fi # Some tests require additional files to work for out-of-tree test runs. - for file in ckdnsrps.sh conftest.py digcomp.pl ditch.pl fromhex.pl kasp.sh packet.pl start.pl stop.pl testcrypto.sh; do + for file in ckdnsrps.sh conftest.py digcomp.pl ditch.pl fromhex.pl kasp.sh packet.pl pytest_custom_markers.py start.pl stop.pl testcrypto.sh; do if [ ! -r "${file}" ]; then cp -a "${srcdir}/${file}" "${builddir}" fi diff --git a/bin/tests/system/timeouts/conftest.py b/bin/tests/system/timeouts/conftest.py deleted file mode 100644 index 3a4375232d..0000000000 --- a/bin/tests/system/timeouts/conftest.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (C) Internet Systems Consortium, Inc. ("ISC") -# -# SPDX-License-Identifier: MPL-2.0 -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, you can obtain one at https://mozilla.org/MPL/2.0/. -# -# See the COPYRIGHT file distributed with this work for additional -# information regarding copyright ownership. - -import os -import pytest - - -def pytest_configure(config): - config.addinivalue_line( - "markers", "long: mark tests that take a long time to run" - ) - - -def pytest_collection_modifyitems(config, items): - # pylint: disable=unused-argument,unused-import,too-many-branches - # pylint: disable=import-outside-toplevel - skip_long_tests = pytest.mark.skip( - reason="need CI_ENABLE_ALL_TESTS environment variable") - if not os.environ.get("CI_ENABLE_ALL_TESTS"): - for item in items: - if "long" in item.keywords: - item.add_marker(skip_long_tests) diff --git a/bin/tests/system/timeouts/tests-tcp.py b/bin/tests/system/timeouts/tests-tcp.py index e883dddda4..74d7cb695f 100644 --- a/bin/tests/system/timeouts/tests-tcp.py +++ b/bin/tests/system/timeouts/tests-tcp.py @@ -26,6 +26,8 @@ import dns.query import dns.rdataclass import dns.rdatatype +import pytest_custom_markers # pylint: disable=import-error + TIMEOUT = 10 @@ -204,7 +206,7 @@ def test_send_timeout(named_port): raise EOFError from e -@pytest.mark.long +@pytest_custom_markers.long_test def test_max_transfer_idle_out(named_port): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.connect(("10.53.0.1", named_port)) @@ -235,7 +237,7 @@ def test_max_transfer_idle_out(named_port): assert soa is None -@pytest.mark.long +@pytest_custom_markers.long_test def test_max_transfer_time_out(named_port): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.connect(("10.53.0.1", named_port))