From f314f1b43278f524ca8472aee39a9cedddd83cdd Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Fri, 22 Dec 2023 15:56:58 +0100 Subject: [PATCH 1/3] Move custom pytest markers into isctest module Keep our pytest code more organized by moving the shared code for custom pytest markers into a dedicated isctest/mark.py module. --- bin/tests/system/isctest/__init__.py | 5 +++++ .../system/{pytest_custom_markers.py => isctest/mark.py} | 0 bin/tests/system/nsupdate/tests_sh_nsupdate.py | 4 ++-- bin/tests/system/qmin/tests_sh_qmin.py | 4 ++-- bin/tests/system/reclimit/tests_sh_reclimit.py | 4 ++-- bin/tests/system/rrl/tests_sh_rrl.py | 4 ++-- bin/tests/system/statschannel/tests_json.py | 4 ++-- bin/tests/system/statschannel/tests_xml.py | 4 ++-- bin/tests/system/timeouts/tests_tcp_timeouts.py | 8 ++++---- 9 files changed, 21 insertions(+), 16 deletions(-) rename bin/tests/system/{pytest_custom_markers.py => isctest/mark.py} (100%) diff --git a/bin/tests/system/isctest/__init__.py b/bin/tests/system/isctest/__init__.py index 3ea82a3f45..047ca7762a 100644 --- a/bin/tests/system/isctest/__init__.py +++ b/bin/tests/system/isctest/__init__.py @@ -14,3 +14,8 @@ from . import instance from . import query from . import rndc from . import log + +# isctest.mark module is intentionally NOT imported, because it relies on +# environment variables which might not be set at the time of import of the +# `isctest` package. To use the marks, manual `import isctest.mark` is needed +# instead. diff --git a/bin/tests/system/pytest_custom_markers.py b/bin/tests/system/isctest/mark.py similarity index 100% rename from bin/tests/system/pytest_custom_markers.py rename to bin/tests/system/isctest/mark.py diff --git a/bin/tests/system/nsupdate/tests_sh_nsupdate.py b/bin/tests/system/nsupdate/tests_sh_nsupdate.py index 5c1a9c7e3d..305f1a191f 100644 --- a/bin/tests/system/nsupdate/tests_sh_nsupdate.py +++ b/bin/tests/system/nsupdate/tests_sh_nsupdate.py @@ -11,12 +11,12 @@ import platform -import pytest_custom_markers +import isctest.mark MAX_RUNS = 2 if platform.system() == "FreeBSD" else 1 # GL#3846 -@pytest_custom_markers.flaky(max_runs=MAX_RUNS) +@isctest.mark.flaky(max_runs=MAX_RUNS) def test_nsupdate(run_tests_sh): run_tests_sh() diff --git a/bin/tests/system/qmin/tests_sh_qmin.py b/bin/tests/system/qmin/tests_sh_qmin.py index 607732232e..0faeebd0e9 100644 --- a/bin/tests/system/qmin/tests_sh_qmin.py +++ b/bin/tests/system/qmin/tests_sh_qmin.py @@ -9,10 +9,10 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. -import pytest_custom_markers +import isctest.mark # The qmin test is inherently unstable, see GL #904 for details. -@pytest_custom_markers.flaky(max_runs=3) +@isctest.mark.flaky(max_runs=3) def test_qmin(run_tests_sh): run_tests_sh() diff --git a/bin/tests/system/reclimit/tests_sh_reclimit.py b/bin/tests/system/reclimit/tests_sh_reclimit.py index 4710027581..447c9488e0 100644 --- a/bin/tests/system/reclimit/tests_sh_reclimit.py +++ b/bin/tests/system/reclimit/tests_sh_reclimit.py @@ -9,10 +9,10 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. -import pytest_custom_markers +import isctest.mark # The reclimit is known to be quite unstable. GL #1587 -@pytest_custom_markers.flaky(max_runs=2) +@isctest.mark.flaky(max_runs=2) def test_reclimit(run_tests_sh): run_tests_sh() diff --git a/bin/tests/system/rrl/tests_sh_rrl.py b/bin/tests/system/rrl/tests_sh_rrl.py index 05b43a6fe1..c32305b5d7 100644 --- a/bin/tests/system/rrl/tests_sh_rrl.py +++ b/bin/tests/system/rrl/tests_sh_rrl.py @@ -9,10 +9,10 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. -import pytest_custom_markers +import isctest.mark # The rrl is known to be quite unstable. GL #172 -@pytest_custom_markers.flaky(max_runs=2) +@isctest.mark.flaky(max_runs=2) def test_rrl(run_tests_sh): run_tests_sh() diff --git a/bin/tests/system/statschannel/tests_json.py b/bin/tests/system/statschannel/tests_json.py index 4f7c4a7d7c..9b52541847 100755 --- a/bin/tests/system/statschannel/tests_json.py +++ b/bin/tests/system/statschannel/tests_json.py @@ -15,12 +15,12 @@ from datetime import datetime import pytest -import pytest_custom_markers +import isctest.mark pytest.register_assert_rewrite("generic") import generic -pytestmark = pytest_custom_markers.have_json_c +pytestmark = isctest.mark.have_json_c requests = pytest.importorskip("requests") diff --git a/bin/tests/system/statschannel/tests_xml.py b/bin/tests/system/statschannel/tests_xml.py index 53296155da..81ac187587 100755 --- a/bin/tests/system/statschannel/tests_xml.py +++ b/bin/tests/system/statschannel/tests_xml.py @@ -16,12 +16,12 @@ import xml.etree.ElementTree as ET import pytest -import pytest_custom_markers +import isctest.mark pytest.register_assert_rewrite("generic") import generic -pytestmark = pytest_custom_markers.have_libxml2 +pytestmark = isctest.mark.have_libxml2 requests = pytest.importorskip("requests") diff --git a/bin/tests/system/timeouts/tests_tcp_timeouts.py b/bin/tests/system/timeouts/tests_tcp_timeouts.py index 2e2a4b47e2..f615340510 100644 --- a/bin/tests/system/timeouts/tests_tcp_timeouts.py +++ b/bin/tests/system/timeouts/tests_tcp_timeouts.py @@ -26,7 +26,7 @@ import dns.query import dns.rdataclass import dns.rdatatype -import pytest_custom_markers # pylint: disable=import-error +import isctest.mark # pylint: disable=import-error TIMEOUT = 10 @@ -185,7 +185,7 @@ def test_long_axfr(named_port): assert soa is not None -@pytest_custom_markers.flaky(max_runs=3) +@isctest.mark.flaky(max_runs=3) def test_send_timeout(named_port): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.connect(("10.53.0.1", named_port)) @@ -212,7 +212,7 @@ def test_send_timeout(named_port): raise EOFError from e -@pytest_custom_markers.long_test +@isctest.mark.long_test def test_max_transfer_idle_out(named_port): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.connect(("10.53.0.1", named_port)) @@ -246,7 +246,7 @@ def test_max_transfer_idle_out(named_port): assert soa is None -@pytest_custom_markers.long_test +@isctest.mark.long_test def test_max_transfer_time_out(named_port): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.connect(("10.53.0.1", named_port)) From a65b654a233d4e1f2d86e22dda8bc8b8bb1164ac Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Fri, 22 Dec 2023 16:11:21 +0100 Subject: [PATCH 2/3] Don't type-check the flaky plugin with mypy Since we execute mypy for bin/tests/system/isctest package, this is now needed because the flaky package doesn't have type hints. --- bin/tests/system/isctest/mark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/tests/system/isctest/mark.py b/bin/tests/system/isctest/mark.py index ba3a9d4894..769abbbb5d 100644 --- a/bin/tests/system/isctest/mark.py +++ b/bin/tests/system/isctest/mark.py @@ -43,7 +43,7 @@ have_json_c = pytest.mark.skipif( try: - import flaky as flaky_pkg + import flaky as flaky_pkg # type: ignore except ModuleNotFoundError: # In case the flaky package is not installed, run the tests as usual # without any attempts to re-run them. From 10827fe96c0842083f2262014241fc399d3b5b36 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Fri, 22 Dec 2023 15:58:27 +0100 Subject: [PATCH 3/3] Allow the fetchlimit test to be re-run The test is known to be unstable due to timing issues. Prevent frequent false positives by allowing the test to be re-run by the flaky pytest plugin. --- bin/tests/system/fetchlimit/tests_sh_fetchlimit.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/tests/system/fetchlimit/tests_sh_fetchlimit.py b/bin/tests/system/fetchlimit/tests_sh_fetchlimit.py index 6df4441b91..04d3e3bf39 100644 --- a/bin/tests/system/fetchlimit/tests_sh_fetchlimit.py +++ b/bin/tests/system/fetchlimit/tests_sh_fetchlimit.py @@ -9,6 +9,9 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. +import isctest.mark + +@isctest.mark.flaky(max_runs=2) def test_fetchlimit(run_tests_sh): run_tests_sh()