Merge branch '3535-fetchlimit-test-flaky' into 'main'

Allow the fetchlimit test to be re-run

Closes #3535

See merge request isc-projects/bind9!8612

bin/tests/system/fetchlimit/tests_sh_fetchlimit.py

@@ -9,6 +9,9 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import isctest.mark
 
+
+@isctest.mark.flaky(max_runs=2)
 def test_fetchlimit(run_tests_sh):
     run_tests_sh()

bin/tests/system/isctest/__init__.py

@@ -14,3 +14,8 @@ from . import instance
 from . import query
 from . import rndc
 from . import log
+
+# isctest.mark module is intentionally NOT imported, because it relies on
+# environment variables which might not be set at the time of import of the
+# `isctest` package. To use the marks, manual `import isctest.mark` is needed
+# instead.

bin/tests/system/isctest/mark.py

@@ -43,7 +43,7 @@ have_json_c = pytest.mark.skipif(
 
 
 try:
-    import flaky as flaky_pkg
+    import flaky as flaky_pkg  # type: ignore
 except ModuleNotFoundError:
     # In case the flaky package is not installed, run the tests as usual
     # without any attempts to re-run them.

bin/tests/system/nsupdate/tests_sh_nsupdate.py

@@ -11,12 +11,12 @@
 
 import platform
 
-import pytest_custom_markers
+import isctest.mark
 
 
 MAX_RUNS = 2 if platform.system() == "FreeBSD" else 1  # GL#3846
 
 
-@pytest_custom_markers.flaky(max_runs=MAX_RUNS)
+@isctest.mark.flaky(max_runs=MAX_RUNS)
 def test_nsupdate(run_tests_sh):
     run_tests_sh()

bin/tests/system/qmin/tests_sh_qmin.py

@@ -9,10 +9,10 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
-import pytest_custom_markers
+import isctest.mark
 
 
 # The qmin test is inherently unstable, see GL #904 for details.
-@pytest_custom_markers.flaky(max_runs=3)
+@isctest.mark.flaky(max_runs=3)
 def test_qmin(run_tests_sh):
     run_tests_sh()

bin/tests/system/reclimit/tests_sh_reclimit.py

@@ -9,10 +9,10 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
-import pytest_custom_markers
+import isctest.mark
 
 
 # The reclimit is known to be quite unstable. GL #1587
-@pytest_custom_markers.flaky(max_runs=2)
+@isctest.mark.flaky(max_runs=2)
 def test_reclimit(run_tests_sh):
     run_tests_sh()

bin/tests/system/rrl/tests_sh_rrl.py

@@ -9,10 +9,10 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
-import pytest_custom_markers
+import isctest.mark
 
 
 # The rrl is known to be quite unstable. GL #172
-@pytest_custom_markers.flaky(max_runs=2)
+@isctest.mark.flaky(max_runs=2)
 def test_rrl(run_tests_sh):
     run_tests_sh()

bin/tests/system/statschannel/tests_json.py

@@ -15,12 +15,12 @@ from datetime import datetime
 
 import pytest
 
-import pytest_custom_markers
+import isctest.mark
 
 pytest.register_assert_rewrite("generic")
 import generic
 
-pytestmark = pytest_custom_markers.have_json_c
+pytestmark = isctest.mark.have_json_c
 requests = pytest.importorskip("requests")
 
 

bin/tests/system/statschannel/tests_xml.py

@@ -16,12 +16,12 @@ import xml.etree.ElementTree as ET
 
 import pytest
 
-import pytest_custom_markers
+import isctest.mark
 
 pytest.register_assert_rewrite("generic")
 import generic
 
-pytestmark = pytest_custom_markers.have_libxml2
+pytestmark = isctest.mark.have_libxml2
 requests = pytest.importorskip("requests")
 
 

bin/tests/system/timeouts/tests_tcp_timeouts.py

@@ -26,7 +26,7 @@ import dns.query
 import dns.rdataclass
 import dns.rdatatype
 
-import pytest_custom_markers  # pylint: disable=import-error
+import isctest.mark  # pylint: disable=import-error
 
 
 TIMEOUT = 10
@@ -185,7 +185,7 @@ def test_long_axfr(named_port):
         assert soa is not None
 
 
-@pytest_custom_markers.flaky(max_runs=3)
+@isctest.mark.flaky(max_runs=3)
 def test_send_timeout(named_port):
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
         sock.connect(("10.53.0.1", named_port))
@@ -212,7 +212,7 @@ def test_send_timeout(named_port):
                 raise EOFError from e
 
 
-@pytest_custom_markers.long_test
+@isctest.mark.long_test
 def test_max_transfer_idle_out(named_port):
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
         sock.connect(("10.53.0.1", named_port))
@@ -246,7 +246,7 @@ def test_max_transfer_idle_out(named_port):
             assert soa is None
 
 
-@pytest_custom_markers.long_test
+@isctest.mark.long_test
 def test_max_transfer_time_out(named_port):
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
         sock.connect(("10.53.0.1", named_port))