From 7a42a6b409944db1792dc399c0d431e7cf1350d5 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Tue, 22 Aug 2023 17:09:59 +0200 Subject: [PATCH 1/7] Skip checkds test on Python<3.7 checkds test requires the capture_output argument for subprocess.run() which was added in Python 3.7. (cherry picked from commit 0361233b3dfbb47fc544a7d4eccff71045156c26) --- bin/tests/system/checkds/tests_checkds.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/tests/system/checkds/tests_checkds.py b/bin/tests/system/checkds/tests_checkds.py index fbd0c74bdc..5f9ae7341c 100755 --- a/bin/tests/system/checkds/tests_checkds.py +++ b/bin/tests/system/checkds/tests_checkds.py @@ -30,6 +30,11 @@ import dns.rdatatype import dns.resolver +pytestmark = pytest.mark.skipif( + sys.version_info < (3, 7), reason="Python >= 3.7 required [GL #3001]" +) + + def has_signed_apex_nsec(zone, response): has_nsec = False has_rrsig = False From 71f96aa1e609fc8e50cc38b5fa1856f9656659e9 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Tue, 22 Aug 2023 17:20:51 +0200 Subject: [PATCH 2/7] Clean up pytest .gitignore file The _last_test_run entry was accidentally added in !8194. It came from a work-in-progress version of the MR and was left there during a rebase. (cherry picked from commit 1b3db25adfefc7385be9a127929f35eb16a70e49) --- bin/tests/system/.gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore index cdf333c5bf..03905e7dd0 100644 --- a/bin/tests/system/.gitignore +++ b/bin/tests/system/.gitignore @@ -2,7 +2,6 @@ .hypothesis .mypy_cache __pycache__ -_last_test_run dig.out* rndc.out* nsupdate.out* From d6c17a45021801f6c51fa50d546d8612aa9cb0f4 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Tue, 22 Aug 2023 17:26:14 +0200 Subject: [PATCH 3/7] ci: fix after_script for out-of-tree tests The commands in after_script run in a new shell, separate from before_script and script commands. Since the pytest.log.txt is for out of tree tests is present in the build directory, switch to it (if set) before running any postprocessing commands. (cherry picked from commit 86043b0bbecb4d5a4273ef669371a9504e15644d) --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 22bf511a92..578deff7b0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -408,6 +408,7 @@ cross-version-config-tests: "$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt - '( ! grep -F "grep: warning:" pytest.out.txt )' after_script: + - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}" - *display_pytest_failures .system_test_legacy: &system_test_legacy From 0c5d718849a3ed969c0ed8d5329a887381e130e5 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Tue, 22 Aug 2023 17:59:16 +0200 Subject: [PATCH 4/7] Disable loadtime check in statschannel test It is better to disable the specific check that causes the test to fail rather than mark the entire test as xfail, which can mask other issues which the test is capable of detecting. (cherry picked from commit 7522583b5756924e697f10633d7d5d476176c594) --- bin/tests/system/statschannel/generic.py | 8 +++++--- bin/tests/system/statschannel/tests_json.py | 2 -- bin/tests/system/statschannel/tests_xml.py | 2 -- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/bin/tests/system/statschannel/generic.py b/bin/tests/system/statschannel/generic.py index 5ff09e2570..bf4b038992 100644 --- a/bin/tests/system/statschannel/generic.py +++ b/bin/tests/system/statschannel/generic.py @@ -34,10 +34,12 @@ def check_refresh(refresh, min_time, max_time): assert refresh <= max_time -def check_loaded(loaded, expected): +def check_loaded(loaded, expected): # pylint: disable=unused-argument # Sanity check the zone timers values - assert loaded == expected - assert loaded < now + # NOTE This check has been disabled due to GL #3983 + # assert loaded == expected + # assert loaded < now + pass def check_zone_timers(loaded, expires, refresh, loaded_exp): diff --git a/bin/tests/system/statschannel/tests_json.py b/bin/tests/system/statschannel/tests_json.py index 6be264f49a..c4599258ea 100755 --- a/bin/tests/system/statschannel/tests_json.py +++ b/bin/tests/system/statschannel/tests_json.py @@ -12,7 +12,6 @@ # information regarding copyright ownership. from datetime import datetime -import os import pytest @@ -83,7 +82,6 @@ def test_zone_timers_primary_json(statsport): ) -@pytest.mark.xfail(reason="GL #3983", strict="LEGACY_TEST_RUNNER" not in os.environ) def test_zone_timers_secondary_json(statsport): generic.test_zone_timers_secondary( fetch_zones_json, diff --git a/bin/tests/system/statschannel/tests_xml.py b/bin/tests/system/statschannel/tests_xml.py index 6375d8a243..7f0b37e846 100755 --- a/bin/tests/system/statschannel/tests_xml.py +++ b/bin/tests/system/statschannel/tests_xml.py @@ -12,7 +12,6 @@ # information regarding copyright ownership. from datetime import datetime -import os import xml.etree.ElementTree as ET import pytest @@ -113,7 +112,6 @@ def test_zone_timers_primary_xml(statsport): ) -@pytest.mark.xfail(reason="GL #3983", strict="LEGACY_TEST_RUNNER" not in os.environ) def test_zone_timers_secondary_xml(statsport): generic.test_zone_timers_secondary( fetch_zones_xml, From 83cde477348bc1fc9eba2835cc4d84b7041e9a29 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Tue, 22 Aug 2023 18:39:51 +0200 Subject: [PATCH 5/7] Allow re-runs of qmin system test The qmin test is inherently unstable. It fails quite often with failure modes described in GL #904. Allow the pytest runner to re-run the test up to 3 times to only detect a more persistent and reproducible failures rather than random noise caused by the nature of the test. (cherry picked from commit be2123a8e9926782dcc40cd93c2482f175c4d368) --- bin/tests/system/qmin/tests_sh_qmin.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/tests/system/qmin/tests_sh_qmin.py b/bin/tests/system/qmin/tests_sh_qmin.py index 2566c78e0f..607732232e 100644 --- a/bin/tests/system/qmin/tests_sh_qmin.py +++ b/bin/tests/system/qmin/tests_sh_qmin.py @@ -9,6 +9,10 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. +import pytest_custom_markers + +# The qmin test is inherently unstable, see GL #904 for details. +@pytest_custom_markers.flaky(max_runs=3) def test_qmin(run_tests_sh): run_tests_sh() From ac465416f9068af66db06d0c9c8e640063b89913 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Tue, 22 Aug 2023 18:46:05 +0200 Subject: [PATCH 6/7] Allow re-runs of reclimit system test The reclimit system test has been unstable and producing false positive results for years (GL #1587). Allow the test to be re-run (once) to reduce the noise it causes. (cherry picked from commit 8c5833fe391035c4756c0242e42ba43c9dfbe894) --- bin/tests/system/reclimit/tests_sh_reclimit.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/tests/system/reclimit/tests_sh_reclimit.py b/bin/tests/system/reclimit/tests_sh_reclimit.py index 38cfbc6e7a..4710027581 100644 --- a/bin/tests/system/reclimit/tests_sh_reclimit.py +++ b/bin/tests/system/reclimit/tests_sh_reclimit.py @@ -9,6 +9,10 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. +import pytest_custom_markers + +# The reclimit is known to be quite unstable. GL #1587 +@pytest_custom_markers.flaky(max_runs=2) def test_reclimit(run_tests_sh): run_tests_sh() From 764161d8cfafbd524975fc02cc92edf95a8fd507 Mon Sep 17 00:00:00 2001 From: Tom Krizek Date: Tue, 22 Aug 2023 18:49:55 +0200 Subject: [PATCH 7/7] Allow re-runs of rrl system test The rrl system test has been unstable and producing false positive results for years (GL #172). Allow the test to be re-run (once) to reduce the noise it causes. (cherry picked from commit 40289d5440a2c39f94a80bc49904c3bc4c8c12f0) --- bin/tests/system/rrl/tests_sh_rrl.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/tests/system/rrl/tests_sh_rrl.py b/bin/tests/system/rrl/tests_sh_rrl.py index 6c8edc55c2..05b43a6fe1 100644 --- a/bin/tests/system/rrl/tests_sh_rrl.py +++ b/bin/tests/system/rrl/tests_sh_rrl.py @@ -9,6 +9,10 @@ # See the COPYRIGHT file distributed with this work for additional # information regarding copyright ownership. +import pytest_custom_markers + +# The rrl is known to be quite unstable. GL #172 +@pytest_custom_markers.flaky(max_runs=2) def test_rrl(run_tests_sh): run_tests_sh()