From d0288545f329966e84bdf410577102ccbd597356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Bal=C3=A1=C5=BEik?= Date: Tue, 23 Apr 2024 15:48:18 +0200 Subject: [PATCH] Disable deadlines for hypothesis tests when running in CI The times it takes to run tests CI vary significantly enough that it makes hypothesis test reach their deadlines and fail randomly marking the tests as flaky. This commit disables the deadlines when running in CI. (cherry picked from commit f55cacbbfd5eeaeb8e521895a0ffab8addeaad6f) --- .gitlab-ci.yml | 2 ++ bin/tests/system/wildcard/tests_wildcard.py | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0779e09464..8376972155 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,6 +55,8 @@ variables: BIND_STRESS_TEST_OS: linux BIND_STRESS_TEST_ARCH: amd64 + HYPOTHESIS_PROFILE: "ci" + default: # Allow all running CI jobs to be automatically canceled when a new # version of a branch is pushed. diff --git a/bin/tests/system/wildcard/tests_wildcard.py b/bin/tests/system/wildcard/tests_wildcard.py index cd5b278d28..f304f966b3 100755 --- a/bin/tests/system/wildcard/tests_wildcard.py +++ b/bin/tests/system/wildcard/tests_wildcard.py @@ -28,6 +28,7 @@ Limitations - untested properties: - special behavior of rdtypes like CNAME """ +import os import pytest pytest.importorskip("dns", minversion="2.0.0") @@ -47,7 +48,7 @@ try: pytest.importorskip("hypothesis") except ValueError: pytest.importorskip("hypothesis", minversion="4.41.2") -from hypothesis import assume, example, given +from hypothesis import assume, example, given, settings from strategies import dns_names, dns_rdatatypes_without_meta import isctest.check @@ -62,6 +63,10 @@ WILDCARD_RDATA = "192.0.2.1" IP_ADDR = "10.53.0.1" TIMEOUT = 5 # seconds, just a sanity check +# Timing of hypothesis tests is flaky in the CI, so we disable deadlines. +settings.register_profile("ci", deadline=None) +settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default")) + @given(name=dns_names(suffix=SUFFIX), rdtype=dns_rdatatypes_without_meta) def test_wildcard_rdtype_mismatch(