From c97f3c80f550320abcbc3bcbaa0a3673c2091c34 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Fri, 11 Jan 2019 16:16:45 +0100 Subject: [PATCH 1/2] Fix checkresults from the future breaking checks --- lib/icinga/checkable-check.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/icinga/checkable-check.cpp b/lib/icinga/checkable-check.cpp index 0b8917b77..09fa901e1 100644 --- a/lib/icinga/checkable-check.cpp +++ b/lib/icinga/checkable-check.cpp @@ -170,8 +170,8 @@ void Checkable::ProcessCheckResult(const CheckResult::Ptr& cr, const MessageOrig long old_attempt = GetCheckAttempt(); bool recovery = false; - /* Ignore check results older than the current one. */ - if (old_cr && cr->GetExecutionStart() < old_cr->GetExecutionStart()) + /* Ignore check results older than the current one, except if the previous check result is from the future. */ + if (old_cr && cr->GetExecutionStart() < old_cr->GetExecutionStart() && old_cr->GetExecutionStart() < now) return; /* The ExecuteCheck function already sets the old state, but we need to do it again From d1fb1a8eda05018c5f665fceeb52569e243a7b65 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Fri, 8 Feb 2019 13:32:13 +0100 Subject: [PATCH 2/2] Refactor conditions and add debug log messages for future crs and skipped crs --- lib/icinga/checkable-check.cpp | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/lib/icinga/checkable-check.cpp b/lib/icinga/checkable-check.cpp index 09fa901e1..2437aa787 100644 --- a/lib/icinga/checkable-check.cpp +++ b/lib/icinga/checkable-check.cpp @@ -170,9 +170,34 @@ void Checkable::ProcessCheckResult(const CheckResult::Ptr& cr, const MessageOrig long old_attempt = GetCheckAttempt(); bool recovery = false; - /* Ignore check results older than the current one, except if the previous check result is from the future. */ - if (old_cr && cr->GetExecutionStart() < old_cr->GetExecutionStart() && old_cr->GetExecutionStart() < now) - return; + /* When we have an check result already (not after fresh start), + * prevent to accept old check results and allow overrides for + * CRs happened in the future. + */ + if (old_cr) { + double currentCRTimestamp = old_cr->GetExecutionStart(); + double newCRTimestamp = cr->GetExecutionStart(); + + /* Our current timestamp may be from the future (wrong server time adjusted again). Allow overrides here. */ + if (currentCRTimestamp > now) { + /* our current CR is from the future, let the new CR override it. */ + Log(LogDebug, "Checkable") + << std::fixed << std::setprecision(6) << "Processing check result for checkable '" << GetName() << "' from " + << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", newCRTimestamp) << " (" << newCRTimestamp + << "). Overriding since ours is from the future at " + << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", currentCRTimestamp) << " (" << currentCRTimestamp << ")."; + } else { + /* Current timestamp is from the past, but the new timestamp is even more in the past. Skip it. */ + if (newCRTimestamp < currentCRTimestamp) { + Log(LogDebug, "Checkable") + << std::fixed << std::setprecision(6) << "Skipping check result for checkable '" << GetName() << "' from " + << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", newCRTimestamp) << " (" << newCRTimestamp + << "). It is in the past compared to ours at " + << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", currentCRTimestamp) << " (" << currentCRTimestamp << ")."; + return; + } + } + } /* The ExecuteCheck function already sets the old state, but we need to do it again * in case this was a passive check result. */