diff --git a/doc/31-Changelog.md b/doc/31-Changelog.md index 9a363c7..988783b 100644 --- a/doc/31-Changelog.md +++ b/doc/31-Changelog.md @@ -13,11 +13,12 @@ Released closed milestones can be found on [GitHub](https://github.com/Icinga/ic ### Bugfixes -## 1.4.1 (pending) +## 1.4.1 (2021-03-10) ### Bugfixes * [#222](https://github.com/Icinga/icinga-powershell-framework/pull/222) Fixes an issue with [Secure.String] arguments for PowerShell plugins, caused by `ConvertTo-IcingaSecureString` Cmdlet not being pre-loaded +* [#224](https://github.com/Icinga/icinga-powershell-framework/issues/224) Fixes "memory leak" on background daemon for registered service checks, by clearing the error stack and manually calling the PowerShell garbage collector to force freeing of memory ## 1.4.0 (2021-03-02) diff --git a/lib/core/logging/Write-IcingaConsoleOutput.psm1 b/lib/core/logging/Write-IcingaConsoleOutput.psm1 index 6fed107..6cb660b 100644 --- a/lib/core/logging/Write-IcingaConsoleOutput.psm1 +++ b/lib/core/logging/Write-IcingaConsoleOutput.psm1 @@ -40,6 +40,11 @@ function Write-IcingaConsoleOutput() return; } + # Never write console output in case the Framework is running as daemon + if ($null -ne $global:IcingaDaemonData -And $null -ne $global:IcingaDaemonData.FrameworkRunningAsDaemon -And $global:IcingaDaemonData.FrameworkRunningAsDaemon -eq $TRUE) { + return; + } + $OutputMessage = $Message; [int]$Index = 0; diff --git a/lib/daemons/ServiceCheckDaemon/Start-IcingaServiceCheckDaemon.psm1 b/lib/daemons/ServiceCheckDaemon/Start-IcingaServiceCheckDaemon.psm1 index e708408..a58a06a 100644 --- a/lib/daemons/ServiceCheckDaemon/Start-IcingaServiceCheckDaemon.psm1 +++ b/lib/daemons/ServiceCheckDaemon/Start-IcingaServiceCheckDaemon.psm1 @@ -66,7 +66,6 @@ function Start-IcingaServiceCheckTask() Use-Icinga -LibOnly -Daemon; $PassedTime = 0; $SortedResult = $null; - $OldData = @{ }; $PerfCache = @{ }; $AverageCalc = @{ }; [int]$MaxTime = 0; @@ -85,8 +84,8 @@ function Start-IcingaServiceCheckTask() $AverageCalc.Add( [string]$index, @{ - 'Interval' = [int]$index; - 'Time' = [int]$index * 60; + 'Interval' = ([int]$index); + 'Time' = ([int]$index * 60); 'Sum' = 0; 'Count' = 0; } @@ -126,19 +125,26 @@ function Start-IcingaServiceCheckTask() if ($PassedTime -ge $Interval) { try { & $CheckCommand @Arguments | Out-Null; + } catch { + # Just for debugging. Not required in production or usable at all + $ErrMsg = $_.Exception.Message; + Write-IcingaConsoleError $ErrMsg; + } - Get-IcingaCheckSchedulerPerfData | Out-Null; - Get-IcingaCheckSchedulerPluginOutput | Out-Null; - + try { $UnixTime = Get-IcingaUnixTime; foreach ($result in $global:Icinga.CheckData[$CheckCommand]['results'].Keys) { [string]$HashIndex = $result; $SortedResult = $global:Icinga.CheckData[$CheckCommand]['results'][$HashIndex].GetEnumerator() | Sort-Object name -Descending; - Add-IcingaHashtableItem -Hashtable $OldData -Key $HashIndex -Value @{ } | Out-Null; Add-IcingaHashtableItem -Hashtable $PerfCache -Key $HashIndex -Value @{ } | Out-Null; foreach ($timeEntry in $SortedResult) { + + if ((Test-Numeric $timeEntry.Value) -eq $FALSE) { + continue; + } + foreach ($calc in $AverageCalc.Keys) { if (($UnixTime - $AverageCalc[$calc].Time) -le [int]$timeEntry.Key) { $AverageCalc[$calc].Sum += $timeEntry.Value; @@ -147,20 +153,20 @@ function Start-IcingaServiceCheckTask() } if (($UnixTime - $MaxTimeInSeconds) -le [int]$timeEntry.Key) { Add-IcingaHashtableItem -Hashtable $PerfCache[$HashIndex] -Key ([string]$timeEntry.Key) -Value ([string]$timeEntry.Value) | Out-Null; - } else { - Add-IcingaHashtableItem -Hashtable $OldData[$HashIndex] -Key $timeEntry -Value $null | Out-Null; } } foreach ($calc in $AverageCalc.Keys) { - $AverageValue = ($AverageCalc[$calc].Sum / $AverageCalc[$calc].Count); - [string]$MetricName = Format-IcingaPerfDataLabel ( - [string]::Format('{0}_{1}', $HashIndex, $AverageCalc[$calc].Interval) - ); + if ($AverageCalc[$calc].Count -ne 0) { + $AverageValue = ($AverageCalc[$calc].Sum / $AverageCalc[$calc].Count); + [string]$MetricName = Format-IcingaPerfDataLabel ( + [string]::Format('{0}_{1}', $HashIndex, $AverageCalc[$calc].Interval) + ); - Add-IcingaHashtableItem ` - -Hashtable $global:Icinga.CheckData[$CheckCommand]['average'] ` - -Key $MetricName -Value $AverageValue -Override | Out-Null; + Add-IcingaHashtableItem ` + -Hashtable $global:Icinga.CheckData[$CheckCommand]['average'] ` + -Key $MetricName -Value $AverageValue -Override | Out-Null; + } $AverageCalc[$calc].Sum = 0; $AverageCalc[$calc].Count = 0; @@ -168,9 +174,16 @@ function Start-IcingaServiceCheckTask() } # Flush data we no longer require in our cache to free memory - foreach ($entry in $OldData.Keys) { - foreach ($key in $OldData[$entry].Keys) { - Remove-IcingaHashtableItem -Hashtable $global:Icinga.CheckData[$CheckCommand]['results'][$entry] -Key $key.Name; + [array]$CheckStores = $global:Icinga.CheckData[$CheckCommand]['results'].Keys; + + foreach ($CheckStore in $CheckStores) { + [string]$CheckKey = $CheckStore; + [array]$CheckTimeStamps = $global:Icinga.CheckData[$CheckCommand]['results'][$CheckKey].Keys; + + foreach ($TimeSample in $CheckTimeStamps) { + if (($UnixTime - $MaxTimeInSeconds) -gt [int]$TimeSample) { + Remove-IcingaHashtableItem -Hashtable $global:Icinga.CheckData[$CheckCommand]['results'][$CheckKey] -Key ([string]$TimeSample); + } } } @@ -178,16 +191,28 @@ function Start-IcingaServiceCheckTask() # Write collected metrics to disk in case we reload the daemon. We will load them back into the module after reload then Set-IcingaCacheData -Space 'sc_daemon' -CacheStore 'checkresult_store' -KeyName $CheckCommand -Value $PerfCache; } catch { - # Todo: Add error reporting / handling + # Just for debugging. Not required in production or usable at all + $ErrMsg = $_.Exception.Message; + Write-IcingaConsoleError 'Failed to handle check result processing: {0}' -Objects $ErrMsg; } + # Cleanup the error stack and remove not required data + $Error.Clear(); + + # Always ensure our check data is cleared regardless of possible + # exceptions which might occur + Get-IcingaCheckSchedulerPerfData | Out-Null; + Get-IcingaCheckSchedulerPluginOutput | Out-Null; + $PassedTime = 0; $SortedResult.Clear(); - $OldData.Clear(); $PerfCache.Clear(); } + $PassedTime += 1; Start-Sleep -Seconds 1; + # Force PowerShell to call the garbage collector to free memory + [System.GC]::Collect(); } }; diff --git a/lib/icinga/plugin/New-IcingaCheck.psm1 b/lib/icinga/plugin/New-IcingaCheck.psm1 index 3a116c3..313a164 100644 --- a/lib/icinga/plugin/New-IcingaCheck.psm1 +++ b/lib/icinga/plugin/New-IcingaCheck.psm1 @@ -63,8 +63,16 @@ function New-IcingaCheck() ); } + # Fix possible error for identical time stamps due to internal exceptions + # and check execution within the same time slot because of this + [string]$TimeIndex = Get-IcingaUnixTime; + + if ($global:Icinga.CheckData[$this.checkcommand]['results'][$this.name].ContainsKey($TimeIndex)) { + return; + } + $global:Icinga.CheckData[$this.checkcommand]['results'][$this.name].Add( - (Get-IcingaUnixTime), + $TimeIndex, $this.value ); }