Fix memory leak on background service daemon

doc/31-Changelog.md

@@ -13,11 +13,12 @@ Released closed milestones can be found on [GitHub](https://github.com/Icinga/ic
 
 ### Bugfixes
 
-## 1.4.1 (pending)
+## 1.4.1 (2021-03-10)
 
 ### Bugfixes
 
 * [#222](https://github.com/Icinga/icinga-powershell-framework/pull/222) Fixes an issue with [Secure.String] arguments for PowerShell plugins, caused by `ConvertTo-IcingaSecureString` Cmdlet not being pre-loaded
+* [#224](https://github.com/Icinga/icinga-powershell-framework/issues/224) Fixes "memory leak" on background daemon for registered service checks, by clearing the error stack and manually calling the PowerShell garbage collector to force freeing of memory
 
 ## 1.4.0 (2021-03-02)
 

lib/core/logging/Write-IcingaConsoleOutput.psm1

@@ -40,6 +40,11 @@ function Write-IcingaConsoleOutput()
         return;
     }
 
+    # Never write console output in case the Framework is running as daemon
+    if ($null -ne $global:IcingaDaemonData -And $null -ne $global:IcingaDaemonData.FrameworkRunningAsDaemon -And $global:IcingaDaemonData.FrameworkRunningAsDaemon -eq $TRUE) {
+        return;
+    }
+
     $OutputMessage = $Message;
     [int]$Index    = 0;
 

lib/daemons/ServiceCheckDaemon/Start-IcingaServiceCheckDaemon.psm1

@@ -66,7 +66,6 @@ function Start-IcingaServiceCheckTask()
         Use-Icinga -LibOnly -Daemon;
         $PassedTime   = 0;
         $SortedResult = $null;
-        $OldData      = @{ };
         $PerfCache    = @{ };
         $AverageCalc  = @{ };
         [int]$MaxTime = 0;
@@ -85,8 +84,8 @@ function Start-IcingaServiceCheckTask()
                 $AverageCalc.Add(
                     [string]$index,
                     @{
-                        'Interval' = [int]$index;
+                        'Interval' = ([int]$index);
-                        'Time'     = [int]$index * 60;
+                        'Time'     = ([int]$index * 60);
                         'Sum'      = 0;
                         'Count'    = 0;
                     }
@@ -126,19 +125,26 @@ function Start-IcingaServiceCheckTask()
             if ($PassedTime -ge $Interval) {
                 try {
                     & $CheckCommand @Arguments | Out-Null;
+                } catch {
+                    # Just for debugging. Not required in production or usable at all
+                    $ErrMsg = $_.Exception.Message;
+                    Write-IcingaConsoleError $ErrMsg;
+                }
 
-                    Get-IcingaCheckSchedulerPerfData | Out-Null;
+                try {
-                    Get-IcingaCheckSchedulerPluginOutput | Out-Null;
-
                     $UnixTime = Get-IcingaUnixTime;
 
                     foreach ($result in $global:Icinga.CheckData[$CheckCommand]['results'].Keys) {
                         [string]$HashIndex = $result;
                         $SortedResult = $global:Icinga.CheckData[$CheckCommand]['results'][$HashIndex].GetEnumerator() | Sort-Object name -Descending;
-                        Add-IcingaHashtableItem -Hashtable $OldData -Key $HashIndex -Value @{ } | Out-Null;
                         Add-IcingaHashtableItem -Hashtable $PerfCache -Key $HashIndex -Value @{ } | Out-Null;
 
                         foreach ($timeEntry in $SortedResult) {
+
+                            if ((Test-Numeric $timeEntry.Value) -eq $FALSE) {
+                                continue;
+                            }
+
                             foreach ($calc in $AverageCalc.Keys) {
                                 if (($UnixTime - $AverageCalc[$calc].Time) -le [int]$timeEntry.Key) {
                                     $AverageCalc[$calc].Sum   += $timeEntry.Value;
@@ -147,20 +153,20 @@ function Start-IcingaServiceCheckTask()
                             }
                             if (($UnixTime - $MaxTimeInSeconds) -le [int]$timeEntry.Key) {
                                 Add-IcingaHashtableItem -Hashtable $PerfCache[$HashIndex] -Key ([string]$timeEntry.Key) -Value ([string]$timeEntry.Value) | Out-Null;
-                            } else {
-                                Add-IcingaHashtableItem -Hashtable $OldData[$HashIndex] -Key $timeEntry -Value $null | Out-Null;
                             }
                         }
 
                         foreach ($calc in $AverageCalc.Keys) {
-                            $AverageValue         = ($AverageCalc[$calc].Sum / $AverageCalc[$calc].Count);
+                            if ($AverageCalc[$calc].Count -ne 0) {
-                            [string]$MetricName   = Format-IcingaPerfDataLabel (
+                                $AverageValue         = ($AverageCalc[$calc].Sum / $AverageCalc[$calc].Count);
-                                [string]::Format('{0}_{1}', $HashIndex, $AverageCalc[$calc].Interval)
+                                [string]$MetricName   = Format-IcingaPerfDataLabel (
-                            );
+                                    [string]::Format('{0}_{1}', $HashIndex, $AverageCalc[$calc].Interval)
+                                );
 
-                            Add-IcingaHashtableItem `
+                                Add-IcingaHashtableItem `
-                                -Hashtable $global:Icinga.CheckData[$CheckCommand]['average'] `
+                                    -Hashtable $global:Icinga.CheckData[$CheckCommand]['average'] `
-                                -Key $MetricName -Value $AverageValue -Override | Out-Null;
+                                    -Key $MetricName -Value $AverageValue -Override | Out-Null;
+                            }
 
                             $AverageCalc[$calc].Sum   = 0;
                             $AverageCalc[$calc].Count = 0;
@@ -168,9 +174,16 @@ function Start-IcingaServiceCheckTask()
                     }
 
                     # Flush data we no longer require in our cache to free memory
-                    foreach ($entry in $OldData.Keys) {
+                    [array]$CheckStores = $global:Icinga.CheckData[$CheckCommand]['results'].Keys;
-                        foreach ($key in $OldData[$entry].Keys) {
+
-                            Remove-IcingaHashtableItem -Hashtable $global:Icinga.CheckData[$CheckCommand]['results'][$entry] -Key $key.Name;
+                    foreach ($CheckStore in $CheckStores) {
+                        [string]$CheckKey       = $CheckStore;
+                        [array]$CheckTimeStamps = $global:Icinga.CheckData[$CheckCommand]['results'][$CheckKey].Keys;
+
+                        foreach ($TimeSample in $CheckTimeStamps) {
+                            if (($UnixTime - $MaxTimeInSeconds) -gt [int]$TimeSample) {
+                                Remove-IcingaHashtableItem -Hashtable $global:Icinga.CheckData[$CheckCommand]['results'][$CheckKey] -Key ([string]$TimeSample);
+                            }
                         }
                     }
 
@@ -178,16 +191,28 @@ function Start-IcingaServiceCheckTask()
                     # Write collected metrics to disk in case we reload the daemon. We will load them back into the module after reload then
                     Set-IcingaCacheData -Space 'sc_daemon' -CacheStore 'checkresult_store' -KeyName $CheckCommand -Value $PerfCache;
                 } catch {
-                    # Todo: Add error reporting / handling
+                    # Just for debugging. Not required in production or usable at all
+                    $ErrMsg = $_.Exception.Message;
+                    Write-IcingaConsoleError 'Failed to handle check result processing: {0}' -Objects $ErrMsg;
                 }
 
+                # Cleanup the error stack and remove not required data
+                $Error.Clear();
+
+                # Always ensure our check data is cleared regardless of possible
+                # exceptions which might occur
+                Get-IcingaCheckSchedulerPerfData | Out-Null;
+                Get-IcingaCheckSchedulerPluginOutput | Out-Null;
+
                 $PassedTime   = 0;
                 $SortedResult.Clear();
-                $OldData.Clear();
                 $PerfCache.Clear();
             }
+
             $PassedTime += 1;
             Start-Sleep -Seconds 1;
+            # Force PowerShell to call the garbage collector to free memory
+            [System.GC]::Collect();
         }
     };
 

lib/icinga/plugin/New-IcingaCheck.psm1

@@ -63,8 +63,16 @@ function New-IcingaCheck()
                 );
             }
 
+            # Fix possible error for identical time stamps due to internal exceptions
+            # and check execution within the same time slot because of this
+            [string]$TimeIndex = Get-IcingaUnixTime;
+
+            if ($global:Icinga.CheckData[$this.checkcommand]['results'][$this.name].ContainsKey($TimeIndex)) {
+                return;
+            }
+
             $global:Icinga.CheckData[$this.checkcommand]['results'][$this.name].Add(
-                (Get-IcingaUnixTime),
+                $TimeIndex,
                 $this.value
             );
         }