Merge pull request #405 from Icinga:fix/garbage_collector_memory_leak

Fix: Icinga for Windows memory leak

In some scenarios, Icinga for Windows can contain a memory leak, caused by a bug of the Garbage Collector in older PowerShell versions.

We resolve this by enforcing a more aggressive approach on memory cleanup and by collecting all objects being used.

doc/100-General/10-Changelog.md

@@ -13,6 +13,7 @@ Released closed milestones can be found on [GitHub](https://github.com/Icinga/ic
 
 ### Bugfixes
 
+* [#379](https://github.com/Icinga/icinga-powershell-framework/issues/379) Fixes memory leak for Icinga for Windows by using a custom function being more aggressive on memory cleanup
 * [#402](https://github.com/Icinga/icinga-powershell-framework/pull/402) Fixes missing address attribute for REST-Api daemon, making it unable to change the listening address
 * [#403](https://github.com/Icinga/icinga-powershell-framework/pull/403) Fixes memory leak on newly EventLog reader for CLI event stream
 * [#407](https://github.com/Icinga/icinga-powershell-framework/pull/407) Removes unnecessary module import inside `Invoke-IcingaNamespaceCmdlets`

lib/core/framework/Read-IcingaWindowsEventLog.psm1

@@ -66,7 +66,7 @@ function Read-IcingaWindowsEventLog()
         }
 
         Start-Sleep -Seconds 1;
-        # Force PowerShell to call the garbage collector to free memory
+        # Force Icinga for Windows Garbage Collection
-        [System.GC]::Collect();
+        Optimize-IcingaForWindowsMemory -ClearErrorStack;
     }
 }

lib/core/tools/Optimize-IcingaForWindowsMemory.psm1

@@ -0,0 +1,60 @@
+<#
+.SYNOPSIS
+    Calls the PowerShell Garbage-Collector to force freeing memory
+    in a more aggressive way. Also fixes an issue on older PowerShell
+    versions on which the Garbage-Collector is not clearing memory
+    properly
+.DESCRIPTION
+    Calls the PowerShell Garbage-Collector to force freeing memory
+    in a more aggressive way. Also fixes an issue on older PowerShell
+    versions on which the Garbage-Collector is not clearing memory
+    properly
+.PARAMETER ClearErrorStack
+    Also clears the current error stack to free additional memory
+.EXAMPLE
+    Optimize-IcingaForWindowsMemory;
+.EXAMPLE
+    Optimize-IcingaForWindowsMemory -ClearErrorStack;
+.NOTES
+    Clears memory used by PowerShell in a more aggressive way
+#>
+function Optimize-IcingaForWindowsMemory()
+{
+    param (
+        [switch]$ClearErrorStack = $FALSE
+    );
+
+    # Clear all errors within our error stack
+    if ($ClearErrorStack) {
+        $Error.Clear();
+    }
+
+    # Force the full collection of memory being used
+    [decimal]$MemoryConsumption = [System.GC]::GetTotalMemory('forcefullcollection');
+    # Collect all generations of objects inside our memory and force the flushing of
+    # objects
+    [System.GC]::Collect([System.GC]::MaxGeneration, [System.GCCollectionMode]::Forced);
+    # Collect the remaining memory for a before/after delta
+    [decimal]$CollectedMemory   = [System.GC]::GetTotalMemory('forcefullcollection');
+    # Just for debugging
+    [decimal]$MemoryDelta       = $MemoryConsumption - $CollectedMemory;
+    [bool]$Negate               = $FALSE;
+
+    if ($MemoryDelta -lt 0) {
+        $MemoryDelta = $MemoryDelta * -1;
+        $Negate      = $TRUE;
+    }
+
+    $MemoryConsumptionMiB = Convert-Bytes -Value $MemoryConsumption -Unit 'MiB';
+    $CollectedMemoryMiB   = Convert-Bytes -Value $CollectedMemory -Unit 'MiB';
+    $MemoryDeltaMiB       = Convert-Bytes -Value $MemoryDelta -Unit 'MiB';
+
+    # Print an optional debug message, allowing us to analyze memory behavior over time
+    Write-IcingaDebugMessage `
+        -Message 'Calling Icinga for Windows Garbage-Collector. Memory overview below (Before, After, Flushed/Added)' `
+        -Objects @(
+            ([string]::Format('{0} MiB', $MemoryConsumptionMiB.Value)),
+            ([string]::Format('{0} MiB', $CollectedMemoryMiB.Value)),
+            ([string]::Format('{1}{0} MiB', $MemoryDeltaMiB.Value, (&{ if ($Negate) { return '-'; } else { return ''; } })))
+        );
+}

lib/daemons/RestAPI/daemon/New-IcingaForWindowsRESTApi.psm1

@@ -123,6 +123,10 @@ function New-IcingaForWindowsRESTApi()
     # being executed. This is required to ensure we will execute
     # the code frequently instead of only once
     while ($TRUE) {
+
+        # Force Icinga for Windows Garbage Collection
+        Optimize-IcingaForWindowsMemory -ClearErrorStack;
+
         $Connection = Open-IcingaTCPClientConnection `
             -Client (New-IcingaTCPClient -Socket $Socket) `
             -Certificate $Certificate;
@@ -156,10 +160,5 @@ function New-IcingaForWindowsRESTApi()
             $ExMsg = $_.Exception.Message;
             Write-IcingaEventMessage -Namespace 'RESTApi' -EvenId 2050 -Objects $ExMsg;
         }
-
-        # Cleanup the error stack and remove not required data
-        $Error.Clear();
-        # Force PowerShell to call the garbage collector to free memory
-        [System.GC]::Collect();
     }
 }

lib/daemons/RestAPI/threads/New-IcingaForWindowsRESTThread.psm1

@@ -36,8 +36,8 @@ function New-IcingaForWindowsRESTThread()
 
             # Read the received message from the stream by using our smart functions
             [string]$RestMessage = Read-IcingaTCPStream -Client $Connection.Client -Stream $Connection.Stream;
-            # Now properly translate the entire rest message to a parseable hashtable
+            # Now properly translate the entire rest message to a parsable hashtable
-            $RESTRequest         = Read-IcingaRestMessage -RestMessage $RestMessage -Connection $Connection;
+            $RESTRequest         = Read-IcingaRESTMessage -RestMessage $RestMessage -Connection $Connection;
 
             if ($null -ne $RESTRequest) {
 
@@ -110,9 +110,7 @@ function New-IcingaForWindowsRESTThread()
             Close-IcingaTCPConnection -Client $Connection.Client;
         }
 
-        # Cleanup the error stack and remove not required data
+        # Force Icinga for Windows Garbage Collection
-        $Error.Clear();
+        Optimize-IcingaForWindowsMemory -ClearErrorStack;
-        # Force PowerShell to call the garbage collector to free memory
-        [System.GC]::Collect();
     }
 }

lib/daemons/ServiceCheckDaemon/Add-IcingaServiceCheckTask.psm1

@@ -143,9 +143,6 @@ function Add-IcingaServiceCheckTask()
                 Write-IcingaConsoleError 'Failed to handle check result processing: {0}' -Objects $ErrMsg;
             }
 
-            # Cleanup the error stack and remove not required data
-            $Error.Clear();
-
             # Always ensure our check data is cleared regardless of possible
             # exceptions which might occur
             Get-IcingaCheckSchedulerPerfData | Out-Null;
@@ -158,7 +155,7 @@ function Add-IcingaServiceCheckTask()
 
         $PassedTime += 1;
         Start-Sleep -Seconds 1;
-        # Force PowerShell to call the garbage collector to free memory
+        # Force Icinga for Windows Garbage Collection
-        [System.GC]::Collect();
+        Optimize-IcingaForWindowsMemory -ClearErrorStack;
     }
 }