From 5ed5f0b5e2bf25672ac5c9025a77eff5e03b98f0 Mon Sep 17 00:00:00 2001 From: Lord Hepipud Date: Wed, 17 Nov 2021 19:06:09 +0100 Subject: [PATCH] Fixes Icinga for Windows memory leak --- doc/100-General/10-Changelog.md | 1 + .../framework/Read-IcingaWindowsEventLog.psm1 | 4 +- .../Optimize-IcingaForWindowsMemory.psm1 | 60 +++++++++++++++++++ .../daemon/New-IcingaForWindowsRESTApi.psm1 | 13 ++-- .../New-IcingaForWindowsRESTThread.psm1 | 10 ++-- .../Add-IcingaServiceCheckTask.psm1 | 7 +-- 6 files changed, 75 insertions(+), 20 deletions(-) create mode 100644 lib/core/tools/Optimize-IcingaForWindowsMemory.psm1 diff --git a/doc/100-General/10-Changelog.md b/doc/100-General/10-Changelog.md index 6a7dac7..eb7f6ae 100644 --- a/doc/100-General/10-Changelog.md +++ b/doc/100-General/10-Changelog.md @@ -13,6 +13,7 @@ Released closed milestones can be found on [GitHub](https://github.com/Icinga/ic ### Bugfixes +* [#379](https://github.com/Icinga/icinga-powershell-framework/issues/379) Fixes memory leak for Icinga for Windows by using a custom function being more aggressive on memory cleanup * [#402](https://github.com/Icinga/icinga-powershell-framework/pull/402) Fixes missing address attribute for REST-Api daemon, making it unable to change the listening address * [#403](https://github.com/Icinga/icinga-powershell-framework/pull/403) Fixes memory leak on newly EventLog reader for CLI event stream * [#407](https://github.com/Icinga/icinga-powershell-framework/pull/407) Removes unnecessary module import inside `Invoke-IcingaNamespaceCmdlets` diff --git a/lib/core/framework/Read-IcingaWindowsEventLog.psm1 b/lib/core/framework/Read-IcingaWindowsEventLog.psm1 index 2adbcfe..cbb350d 100644 --- a/lib/core/framework/Read-IcingaWindowsEventLog.psm1 +++ b/lib/core/framework/Read-IcingaWindowsEventLog.psm1 @@ -66,7 +66,7 @@ function Read-IcingaWindowsEventLog() } Start-Sleep -Seconds 1; - # Force PowerShell to call the garbage collector to free memory - [System.GC]::Collect(); + # Force Icinga for Windows Garbage Collection + Optimize-IcingaForWindowsMemory -ClearErrorStack; } } diff --git a/lib/core/tools/Optimize-IcingaForWindowsMemory.psm1 b/lib/core/tools/Optimize-IcingaForWindowsMemory.psm1 new file mode 100644 index 0000000..d37766a --- /dev/null +++ b/lib/core/tools/Optimize-IcingaForWindowsMemory.psm1 @@ -0,0 +1,60 @@ +<# +.SYNOPSIS + Calls the PowerShell Garbage-Collector to force freeing memory + in a more aggressive way. Also fixes an issue on older PowerShell + versions on which the Garbage-Collector is not clearing memory + properly +.DESCRIPTION + Calls the PowerShell Garbage-Collector to force freeing memory + in a more aggressive way. Also fixes an issue on older PowerShell + versions on which the Garbage-Collector is not clearing memory + properly +.PARAMETER ClearErrorStack + Also clears the current error stack to free additional memory +.EXAMPLE + Optimize-IcingaForWindowsMemory; +.EXAMPLE + Optimize-IcingaForWindowsMemory -ClearErrorStack; +.NOTES + Clears memory used by PowerShell in a more aggressive way +#> +function Optimize-IcingaForWindowsMemory() +{ + param ( + [switch]$ClearErrorStack = $FALSE + ); + + # Clear all errors within our error stack + if ($ClearErrorStack) { + $Error.Clear(); + } + + # Force the full collection of memory being used + [decimal]$MemoryConsumption = [System.GC]::GetTotalMemory('forcefullcollection'); + # Collect all generations of objects inside our memory and force the flushing of + # objects + [System.GC]::Collect([System.GC]::MaxGeneration, [System.GCCollectionMode]::Forced); + # Collect the remaining memory for a before/after delta + [decimal]$CollectedMemory = [System.GC]::GetTotalMemory('forcefullcollection'); + # Just for debugging + [decimal]$MemoryDelta = $MemoryConsumption - $CollectedMemory; + [bool]$Negate = $FALSE; + + if ($MemoryDelta -lt 0) { + $MemoryDelta = $MemoryDelta * -1; + $Negate = $TRUE; + } + + $MemoryConsumptionMiB = Convert-Bytes -Value $MemoryConsumption -Unit 'MiB'; + $CollectedMemoryMiB = Convert-Bytes -Value $CollectedMemory -Unit 'MiB'; + $MemoryDeltaMiB = Convert-Bytes -Value $MemoryDelta -Unit 'MiB'; + + # Print an optional debug message, allowing us to analyze memory behavior over time + Write-IcingaDebugMessage ` + -Message 'Calling Icinga for Windows Garbage-Collector. Memory overview below (Before, After, Flushed/Added)' ` + -Objects @( + ([string]::Format('{0} MiB', $MemoryConsumptionMiB.Value)), + ([string]::Format('{0} MiB', $CollectedMemoryMiB.Value)), + ([string]::Format('{1}{0} MiB', $MemoryDeltaMiB.Value, (&{ if ($Negate) { return '-'; } else { return ''; } }))) + ); +} diff --git a/lib/daemons/RestAPI/daemon/New-IcingaForWindowsRESTApi.psm1 b/lib/daemons/RestAPI/daemon/New-IcingaForWindowsRESTApi.psm1 index 49d360a..1359ad2 100644 --- a/lib/daemons/RestAPI/daemon/New-IcingaForWindowsRESTApi.psm1 +++ b/lib/daemons/RestAPI/daemon/New-IcingaForWindowsRESTApi.psm1 @@ -73,7 +73,7 @@ function New-IcingaForWindowsRESTApi() [bool]$Success = Add-IcingaHashtableItem -Hashtable $RestDaemon.RegisteredEndpoints ` -Key $entry.Alias ` -Value $entry.Command; - + if ($Success -eq $FALSE) { Write-IcingaEventMessage ` -EventId 2100 ` @@ -89,7 +89,7 @@ function New-IcingaForWindowsRESTApi() [bool]$Success = Add-IcingaHashtableItem -Hashtable $RestDaemon.CommandAliases ` -Key $component ` -Value $entry[$component]; - + if ($Success -eq $FALSE) { Write-IcingaEventMessage ` -EventId 2101 ` @@ -123,6 +123,10 @@ function New-IcingaForWindowsRESTApi() # being executed. This is required to ensure we will execute # the code frequently instead of only once while ($TRUE) { + + # Force Icinga for Windows Garbage Collection + Optimize-IcingaForWindowsMemory -ClearErrorStack; + $Connection = Open-IcingaTCPClientConnection ` -Client (New-IcingaTCPClient -Socket $Socket) ` -Certificate $Certificate; @@ -156,10 +160,5 @@ function New-IcingaForWindowsRESTApi() $ExMsg = $_.Exception.Message; Write-IcingaEventMessage -Namespace 'RESTApi' -EvenId 2050 -Objects $ExMsg; } - - # Cleanup the error stack and remove not required data - $Error.Clear(); - # Force PowerShell to call the garbage collector to free memory - [System.GC]::Collect(); } } diff --git a/lib/daemons/RestAPI/threads/New-IcingaForWindowsRESTThread.psm1 b/lib/daemons/RestAPI/threads/New-IcingaForWindowsRESTThread.psm1 index adb19ea..c1ae195 100644 --- a/lib/daemons/RestAPI/threads/New-IcingaForWindowsRESTThread.psm1 +++ b/lib/daemons/RestAPI/threads/New-IcingaForWindowsRESTThread.psm1 @@ -36,8 +36,8 @@ function New-IcingaForWindowsRESTThread() # Read the received message from the stream by using our smart functions [string]$RestMessage = Read-IcingaTCPStream -Client $Connection.Client -Stream $Connection.Stream; - # Now properly translate the entire rest message to a parseable hashtable - $RESTRequest = Read-IcingaRestMessage -RestMessage $RestMessage -Connection $Connection; + # Now properly translate the entire rest message to a parsable hashtable + $RESTRequest = Read-IcingaRESTMessage -RestMessage $RestMessage -Connection $Connection; if ($null -ne $RESTRequest) { @@ -110,9 +110,7 @@ function New-IcingaForWindowsRESTThread() Close-IcingaTCPConnection -Client $Connection.Client; } - # Cleanup the error stack and remove not required data - $Error.Clear(); - # Force PowerShell to call the garbage collector to free memory - [System.GC]::Collect(); + # Force Icinga for Windows Garbage Collection + Optimize-IcingaForWindowsMemory -ClearErrorStack; } } diff --git a/lib/daemons/ServiceCheckDaemon/Add-IcingaServiceCheckTask.psm1 b/lib/daemons/ServiceCheckDaemon/Add-IcingaServiceCheckTask.psm1 index 8649ea9..33fc4f8 100644 --- a/lib/daemons/ServiceCheckDaemon/Add-IcingaServiceCheckTask.psm1 +++ b/lib/daemons/ServiceCheckDaemon/Add-IcingaServiceCheckTask.psm1 @@ -143,9 +143,6 @@ function Add-IcingaServiceCheckTask() Write-IcingaConsoleError 'Failed to handle check result processing: {0}' -Objects $ErrMsg; } - # Cleanup the error stack and remove not required data - $Error.Clear(); - # Always ensure our check data is cleared regardless of possible # exceptions which might occur Get-IcingaCheckSchedulerPerfData | Out-Null; @@ -158,7 +155,7 @@ function Add-IcingaServiceCheckTask() $PassedTime += 1; Start-Sleep -Seconds 1; - # Force PowerShell to call the garbage collector to free memory - [System.GC]::Collect(); + # Force Icinga for Windows Garbage Collection + Optimize-IcingaForWindowsMemory -ClearErrorStack; } }