upstream/icinga-powershell-framework
1
0
Fork 0
mirror of https://github.com/Icinga/icinga-powershell-framework.git synced 2025-12-20 23:00:35 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix memory leak on background service daemon

Browse source
This commit is contained in:
Lord Hepipud 2021-03-08 14:15:50 +01:00
parent 20a4c8d4d3
commit 637d06e05d
4 changed files with 62 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
doc/31-Changelog.md
View file

@ -13,11 +13,12 @@ Released closed milestones can be found on [GitHub](https://github.com/Icinga/ic
### Bugfixes
## 1.4.1 (pending)
## 1.4.1 (2021-03-10)
### Bugfixes
* [#222](https://github.com/Icinga/icinga-powershell-framework/pull/222) Fixes an issue with [Secure.String] arguments for PowerShell plugins, caused by `ConvertTo-IcingaSecureString` Cmdlet not being pre-loaded
* [#224](https://github.com/Icinga/icinga-powershell-framework/issues/224) Fixes "memory leak" on background daemon for registered service checks, by clearing the error stack and manually calling the PowerShell garbage collector to force freeing of memory
## 1.4.0 (2021-03-02)

5
lib/core/logging/Write-IcingaConsoleOutput.psm1
View file

@ -40,6 +40,11 @@ function Write-IcingaConsoleOutput()
return;
}
# Never write console output in case the Framework is running as daemon
if ($null -ne $global:IcingaDaemonData -And $null -ne $global:IcingaDaemonData.FrameworkRunningAsDaemon -And $global:IcingaDaemonData.FrameworkRunningAsDaemon -eq $TRUE) {
return;
}
$OutputMessage = $Message;
[int]$Index = 0;

67
lib/daemons/ServiceCheckDaemon/Start-IcingaServiceCheckDaemon.psm1
View file

@ -66,7 +66,6 @@ function Start-IcingaServiceCheckTask()
Use-Icinga -LibOnly -Daemon;
$PassedTime = 0;
$SortedResult = $null;
$OldData = @{ };
$PerfCache = @{ };
$AverageCalc = @{ };
[int]$MaxTime = 0;
@ -85,8 +84,8 @@ function Start-IcingaServiceCheckTask()
$AverageCalc.Add(
[string]$index,
@{
'Interval' = [int]$index;
'Time' = [int]$index * 60;
'Interval' = ([int]$index);
'Time' = ([int]$index * 60);
'Sum' = 0;
'Count' = 0;
}
@ -126,19 +125,26 @@ function Start-IcingaServiceCheckTask()
if ($PassedTime -ge $Interval) {
try {
& $CheckCommand @Arguments | Out-Null;
} catch {
# Just for debugging. Not required in production or usable at all
$ErrMsg = $_.Exception.Message;
Write-IcingaConsoleError $ErrMsg;
}
Get-IcingaCheckSchedulerPerfData | Out-Null;
Get-IcingaCheckSchedulerPluginOutput | Out-Null;
try {
$UnixTime = Get-IcingaUnixTime;
foreach ($result in $global:Icinga.CheckData[$CheckCommand]['results'].Keys) {
[string]$HashIndex = $result;
$SortedResult = $global:Icinga.CheckData[$CheckCommand]['results'][$HashIndex].GetEnumerator() | Sort-Object name -Descending;
Add-IcingaHashtableItem -Hashtable $OldData -Key $HashIndex -Value @{ } | Out-Null;
Add-IcingaHashtableItem -Hashtable $PerfCache -Key $HashIndex -Value @{ } | Out-Null;
foreach ($timeEntry in $SortedResult) {
if ((Test-Numeric $timeEntry.Value) -eq $FALSE) {
continue;
}
foreach ($calc in $AverageCalc.Keys) {
if (($UnixTime - $AverageCalc[$calc].Time) -le [int]$timeEntry.Key) {
$AverageCalc[$calc].Sum += $timeEntry.Value;
@ -147,20 +153,20 @@ function Start-IcingaServiceCheckTask()
}
if (($UnixTime - $MaxTimeInSeconds) -le [int]$timeEntry.Key) {
Add-IcingaHashtableItem -Hashtable $PerfCache[$HashIndex] -Key ([string]$timeEntry.Key) -Value ([string]$timeEntry.Value) | Out-Null;
} else {
Add-IcingaHashtableItem -Hashtable $OldData[$HashIndex] -Key $timeEntry -Value $null | Out-Null;
}
}
foreach ($calc in $AverageCalc.Keys) {
$AverageValue = ($AverageCalc[$calc].Sum / $AverageCalc[$calc].Count);
[string]$MetricName = Format-IcingaPerfDataLabel (
[string]::Format('{0}_{1}', $HashIndex, $AverageCalc[$calc].Interval)
);
if ($AverageCalc[$calc].Count -ne 0) {
$AverageValue = ($AverageCalc[$calc].Sum / $AverageCalc[$calc].Count);
[string]$MetricName = Format-IcingaPerfDataLabel (
[string]::Format('{0}_{1}', $HashIndex, $AverageCalc[$calc].Interval)
);
Add-IcingaHashtableItem `
-Hashtable $global:Icinga.CheckData[$CheckCommand]['average'] `
-Key $MetricName -Value $AverageValue -Override | Out-Null;
Add-IcingaHashtableItem `
-Hashtable $global:Icinga.CheckData[$CheckCommand]['average'] `
-Key $MetricName -Value $AverageValue -Override | Out-Null;
}
$AverageCalc[$calc].Sum = 0;
$AverageCalc[$calc].Count = 0;
@ -168,9 +174,16 @@ function Start-IcingaServiceCheckTask()
}
# Flush data we no longer require in our cache to free memory
foreach ($entry in $OldData.Keys) {
foreach ($key in $OldData[$entry].Keys) {
Remove-IcingaHashtableItem -Hashtable $global:Icinga.CheckData[$CheckCommand]['results'][$entry] -Key $key.Name;
[array]$CheckStores = $global:Icinga.CheckData[$CheckCommand]['results'].Keys;
foreach ($CheckStore in $CheckStores) {
[string]$CheckKey = $CheckStore;
[array]$CheckTimeStamps = $global:Icinga.CheckData[$CheckCommand]['results'][$CheckKey].Keys;
foreach ($TimeSample in $CheckTimeStamps) {
if (($UnixTime - $MaxTimeInSeconds) -gt [int]$TimeSample) {
Remove-IcingaHashtableItem -Hashtable $global:Icinga.CheckData[$CheckCommand]['results'][$CheckKey] -Key ([string]$TimeSample);
}
}
}
@ -178,16 +191,28 @@ function Start-IcingaServiceCheckTask()
# Write collected metrics to disk in case we reload the daemon. We will load them back into the module after reload then
Set-IcingaCacheData -Space 'sc_daemon' -CacheStore 'checkresult_store' -KeyName $CheckCommand -Value $PerfCache;
} catch {
# Todo: Add error reporting / handling
# Just for debugging. Not required in production or usable at all
$ErrMsg = $_.Exception.Message;
Write-IcingaConsoleError 'Failed to handle check result processing: {0}' -Objects $ErrMsg;
}
# Cleanup the error stack and remove not required data
$Error.Clear();
# Always ensure our check data is cleared regardless of possible
# exceptions which might occur
Get-IcingaCheckSchedulerPerfData | Out-Null;
Get-IcingaCheckSchedulerPluginOutput | Out-Null;
$PassedTime = 0;
$SortedResult.Clear();
$OldData.Clear();
$PerfCache.Clear();
}
$PassedTime += 1;
Start-Sleep -Seconds 1;
# Force PowerShell to call the garbage collector to free memory
[System.GC]::Collect();
}
};

10
lib/icinga/plugin/New-IcingaCheck.psm1
View file

@ -63,8 +63,16 @@ function New-IcingaCheck()
);
}
# Fix possible error for identical time stamps due to internal exceptions
# and check execution within the same time slot because of this
[string]$TimeIndex = Get-IcingaUnixTime;
if ($global:Icinga.CheckData[$this.checkcommand]['results'][$this.name].ContainsKey($TimeIndex)) {
return;
}
$global:Icinga.CheckData[$this.checkcommand]['results'][$this.name].Add(
(Get-IcingaUnixTime),
$TimeIndex,
$this.value
);
}