upstream/icinga-powershell-framework
1
0
Fork 0
mirror of https://github.com/Icinga/icinga-powershell-framework.git synced 2025-12-20 23:00:35 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Adds feature to detect frozen threads

Browse source
This commit is contained in:
Lord Hepipud 2022-03-18 20:58:56 +01:00
parent 722b8ba838
commit e4ddbea4d6
13 changed files with 186 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
doc/100-General/10-Changelog.md
View file

@ -25,6 +25,7 @@ Released closed milestones can be found on [GitHub](https://github.com/Icinga/ic
* [#469](https://github.com/Icinga/icinga-powershell-framework/pull/469) Improves plugin doc generator to allow multi-lines in code examples and updates plugin overview as table, adding a short description on what the plugin is for
* [#495](https://github.com/Icinga/icinga-powershell-framework/pull/495) Adds feature to check the sign status for the local Icinga Agent certificate and notifying the user, in case the certificate is not yet signed by the Icinga CA
* [#496](https://github.com/Icinga/icinga-powershell-framework/pull/496) Improves REST-Api default timeout for internal plugin execution calls from 30s to 120s
* [#498](https://github.com/Icinga/icinga-powershell-framework/pull/498) Adds feature for thread queuing optimisation and frozen thread detection for REST calls
## 1.8.0 (2022-02-08)

2
lib/core/framework/New-IcingaEnvironmentVariable.psm1
View file

@ -50,6 +50,7 @@ function New-IcingaEnvironmentVariable()
$Global:Icinga.Public.Add('Daemons', @{ });
$Global:Icinga.Public.Add('Threads', @{ });
$Global:Icinga.Public.Add('ThreadPools', @{ });
$Global:Icinga.Public.Add('ThreadAliveHousekeeping', @{ });
}
# Session specific configuration which should never be modified by users!
@ -60,5 +61,6 @@ function New-IcingaEnvironmentVariable()
$Global:Icinga.Protected.Add('JEAContext', $FALSE);
$Global:Icinga.Protected.Add('RunAsDaemon', $FALSE);
$Global:Icinga.Protected.Add('Minimal', $FALSE);
$Global:Icinga.Protected.Add('ThreadName', '');
}
}

6
lib/core/logging/Icinga_EventLog_Enums.psm1
View file

@ -110,6 +110,12 @@ if ($null -eq $IcingaEventLogEnums -Or $IcingaEventLogEnums.ContainsKey('Framewo
'Details' = 'The local Icinga Agent certificate seems not to be signed by our Icinga CA yet. Using this certificate for the REST-Api as example might not work yet. Please check the state of the certificate and complete the signing process if required [IWKB000013]';
'EventId' = 1506;
};
1507 = @{
'EntryType' = 'Error';
'Message' = 'An internal threading error occurred. A frozen thread was detected';
'Details' = 'One of the internal Icinga for Windows threads was being active but not responding for at least 5 minutes. The frozen thread has been terminated and restarted.';
'EventId' = 1507;
};
1550 = @{
'EntryType' = 'Error';
'Message' = 'Unsupported web authentication used';

62
lib/core/thread/New-IcingaThreadInstance.psm1
View file

@ -1,24 +1,40 @@
function New-IcingaThreadInstance()
{
param (
[string]$Name,
$ThreadPool,
[ScriptBlock]$ScriptBlock,
[string]$Command,
[hashtable]$CmdParameters,
[array]$Arguments,
[Switch]$Start
[string]$Name = '',
[string]$ThreadName = $null,
$ThreadPool = $null,
[ScriptBlock]$ScriptBlock = $null,
[string]$Command = '',
[hashtable]$CmdParameters = @{ },
[array]$Arguments = @(),
[Switch]$Start = $FALSE,
[switch]$CheckAliveState = $FALSE
);
$CallStack = Get-PSCallStack;
$SourceCommand = $CallStack[1].Command;
if ([string]::IsNullOrEmpty($ThreadName)) {
$CallStack = Get-PSCallStack;
$SourceCommand = $CallStack[1].Command;
if ([string]::IsNullOrEmpty($Name)) {
$Name = New-IcingaThreadHash -ShellScript $ScriptBlock -Arguments $Arguments;
if ([string]::IsNullOrEmpty($Name)) {
$Name = New-IcingaThreadHash -ShellScript $ScriptBlock -Arguments $Arguments;
}
$ThreadName = [string]::Format('{0}::{1}::{2}::0', $SourceCommand, $Command, $Name);
[int]$ThreadIndex = 0;
while ($TRUE) {
if ($Global:Icinga.Public.Threads.ContainsKey($ThreadName) -eq $FALSE) {
break;
}
$ThreadIndex += 1;
$ThreadName = [string]::Format('{0}::{1}::{2}::{3}', $SourceCommand, $Command, $Name, $ThreadIndex);
}
}
$ThreadName = [string]::Format('{0}::{1}::{2}::0', $SourceCommand, $Command, $Name);
Write-IcingaDebugMessage -Message (
[string]::Format(
'Creating new thread instance {0}{1}Arguments:{1}{2}',
@ -51,6 +67,9 @@ function New-IcingaThreadInstance()
[void]$Shell.AddParameter('JeaEnabled', $Global:Icinga.Protected.JEAContext);
}
[void]$Shell.AddCommand('Set-IcingaEnvironmentThreadName');
[void]$Shell.AddParameter('ThreadName', $ThreadName);
[void]$Shell.AddCommand($Command);
$CodeHash = $Command;
@ -94,16 +113,13 @@ function New-IcingaThreadInstance()
Add-Member -InputObject $Thread -MemberType NoteProperty -Name Started -Value $FALSE;
}
[int]$ThreadIndex = 0;
$Global:Icinga.Public.Threads.Add($ThreadName, $Thread);
while ($TRUE) {
if ($Global:Icinga.Public.Threads.ContainsKey($ThreadName) -eq $FALSE) {
$Global:Icinga.Public.Threads.Add($ThreadName, $Thread);
break;
}
$ThreadIndex += 1;
$ThreadName = [string]::Format('{0}::{1}::{2}::{3}', $SourceCommand, $Command, $Name, $ThreadIndex);
if ($CheckAliveState) {
Set-IcingaForWindowsThreadAlive `
-ThreadName $ThreadName `
-ThreadCmd $Command `
-ThreadArgs $CmdParameters `
-ThreadPool $ThreadPool;
}
}

2
lib/core/thread/Remove-IcingaThread.psm1
View file

@ -1,7 +1,7 @@
function Remove-IcingaThread()
{
param(
[string]$Thread
[string]$Thread = ''
);
if ([string]::IsNullOrEmpty($Thread)) {

8
lib/core/thread/Set-IcingaEnvironmentThreadName.psm1 Normal file
View file

@ -0,0 +1,8 @@
function Set-IcingaEnvironmentThreadName()
{
param (
[string]$ThreadName = ''
);
$Global:Icinga.Protected.ThreadName = $ThreadName;
}

4
lib/core/thread/Stop-IcingaThread.psm1
View file

@ -1,7 +1,7 @@
function Stop-IcingaThread()
{
param(
[string]$Thread
param (
[string]$Thread = ''
);
if ([string]::IsNullOrEmpty($Thread)) {

3
lib/daemon/Add-IcingaForWindowsDaemon.psm1
View file

@ -21,5 +21,8 @@ function Add-IcingaForWindowsDaemon()
while ($TRUE) {
Start-Sleep -Seconds 1;
# Handle possible threads being frozen
Suspend-IcingaForWindowsFrozenThreads;
}
}

39
lib/daemon/Set-IcingaForWindowsThreadAlive.psm1 Normal file
View file

@ -0,0 +1,39 @@
function Set-IcingaForWindowsThreadAlive()
{
param (
[string]$ThreadName = '',
[string]$ThreadCmd = '',
$ThreadPool = $null,
[hashtable]$ThreadArgs = @{ },
[switch]$Active = $FALSE,
[hashtable]$TerminateAction = @{ }
);
if ([string]::IsNullOrEmpty($ThreadName)) {
return;
}
if ($Global:Icinga.Public.ThreadAliveHousekeeping.ContainsKey($ThreadName) -eq $FALSE) {
if ($null -eq $ThreadPool) {
return;
}
$Global:Icinga.Public.ThreadAliveHousekeeping.Add(
$ThreadName,
@{
'LastSeen' = [DateTime]::Now;
'Command' = $ThreadCmd;
'Arguments' = $ThreadArgs;
'ThreadPool' = $ThreadPool;
'Active' = [bool]$Active;
'TerminateAction' = $TerminateAction;
}
);
return;
}
$Global:Icinga.Public.ThreadAliveHousekeeping[$ThreadName].LastSeen = [DateTime]::Now;
$Global:Icinga.Public.ThreadAliveHousekeeping[$ThreadName].Active = [bool]$Active;
$Global:Icinga.Public.ThreadAliveHousekeeping[$ThreadName].TerminateAction = $TerminateAction;
}

51
lib/daemon/Suspend-IcingaForWindowsFrozenThreads.psm1 Normal file
View file

@ -0,0 +1,51 @@
function Suspend-IcingaForWindowsFrozenThreads()
{
try {
[array]$ConfiguredThreads = $Global:Icinga.Public.ThreadAliveHousekeeping.Keys;
foreach ($thread in $ConfiguredThreads) {
$ThreadConfig = $Global:Icinga.Public.ThreadAliveHousekeeping[$thread];
# Only check active threads
if ($ThreadConfig.Active -eq $FALSE) {
continue;
}
# Check if the thread is active and not doing something for 5 minutes
if (([DateTime]::Now - $ThreadConfig.LastSeen).TotalSeconds -lt 300) {
continue;
}
# If it does, kill the thread
Remove-IcingaThread -Thread $thread;
if ($ThreadConfig.TerminateAction.Count -ne 0) {
$TerminateArguments = @{ };
if ($ThreadConfig.TerminateAction.ContainsKey('Arguments')) {
$TerminateArguments = $ThreadConfig.TerminateAction.Arguments;
}
if ($ThreadConfig.TerminateAction.ContainsKey('Command')) {
$TerminateCmd = $ThreadConfig.TerminateAction.Command;
if ([string]::IsNullOrEmpty($TerminateCmd) -eq $FALSE) {
& $TerminateCmd @TerminateArguments | Out-Null;
}
}
}
# Now restart it
New-IcingaThreadInstance `
-ThreadName $thread `
-ThreadPool $ThreadConfig.ThreadPool `
-Command $ThreadConfig.Command `
-CmdParameters $ThreadConfig.Arguments `
-Start `
-CheckAliveState;
Write-IcingaEventMessage -EventId 1507 -Namespace 'Framework' -Objects $thread;
}
} catch {
# Nothing to do here
}
}

24
lib/daemons/RestAPI/threads/Get-IcingaNextRESTApiThreadId.psm1
View file

@ -1,5 +1,29 @@
function Get-IcingaNextRESTApiThreadId()
{
# Improve our thread management by distributing new REST requests to a non-active thread
[array]$ConfiguredThreads = $Global:Icinga.Public.ThreadAliveHousekeeping.Keys;
foreach ($thread in $ConfiguredThreads) {
if ($thread.ToLower() -NotLike 'Start-IcingaForWindowsRESTThread::New-IcingaForWindowsRESTThread::CheckThread::*') {
continue;
}
$ThreadConfig = $Global:Icinga.Public.ThreadAliveHousekeeping[$thread];
# If our thread is busy, skip this one and check for another one
if ($ThreadConfig.Active) {
continue;
}
$ThreadIndex = $thread.Replace('Start-IcingaForWindowsRESTThread::New-IcingaForWindowsRESTThread::CheckThread::', '');
if (Test-Numeric $ThreadIndex) {
$Global:Icinga.Public.Daemons.RESTApi.LastThreadId = [int]$ThreadIndex;
return ([int]$ThreadIndex)
}
}
# In case we are not having any spare thread left, distribute the thread to the next thread in our list
[int]$ConcurrentThreads = $Global:Icinga.Public.Daemons.RESTApi.TotalThreads - 1;
[int]$LastThreadId = $Global:Icinga.Public.Daemons.RESTApi.LastThreadId + 1;

9
lib/daemons/RestAPI/threads/New-IcingaForWindowsRESTThread.psm1
View file

@ -1,6 +1,6 @@
function New-IcingaForWindowsRESTThread()
{
param(
param (
$RequireAuth,
$ThreadId
);
@ -69,6 +69,9 @@ function New-IcingaForWindowsRESTThread()
}
}
# Set our thread being active
Set-IcingaForWindowsThreadAlive -ThreadName $Global:Icinga.Protected.ThreadName -Active -TerminateAction @{ 'Command' = 'Close-IcingaTCPConnection'; 'Arguments' = @{ 'Client' = $Connection.Client } };
# We should remove clients from the blacklist who are sending valid requests
Remove-IcingaRESTClientBlacklist -Client $Connection.Client -ClientList $Global:Icinga.Public.Daemons.RESTApi.ClientBlacklist;
switch (Get-IcingaRESTPathElement -Request $RESTRequest -Index 0) {
@ -85,6 +88,10 @@ function New-IcingaForWindowsRESTThread()
) -Stream $Connection.Stream;
};
}
# set our thread no longer be active. We do this, because below there is no way we can
# actually get stuck on a endless loop, caused by external modules
Set-IcingaForWindowsThreadAlive -ThreadName $Global:Icinga.Protected.ThreadName;
}
} catch {
$ExMsg = $_.Exception.Message;

3
lib/daemons/RestAPI/threads/Start-IcingaForWindowsRESTThread.psm1
View file

@ -15,5 +15,6 @@ function Start-IcingaForWindowsRESTThread()
'RequireAuth' = $RequireAuth;
'ThreadId' = $ThreadId;
} `
-Start;
-Start `
-CheckAliveState;
}