Merge pull request #498 from Icinga:feature/thread_keep_alive_and_housekeeping_on_freeze

Feature: Add thread queuing optimisation and frozen thread detection

Adds feature to check for frozen threads on REST-Api, ensuring that non-responding threads are killed after 5 minutes without progress and restartet.
Also improves queing of REST-Api tasks into threads, which now prioritizes to check for inactive threads first to enque new calls and falls back to old behaviour, in case all threads are busy.

doc/100-General/10-Changelog.md

@@ -25,6 +25,7 @@ Released closed milestones can be found on [GitHub](https://github.com/Icinga/ic
 * [#469](https://github.com/Icinga/icinga-powershell-framework/pull/469) Improves plugin doc generator to allow multi-lines in code examples and updates plugin overview as table, adding a short description on what the plugin is for
 * [#495](https://github.com/Icinga/icinga-powershell-framework/pull/495) Adds feature to check the sign status for the local Icinga Agent certificate and notifying the user, in case the certificate is not yet signed by the Icinga CA
 * [#496](https://github.com/Icinga/icinga-powershell-framework/pull/496) Improves REST-Api default timeout for internal plugin execution calls from 30s to 120s
+* [#498](https://github.com/Icinga/icinga-powershell-framework/pull/498) Adds feature for thread queuing optimisation and frozen thread detection for REST calls
 
 ## 1.8.0 (2022-02-08)
 

lib/core/framework/New-IcingaEnvironmentVariable.psm1

@@ -50,6 +50,7 @@ function New-IcingaEnvironmentVariable()
         $Global:Icinga.Public.Add('Daemons', @{ });
         $Global:Icinga.Public.Add('Threads', @{ });
         $Global:Icinga.Public.Add('ThreadPools', @{ });
+        $Global:Icinga.Public.Add('ThreadAliveHousekeeping', @{ });
     }
 
     # Session specific configuration which should never be modified by users!
@@ -60,5 +61,6 @@ function New-IcingaEnvironmentVariable()
         $Global:Icinga.Protected.Add('JEAContext', $FALSE);
         $Global:Icinga.Protected.Add('RunAsDaemon', $FALSE);
         $Global:Icinga.Protected.Add('Minimal', $FALSE);
+        $Global:Icinga.Protected.Add('ThreadName', '');
     }
 }

lib/core/logging/Icinga_EventLog_Enums.psm1

@@ -110,6 +110,12 @@ if ($null -eq $IcingaEventLogEnums -Or $IcingaEventLogEnums.ContainsKey('Framewo
                 'Details'   = 'The local Icinga Agent certificate seems not to be signed by our Icinga CA yet. Using this certificate for the REST-Api as example might not work yet. Please check the state of the certificate and complete the signing process if required [IWKB000013]';
                 'EventId'   = 1506;
             };
+            1507 = @{
+                'EntryType' = 'Error';
+                'Message'   = 'An internal threading error occurred. A frozen thread was detected';
+                'Details'   = 'One of the internal Icinga for Windows threads was being active but not responding for at least 5 minutes. The frozen thread has been terminated and restarted.';
+                'EventId'   = 1507;
+            };
             1550 = @{
                 'EntryType' = 'Error';
                 'Message'   = 'Unsupported web authentication used';

lib/core/thread/New-IcingaThreadInstance.psm1

@@ -1,24 +1,40 @@
 function New-IcingaThreadInstance()
 {
     param (
-        [string]$Name,
-        $ThreadPool,
-        [ScriptBlock]$ScriptBlock,
-        [string]$Command,
-        [hashtable]$CmdParameters,
-        [array]$Arguments,
-        [Switch]$Start
+        [string]$Name             = '',
+        [string]$ThreadName       = $null,
+        $ThreadPool               = $null,
+        [ScriptBlock]$ScriptBlock = $null,
+        [string]$Command          = '',
+        [hashtable]$CmdParameters = @{ },
+        [array]$Arguments         = @(),
+        [Switch]$Start            = $FALSE,
+        [switch]$CheckAliveState  = $FALSE
     );
 
-    $CallStack     = Get-PSCallStack;
-    $SourceCommand = $CallStack[1].Command;
+    if ([string]::IsNullOrEmpty($ThreadName)) {
+        $CallStack     = Get-PSCallStack;
+        $SourceCommand = $CallStack[1].Command;
 
-    if ([string]::IsNullOrEmpty($Name)) {
-        $Name = New-IcingaThreadHash -ShellScript $ScriptBlock -Arguments $Arguments;
+        if ([string]::IsNullOrEmpty($Name)) {
+            $Name = New-IcingaThreadHash -ShellScript $ScriptBlock -Arguments $Arguments;
+        }
+
+        $ThreadName = [string]::Format('{0}::{1}::{2}::0', $SourceCommand, $Command, $Name);
+
+        [int]$ThreadIndex = 0;
+
+        while ($TRUE) {
+
+            if ($Global:Icinga.Public.Threads.ContainsKey($ThreadName) -eq $FALSE) {
+                break;
+            }
+
+            $ThreadIndex += 1;
+            $ThreadName   = [string]::Format('{0}::{1}::{2}::{3}', $SourceCommand, $Command, $Name, $ThreadIndex);
+        }
     }
 
-    $ThreadName = [string]::Format('{0}::{1}::{2}::0', $SourceCommand, $Command, $Name);
-
     Write-IcingaDebugMessage -Message (
         [string]::Format(
             'Creating new thread instance {0}{1}Arguments:{1}{2}',
@@ -51,6 +67,9 @@ function New-IcingaThreadInstance()
             [void]$Shell.AddParameter('JeaEnabled', $Global:Icinga.Protected.JEAContext);
         }
 
+        [void]$Shell.AddCommand('Set-IcingaEnvironmentThreadName');
+        [void]$Shell.AddParameter('ThreadName', $ThreadName);
+
         [void]$Shell.AddCommand($Command);
 
         $CodeHash = $Command;
@@ -94,16 +113,13 @@ function New-IcingaThreadInstance()
         Add-Member -InputObject $Thread -MemberType NoteProperty -Name Started -Value $FALSE;
     }
 
-    [int]$ThreadIndex = 0;
+    $Global:Icinga.Public.Threads.Add($ThreadName, $Thread);
 
-    while ($TRUE) {
-
-        if ($Global:Icinga.Public.Threads.ContainsKey($ThreadName) -eq $FALSE) {
-            $Global:Icinga.Public.Threads.Add($ThreadName, $Thread);
-            break;
-        }
-
-        $ThreadIndex += 1;
-        $ThreadName   = [string]::Format('{0}::{1}::{2}::{3}', $SourceCommand, $Command, $Name, $ThreadIndex);
+    if ($CheckAliveState) {
+        Set-IcingaForWindowsThreadAlive `
+            -ThreadName $ThreadName `
+            -ThreadCmd $Command `
+            -ThreadArgs $CmdParameters `
+            -ThreadPool $ThreadPool;
     }
 }

lib/core/thread/Remove-IcingaThread.psm1

@@ -1,7 +1,7 @@
 function Remove-IcingaThread()
 {
     param(
-        [string]$Thread
+        [string]$Thread = ''
     );
 
     if ([string]::IsNullOrEmpty($Thread)) {

lib/core/thread/Set-IcingaEnvironmentThreadName.psm1

@@ -0,0 +1,8 @@
+function Set-IcingaEnvironmentThreadName()
+{
+    param (
+        [string]$ThreadName = ''
+    );
+
+    $Global:Icinga.Protected.ThreadName = $ThreadName;
+}

lib/core/thread/Stop-IcingaThread.psm1

@@ -1,7 +1,7 @@
 function Stop-IcingaThread()
 {
-    param(
-        [string]$Thread
+    param (
+        [string]$Thread = ''
     );
 
     if ([string]::IsNullOrEmpty($Thread)) {

lib/daemon/Add-IcingaForWindowsDaemon.psm1

@@ -21,5 +21,8 @@ function Add-IcingaForWindowsDaemon()
 
     while ($TRUE) {
         Start-Sleep -Seconds 1;
+
+        # Handle possible threads being frozen
+        Suspend-IcingaForWindowsFrozenThreads;
     }
 }

lib/daemon/Set-IcingaForWindowsThreadAlive.psm1

@@ -0,0 +1,39 @@
+function Set-IcingaForWindowsThreadAlive()
+{
+    param (
+        [string]$ThreadName         = '',
+        [string]$ThreadCmd          = '',
+        $ThreadPool                 = $null,
+        [hashtable]$ThreadArgs      = @{ },
+        [switch]$Active             = $FALSE,
+        [hashtable]$TerminateAction = @{ }
+    );
+
+    if ([string]::IsNullOrEmpty($ThreadName)) {
+        return;
+    }
+
+    if ($Global:Icinga.Public.ThreadAliveHousekeeping.ContainsKey($ThreadName) -eq $FALSE) {
+        if ($null -eq $ThreadPool) {
+            return;
+        }
+
+        $Global:Icinga.Public.ThreadAliveHousekeeping.Add(
+            $ThreadName,
+            @{
+                'LastSeen'        = [DateTime]::Now;
+                'Command'         = $ThreadCmd;
+                'Arguments'       = $ThreadArgs;
+                'ThreadPool'      = $ThreadPool;
+                'Active'          = [bool]$Active;
+                'TerminateAction' = $TerminateAction;
+            }
+        );
+
+        return;
+    }
+
+    $Global:Icinga.Public.ThreadAliveHousekeeping[$ThreadName].LastSeen        = [DateTime]::Now;
+    $Global:Icinga.Public.ThreadAliveHousekeeping[$ThreadName].Active          = [bool]$Active;
+    $Global:Icinga.Public.ThreadAliveHousekeeping[$ThreadName].TerminateAction = $TerminateAction;
+}

lib/daemon/Suspend-IcingaForWindowsFrozenThreads.psm1

@@ -0,0 +1,51 @@
+function Suspend-IcingaForWindowsFrozenThreads()
+{
+    try {
+        [array]$ConfiguredThreads = $Global:Icinga.Public.ThreadAliveHousekeeping.Keys;
+
+        foreach ($thread in $ConfiguredThreads) {
+            $ThreadConfig = $Global:Icinga.Public.ThreadAliveHousekeeping[$thread];
+
+            # Only check active threads
+            if ($ThreadConfig.Active -eq $FALSE) {
+                continue;
+            }
+
+            # Check if the thread is active and not doing something for 5 minutes
+            if (([DateTime]::Now - $ThreadConfig.LastSeen).TotalSeconds -lt 300) {
+                continue;
+            }
+
+            # If it does, kill the thread
+            Remove-IcingaThread -Thread $thread;
+
+            if ($ThreadConfig.TerminateAction.Count -ne 0) {
+                $TerminateArguments = @{ };
+                if ($ThreadConfig.TerminateAction.ContainsKey('Arguments')) {
+                    $TerminateArguments = $ThreadConfig.TerminateAction.Arguments;
+                }
+
+                if ($ThreadConfig.TerminateAction.ContainsKey('Command')) {
+                    $TerminateCmd = $ThreadConfig.TerminateAction.Command;
+
+                    if ([string]::IsNullOrEmpty($TerminateCmd) -eq $FALSE) {
+                        & $TerminateCmd @TerminateArguments | Out-Null;
+                    }
+                }
+            }
+
+            # Now restart it
+            New-IcingaThreadInstance `
+                -ThreadName $thread `
+                -ThreadPool $ThreadConfig.ThreadPool `
+                -Command $ThreadConfig.Command `
+                -CmdParameters $ThreadConfig.Arguments `
+                -Start `
+                -CheckAliveState;
+
+            Write-IcingaEventMessage -EventId 1507 -Namespace 'Framework' -Objects $thread;
+        }
+    } catch {
+        # Nothing to do here
+    }
+}

lib/daemons/RestAPI/threads/Get-IcingaNextRESTApiThreadId.psm1

@@ -1,5 +1,29 @@
 function Get-IcingaNextRESTApiThreadId()
 {
+    # Improve our thread management by distributing new REST requests to a non-active thread
+    [array]$ConfiguredThreads = $Global:Icinga.Public.ThreadAliveHousekeeping.Keys;
+
+    foreach ($thread in $ConfiguredThreads) {
+        if ($thread.ToLower() -NotLike 'Start-IcingaForWindowsRESTThread::New-IcingaForWindowsRESTThread::CheckThread::*') {
+            continue;
+        }
+
+        $ThreadConfig = $Global:Icinga.Public.ThreadAliveHousekeeping[$thread];
+
+        # If our thread is busy, skip this one and check for another one
+        if ($ThreadConfig.Active) {
+            continue;
+        }
+
+        $ThreadIndex = $thread.Replace('Start-IcingaForWindowsRESTThread::New-IcingaForWindowsRESTThread::CheckThread::', '');
+
+        if (Test-Numeric $ThreadIndex) {
+            $Global:Icinga.Public.Daemons.RESTApi.LastThreadId = [int]$ThreadIndex;
+            return ([int]$ThreadIndex)
+        }
+    }
+
+    # In case we are not having any spare thread left, distribute the thread to the next thread in our list
     [int]$ConcurrentThreads = $Global:Icinga.Public.Daemons.RESTApi.TotalThreads - 1;
     [int]$LastThreadId      = $Global:Icinga.Public.Daemons.RESTApi.LastThreadId + 1;
 

lib/daemons/RestAPI/threads/New-IcingaForWindowsRESTThread.psm1

@@ -1,6 +1,6 @@
 function New-IcingaForWindowsRESTThread()
 {
-    param(
+    param (
         $RequireAuth,
         $ThreadId
     );
@@ -69,6 +69,9 @@ function New-IcingaForWindowsRESTThread()
                     }
                 }
 
+                # Set our thread being active
+                Set-IcingaForWindowsThreadAlive -ThreadName $Global:Icinga.Protected.ThreadName -Active -TerminateAction @{ 'Command' = 'Close-IcingaTCPConnection'; 'Arguments' = @{ 'Client' = $Connection.Client } };
+
                 # We should remove clients from the blacklist who are sending valid requests
                 Remove-IcingaRESTClientBlacklist -Client $Connection.Client -ClientList $Global:Icinga.Public.Daemons.RESTApi.ClientBlacklist;
                 switch (Get-IcingaRESTPathElement -Request $RESTRequest -Index 0) {
@@ -85,6 +88,10 @@ function New-IcingaForWindowsRESTThread()
                         ) -Stream $Connection.Stream;
                     };
                 }
+
+                # set our thread no longer be active. We do this, because below there is no way we can
+                # actually get stuck on a endless loop, caused by external modules
+                Set-IcingaForWindowsThreadAlive -ThreadName $Global:Icinga.Protected.ThreadName;
             }
         } catch {
             $ExMsg = $_.Exception.Message;

lib/daemons/RestAPI/threads/Start-IcingaForWindowsRESTThread.psm1

@@ -15,5 +15,6 @@ function Start-IcingaForWindowsRESTThread()
             'RequireAuth' = $RequireAuth;
             'ThreadId'    = $ThreadId;
         } `
-        -Start;
+        -Start `
+        -CheckAliveState;
 }