Merge pull request #204 from Icinga:feature/forward_checks_to_internal_api

Feature: Adds experimental feature for internal API checks

Adds experimental feature to forward checks executed by the Icinga Agent to an internal REST-Api, to reduce the performance impact on systems with lower ressources available
This commit is contained in:
Lord Hepipud 2021-02-24 13:05:51 +01:00 committed by GitHub
commit 96afdc7188
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 510 additions and 77 deletions

View file

@ -26,6 +26,10 @@ Released closed milestones can be found on [GitHub](https://github.com/Icinga/ic
* [#206](https://github.com/Icinga/icinga-powershell-framework/pull/206) Fixes background service check daemon for collecting metrics over time which will no longer share data between configured checks which might cause higher CPU load and a possible memory leak
* [#208](https://github.com/Icinga/icinga-powershell-framework/pull/208) Fixes `Convert-IcingaPluginThresholds` which sometimes did not return proper numeric usable values for our internal functions, causing issues on plugin calls. In addition the function now also supports the handling for % units.
### Experimental
* [#204](https://github.com/Icinga/icinga-powershell-framework/pull/204) Adds experimental feature to forward checks executed by the Icinga Agent to an internal REST-Api, to reduce the performance impact on systems with lower resources available
## 1.3.1 (2021-02-04)
[Issue and PRs](https://github.com/Icinga/icinga-powershell-framework/milestone/12?closed=1)

View file

@ -15,7 +15,19 @@
'.\lib\core\logging\Write-IcingaConsoleOutput.psm1',
'.\lib\core\logging\Write-IcingaConsoleNotice.psm1',
'.\lib\core\logging\Write-IcingaConsoleWarning.psm1',
'.\lib\core\tools\Read-IcingaFileContent.psm1'
'.\lib\core\tools\Read-IcingaFileContent.psm1',
'.\lib\core\framework\Invoke-IcingaInternalServiceCall.psm1',
'.\lib\core\framework\Get-IcingaFrameworkApiChecks.psm1',
'.\lib\daemon\Get-IcingaBackgroundDaemons.psm1',
'.\lib\webserver\Enable-IcingaUntrustedCertificateValidation.psm1',
'.\lib\core\logging\Write-IcingaEventMessage.psm1',
'.\lib\icinga\plugin\Exit-IcingaExecutePlugin.psm1',
'.\lib\icinga\exception\Exit-IcingaPluginNotInstalled.psm1',
'.\lib\icinga\exception\Exit-IcingaThrowException.psm1',
'.\lib\web\Disable-IcingaProgressPreference.psm1',
'.\lib\core\tools\New-IcingaNewLine.psm1',
'.\lib\core\logging\Write-IcingaConsolePlain.psm1',
'.\lib\core\tools\Test-IcingaFunction.psm1'
)
FunctionsToExport = @(
'Use-Icinga',
@ -39,7 +51,19 @@
'Write-IcingaConsoleOutput',
'Write-IcingaConsoleNotice',
'Write-IcingaConsoleWarning',
'Read-IcingaFileContent'
'Read-IcingaFileContent',
'Invoke-IcingaInternalServiceCall',
'Get-IcingaFrameworkApiChecks',
'Get-IcingaBackgroundDaemons',
'Enable-IcingaUntrustedCertificateValidation',
'Write-IcingaEventMessage',
'Exit-IcingaExecutePlugin',
'Exit-IcingaPluginNotInstalled',
'Exit-IcingaThrowException',
'Disable-IcingaProgressPreference',
'New-IcingaNewLine',
'Write-IcingaConsolePlain',
'Test-IcingaFunction'
)
CmdletsToExport = @('*')
VariablesToExport = '*'

View file

@ -13,9 +13,21 @@ function Use-Icinga()
param(
[switch]$LibOnly = $FALSE,
[switch]$Daemon = $FALSE,
[switch]$DebugMode = $FALSE
[switch]$DebugMode = $FALSE,
[switch]$Minimal = $FALSE
);
Disable-IcingaProgressPreference;
if ($Minimal) {
# If we load the minimal Framework files, we have to ensure our enums are loaded
Import-Module ([string]::Format('{0}\lib\icinga\exception\Icinga_IcingaExceptionEnums.psm1', $PSScriptRoot)) -Global;
Import-Module ([string]::Format('{0}\lib\icinga\enums\Icinga_IcingaEnums.psm1', $PSScriptRoot)) -Global;
Import-Module ([string]::Format('{0}\lib\core\logging\Icinga_EventLog_Enums.psm1', $PSScriptRoot)) -Global;
return;
}
# Ensure we autoload the Icinga Plugin collection, provided by the external
# module 'icinga-powershell-plugins'
if (Get-Command 'Use-IcingaPlugins' -ErrorAction SilentlyContinue) {
@ -61,7 +73,6 @@ function Use-Icinga()
}
}
New-IcingaPerformanceCounterCache;
Disable-IcingaProgressPreference;
# Enable DebugMode in case it is enabled in our config
if (Get-IcingaFrameworkDebugMode) {

View file

@ -0,0 +1,53 @@
<#
.SYNOPSIS
Adds a Cmdlet to an REST-Api endpoint which is either whitelisted or blacklisted.
Whitelisted Cmdlets can be executed over API endpoints, blacklisted not.
Use '*' for wildcard matches
.DESCRIPTION
Adds a Cmdlet to an REST-Api endpoint which is either whitelisted or blacklisted.
Whitelisted Cmdlets can be executed over API endpoints, blacklisted not.
Use '*' for wildcard matches
.FUNCTIONALITY
Enables or disables Cmdlets for REST-Api endpoints
.EXAMPLE
PS>Add-IcingaFrameworkApiCommand -Command 'Invoke-IcingaCheck*' -Endpoint 'checker';
.EXAMPLE
PS>Add-IcingaFrameworkApiCommand -Command 'Invoke-IcingaCheck*' -Endpoint 'checker' -Blacklist;
.LINK
https://github.com/Icinga/icinga-powershell-framework
#>
function Add-IcingaFrameworkApiCommand()
{
param (
[string]$Command = '',
[string]$Endpoint = '',
[switch]$Blacklist = $FALSE
);
if ([string]::IsNullOrEmpty($Command) -Or [string]::IsNullOrEmpty($Endpoint)) {
return;
}
$Commands = $null;
$ConfigPath = ([string]::Format('Framework.RESTApiCommands.{0}.Whitelist', $Endpoint));
[array]$Values = @();
if ($Blacklist) {
$ConfigPath = ([string]::Format('Framework.RESTApiCommands.{0}.Blacklist', $Endpoint));
}
$Commands = Get-IcingaPowerShellConfig -Path $ConfigPath;
if ((Test-IcingaPowerShellConfigItem -ConfigObject $Commands -ConfigKey $Command)) {
return;
}
if ($null -ne $Commands) {
$Values = $Commands;
}
$Values += $Command;
Set-IcingaPowerShellConfig -Path $ConfigPath -Value $Values;
}

View file

@ -0,0 +1,19 @@
<#
.SYNOPSIS
Disables the feature to forward all executed checks to an internal
installed API to run them within a daemon
.DESCRIPTION
Disables the feature to forward all executed checks to an internal
installed API to run them within a daemon
.FUNCTIONALITY
Disables the Icinga for Windows Api checks forwarded
.EXAMPLE
PS>Disable-IcingaFrameworkApiChecks;
.LINK
https://github.com/Icinga/icinga-powershell-framework
#>
function Disable-IcingaFrameworkApiChecks()
{
Set-IcingaPowerShellConfig -Path 'Framework.Experimental.UseApiChecks' -Value $FALSE;
}

View file

@ -0,0 +1,21 @@
<#
.SYNOPSIS
Enables the feature to forward all executed checks to an internal
installed API to run them within a daemon
.DESCRIPTION
Enables the feature to forward all executed checks to an internal
installed API to run them within a daemon
.FUNCTIONALITY
Enables the Icinga for Windows Api checks forwarded
.EXAMPLE
PS>Enable-IcingaFrameworkApiChecks;
.LINK
https://github.com/Icinga/icinga-powershell-framework
#>
function Enable-IcingaFrameworkApiChecks()
{
Set-IcingaPowerShellConfig -Path 'Framework.Experimental.UseApiChecks' -Value $TRUE;
Write-IcingaConsoleWarning 'Experimental Feature: Please ensure to install the packages "icinga-powershell-restapi" and "icinga-powershell-apichecks", install the Icinga for Windows background service and also register the daemon with "Register-IcingaBackgroundDaemon -Command {0}". Afterwards all services will be executed by the background daemon in case it is running.' -Objects "'Start-IcingaWindowsRESTApi'";
}

View file

@ -0,0 +1,28 @@
<#
.SYNOPSIS
Fetches the current enable/disable state of the feature
for executing checks of the internal REST-Api
.DESCRIPTION
Fetches the current enable/disable state of the feature
for executing checks of the internal REST-Api
.FUNCTIONALITY
Get the current API check execution configuration of the
Icinga PowerShell Framework
.EXAMPLE
PS>Get-IcingaFrameworkApiChecks;
.LINK
https://github.com/Icinga/icinga-powershell-framework
.OUTPUTS
System.Boolean
#>
function Get-IcingaFrameworkApiChecks()
{
$CodeCaching = Get-IcingaPowerShellConfig -Path 'Framework.Experimental.UseApiChecks';
if ($null -eq $CodeCaching) {
return $FALSE;
}
return $CodeCaching;
}

View file

@ -0,0 +1,121 @@
function Invoke-IcingaInternalServiceCall()
{
param (
[string]$Command = '',
[array]$Arguments = @()
);
# If our Framework is running as daemon, never call our api
if ($global:IcingaDaemonData.FrameworkRunningAsDaemon) {
return;
}
# If the API forward feature is disabled, do nothing
if ((Get-IcingaFrameworkApiChecks) -eq $FALSE) {
return;
}
# Test our Icinga for Windows service. If the service is not installed or not running, execute the plugin locally
$IcingaForWindowsService = (Get-Service 'icingapowershell' -ErrorAction SilentlyContinue);
if ($null -eq $IcingaForWindowsService -Or $IcingaForWindowsService.Status -ne 'Running') {
return;
}
# In case the REST-Api module ist not configured, do nothing
$BackgroundDaemons = Get-IcingaBackgroundDaemons;
if ($null -eq $BackgroundDaemons -Or $BackgroundDaemons.ContainsKey('Start-IcingaWindowsRESTApi') -eq $FALSE) {
return;
}
# If neither 'icinga-powershell-restapi' or 'icinga-powershell-apichecks' is installed, execute the plugin locally
if ((Test-IcingaFunction 'Invoke-IcingaApiChecksRESTCall') -eq $FALSE -Or (Test-IcingaFunction 'Start-IcingaWindowsRESTApi') -eq $FALSE) {
return;
}
$RestApiPort = 5668;
[int]$Timeout = 30;
$Daemon = $BackgroundDaemons['Start-IcingaWindowsRESTApi'];
# Fetch our deamon configuration
if ($Daemon.ContainsKey('-Port')) {
$RestApiPort = $Daemon['-Port'];
} elseif ($Daemon.ContainsKey('Port')) {
$RestApiPort = $Daemon['Port'];
}
if ($Daemon.ContainsKey('-Timeout')) {
$Timeout = $Daemon['-Timeout'];
} elseif ($Daemon.ContainsKey('Timeout')) {
$Timeout = $Daemon['Timeout'];
}
Enable-IcingaUntrustedCertificateValidation -SuppressMessages;
[hashtable]$CommandArguments = @{ };
[hashtable]$DebugArguments = @{ };
[hashtable]$ConvertedArgs = @{ };
[int]$ArgumentIndex = 0;
# Resolve our array arguments provided by $args and build proper check arguments
while ($ArgumentIndex -lt $Arguments.Count) {
$Value = $Arguments[$ArgumentIndex];
[string]$Argument = [string]$Value;
$ArgumentValue = $null;
if ($Value[0] -eq '-') {
if (($ArgumentIndex + 1) -lt $Arguments.Count) {
[string]$NextValue = $Arguments[$ArgumentIndex + 1];
if ($NextValue[0] -eq '-') {
$ArgumentValue = $TRUE;
} else {
$ArgumentValue = $Arguments[$ArgumentIndex + 1];
}
} else {
$ArgumentValue = $TRUE;
}
} else {
$ArgumentIndex += 1;
continue;
}
$Argument = $Argument.Replace('-', '');
$ConvertedArgs.Add($Argument, $ArgumentValue);
$ArgumentIndex += 1;
}
# Now queue the check inside our REST-Api
try {
$ApiResult = Invoke-WebRequest -Method POST -UseBasicParsing -Uri ([string]::Format('https://localhost:{0}/v1/checker?command={1}', $RestApiPort, $Command)) -Body ($CommandArguments | ConvertTo-Json -Depth 100) -ContentType 'application/json' -TimeoutSec $Timeout;
} catch {
# Something went wrong -> fallback to local execution
$ExMsg = $_.Exception.message;
# Fallback to execute plugin locally
Write-IcingaEventMessage -Namespace 'Framework' -EventId 1553 -Objects $ExMsg, $Command, $DebugArguments;
return;
}
# Resolve our result from the API
$IcingaResult = ConvertFrom-Json -InputObject $ApiResult;
$IcingaCR = '';
# In case we didn't receive a check result, fallback to local execution
if ($null -eq $IcingaResult.$Command.checkresult) {
Write-IcingaEventMessage -Namespace 'Framework' -EventId 1553 -Objects 'The check result for the executed command was empty', $Command, $DebugArguments;
return;
}
$IcingaCR = ($IcingaResult.$Command.checkresult.Replace("`r`n", "`n"));
if ($IcingaResult.$Command.perfdata.Count -ne 0) {
$IcingaCR += ' | ';
foreach ($perfdata in $IcingaResult.$Command.perfdata) {
$IcingaCR += $perfdata;
}
}
# Print our response and exit with the provide exit code
Write-IcingaConsolePlain $IcingaCR;
exit $IcingaResult.$Command.exitcode;
}

View file

@ -0,0 +1,49 @@
<#
.SYNOPSIS
Removes a Cmdlet from an REST-Api endpoints whitelist or blacklist.
.DESCRIPTION
Removes a Cmdlet from an REST-Api endpoints whitelist or blacklist.
.FUNCTIONALITY
Removes Cmdlets for REST-Api endpoints
.EXAMPLE
PS>Remove-IcingaFrameworkApiCommand -Command 'Invoke-IcingaCheck*' -Endpoint 'checker';
.EXAMPLE
PS>Add-IcingaFrameworkApiCommand -Command 'Invoke-IcingaCheck*' -Endpoint 'checker' -Blacklist;
.LINK
https://github.com/Icinga/icinga-powershell-framework
#>
function Remove-IcingaFrameworkApiCommand()
{
param (
[string]$Command = '',
[string]$Endpoint = '',
[switch]$Blacklist = $FALSE
);
if ([string]::IsNullOrEmpty($Command) -Or [string]::IsNullOrEmpty($Endpoint)) {
return;
}
$Commands = $null;
$ConfigPath = ([string]::Format('Framework.RESTApiCommands.{0}.Whitelist', $Endpoint));
[array]$Values = @();
if ($Blacklist) {
$ConfigPath = ([string]::Format('Framework.RESTApiCommands.{0}.Blacklist', $Endpoint));
}
$Commands = Get-IcingaPowerShellConfig -Path $ConfigPath;
if ($null -eq $Commands) {
return;
}
foreach ($element in $Commands) {
if ($element.ToLower() -ne $Command.ToLower()) {
$Values += $element;
}
}
Set-IcingaPowerShellConfig -Path $ConfigPath -Value $Values;
}

View file

@ -0,0 +1,46 @@
<#
.SYNOPSIS
Tests if a Cmdlet for a given endpoint is configured as whitelist or blacklist.
This function will return True if the command is whitelisted and False if it is
blacklisted. If the Cmdlet is not added anywhere, the function will return False as well.
.DESCRIPTION
Tests if a Cmdlet for a given endpoint is configured as whitelist or blacklist.
This function will return True if the command is whitelisted and False if it is
blacklisted. If the Cmdlet is not added anywhere, the function will return False as well.
.FUNCTIONALITY
Tests if a Cmdlet is allowed to be executed over the REST-Api
.EXAMPLE
PS>Test-IcingaFrameworkApiCommand -Command 'Invoke-IcingaCheckCPU' -Endpoint 'checker';
.LINK
https://github.com/Icinga/icinga-powershell-framework
#>
function Test-IcingaFrameworkApiCommand()
{
param (
[string]$Command = '',
[string]$Endpoint = ''
);
if ([string]::IsNullOrEmpty($Command) -Or [string]::IsNullOrEmpty($Endpoint)) {
return $FALSE;
}
$WhiteList = Get-IcingaPowerShellConfig -Path ([string]::Format('Framework.RESTApiCommands.{0}.Whitelist', $Endpoint));
$Blacklist = Get-IcingaPowerShellConfig -Path ([string]::Format('Framework.RESTApiCommands.{0}.Blacklist', $Endpoint));
foreach ($entry in $Blacklist) {
if ($Command.ToLower() -like $entry.ToLower()) {
return $FALSE;
}
}
foreach ($entry in $WhiteList) {
if ($Command.ToLower() -like $entry.ToLower()) {
return $TRUE;
}
}
# If the command is not configured, always return false
return $FALSE;
}

View file

@ -5,6 +5,7 @@
# Example usage:
# $IcingaEventLogEnums[2000]
#>
if ($null -eq $IcingaEventLogEnums -Or $IcingaEventLogEnums.ContainsKey('Framework') -eq $FALSE) {
[hashtable]$IcingaEventLogEnums += @{
'Framework' = @{
1000 = @{
@ -15,7 +16,7 @@
};
1500 = @{
'EntryType' = 'Error';
'Message' = 'Failed to securely establish a communiation between this server and the client';
'Message' = 'Failed to securely establish a communication between this server and the client';
'Details' = 'A client connection could not be established to this server. This issue is mostly caused by using Self-Signed/Icinga 2 Agent certificates for the server and the client not trusting the certificate. To resolve this issue, either use trusted certificates signed by your trusted CA or setup the client to accept untrusted certificates';
'EventId' = 1500;
};
@ -43,6 +44,12 @@
'Details' = 'Provided user credentials encoded as base64 could not be converted to domain, user and password objects.';
'EventId' = 1552;
};
1553 = @{
'EntryType' = 'Error';
'Message' = 'Failed to query Icinga check over internal REST-Api check handler';
'Details' = 'A service check could not be executed by using the internal REST-Api check handler. The check either ran into a timeout or could not be processed. Maybe the check was not registered to be allowed for being executed. Further details can be found below.';
'EventId' = 1553;
};
1560 = @{
'EntryType' = 'Error';
'Message' = 'Failed to test user login as no Principal Context could be established';
@ -57,5 +64,6 @@
};
}
};
}
Export-ModuleMember -Variable @( 'IcingaEventLogEnums' );

View file

@ -0,0 +1,13 @@
function Add-IcingaArrayListItem()
{
param (
[System.Collections.ArrayList]$Array,
$Element
);
if ($null -eq $Array -Or $null -eq $Element) {
return;
}
$Array.Add($Element) | Out-Null;
}

View file

@ -123,7 +123,7 @@ function Get-IcingaCheckCommandConfig()
'arguments' = @{
# Set the Command handling for every check command
'-C' = @{
'value' = [string]::Format('try {{ Use-Icinga; }} catch {{ Write-Output {1}The Icinga PowerShell Framework is either not installed on the system or not configured properly. Please check https://icinga.com/docs/windows for further details{1}; exit 3; }}; Exit-IcingaPluginNotInstalled {1}{0}{1}; exit {0}', $Data.Name, "'");
'value' = [string]::Format('try {{ Use-Icinga -Minimal; }} catch {{ Write-Output {1}The Icinga PowerShell Framework is either not installed on the system or not configured properly. Please check https://icinga.com/docs/windows for further details{1}; exit 3; }}; Exit-IcingaExecutePlugin -Command {1}{0}{1} ', $Data.Name, "'");
'order' = '0';
}
}
@ -373,7 +373,7 @@ function Get-IcingaCheckCommandConfig()
Write-IcingaConsoleNotice "- '$check'";
}
Write-IcingaConsoleNotice "JSON export created in '${OutDirectory}'"
Write-IcingaConsoleWarning 'By using this generated check command configuration you will require the Icinga PowerShell Framework 1.2.0 or later to be installed on ALL monitored machines!';
Write-IcingaConsoleWarning 'By using this generated check command configuration you will require the Icinga PowerShell Framework 1.4.0 or later to be installed on ALL monitored machines!';
return;
}
@ -381,7 +381,7 @@ function Get-IcingaCheckCommandConfig()
foreach ($check in $CheckName) {
Write-IcingaConsoleNotice "- '$check'"
}
Write-IcingaConsoleWarning 'By using this generated check command configuration you will require the Icinga PowerShell Framework 1.2.0 or later to be installed on ALL monitored machines!';
Write-IcingaConsoleWarning 'By using this generated check command configuration you will require the Icinga PowerShell Framework 1.4.0 or later to be installed on ALL monitored machines!';
Write-IcingaConsoleNotice '############################################################';
return $output;

View file

@ -67,6 +67,7 @@
# Example usage:
# $IcingaEnums.IcingaExitCode.Ok
#>
if ($null -eq $IcingaEnums) {
[hashtable]$IcingaEnums = @{
IcingaExitCode = $IcingaExitCode;
IcingaExitCodeText = $IcingaExitCodeText;
@ -76,5 +77,6 @@
ServiceStartupTypeName = $ServiceStartupTypeName;
ServiceWmiStartupType = $ServiceWmiStartupType;
}
}
Export-ModuleMember -Variable @( 'IcingaEnums' );

View file

@ -98,7 +98,7 @@ function Exit-IcingaThrowException()
$ExceptionTypeString
);
if ($global:IcingaDaemonData.FrameworkRunningAsDaemon -eq $FALSE) {
if ($null -eq $global:IcingaDaemonData -Or $global:IcingaDaemonData.FrameworkRunningAsDaemon -eq $FALSE) {
Write-IcingaConsolePlain $OutputMessage;
exit $IcingaEnums.IcingaExitCode.Unknown;
}

View file

@ -53,11 +53,14 @@
# Example usage:
# $IcingaException.Inputs.PerformanceCounter
#>
if ($null -eq $IcingaExceptions) {
[hashtable]$IcingaExceptions = @{
Permission = $Permission;
Inputs = $Inputs;
Configuration = $Configuration;
Connection = $Connection;
}
}
Export-ModuleMember -Variable @( 'IcingaExceptions' );

View file

@ -0,0 +1,23 @@
function Exit-IcingaExecutePlugin()
{
param (
[string]$Command = ''
);
Exit-IcingaPluginNotInstalled -Command $Command;
Invoke-IcingaInternalServiceCall -Command $Command -Arguments $args;
try {
# Load the entire framework now, as we require to execute plugins locally
if ($null -eq $global:IcingaDaemonData) {
Use-Icinga;
}
exit (& $Command @args);
} catch {
$ExMsg = $_.Exception.Message;
Write-IcingaConsolePlain '[UNKNOWN]: {0}{1}{1}CheckCommand: {2}{1}Arguments: {3}' -Objects $ExMsg, (New-IcingaNewLine), $Command, $args;
exit 3;
}
}

View file

@ -1,5 +1,9 @@
function Enable-IcingaUntrustedCertificateValidation()
{
param (
[switch]$SuppressMessages = $FALSE
);
try {
# There is no other way as to use C# for this specific
# case to configure the certificate validation check
@ -16,8 +20,12 @@ function Enable-IcingaUntrustedCertificateValidation()
[System.Net.ServicePointManager]::CertificatePolicy = New-Object IcingaUntrustedCertificateValidation;
if ($SuppressMessages -eq $FALSE) {
Write-IcingaConsoleNotice 'Successfully enabled untrusted certificate validation for this shell instance';
}
} catch {
if ($SuppressMessages -eq $FALSE) {
Write-IcingaConsoleError -Message 'Failed to enable untrusted certificate policy: {0}' -Objects $_.Exception.Message;
}
}
}