Merge pull request #526 from Icinga:feature/new_perfdata_handling

Feature: Rewrite PerfData Labels for multi output

Performance data should be created with some more adjustments to ensure we can properly create graphcs by using InfluxDB and Grafana as example

doc/100-General/10-Changelog.md

@@ -28,6 +28,7 @@ Released closed milestones can be found on [GitHub](https://github.com/Icinga/ic
 ### Enhancements
 
 * [#40](https://github.com/Icinga/icinga-powershell-framework/issues/40) Adds support to set service recovery for the Icinga Agent and Icinga for Windows service, to restart them in case of a crash or error
+* [#485](https://github.com/Icinga/icinga-powershell-framework/issues/485) Adds new style for performance data labels, to use the multi output format, allowing for better filtering and visualisation with InfluxDB and Grafana
 * [#525](https://github.com/Icinga/icinga-powershell-framework/pull/525) Adds new developer mode for `icinga` command and improved cache handling, to ensure within `-DeveloperMode` and inside a VS Code environment, the framework cache file is never overwritten, while still all functions are loaded and imported.
 * [#531](https://github.com/Icinga/icinga-powershell-framework/pull/531) Adds `Test-IcingaStateFile` and `Repair-IcingaStateFile`, which is integrated into `Test-IcingaAgent`, to ensure the Icinga Agent state file is healthy and not corrupt, causing the Icinga Agent to fail on start
 * [#534](https://github.com/Icinga/icinga-powershell-framework/pull/534) Improves Icinga and Director configuration generator, by wrapping PowerShell arrays inside `@()` instead of simply writing them comma separated

lib/core/framework/Get-IcingaCheckSchedulerPerfData.psm1

@@ -18,7 +18,7 @@
 function Get-IcingaCheckSchedulerPerfData()
 {
     $PerfData                                               = $Global:Icinga.Private.Scheduler.PerformanceData;
-    $Global:Icinga.Private.Scheduler.PerformanceData = @();
+    [array]$Global:Icinga.Private.Scheduler.PerformanceData = @();
 
     return $PerfData;
 }

lib/core/framework/Get-IcingaCheckSchedulerPluginOutput.psm1

@@ -17,8 +17,12 @@
 
 function Get-IcingaCheckSchedulerPluginOutput()
 {
+    if ($Global:Icinga.Private.Scheduler.CheckResults.Count -eq 0) {
+        return @();
+    }
+
     $CheckResult                                         = [string]::Join("`r`n", $Global:Icinga.Private.Scheduler.CheckResults);
-    $Global:Icinga.Private.Scheduler.CheckResults = @();
+    [array]$Global:Icinga.Private.Scheduler.CheckResults = @();
 
     return $CheckResult;
 }

lib/core/framework/Invoke-IcingaForWindowsMigration.psm1

@@ -39,4 +39,41 @@ function Invoke-IcingaForWindowsMigration()
             Restart-IcingaService -Service 'icingapowershell';
         }
     }
+
+    if (Test-IcingaForWindowsMigration -MigrationVersion (New-IcingaVersionObject -Version '1.10.0')) {
+        Write-IcingaConsoleNotice 'Applying pending migrations required for Icinga for Windows v1.10.0';
+
+        $ServiceStatus = (Get-Service 'icingapowershell' -ErrorAction SilentlyContinue).Status;
+
+        if ($ServiceStatus -eq 'Running') {
+            Stop-IcingaWindowsService;
+        }
+
+        # Convert the time intervals for the background daemon services from the previous index handling
+        # 1, 3, 5, 15 as example to 1m, 3m, 5m, 15m
+        $BackgroundServices = Get-IcingaPowerShellConfig -Path 'BackgroundDaemon.RegisteredServices';
+        [hashtable]$Output = @{ };
+
+        foreach ($service in $BackgroundServices.PSObject.Properties) {
+            [array]$ConvertedTimeIndex = @();
+
+            foreach ($interval in $service.Value.TimeIndexes) {
+                if (Test-Numeric $interval) {
+                    $ConvertedTimeIndex += [string]::Format('{0}m', $interval);
+                } else {
+                    $ConvertedTimeIndex = $interval;
+                }
+            }
+
+            $service.Value.TimeIndexes = $ConvertedTimeIndex;
+        }
+
+        Set-IcingaPowerShellConfig -Path 'BackgroundDaemon.RegisteredServices' -Value $BackgroundServices;
+
+        Set-IcingaForWindowsMigration -MigrationVersion (New-IcingaVersionObject -Version '1.10.0');
+
+        if ($ServiceStatus -eq 'Running') {
+            Restart-IcingaWindowsService -Service 'icingapowershell';
+        }
+    }
 }

lib/core/framework/Invoke-IcingaInternalServiceCall.psm1

@@ -76,10 +76,7 @@ function Invoke-IcingaInternalServiceCall()
     $IcingaCR = ($IcingaResult.$Command.checkresult.Replace("`r`n", "`n"));
 
     if ($IcingaResult.$Command.perfdata.Count -ne 0) {
-        $IcingaCR += ' | ';
+        $IcingaCR = [string]::Format('{0}{1}| {2}', $IcingaCR, "`r`n", ([string]::Join(' ', $IcingaResult.$Command.perfdata)));
-        foreach ($perfdata in $IcingaResult.$Command.perfdata) {
-            $IcingaCR += $perfdata;
-        }
     }
 
     if ($NoExit) {

lib/core/perfcounter/Get-IcingaPerformanceCounterDetails.psm1

@@ -0,0 +1,45 @@
+function Get-IcingaPerformanceCounterDetails()
+{
+    param (
+        [string]$Counter = $null
+    );
+
+    [hashtable]$RetValue = @{
+        'RawCounter'      = $Counter;
+        'HasValue'        = $TRUE;
+        'HasInstance'     = $FALSE;
+        'Category'        = '';
+        'Instance'        = '';
+        'Counter'         = '';
+        'CounterInstance' = '';
+    }
+
+    if ([string]::IsNullOrEmpty($Counter)) {
+        $RetValue.HasValue = $FALSE;
+
+        return $RetValue;
+    }
+
+    [array]$CounterElements = $Counter.Split('\');
+    [string]$Instance       = '';
+    [string]$Category       = '';
+    [bool]$HasInstance      = $FALSE;
+
+    if ($CounterElements[1].Contains('(') -And $CounterElements[1].Contains(')')) {
+        $HasInstance              = $TRUE;
+        [int]$StartIndex          = $CounterElements[1].IndexOf('(') + 1;
+        [int]$EndIndex            = $CounterElements[1].Length - $StartIndex - 1;
+        $Instance                 = $CounterElements[1].Substring($StartIndex, $EndIndex);
+        $RetValue.HasInstance     = $HasInstance;
+        $Category                 = $CounterElements[1].Substring(0, $CounterElements[1].IndexOf('('));
+        $RetValue.CounterInstance = [string]::Format('{0}_{1}', $Instance, $CounterElements[2]);
+    } else {
+        $Category = $CounterElements[1];
+    }
+
+    $RetValue.Category = $Category;
+    $RetValue.Instance = $Instance;
+    $RetValue.Counter  = $CounterElements[2];
+
+    return $RetValue;
+}

lib/core/tools/ConvertTo-IcingaNumericTimeIndex.psm1

@@ -0,0 +1,24 @@
+function ConvertTo-IcingaNumericTimeIndex()
+{
+    param (
+        [int]$TimeValue = 0
+    );
+
+    if ($TimeValue -lt 60) {
+        return ([string]::Format('{0}s', $TimeValue));
+    }
+
+    [decimal]$Minutes = $TimeValue / 60;
+    [decimal]$Seconds = $Minutes - [math]::Truncate($Minutes);
+    [decimal]$Minutes = [math]::Truncate($Minutes);
+    [decimal]$Seconds = [math]::Round(60 * $Seconds, 0);
+
+    $TimeIndex = New-Object -TypeName 'System.Text.StringBuilder';
+    $TimeIndex.Append([string]::Format('{0}m', $Minutes)) | Out-Null;
+
+    if ($Seconds -ne 0) {
+        $TimeIndex.Append([string]::Format('{0}s', $Seconds)) | Out-Null;
+    }
+
+    return $TimeIndex.ToString();
+}

lib/core/tools/Format-IcingaPerfDataLabel.psm1

@@ -1,14 +1,20 @@
 function Format-IcingaPerfDataLabel()
 {
     param(
-        $PerfData
+        $PerfData,
+        [switch]$MultiOutput = $FALSE
     );
 
+    if ($MultiOutput) {
+        return (($PerfData) -Replace '[\W]', '');
+    }
+
     $Output = ((($PerfData) -Replace ' ', '_') -Replace '[\W]', '');
 
     while ($Output.Contains('__')) {
         $Output = $Output.Replace('__', '_');
     }
+
     # Remove all special characters and spaces on label names
     return $Output;
 }

lib/daemons/ServiceCheckDaemon/task/Add-IcingaServiceCheckTask.psm1

@@ -17,8 +17,10 @@ function Add-IcingaServiceCheckTask()
     # Read our check result store data from disk for this service check
     Read-IcingaCheckResultStore -CheckCommand $CheckCommand;
 
+    [int]$CheckInterval   = ConvertTo-Seconds $Interval;
+
     while ($TRUE) {
-        if ($Global:Icinga.Private.Daemons.ServiceCheck.PassedTime -lt $Interval) {
+        if ($Global:Icinga.Private.Daemons.ServiceCheck.PassedTime -lt $CheckInterval) {
             $Global:Icinga.Private.Daemons.ServiceCheck.PassedTime += 1;
             Start-Sleep -Seconds 1;
 
@@ -76,11 +78,8 @@ function Add-IcingaServiceCheckTask()
                 foreach ($calc in $Global:Icinga.Private.Daemons.ServiceCheck.AverageCalculation.Keys) {
                     if ($Global:Icinga.Private.Daemons.ServiceCheck.AverageCalculation[$calc].Count -ne 0) {
                         $AverageValue            = ($Global:Icinga.Private.Daemons.ServiceCheck.AverageCalculation[$calc].Sum / $Global:Icinga.Private.Daemons.ServiceCheck.AverageCalculation[$calc].Count);
-                        [string]$MetricName   = Format-IcingaPerfDataLabel (
+                        [string]$MetricMultiName = [string]::Format('::{0}::Interval{1}', (Format-IcingaPerfDataLabel -PerfData $HashIndex -MultiOutput), $Global:Icinga.Private.Daemons.ServiceCheck.AverageCalculation[$calc].Time);
-                            [string]::Format('{0}_{1}', $HashIndex, $Global:Icinga.Private.Daemons.ServiceCheck.AverageCalculation[$calc].Interval)
+                        $Global:Icinga.Private.Scheduler.CheckData[$CheckCommand]['average'] | Add-Member -MemberType NoteProperty -Name $MetricMultiName -Value $AverageValue -Force;
-                        );
-
-                        $Global:Icinga.Private.Scheduler.CheckData[$CheckCommand]['average'] | Add-Member -MemberType NoteProperty -Name $MetricName -Value $AverageValue -Force;
                     }
 
                     $Global:Icinga.Private.Daemons.ServiceCheck.AverageCalculation[$calc].Sum   = 0;

lib/daemons/ServiceCheckDaemon/tools/New-IcingaServiceCheckDaemonEnvironment.psm1

@@ -26,7 +26,7 @@ function New-IcingaServiceCheckDaemonEnvironment()
 
     foreach ($index in $TimeIndexes) {
         # Only allow numeric index values
-        if ((Test-Numeric $index) -eq $FALSE) {
+        if ((Test-Numeric (ConvertTo-Seconds $index)) -eq $FALSE) {
             Write-IcingaEventMessage -EventId 1450 -Namespace 'Framework' -Objects $CheckCommand, ($Arguments | Out-String), ($TimeIndexes | Out-String), $index;
             continue;
         }
@@ -34,15 +34,16 @@ function New-IcingaServiceCheckDaemonEnvironment()
             $Global:Icinga.Private.Daemons.ServiceCheck.AverageCalculation.Add(
                 [string]$index,
                 @{
-                    'Interval' = ([int]$index);
+                    'Interval'    = ([int]($index -Replace '[^0-9]', ''));
-                    'Time'     = ([int]$index * 60);
+                    'RawInterval' = $index;
+                    'Time'        = (ConvertTo-Seconds $index);
                     'Sum'         = 0;
                     'Count'       = 0;
                 }
             );
         }
-        if ($Global:Icinga.Private.Daemons.ServiceCheck.MaxTime -le [int]$index) {
+        if ($Global:Icinga.Private.Daemons.ServiceCheck.MaxTime -le (ConvertTo-Seconds $index)) {
-            $Global:Icinga.Private.Daemons.ServiceCheck.MaxTime = [int]$index;
+            $Global:Icinga.Private.Daemons.ServiceCheck.MaxTime = (ConvertTo-Seconds $index);
         }
     }
 

lib/icinga/plugin/Compare-IcingaPluginThresholds.psm1

@@ -10,6 +10,7 @@ function Compare-IcingaPluginThresholds()
         [string]$Unit           = '',
         $ThresholdCache         = $null,
         [string]$CheckName      = '',
+        [string]$PerfDataLabel  = '',
         [hashtable]$Translation = @{ },
         $Minium                 = $null,
         $Maximum                = $null,
@@ -44,6 +45,7 @@ function Compare-IcingaPluginThresholds()
     $IcingaThresholds | Add-Member -MemberType NoteProperty -Name 'MaxRangeValue'   -Value $null;
     $IcingaThresholds | Add-Member -MemberType NoteProperty -Name 'PercentValue'    -Value '';
     $IcingaThresholds | Add-Member -MemberType NoteProperty -Name 'TimeSpan'        -Value '';
+    $IcingaThresholds | Add-Member -MemberType NoteProperty -Name 'TimeSpanOutput'  -Value '';
     $IcingaThresholds | Add-Member -MemberType NoteProperty -Name 'InRange'         -Value $TRUE;
     $IcingaThresholds | Add-Member -MemberType NoteProperty -Name 'Message'         -Value '';
     $IcingaThresholds | Add-Member -MemberType NoteProperty -Name 'Range'           -Value '';
@@ -61,14 +63,34 @@ function Compare-IcingaPluginThresholds()
 
     if ([string]::IsNullOrEmpty($TimeInterval) -eq $FALSE -And $null -ne $ThresholdCache) {
         $TimeSeconds            = ConvertTo-Seconds $TimeInterval;
+        $IntervalLabelName      = (Format-IcingaPerfDataLabel -PerfData $CheckName);
+        $IntervalMultiLabelName = (Format-IcingaPerfDataLabel -PerfData $CheckName -MultiOutput);
+
+        if ([string]::IsNullOrEmpty($PerfDataLabel) -eq $FALSE) {
+            $IntervalLabelName      = $PerfDataLabel;
+            $IntervalMultiLabelName = $PerfDataLabel;
+        }
+
         $MinuteInterval      = [math]::round(([TimeSpan]::FromSeconds($TimeSeconds)).TotalMinutes, 0);
-        $CheckPerfDataLabel = [string]::Format('{0}_{1}', (Format-IcingaPerfDataLabel $CheckName), $MinuteInterval);
+        $CheckPerfDataLabel  = [string]::Format('{0}_{1}', $IntervalLabelName, $MinuteInterval);
+        $MultiPerfDataLabel  = [string]::Format('::{0}::Interval{1}', $IntervalMultiLabelName, $TimeSeconds);
+        [bool]$FoundInterval = $FALSE;
 
         if ($null -ne $ThresholdCache.$CheckPerfDataLabel) {
             $InputValue                      = $ThresholdCache.$CheckPerfDataLabel;
             $InputValue                      = [math]::round([decimal]$InputValue, 6);
+            $IcingaThresholds.TimeSpanOutput = $MinuteInterval;
             $IcingaThresholds.TimeSpan       = $MinuteInterval;
-        } else {
+            $FoundInterval                   = $TRUE;
+        }
+        if ($null -ne $ThresholdCache.$MultiPerfDataLabel) {
+            $InputValue                      = $ThresholdCache.$MultiPerfDataLabel;
+            $InputValue                      = [math]::round([decimal]$InputValue, 6);
+            $IcingaThresholds.TimeSpanOutput = $MinuteInterval;
+            $IcingaThresholds.TimeSpan       = $TimeSeconds;
+            $FoundInterval                   = $TRUE;
+        }
+        if ($FoundInterval -eq $FALSE) {
             $IcingaThresholds.HasError     = $TRUE;
             $IcingaThresholds.ErrorMessage = [string]::Format(
                 'The provided time interval "{0}" which translates to "{1}m" in your "-ThresholdInterval" argument does not exist',

lib/icinga/plugin/New-IcingaCheck.psm1

@@ -5,6 +5,9 @@ function New-IcingaCheck()
         $Value              = $null,
         $BaseValue          = $null,
         $Unit               = '',
+        $MetricIndex        = 'default',
+        $MetricName         = '',
+        $MetricTemplate     = '',
         [string]$Minimum    = '',
         [string]$Maximum    = '',
         $ObjectExists       = -1,
@@ -21,6 +24,9 @@ function New-IcingaCheck()
     $IcingaCheck | Add-Member -MemberType NoteProperty -Name 'Value'             -Value $Value;
     $IcingaCheck | Add-Member -MemberType NoteProperty -Name 'BaseValue'         -Value $BaseValue;
     $IcingaCheck | Add-Member -MemberType NoteProperty -Name 'Unit'              -Value $Unit;
+    $IcingaCheck | Add-Member -MemberType NoteProperty -Name 'MetricIndex'       -Value $MetricIndex;
+    $IcingaCheck | Add-Member -MemberType NoteProperty -Name 'MetricName'        -Value $MetricName;
+    $IcingaCheck | Add-Member -MemberType NoteProperty -Name 'MetricTemplate'    -Value $MetricTemplate;
     $IcingaCheck | Add-Member -MemberType NoteProperty -Name 'Minimum'           -Value $Minimum;
     $IcingaCheck | Add-Member -MemberType NoteProperty -Name 'Maximum'           -Value $Maximum;
     $IcingaCheck | Add-Member -MemberType NoteProperty -Name 'ObjectExists'      -Value $ObjectExists;
@@ -114,7 +120,7 @@ function New-IcingaCheck()
             $TimeSpan = [string]::Format(
                 '{0}({1}m avg.)',
                 (&{ if ([string]::IsNullOrEmpty($PluginThresholds)) { return ''; } else { return ' ' } }),
-                $this.__ThresholdObject.TimeSpan
+                $this.__ThresholdObject.TimeSpanOutput
             );
         }
 
@@ -138,14 +144,15 @@ function New-IcingaCheck()
 
     # __GetTimeSpanThreshold(0, 'Core_30_20', 'Core_30')
     $IcingaCheck | Add-Member -MemberType ScriptMethod -Force -Name '__GetTimeSpanThreshold' -Value {
-        param ($TimeSpanLabel, $Label);
+        param ($TimeSpanLabel, $Label, $MultiOutput);
 
         [hashtable]$TimeSpans = @{
             'Warning'  = '';
             'Critical' = '';
+            'Interval' = '';
         }
 
-        [string]$LabelName = (Format-IcingaPerfDataLabel $this.Name);
+        [string]$LabelName = (Format-IcingaPerfDataLabel -PerfData $this.Name -MultiOutput:$MultiOutput);
         if ([string]::IsNullOrEmpty($this.LabelName) -eq $FALSE) {
             $LabelName = $this.LabelName;
         }
@@ -154,7 +161,7 @@ function New-IcingaCheck()
             return $TimeSpans;
         }
 
-        $TimeSpan = $TimeSpanLabel.Replace($Label, '').Replace('_', '');
+        $TimeSpan = $TimeSpanLabel.Replace($Label, '').Replace('_', '').Replace('::Interval', '').Replace('::', '');
 
         if ($null -ne $this.__WarningValue -And [string]::IsNullOrEmpty($this.__WarningValue.TimeSpan) -eq $FALSE -And $this.__WarningValue.TimeSpan -eq $TimeSpan) {
             $TimeSpans.Warning = $this.__WarningValue.IcingaThreshold;
@@ -162,6 +169,7 @@ function New-IcingaCheck()
         if ($null -ne $this.__CriticalValue -And [string]::IsNullOrEmpty($this.__CriticalValue.TimeSpan) -eq $FALSE -And $this.__CriticalValue.TimeSpan -eq $TimeSpan) {
             $TimeSpans.Critical = $this.__CriticalValue.IcingaThreshold;
         }
+        $TimeSpans.Interval = $TimeSpan;
 
         return $TimeSpans;
     }
@@ -179,7 +187,8 @@ function New-IcingaCheck()
             return;
         }
 
-        [string]$LabelName = (Format-IcingaPerfDataLabel $this.Name);
+        [string]$LabelName      = (Format-IcingaPerfDataLabel -PerfData $this.Name);
+        [string]$MultiLabelName = (Format-IcingaPerfDataLabel -PerfData $this.Name -MultiOutput);
         $value                  = ConvertTo-Integer -Value $this.__ThresholdObject.RawValue -NullAsEmpty;
         $warning                = '';
         $critical               = '';
@@ -195,6 +204,7 @@ function New-IcingaCheck()
 
         if ([string]::IsNullOrEmpty($this.LabelName) -eq $FALSE) {
             $LabelName      = $this.LabelName;
+            $MultiLabelName = $this.LabelName;
         }
 
         if ([string]::IsNullOrEmpty($this.Minimum) -And [string]::IsNullOrEmpty($this.Maximum)) {
@@ -211,8 +221,18 @@ function New-IcingaCheck()
             }
         }
 
+        $PerfDataTemplate = ($this.__CheckCommand.Replace('Invoke-IcingaCheck', ''));
+
+        if ([string]::IsNullOrEmpty($this.MetricTemplate) -eq $FALSE) {
+            $PerfDataTemplate = $this.MetricTemplate;
+        }
+
         $this.__CheckPerfData = @{
+            'index'      = $this.MetricIndex;
+            'name'       = $this.MetricName;
+            'template'   = $PerfDataTemplate;
             'label'      = $LabelName;
+            'multilabel' = $MultiLabelName;
             'perfdata'   = '';
             'unit'       = $this.__ThresholdObject.PerfUnit;
             'value'      = (Format-IcingaPerfDataValue $value);
@@ -256,6 +276,20 @@ function New-IcingaCheck()
         }
     }
 
+    $IcingaCheck | Add-Member -MemberType ScriptMethod -Name '__ValidateMetricsName' -Value {
+        if ([string]::IsNullOrEmpty($this.MetricIndex)) {
+            Write-IcingaConsoleError -Message 'The metric index has no default value for the check object "{0}"' -Objects $this.Name;
+        } else {
+            $this.MetricIndex = Format-IcingaPerfDataLabel -PerfData $this.MetricIndex -MultiOutput;
+        }
+
+        if ([string]::IsNullOrEmpty($this.MetricName)) {
+            $this.MetricName = Format-IcingaPerfDataLabel -PerfData $this.Name -MultiOutput;
+        } else {
+            $this.MetricName = Format-IcingaPerfDataLabel -PerfData $this.MetricName -MultiOutput;
+        }
+    }
+
     $IcingaCheck | Add-Member -MemberType ScriptMethod -Name '__ConvertMinMax' -Value {
         if ([string]::IsNullOrEmpty($this.Unit) -eq $FALSE) {
             if ([string]::IsNullOrEmpty($this.Minimum) -eq $FALSE) {
@@ -298,9 +332,14 @@ function New-IcingaCheck()
         # Fix possible error for identical time stamps due to internal exceptions
         # and check execution within the same time slot because of this
         [string]$TimeIndex = Get-IcingaUnixTime;
+        [string]$LabelName = $this.Name;
 
-        Add-IcingaHashtableItem -Hashtable $global:Icinga.Private.Scheduler.CheckData[$this.__CheckCommand]['results'] -Key $this.Name -Value @{ } | Out-Null;
+        if ([string]::IsNullOrEmpty($this.LabelName) -eq $FALSE) {
-        Add-IcingaHashtableItem -Hashtable $global:Icinga.Private.Scheduler.CheckData[$this.__CheckCommand]['results'][$this.Name] -Key $TimeIndex -Value $this.Value -Override | Out-Null;
+            $LabelName = $this.LabelName;
+        }
+
+        Add-IcingaHashtableItem -Hashtable $global:Icinga.Private.Scheduler.CheckData[$this.__CheckCommand]['results'] -Key $LabelName -Value @{ } | Out-Null;
+        Add-IcingaHashtableItem -Hashtable $global:Icinga.Private.Scheduler.CheckData[$this.__CheckCommand]['results'][$LabelName] -Key $TimeIndex -Value $this.Value -Override | Out-Null;
     }
 
     $IcingaCheck | Add-Member -MemberType ScriptMethod -Name 'SetOk' -Value {
@@ -399,6 +438,7 @@ function New-IcingaCheck()
             '-BaseValue'      = $this.BaseValue;
             '-Unit'           = $this.Unit;
             '-CheckName'      = $this.__GetName();
+            '-PerfDataLabel'  = $this.LabelName;
             '-ThresholdCache' = (Get-IcingaThresholdCache -CheckCommand $this.__CheckCommand);
             '-Translation'    = $this.Translation;
             '-TimeInterval'   = $this.__TimeInterval;
@@ -936,6 +976,7 @@ function New-IcingaCheck()
 
     $IcingaCheck.__ValidateObject();
     $IcingaCheck.__ValidateUnit();
+    $IcingaCheck.__ValidateMetricsName();
     $IcingaCheck.__SetCurrentExecutionTime();
     $IcingaCheck.__AddCheckDataToCache();
     $IcingaCheck.__SetInternalTimeInterval();

lib/icinga/plugin/New-IcingaCheckPackage.psm1

@@ -362,12 +362,12 @@ function New-IcingaCheckPackage()
 
     # __GetTimeSpanThreshold(0, 'Core_30_20', 'Core_30')
     $IcingaCheckPackage | Add-Member -MemberType ScriptMethod -Force -Name '__GetTimeSpanThreshold' -Value {
-        param ($TimeSpanLabel, $Label);
+        param ($TimeSpanLabel, $Label, $MultiOutput);
 
         foreach ($check in $this.__Checks) {
-            $Result = $check.__GetTimeSpanThreshold($TimeSpanLabel, $Label);
+            $Result = $check.__GetTimeSpanThreshold($TimeSpanLabel, $Label, $MultiOutput);
 
-            if ([string]::IsNullOrEmpty($Result) -eq $FALSE) {
+            if ([string]::IsNullOrEmpty($Result.Interval) -eq $FALSE) {
                 return $Result;
             }
         }
@@ -375,6 +375,7 @@ function New-IcingaCheckPackage()
         return @{
             'Warning'  = '';
             'Critical' = '';
+            'Interval' = '';
         };
     }
 

lib/icinga/plugin/New-IcingaCheckResult.psm1

@@ -11,6 +11,10 @@ function New-IcingaCheckResult()
     $IcingaCheckResult | Add-Member -MemberType NoteProperty -Name 'NoPerfData' -Value $NoPerfData;
 
     $IcingaCheckResult | Add-Member -MemberType ScriptMethod -Name 'Compile' -Value {
+        # Always ensure our cache is cleared before compiling new check data
+        Get-IcingaCheckSchedulerPluginOutput | Out-Null;
+        Get-IcingaCheckSchedulerPerfData | Out-Null;
+
         if ($null -eq $this.Check) {
             return $IcingaEnums.IcingaExitCode.Unknown;
         }

lib/icinga/plugin/New-IcingaPerformanceDataEntry.psm1

@@ -5,20 +5,27 @@ function New-IcingaPerformanceDataEntry()
         $Label               = $null,
         $Value               = $null,
         $Warning             = $null,
-        $Critical       = $null
+        $Critical            = $null,
+        [hashtable]$PerfData = @{ },
+        [string]$Interval    = ''
     );
 
     if ($null -eq $PerfDataObject) {
-        return '';
+        return $PerfData;
     }
 
+    [string]$MetricIndex   = $PerfDataObject.index;
+    [string]$MetricName    = $PerfDataObject.name;
     [string]$LabelName     = $PerfDataObject.label;
+    [string]$Template      = $PerfDataObject.template;
     [string]$PerfValue     = $PerfDataObject.value;
     [string]$WarningValue  = $PerfDataObject.warning;
     [string]$CriticalValue = $PerfDataObject.critical;
 
     if ([string]::IsNullOrEmpty($Label) -eq $FALSE) {
         $LabelName      = $Label;
+        $MetricInterval = $Label.Split('::')[-1];
+        $MetricName     = [string]::Format('{0}::{1}', $MetricName, $MetricInterval);
     }
     if ([string]::IsNullOrEmpty($Value) -eq $FALSE) {
         $PerfValue = $Value;
@@ -26,7 +33,7 @@ function New-IcingaPerformanceDataEntry()
 
     # Override our warning/critical values only if the label does not match.
     # Eg. Core_1 not matching Core_1_5 - this is only required for time span checks
-    if ([string]::IsNullOrEmpty($Label) -eq $FALSE -And $Label -ne $PerfDataObject.label) {
+    if ([string]::IsNullOrEmpty($Label) -eq $FALSE -And [string]::IsNullOrEmpty($Interval) -eq $FALSE -And $Label.Contains([string]::Format('::Interval{0}', $Interval))) {
         $WarningValue  = $Warning;
         $CriticalValue = $Critical;
     }
@@ -41,16 +48,47 @@ function New-IcingaPerformanceDataEntry()
         $maximum = [string]::Format(';{0}', $PerfDataObject.maximum);
     }
 
-    return (
+    [string]$MultiLabelName = '';
-        [string]::Format(
+    $LabelName              = [string]::Format('{0}::ifw_{1}::{2}', $MetricIndex, $Template, $MetricName).Replace('::::', '::');
-            "'{0}'={1}{2};{3};{4}{5}{6} ",
+
-            $LabelName.ToLower(),
+    if ($LabelName.Contains('::Interval') -eq $FALSE) {
+        if ($PerfData.ContainsKey($LabelName) -eq $FALSE) {
+            $PerfData.Add(
+                $LabelName,
+                @{
+                    'Index'  = '';
+                    'Values' = @()
+                }
+            );
+        }
+    }
+
+    if ([string]::IsNullOrEmpty($LabelName) -eq $FALSE -And $LabelName.Contains('::Interval')) {
+        $IntervalName   = $LabelName.Split('::')[-1];
+        $LabelInterval  = $IntervalName.Replace('Interval', '');
+        $MetricName     = $LabelName.Split('::')[4];
+        $MultiLabelName = [string]::Format('{0}{1}', $MetricName, (ConvertTo-IcingaNumericTimeIndex -TimeValue $LabelInterval));
+        $LabelName      = [string]::Format('{0}::ifw_{1}::{2}', $MetricIndex, $Template, $MetricName);
+    } else {
+        $MultiLabelName = $LabelName;
+    }
+
+    $PerfDataOutput = [string]::Format(
+        "'{0}'={1}{2};{3};{4}{5}{6}",
+        $MultiLabelName.ToLower(),
         (Format-IcingaPerfDataValue $PerfValue),
         $PerfDataObject.unit,
         (Format-IcingaPerfDataValue $WarningValue),
         (Format-IcingaPerfDataValue $CriticalValue),
         (Format-IcingaPerfDataValue $minimum),
         (Format-IcingaPerfDataValue $maximum)
-        )
     );
+
+    if ($MultiLabelName.Contains('::ifw_')) {
+        $PerfData[$LabelName].Index = $PerfDataOutput;
+    } else {
+        $PerfData[$LabelName].Values += $PerfDataOutput;
+    }
+
+    return $PerfData;
 }

lib/icinga/plugin/Write-IcingaPluginOutput.psm1

@@ -11,6 +11,6 @@ function Write-IcingaPluginOutput()
         Write-IcingaConsolePlain $Output;
     } else {
         # New behavior with local thread separated results
-        $global:Icinga.Private.Scheduler.CheckResults += $Output;
+        [array]$Global:Icinga.Private.Scheduler.CheckResults += $Output;
     }
 }

lib/icinga/plugin/Write-IcingaPluginPerfData.psm1

@@ -25,58 +25,60 @@ function Write-IcingaPluginPerfData()
         $CheckResultCache = New-Object PSCustomObject;
     }
 
+    Get-IcingaPluginPerfDataContent -PerfData $PerformanceData -CheckResultCache $CheckResultCache -IcingaCheck $IcingaCheck;
+
     if ($Global:Icinga.Protected.RunAsDaemon -eq $FALSE -And $Global:Icinga.Protected.JEAContext -eq $FALSE) {
-        [string]$PerfDataOutput = (Get-IcingaPluginPerfDataContent -PerfData $PerformanceData -CheckResultCache $CheckResultCache -IcingaCheck $IcingaCheck);
+        if ($Global:Icinga.Private.Scheduler.PerformanceData.Count -ne 0) {
-        Write-IcingaConsolePlain ([string]::Format('| {0}', $PerfDataOutput));
+            Write-IcingaConsolePlain ([string]::Format('| {0}', ([string]::Join(' ', $Global:Icinga.Private.Scheduler.PerformanceData))));
-    } else {
+        }
-        [void](Get-IcingaPluginPerfDataContent -PerfData $PerformanceData -CheckResultCache $CheckResultCache -AsObject $TRUE -IcingaCheck $IcingaCheck);
     }
 }
 
 function Get-IcingaPluginPerfDataContent()
 {
-    param(
+    param (
         $PerfData,
         $CheckResultCache,
-        [bool]$AsObject = $FALSE,
         $IcingaCheck      = $null
     );
 
-    [string]$PerfDataOutput = '';
+    [hashtable]$CompiledPerfData = @{ };
 
     foreach ($package in $PerfData.Keys) {
         $data = $PerfData[$package];
         if ($data.package) {
-            $PerfDataOutput += (Get-IcingaPluginPerfDataContent -PerfData $data.perfdata -CheckResultCache $CheckResultCache -AsObject $AsObject -IcingaCheck $IcingaCheck);
+            (Get-IcingaPluginPerfDataContent -PerfData $data.perfdata -CheckResultCache $CheckResultCache -IcingaCheck $IcingaCheck);
         } else {
+            $CompiledPerfData = New-IcingaPerformanceDataEntry -PerfDataObject $data -PerfData $CompiledPerfData;
+
             foreach ($checkresult in $CheckResultCache.PSobject.Properties) {
 
                 $SearchPattern      = [string]::Format('{0}_', $data.label);
+                $SearchPatternMulti = [string]::Format('::{0}::Interval', $data.multilabel);
                 $SearchEntry        = $checkresult.Name;
-                if ($SearchEntry -like "$SearchPattern*") {
-                    $TimeSpan  = $IcingaCheck.__GetTimeSpanThreshold($SearchEntry, $data.label);
 
-                    $cachedresult = (New-IcingaPerformanceDataEntry -PerfDataObject $data -Label $SearchEntry -Value $checkresult.Value -Warning $TimeSpan.Warning -Critical $TimeSpan.Critical);
+                if ($SearchEntry -like "$SearchPatternMulti*") {
-
+                    $TimeSpan          = $IcingaCheck.__GetTimeSpanThreshold($SearchEntry, $data.multilabel, $TRUE);
-                    if ($AsObject) {
+                    $CompiledPerfData = New-IcingaPerformanceDataEntry -PerfDataObject $data -Label $SearchEntry -Value $checkresult.Value -Warning $TimeSpan.Warning -Critical $TimeSpan.Critical -Interval $TimeSpan.Interval -PerfData $CompiledPerfData;
-                        # New behavior with local thread separated results
+                }
-                        $global:Icinga.Private.Scheduler.PerformanceData += $cachedresult;
             }
-                    $PerfDataOutput += $cachedresult;
         }
     }
 
-            $compiledPerfData = (New-IcingaPerformanceDataEntry $data);
+    foreach ($entry in $CompiledPerfData.Keys) {
+        $entryValue            = $CompiledPerfData[$entry];
+        [string]$PerfDataLabel = $entryValue.Index;
 
-            if ($AsObject) {
+        if ($entryValue.Values.Count -ne 0) {
-                # New behavior with local thread separated results
+            [string]$PerfDataLabel = [string]::Format('{0} {1}', $entryValue.Index, ([string]::Join(' ', ($entryValue.Values | Sort-Object))));
-                $global:Icinga.Private.Scheduler.PerformanceData += $compiledPerfData;
-            }
-            $PerfDataOutput += $compiledPerfData;
-        }
         }
 
-    return $PerfDataOutput;
+        [array]$global:Icinga.Private.Scheduler.PerformanceData += $PerfDataLabel;
+    }
+
+    [array]$global:Icinga.Private.Scheduler.PerformanceData = $Global:Icinga.Private.Scheduler.PerformanceData | Sort-Object @{
+        expression = { $_.Substring(1, $_.IndexOf('::') - 1) -As [int] }
+    };
 }
 
 Export-ModuleMember -Function @( 'Write-IcingaPluginPerfData' );