Merge pull request #142 from Icinga/feature/add_code_cache_for_faster_framework_loading

Experimental: Adds code caching for faster framework loading

## Current Situation

Currently the entire Framework is taking a huge amount of time to load the files. This causes several issues:

* Slow loading increases the runtime of checks
* High CPU usage for the initial loading of the Framework
* High CPU usage over a longer period of time, cause more impcact on the systems

## Possible solutions

### Add caching

To reduce the impact for the Framework loading, we could add a cache file containing all Cmdlets, Enums and Functions allowing us to import file on initialization instead of having of to search for all `.psm1` files and load them one by one

### Use Nested Modules for PowerShell Plugins

PowerShell plugins right now are using `Use-IcingaPlugins` which searchs for all `.psm1` files inside the plugin folder to load them. We should use `NestedModules` here, as the overall impact is lower. On Framework side we can't do this how ever without loading times to explode. Plugins are tracked [at issue #87 here](https://github.com/Icinga/icinga-powershell-plugins/pull/87)

## Current Status

**Mitigated**

## Usage

You can enable/disable this feature by using `Enable-IcingaFrameworkCodeCache` and `Disable-IcingaFrameworkCodeCache`. Updating the cache is done with `Write-IcingaFrameworkCodeCache`

doc/31-Changelog.md

@@ -18,6 +18,7 @@ Released closed milestones can be found on [GitHub](https://github.com/Icinga/ic
 * [#139](https://github.com/Icinga/icinga-powershell-framework/pull/139) Add Cmdlet `Start-IcingaShellAsUser` to open an Icinga Shell as different user for testing
 * [#141](https://github.com/Icinga/icinga-powershell-framework/pull/141) Adds Cmdlet `Convert-IcingaPluginThresholds` as generic approach to convert Icinga Thresholds with units to the lowest unit of this type.
 * [#134](https://github.com/Icinga/icinga-powershell-framework/pull/134) Adds Cmdlet `Test-IcingaWindowsInformation` to check if a WMI class exist and if we can fetch data from it. In addition we add support for binary value comparison with the new Cmdlet `Test-IcingaBinaryOperator`
+* [#142](https://github.com/Icinga/icinga-powershell-framework/pull/142) **Experimental:** Adds feature to cache the Framework code into a single file to speed up the entire loading process, mitigating the impact on performance on systems with few CPU cores. You enable disables this feature by using `Enable-IcingaFrameworkCodeCache` and `Disable-IcingaFrameworkCodeCache`. Updating the cache is done with `Write-IcingaFrameworkCodeCache`
 * [#149](https://github.com/Icinga/icinga-powershell-framework/pull/149) Adds support to add Wmi permissions for a specific user and namespace with `Add-IcingaWmiPermissions`. In addition you can remove users from Wmi namespaces by using `Remove-IcingaWmiPermissions`
 
 ### Bugfixes

icinga-powershell-framework.psd1

@@ -7,10 +7,21 @@
     Copyright         = '(c) 2020 Icinga GmbH | MIT'
     Description       = 'Icinga for Windows module which allows to entirely monitor the Windows Host system.'
     PowerShellVersion = '4.0'
+    NestedModules     = @(
+        '.\lib\core\framework\Get-IcingaFrameworkCodeCache.psm1',
+        '.\lib\config\Get-IcingaPowerShellConfig.psm1',
+        '.\lib\config\Read-IcingaPowerShellConfig.psm1',
+        '.\lib\config\Test-IcingaPowerShellConfigItem.psm1',
+        '.\lib\core\logging\Write-IcingaConsoleOutput.psm1',
+        '.\lib\core\logging\Write-IcingaConsoleNotice.psm1',
+        '.\lib\core\logging\Write-IcingaConsoleWarning.psm1'
+    )
     FunctionsToExport = @(
         'Use-Icinga',
         'Invoke-IcingaCommand',
         'Import-IcingaLib',
+        'Get-IcingaFrameworkCodeCacheFile',
+        'Write-IcingaFrameworkCodeCache',
         'Publish-IcingaModuleManifest',
         'Publish-IcingaEventlogDocumentation',
         'Get-IcingaPluginDir',
@@ -19,9 +30,16 @@
         'Get-IcingaPowerShellConfigDir',
         'Get-IcingaFrameworkRootPath',
         'Get-IcingaPowerShellModuleFile',
-        'Start-IcingaShellAsUser'
+        'Start-IcingaShellAsUser',
+        'Get-IcingaPowerShellConfig',
+        'Get-IcingaFrameworkCodeCache',
+        'Read-IcingaPowerShellConfig',
+        'Test-IcingaPowerShellConfigItem',
+        'Write-IcingaConsoleOutput',
+        'Write-IcingaConsoleNotice',
+        'Write-IcingaConsoleWarning'
     )
-    CmdletsToExport   = @()
+    CmdletsToExport   = @('*')
     VariablesToExport = '*'
     AliasesToExport   = @( 'icinga' )
     PrivateData       = @{

icinga-powershell-framework.psm1

@@ -22,6 +22,10 @@ function Use-Icinga()
         Use-IcingaPlugins;
     }
 
+    if ((Test-Path (Get-IcingaFrameworkCodeCacheFile)) -eq $FALSE -And (Get-IcingaFrameworkCodeCache)) {
+        Write-IcingaFrameworkCodeCache;
+    }
+
     # This function will allow us to load this entire module including possible
     # actions, making it available within our shell environment
     # First load our custom modules
@@ -78,6 +82,20 @@ function Use-Icinga()
     }
 }
 
+function Get-IcingaFrameworkCodeCacheFile()
+{
+    return (Join-Path -Path (Get-IcingaCacheDir) -ChildPath 'framework_cache.psm1');
+}
+
+function Write-IcingaFrameworkCodeCache()
+{
+    if (Get-IcingaFrameworkCodeCache) {
+        Import-IcingaLib '\' -Init -CompileCache;
+    } else {
+        Write-IcingaConsoleNotice 'The experimental code caching feature is currently not enabled. You can enable it with "Enable-IcingaFrameworkCodeCache"';
+    }
+}
+
 function Import-IcingaLib()
 {
     param(
@@ -87,15 +105,27 @@ function Import-IcingaLib()
         [Switch]$ForceReload,
         [switch]$Init,
         [switch]$Custom,
-        [switch]$WriteManifests
+        [switch]$WriteManifests,
+        [switch]$CompileCache
     );
 
+    
     # This is just to only allow a global loading of the module. Import-IcingaLib is ignored on every other
     # location. It is just there to give a basic idea within commands, of which functions are used
     if ($Init -eq $FALSE) {
         return;
     }
     
+    $CacheFile = Get-IcingaFrameworkCodeCacheFile;
+
+    if ($Custom -eq $FALSE -And $CompileCache -eq $FALSE -And (Test-Path $CacheFile) -And (Get-IcingaFrameworkCodeCache)) {
+        Import-Module $CacheFile -Global;
+        return;
+    }
+
+    [array]$ImportModules = @();
+    [array]$RemoveModules = @();
+
     if ($Custom) {
         [string]$directory  = Join-Path -Path $PSScriptRoot -ChildPath 'custom\';
     } else {
@@ -116,11 +146,11 @@ function Import-IcingaLib()
 
                 if ($ListOfLoadedModules -like "*$moduleName*") {
                     if ($ForceReload) {
-                        Remove-Module -Name $moduleName
+                        $RemoveModules += $moduleName;
-                        Import-Module ([string]::Format('{0}', $modulePath)) -Global;
                     }
+                    $ImportModules += $modulePath;
                 } else {
-                    Import-Module ([string]::Format('{0}', $modulePath)) -Global;
+                    $ImportModules += $modulePath;
                     if ($WriteManifests) {
                         Publish-IcingaModuleManifest -Module $moduleName;
                     }
@@ -132,15 +162,35 @@ function Import-IcingaLib()
 
         if ($ForceReload) {
             if ($ListOfLoadedModules -Like "*$moduleName*") {
-                Remove-Module -Name $moduleName;
+                $RemoveModules += $moduleName;
             }
         }
 
-        Import-Module ([string]::Format('{0}.psm1', $module)) -Global;
+        $ImportModules += ([string]::Format('{0}.psm1', $module));
         if ($WriteManifests) {
             Publish-IcingaModuleManifest -Module $moduleName;
         }
     }
+
+    if ($RemoveModules.Count -ne 0) {
+        Remove-Module $RemoveModules;
+    }
+
+    if ($ImportModules.Count -ne 0) {
+
+        if ($CompileCache) {
+            $CacheContent = '';
+            foreach ($module in $ImportModules) {
+                $Content      = Get-Content $module -Raw;
+                $CacheContent += $Content + "`r`n";
+            }
+
+            $CacheContent += $Content + "Export-ModuleMember -Function @( '*' )";
+            Set-Content -Path $CacheFile -Value $CacheContent;
+        } else {
+            Import-Module $ImportModules -Global;
+        }
+    }
 }
 
 function Publish-IcingaModuleManifest()
@@ -279,6 +329,9 @@ function Invoke-IcingaCommand()
         Write-Output ([string]::Format('** Icinga PowerShell Framework {0}', $IcingaFrameworkData.PrivateData.Version));
         Write-Output ([string]::Format('** Copyright {0}', $IcingaFrameworkData.Copyright));
         Write-Output ([string]::Format('** User environment {0}\{1}', $env:USERDOMAIN, $env:USERNAME));
+        if (Get-IcingaFrameworkCodeCache) {
+            Write-Output ([string]::Format('** Warning: Icinga Framework Code Caching is enabled'));
+        }
         Write-Output '******************************************************';
     }
 

lib/config/Read-IcingaPowerShellConfig.psm1

@@ -25,13 +25,13 @@ function Read-IcingaPowerShellConfig()
     }
 
     if (-Not (Test-Path $ConfigFile)) {
-        return (New-Object -TypeName PSOBject);
+        return (New-Object -TypeName PSObject);
     }
 
     [string]$Content = Get-Content -Path $ConfigFile;
 
     if ([string]::IsNullOrEmpty($Content)) {
-        return (New-Object -TypeName PSOBject);
+        return (New-Object -TypeName PSObject);
     }
 
     return (ConvertFrom-Json -InputObject $Content);

lib/core/framework/Disable-IcingaFrameworkCodeCache.psm1

@@ -0,0 +1,19 @@
+<#
+.SYNOPSIS
+   Disables the experimental feature to cache all functions into a single file,
+   allowing quicker loading times of the Icinga PowerShell Framework
+.DESCRIPTION
+   Disables the experimental feature to cache all functions into a single file,
+   allowing quicker loading times of the Icinga PowerShell Framework
+.FUNCTIONALITY
+   Experimental: Disables the Icinga for Windows code caching
+.EXAMPLE
+   PS>Disable-IcingaFrameworkCodeCache;
+.LINK
+   https://github.com/Icinga/icinga-powershell-framework
+#>
+
+function Disable-IcingaFrameworkCodeCache()
+{
+    Set-IcingaPowerShellConfig -Path 'Framework.Experimental.CodeCaching' -Value $FALSE;
+}

lib/core/framework/Enable-IcingaFrameworkCodeCache.psm1

@@ -0,0 +1,21 @@
+<#
+.SYNOPSIS
+   Enables the experimental feature to cache all functions into a single file,
+   allowing quicker loading times of the Icinga PowerShell Framework
+.DESCRIPTION
+   Enables the experimental feature to cache all functions into a single file,
+   allowing quicker loading times of the Icinga PowerShell Framework
+.FUNCTIONALITY
+   Experimental: Enables the Icinga for Windows code caching
+.EXAMPLE
+   PS>Enable-IcingaFrameworkCodeCache;
+.LINK
+   https://github.com/Icinga/icinga-powershell-framework
+#>
+
+function Enable-IcingaFrameworkCodeCache()
+{
+    Set-IcingaPowerShellConfig -Path 'Framework.Experimental.CodeCaching' -Value $TRUE;
+
+    Write-IcingaConsoleWarning 'This is an experimental feature and might cause some side effects during usage. Please use this function with caution. Please run "Write-IcingaFrameworkCodeCache" in addition to ensure your cache is updated. This should be done after each update of the Framework in case the feature was disabled during the update run.';
+}

lib/core/framework/Get-IcingaFrameworkCodeCache.psm1

@@ -0,0 +1,28 @@
+<#
+.SYNOPSIS
+   Fetches the current enable/disable state of the experimental feature
+   for caching the Icinga PowerShell Framework code
+.DESCRIPTION
+   Fetches the current enable/disable state of the experimental feature
+   for caching the Icinga PowerShell Framework code
+.FUNCTIONALITY
+   Experimental: Get the current code caching configuration of the
+   Icinga PowerShell Framework
+.EXAMPLE
+   PS>Get-IcingaFrameworkCodeCache;
+.LINK
+   https://github.com/Icinga/icinga-powershell-framework
+.OUTPUTS
+   System.Boolean
+#>
+
+function Get-IcingaFrameworkCodeCache()
+{
+    $CodeCaching = Get-IcingaPowerShellConfig -Path 'Framework.Experimental.CodeCaching';
+
+    if ($null -eq $CodeCaching) {
+        return $FALSE;
+    }
+
+    return $CodeCaching;
+}

lib/core/framework/Install-IcingaFrameworkUpdate.psm1

@@ -91,6 +91,9 @@ function Install-IcingaFrameworkUpdate()
     Start-Sleep -Seconds 1;
     Remove-ItemSecure -Path $Archive.Directory -Recurse -Force | Out-Null;
 
+    Write-IcingaConsoleNotice 'Updating Framework cache file';
+    Write-IcingaFrameworkCodeCache;
+
     Write-IcingaConsoleNotice 'Framework update has been completed. Please start a new PowerShell instance now to complete the update';
 
     Test-IcingaAgent;