From 0f84ce0470014c05d1b5fcc5ca6b958c31a02ef4 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 13 Feb 2020 18:18:58 +0100 Subject: [PATCH 01/75] Consider a JsonRpcConnection being seen on a single byte of TLS payload, not only a whole message --- lib/base/tlsstream.hpp | 40 ++++++++++++++++++++++++++++++-- lib/remote/jsonrpcconnection.cpp | 2 ++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/lib/base/tlsstream.hpp b/lib/base/tlsstream.hpp index 70a459114..05b3550d5 100644 --- a/lib/base/tlsstream.hpp +++ b/lib/base/tlsstream.hpp @@ -8,17 +8,53 @@ #include "base/stream.hpp" #include "base/tlsutility.hpp" #include "base/fifo.hpp" +#include "base/utility.hpp" +#include #include #include #include #include #include +#include #include #include namespace icinga { +template +class SeenStream : public ARS +{ +public: + template + SeenStream(Args&&... args) : ARS(std::forward(args)...) + { + m_Seen.store(nullptr); + } + + template + auto async_read_some(Args&&... args) -> decltype(((ARS*)nullptr)->async_read_some(std::forward(args)...)) + { + { + auto seen (m_Seen.load()); + + if (seen) { + *seen = Utility::GetTime(); + } + } + + return ((ARS*)this)->async_read_some(std::forward(args)...); + } + + inline void SetSeen(double* seen) + { + m_Seen.store(seen); + } + +private: + std::atomic m_Seen; +}; + struct UnbufferedAsioTlsStreamParams { boost::asio::io_context& IoContext; @@ -26,14 +62,14 @@ struct UnbufferedAsioTlsStreamParams const String& Hostname; }; -typedef boost::asio::ssl::stream AsioTcpTlsStream; +typedef SeenStream> AsioTcpTlsStream; class UnbufferedAsioTlsStream : public AsioTcpTlsStream { public: inline UnbufferedAsioTlsStream(UnbufferedAsioTlsStreamParams& init) - : stream(init.IoContext, init.SslContext), m_VerifyOK(true), m_Hostname(init.Hostname) + : AsioTcpTlsStream(init.IoContext, init.SslContext), m_VerifyOK(true), m_Hostname(init.Hostname) { } diff --git a/lib/remote/jsonrpcconnection.cpp b/lib/remote/jsonrpcconnection.cpp index b6d1d41e6..5ee2b645b 100644 --- a/lib/remote/jsonrpcconnection.cpp +++ b/lib/remote/jsonrpcconnection.cpp @@ -60,6 +60,8 @@ void JsonRpcConnection::Start() void JsonRpcConnection::HandleIncomingMessages(boost::asio::yield_context yc) { + m_Stream->next_layer().SetSeen(&m_Seen); + for (;;) { String message; From 85d757b2c273995e20372c48e15f96fda9f45591 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Wed, 25 Sep 2019 12:50:47 +0200 Subject: [PATCH 02/75] Chocolatey: Update package metadata to fix dependencies --- choco/icinga2.nuspec.cmake | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/choco/icinga2.nuspec.cmake b/choco/icinga2.nuspec.cmake index 46225b556..1e3586799 100755 --- a/choco/icinga2.nuspec.cmake +++ b/choco/icinga2.nuspec.cmake @@ -10,17 +10,20 @@ Icinga GmbH Icinga GmbH icinga2 - Monitoring Agent for Windows - Icinga 2 is an open source monitoring platform which notifies users about host and service outages. + Icinga is an open source monitoring platform which notifies users about host and service outages. https://icinga.com/ - icinga2 agent monitoring admin - https://icinga.com/resources/faq/ + icinga2 icinga agent monitoring admin + https://github.com/Icinga/icinga2/blob/master/COPYING https://github.com/Icinga/icinga2/blob/master/ChangeLog - https://docs.icinga.com/icinga2/ + https://icinga.com/docs/icinga2/latest/ https://github.com/Icinga/icinga2/issues https://github.com/Icinga/icinga2 https://github.com/Icinga/icinga2 false - https://icinga.com/wp-content/uploads/2015/05/icinga_icon_128x128.png + https://raw.githubusercontent.com/Icinga/icinga2/master/icinga-app/icinga.ico + + + From 44d1cefbf7534abd66e51659696a36514d2af85f Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Thu, 26 Sep 2019 13:37:09 +0200 Subject: [PATCH 03/75] ChocoInstall: Prefer short release version download, not long x.y.z.a --- CMakeLists.txt | 4 +++- choco/chocolateyInstall.ps1.cmake | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1eefc9938..5ab01f550 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -119,7 +119,9 @@ endif() # NuGet on Windows requires a semantic versioning, example: 2.10.4.123 (only 4 element, only numeric) string(REGEX REPLACE "-([0-9]+).*$" ".\\1" ICINGA2_VERSION_SAFE "${ICINGA2_VERSION}") string(REGEX REPLACE "-[^\\.]*(.*)$" "\\1" ICINGA2_VERSION_SAFE "${ICINGA2_VERSION_SAFE}") -message(STATUS "ICINGA2_VERSION_SAFE=${ICINGA2_VERSION_SAFE}") +string(REGEX REPLACE "^([0-9]+\\.[0-9]+\\.[0-9]+)[\\.]?[0-9]*" "\\1" CHOCO_VERSION_SHORT "${ICINGA2_VERSION_SAFE}") + +message(STATUS "ICINGA2_VERSION_SAFE=${ICINGA2_VERSION_SAFE} CHOCO_VERSION_SHORT=${CHOCO_VERSION_SHORT}") if(WIN32) set(Boost_USE_STATIC_LIBS ON) diff --git a/choco/chocolateyInstall.ps1.cmake b/choco/chocolateyInstall.ps1.cmake index ab1660324..8a524419f 100755 --- a/choco/chocolateyInstall.ps1.cmake +++ b/choco/chocolateyInstall.ps1.cmake @@ -1,7 +1,7 @@ $packageName = 'icinga2' $installerType = 'msi' -$url32 = 'https://packages.icinga.com/windows/Icinga2-v${ICINGA2_VERSION_SAFE}-x86.msi' -$url64 = 'https://packages.icinga.com/windows/Icinga2-v${ICINGA2_VERSION_SAFE}-x86_64.msi' +$url32 = 'https://packages.icinga.com/windows/Icinga2-v${CHOCO_VERSION_SHORT}-x86.msi' +$url64 = 'https://packages.icinga.com/windows/Icinga2-v${CHOCO_VERSION_SHORT}-x86_64.msi' $silentArgs = '/qn /norestart' $validExitCodes = @(0) From d6048f6fe0a4590df4b07eba11674ee77439b13d Mon Sep 17 00:00:00 2001 From: Michael Insel Date: Mon, 7 Oct 2019 19:36:44 +0200 Subject: [PATCH 04/75] Chocolatey: Update package metadata to correct dependencies --- choco/icinga2.nuspec.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/choco/icinga2.nuspec.cmake b/choco/icinga2.nuspec.cmake index 1e3586799..d0699f24b 100755 --- a/choco/icinga2.nuspec.cmake +++ b/choco/icinga2.nuspec.cmake @@ -22,7 +22,7 @@ false https://raw.githubusercontent.com/Icinga/icinga2/master/icinga-app/icinga.ico - + From d051d71b75490ac54c149eb6d8fd9bb5d6d9fbab Mon Sep 17 00:00:00 2001 From: Michael Insel Date: Thu, 10 Oct 2019 18:39:33 +0200 Subject: [PATCH 05/75] ChocoInstall: Update script to use checksum verification --- CMakeLists.txt | 2 +- choco/chocolateyInstall.ps1.cmake | 24 ++++++++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ab01f550..046f23cf2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -520,4 +520,4 @@ if(WIN32) ) endif() -include(CPack) +include(CPack) \ No newline at end of file diff --git a/choco/chocolateyInstall.ps1.cmake b/choco/chocolateyInstall.ps1.cmake index 8a524419f..26fe2396a 100755 --- a/choco/chocolateyInstall.ps1.cmake +++ b/choco/chocolateyInstall.ps1.cmake @@ -1,8 +1,20 @@ -$packageName = 'icinga2' -$installerType = 'msi' -$url32 = 'https://packages.icinga.com/windows/Icinga2-v${CHOCO_VERSION_SHORT}-x86.msi' +$packageName= 'icinga2' +$toolsDir = "$(Split-Path -Parent $MyInvocation.MyCommand.Definition)" +$url = 'https://packages.icinga.com/windows/Icinga2-v${CHOCO_VERSION_SHORT}-x86.msi' $url64 = 'https://packages.icinga.com/windows/Icinga2-v${CHOCO_VERSION_SHORT}-x86_64.msi' -$silentArgs = '/qn /norestart' -$validExitCodes = @(0) -Install-ChocolateyPackage "$packageName" "$installerType" "$silentArgs" "$url32" "$url64" -validExitCodes $validExitCodes +$packageArgs = @{ + packageName = $packageName + fileType = 'msi' + url = $url + url64bit = $url64 + silentArgs = "/qn /norestart" + validExitCodes= @(0) + softwareName = 'Icinga 2*' + checksum = '' # TODO: Add checksum + checksumType = 'sha256' + checksum64 = '' # TODO: Add checksum + checksumType64= 'sha256' +} + +Install-ChocolateyPackage @packageArgs \ No newline at end of file From feed379229a5ba7cb5bd4b4d9808cfe59683abb7 Mon Sep 17 00:00:00 2001 From: Michael Insel Date: Fri, 11 Oct 2019 18:42:48 +0200 Subject: [PATCH 06/75] Chocolatey: Add build script for package --- choco/CMakeLists.txt | 10 +---- ...e => chocolateyInstall.ps1.template.cmake} | 38 ++++++++--------- tools/win32/build-choco.ps1 | 42 +++++++++++++++++++ 3 files changed, 62 insertions(+), 28 deletions(-) rename choco/{chocolateyInstall.ps1.cmake => chocolateyInstall.ps1.template.cmake} (79%) mode change 100755 => 100644 create mode 100644 tools/win32/build-choco.ps1 diff --git a/choco/CMakeLists.txt b/choco/CMakeLists.txt index d7b90bb47..fb147a15c 100644 --- a/choco/CMakeLists.txt +++ b/choco/CMakeLists.txt @@ -1,14 +1,6 @@ # Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ if(WIN32) - find_program(CHOCO_BINARY choco) - configure_file(icinga2.nuspec.cmake icinga2.nuspec) - configure_file(chocolateyInstall.ps1.cmake chocolateyInstall.ps1) - - add_custom_target(choco-pkg ALL - COMMAND choco pack - COMMAND ${CMAKE_COMMAND} -E rename ${CMAKE_CURRENT_BINARY_DIR}/icinga2.${ICINGA2_VERSION_SAFE}.nupkg ${CMAKE_CURRENT_BINARY_DIR}/icinga2.nupkg - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/icinga2.nuspec ${CMAKE_CURRENT_BINARY_DIR}/chocolateyInstall.ps1 chocolateyUninstall.ps1 - ) + configure_file(chocolateyInstall.ps1.template.cmake chocolateyInstall.ps1.template) endif() diff --git a/choco/chocolateyInstall.ps1.cmake b/choco/chocolateyInstall.ps1.template.cmake old mode 100755 new mode 100644 similarity index 79% rename from choco/chocolateyInstall.ps1.cmake rename to choco/chocolateyInstall.ps1.template.cmake index 26fe2396a..424a73778 --- a/choco/chocolateyInstall.ps1.cmake +++ b/choco/chocolateyInstall.ps1.template.cmake @@ -1,20 +1,20 @@ -$packageName= 'icinga2' -$toolsDir = "$(Split-Path -Parent $MyInvocation.MyCommand.Definition)" -$url = 'https://packages.icinga.com/windows/Icinga2-v${CHOCO_VERSION_SHORT}-x86.msi' -$url64 = 'https://packages.icinga.com/windows/Icinga2-v${CHOCO_VERSION_SHORT}-x86_64.msi' - -$packageArgs = @{ - packageName = $packageName - fileType = 'msi' - url = $url - url64bit = $url64 - silentArgs = "/qn /norestart" - validExitCodes= @(0) - softwareName = 'Icinga 2*' - checksum = '' # TODO: Add checksum - checksumType = 'sha256' - checksum64 = '' # TODO: Add checksum - checksumType64= 'sha256' -} - +$packageName= 'icinga2' +$toolsDir = "$(Split-Path -Parent $MyInvocation.MyCommand.Definition)" +$url = 'https://packages.icinga.com/windows/Icinga2-v${CHOCO_VERSION_SHORT}-x86.msi' +$url64 = 'https://packages.icinga.com/windows/Icinga2-v${CHOCO_VERSION_SHORT}-x86_64.msi' + +$packageArgs = @{ + packageName = $packageName + fileType = 'msi' + url = $url + url64bit = $url64 + silentArgs = "/qn /norestart" + validExitCodes= @(0) + softwareName = 'Icinga 2*' + checksum = '%CHOCO_32BIT_CHECKSUM%' + checksumType = 'sha256' + checksum64 = '%CHOCO_64BIT_CHECKSUM%' + checksumType64= 'sha256' +} + Install-ChocolateyPackage @packageArgs \ No newline at end of file diff --git a/tools/win32/build-choco.ps1 b/tools/win32/build-choco.ps1 new file mode 100644 index 000000000..24fa5f498 --- /dev/null +++ b/tools/win32/build-choco.ps1 @@ -0,0 +1,42 @@ +Set-PsDebug -Trace 1 + +if(-not (Test-Path "$($env:ProgramData)\chocolatey\choco.exe")) { + throw "Could not find Choco executable. Abort." +} + +if (-not (Test-Path env:ICINGA2_BUILDPATH)) { + $env:ICINGA2_BUILDPATH = '.\debug' +} + +if(-not (Test-Path "$($env:ICINGA2_BUILDPATH)\choco\chocolateyInstall.ps1.template")) { + throw "Could not find Chocolatey install script template. Abort." +} + +$chocoInstallScriptTemplatePath = "$($env:ICINGA2_BUILDPATH)\choco\chocolateyInstall.ps1.template" +$chocoInstallScript = Get-Content $chocoInstallScriptTemplatePath + +if(-not (Test-Path "$($env:ICINGA2_BUILDPATH)\*-x86.msi")) { + throw "Could not find Icinga 2 32 bit MSI package. Abort." +} + +$hashMSIpackage32 = Get-FileHash "$($env:ICINGA2_BUILDPATH)\*-x86.msi" +Write-Output "File Hash for 32 bit MSI package: $($hashMSIpackage32.Hash)." + +if(-not (Test-Path "$($env:ICINGA2_BUILDPATH)\*-x86_64.msi")) { + throw "Could not find Icinga 2 64 bit MSI package. Abort." +} + +$hashMSIpackage64 = Get-FileHash "$($env:ICINGA2_BUILDPATH)\*-x86_64.msi" +Write-Output "File Hash for 32 bit MSI package: $($hashMSIpackage64.Hash)" + +$chocoInstallScript = $chocoInstallScript.Replace("%CHOCO_32BIT_CHECKSUM%", "$($hashMSIpackage32.Hash)") +$chocoInstallScript = $chocoInstallScript.Replace("%CHOCO_64BIT_CHECKSUM%", "$($hashMSIpackage64.Hash)") +Write-Output $chocoInstallScript + +Set-Content -Path "$($env:ICINGA2_BUILDPATH)\choco\chocolateyInstall.ps1" -Value $chocoInstallScript + +cd "$($env:ICINGA2_BUILDPATH)\choco" +& "$($env:ProgramData)\chocolatey\choco.exe" "pack" +cd "..\.." + +Move-Item -Path "$($env:ICINGA2_BUILDPATH)\choco\*.nupkg" -Destination "$($env:ICINGA2_BUILDPATH)" \ No newline at end of file From 5e7a675009c8e37a37f9c80d1b612daaa4f51db4 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 18 Mar 2020 11:58:27 +0100 Subject: [PATCH 07/75] JsonRpcConnection#HandleAndWriteHeartbeats(): check !!#m_Endpoint --- lib/remote/jsonrpcconnection-heartbeat.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/remote/jsonrpcconnection-heartbeat.cpp b/lib/remote/jsonrpcconnection-heartbeat.cpp index 993877e4e..ee29d47e1 100644 --- a/lib/remote/jsonrpcconnection-heartbeat.cpp +++ b/lib/remote/jsonrpcconnection-heartbeat.cpp @@ -28,9 +28,17 @@ void JsonRpcConnection::HandleAndWriteHeartbeats(boost::asio::yield_context yc) } if (m_NextHeartbeat != 0 && m_NextHeartbeat < Utility::GetTime()) { - Log(LogWarning, "JsonRpcConnection") - << "Client for endpoint '" << m_Endpoint->GetName() << "' has requested " - << "heartbeat message but hasn't responded in time. Closing connection."; + { + Log logMsg (LogWarning, "JsonRpcConnection"); + + if (m_Endpoint) { + logMsg << "Client for endpoint '" << m_Endpoint->GetName() << "'"; + } else { + logMsg << "Anonymous client"; + } + + logMsg << " has requested heartbeat message but hasn't responded in time. Closing connection."; + } Disconnect(); break; From 3617f05836191506dd1f13cbb162fe255bfa2d63 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 28 Apr 2020 17:34:00 +0200 Subject: [PATCH 08/75] icinga2 pki save-cert: allow to specify --key and --cert refs #7992 --- lib/cli/pkisavecertcommand.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/cli/pkisavecertcommand.cpp b/lib/cli/pkisavecertcommand.cpp index a946396f4..befd0eead 100644 --- a/lib/cli/pkisavecertcommand.cpp +++ b/lib/cli/pkisavecertcommand.cpp @@ -29,6 +29,10 @@ void PKISaveCertCommand::InitParameters(boost::program_options::options_descript ("trustedcert", po::value(), "Trusted certificate file path (output)") ("host", po::value(), "Parent Icinga instance to fetch the public TLS certificate from") ("port", po::value()->default_value("5665"), "Icinga 2 port"); + + hiddenDesc.add_options() + ("key", po::value()) + ("cert", po::value()); } std::vector PKISaveCertCommand::GetArgumentSuggestions(const String& argument, const String& word) const From f21b60e390514a4331c7ae629426238e680fc707 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 15 May 2020 15:33:37 +0200 Subject: [PATCH 09/75] StreamLogger#Flush(): lock self ... just to be sure. --- lib/base/streamlogger.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/base/streamlogger.cpp b/lib/base/streamlogger.cpp index 146fe3c91..cd5e79981 100644 --- a/lib/base/streamlogger.cpp +++ b/lib/base/streamlogger.cpp @@ -41,6 +41,8 @@ void StreamLogger::FlushLogTimerHandler() void StreamLogger::Flush() { + ObjectLock oLock (this); + if (m_Stream) m_Stream->flush(); } From 590b2eeb26a82294ff71b29a3a95d816db228af3 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 26 May 2020 16:33:13 +0200 Subject: [PATCH 10/75] GitHub actions: build Raspbian packages --- .github/workflows/packages.yml | 76 ++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 9f9d043fa..a2dec63bc 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -166,3 +166,79 @@ jobs: with: name: '${{ matrix.distro.name }}-${{ matrix.distro.release }}-packages' path: rpm-icinga2/build + raspbian: + name: Raspbian + + strategy: + matrix: + codename: + - buster + + runs-on: ubuntu-latest + + steps: + - name: Checkout HEAD + uses: actions/checkout@v1 + + - name: qemu-user-static + run: | + set -exo pipefail + sudo apt-get update + DEBIAN_FRONTEND=noninteractive sudo apt-get install -y qemu-user-static + + - name: raspbian-icinga2 + run: | + set -exo pipefail + git clone https://git.icinga.com/packaging/raspbian-icinga2.git + chmod o+w raspbian-icinga2 + + - name: Restore/backup ccache + id: ccache + uses: actions/cache@v1 + with: + path: raspbian-icinga2/ccache + key: 'raspbian/${{ matrix.codename }}-ccache' + + - name: Binary + run: | + set -exo pipefail + git checkout -B master + if [ -e raspbian-icinga2/ccache ]; then + chmod -R o+w raspbian-icinga2/ccache + fi + docker run --rm \ + -v "$(pwd)/raspbian-icinga2:/raspbian-icinga2" \ + -v "$(pwd)/.git:/icinga2.git:ro" \ + -w /raspbian-icinga2 \ + -e ICINGA_BUILD_PROJECT=icinga2 \ + -e ICINGA_BUILD_TYPE=snapshot \ + -e UPSTREAM_GIT_URL=file:///icinga2.git \ + -e ICINGA_BUILD_DEB_DEFAULT_ARCH=armhf \ + registry.icinga.com/build-docker/raspbian/${{ matrix.codename }} \ + icinga-build-package + +# Setting up icinga2-bin (2.12.0+rc1.25.g5d1c82a3d.20200526.0754+buster-0) ... +# enabling default icinga2 features +# qemu:handle_cpu_signal received signal outside vCPU context @ pc=0x6015c75c +# qemu:handle_cpu_signal received signal outside vCPU context @ pc=0x6015c75c +# qemu:handle_cpu_signal received signal outside vCPU context @ pc=0x600016ea +# dpkg: error processing package icinga2-bin (--configure): +# installed icinga2-bin package post-installation script subprocess returned error exit status 127 +# +# - name: Test +# run: | +# set -exo pipefail +# docker run --rm \ +# -v "$(pwd)/raspbian-icinga2:/raspbian-icinga2" \ +# -w /raspbian-icinga2 \ +# -e ICINGA_BUILD_PROJECT=icinga2 \ +# -e ICINGA_BUILD_TYPE=snapshot \ +# -e ICINGA_BUILD_DEB_DEFAULT_ARCH=armhf \ +# registry.icinga.com/build-docker/raspbian/${{ matrix.codename }} \ +# icinga-build-test + + - name: Artifacts + uses: actions/upload-artifact@v1 + with: + name: 'raspbian-${{ matrix.codename }}-packages' + path: raspbian-icinga2/build From d38f4e78c99454d21bc9f10c34c5af6b96ba1027 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 5 Jun 2020 09:57:34 +0200 Subject: [PATCH 11/75] Chocolatey: Use '.\build' as default build directory --- tools/win32/build-choco.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/win32/build-choco.ps1 b/tools/win32/build-choco.ps1 index 24fa5f498..32138bdf2 100644 --- a/tools/win32/build-choco.ps1 +++ b/tools/win32/build-choco.ps1 @@ -5,7 +5,7 @@ if(-not (Test-Path "$($env:ProgramData)\chocolatey\choco.exe")) { } if (-not (Test-Path env:ICINGA2_BUILDPATH)) { - $env:ICINGA2_BUILDPATH = '.\debug' + $env:ICINGA2_BUILDPATH = '.\build' } if(-not (Test-Path "$($env:ICINGA2_BUILDPATH)\choco\chocolateyInstall.ps1.template")) { From 2f0f2e8c355b75fa4407d23f85feea037d2bc4b6 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 8 Jun 2020 15:14:29 +0200 Subject: [PATCH 12/75] prepare-dirs: combine mkdir and chmod --- etc/initsystem/prepare-dirs.cmake | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/etc/initsystem/prepare-dirs.cmake b/etc/initsystem/prepare-dirs.cmake index 99dc60223..4cef83193 100644 --- a/etc/initsystem/prepare-dirs.cmake +++ b/etc/initsystem/prepare-dirs.cmake @@ -26,12 +26,10 @@ getent group $ICINGA2_GROUP >/dev/null 2>&1 || (echo "Icinga group '$ICINGA2_GRO getent group $ICINGA2_COMMAND_GROUP >/dev/null 2>&1 || (echo "Icinga command group '$ICINGA2_COMMAND_GROUP' does not exist. Exiting." && exit 6) if [ ! -e "$ICINGA2_INIT_RUN_DIR" ]; then - mkdir "$ICINGA2_INIT_RUN_DIR" - mkdir "$ICINGA2_INIT_RUN_DIR"/cmd + mkdir -m 755 "$ICINGA2_INIT_RUN_DIR" + mkdir -m 2750 "$ICINGA2_INIT_RUN_DIR"/cmd fi -chmod 755 "$ICINGA2_INIT_RUN_DIR" -chmod 2750 "$ICINGA2_INIT_RUN_DIR"/cmd chown -R $ICINGA2_USER:$ICINGA2_COMMAND_GROUP "$ICINGA2_INIT_RUN_DIR" test -e "$ICINGA2_LOG_DIR" || install -m 750 -o $ICINGA2_USER -g $ICINGA2_COMMAND_GROUP -d "$ICINGA2_LOG_DIR" From 112fb072c1888de7b67fb732a2ed1d3809de6100 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 3 Jun 2020 13:56:20 +0200 Subject: [PATCH 13/75] Docs: Move Icinga DB installation down (until it's ready for production) --- doc/02-installation.md | 96 ++++++++++++++++++++++++------------------ 1 file changed, 54 insertions(+), 42 deletions(-) diff --git a/doc/02-installation.md b/doc/02-installation.md index 0ce756e3e..aee82acc4 100644 --- a/doc/02-installation.md +++ b/doc/02-installation.md @@ -607,50 +607,16 @@ $ nano /etc/icinga2/conf.d/templates.conf Icinga 2 can be used with Icinga Web 2 and a variety of modules. This chapter explains how to set up Icinga Web 2. -Either Icinga DB or the DB IDO (Database Icinga Data Output) feature for Icinga 2 takes care of +The DB IDO (Database Icinga Data Output) feature for Icinga 2 takes care of exporting all configuration and status information into a database. -Please choose whether to install [Icinga DB](02-installation.md#configuring-icinga-db) (MySQL only) -or DB IDO ([MySQL](02-installation.md#configuring-db-ido-mysql) or -[PostgreSQL](02-installation.md#configuring-db-ido-postgresql)). -It's recommended to use the newer Icinga DB feature, if you don't need PostgreSQL. - -### Configuring Icinga DB - -First, make sure to setup Icinga DB itself and its database backends (Redis and MySQL) by following the [installation instructions](https://icinga.com/docs/icingadb/latest/doc/02-Installation/). - -#### Enabling the Icinga DB feature - -Icinga 2 provides a configuration file that is installed in -`/etc/icinga2/features-available/icingadb.conf`. You can update -the Redis credentials in this file. - -All available attributes are explained in the -[IcingaDB object](09-object-types.md#objecttype-icingadb) -chapter. - -You can enable the `icingadb` feature configuration file using -`icinga2 feature enable`: - -``` -# icinga2 feature enable icingadb -Module 'icingadb' was enabled. -Make sure to restart Icinga 2 for these changes to take effect. -``` - -Restart Icinga 2. - -``` -systemctl restart icinga2 -``` - -Alpine Linux: - -``` -rc-service icinga2 restart -``` - -Continue with the [webserver setup](02-installation.md#icinga2-user-interface-webserver). +> **Note** +> +> We're currently working on a new data backend called Icinga DB. +> If you want to try the latest release candidate skip to +> the [Icinga DB Chapter](02-installation.md#icingadb). +> Please keep in mind, that this version is not ready for use in +> production and currently only supports MySQL. ### Configuring DB IDO MySQL @@ -1168,3 +1134,49 @@ PostgreSQL: * [Documentation](https://www.postgresql.org/docs/9.3/static/backup.html) +## Icinga DB + +Icinga DB is a new data backend currently in development. +It's purpose is to synchronise data between Icinga 2 (Redis) and Icinga Web 2 (MySQL), some day replacing the IDO. +Don't worry, we won't drop support on the IDO any time soon. + +> **Note** +> Icinga DB is not ready to be used in production +> and should only be used for testing purposes. + +### Configuring Icinga DB + +First, make sure to setup Icinga DB itself and its database backends (Redis and MySQL) by following the [installation instructions](https://icinga.com/docs/icingadb/latest/doc/02-Installation/). + +#### Enabling the Icinga DB feature + +Icinga 2 provides a configuration file that is installed in +`/etc/icinga2/features-available/icingadb.conf`. You can update +the Redis credentials in this file. + +All available attributes are explained in the +[IcingaDB object](09-object-types.md#objecttype-icingadb) +chapter. + +You can enable the `icingadb` feature configuration file using +`icinga2 feature enable`: + +``` +# icinga2 feature enable icingadb +Module 'icingadb' was enabled. +Make sure to restart Icinga 2 for these changes to take effect. +``` + +Restart Icinga 2. + +``` +systemctl restart icinga2 +``` + +Alpine Linux: + +``` +rc-service icinga2 restart +``` + +Continue with the [webserver setup](02-installation.md#icinga2-user-interface-webserver). From 48fe2813da699ed372beac502807d63d9b7a4aca Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 9 Jun 2020 11:21:05 +0200 Subject: [PATCH 14/75] Deprecate Livestatus --- doc/09-object-types.md | 5 + doc/14-features.md | 413 +++++++++++++++++++++-------------------- 2 files changed, 214 insertions(+), 204 deletions(-) diff --git a/doc/09-object-types.md b/doc/09-object-types.md index 2c2c6f9cc..ebbf2e546 100644 --- a/doc/09-object-types.md +++ b/doc/09-object-types.md @@ -1661,6 +1661,11 @@ require the [CompatLogger](09-object-types.md#objecttype-compatlogger) feature e pointing to the log files using the `compat_log_path` configuration attribute. This configuration object is available as [livestatus feature](14-features.md#setting-up-livestatus). +> **Note** +> +> This feature is DEPRECATED and will be removed in future releases. +> Check the [roadmap](https://github.com/Icinga/icinga2/milestones). + Examples: ``` diff --git a/doc/14-features.md b/doc/14-features.md index aa7572e3f..dc2262ef6 100644 --- a/doc/14-features.md +++ b/doc/14-features.md @@ -915,210 +915,6 @@ is running on. -## Livestatus - -The [MK Livestatus](https://mathias-kettner.de/checkmk_livestatus.html) project -implements a query protocol that lets users query their Icinga instance for -status information. It can also be used to send commands. - -The Livestatus component that is distributed as part of Icinga 2 is a -re-implementation of the Livestatus protocol which is compatible with MK -Livestatus. - -> **Tip** -> -> Only install the Livestatus feature if your web interface or addon requires -> you to do so. -> [Icinga Web 2](02-installation.md#setting-up-icingaweb2) does not need -> Livestatus. - -Details on the available tables and attributes with Icinga 2 can be found -in the [Livestatus Schema](24-appendix.md#schema-livestatus) section. - -You can enable Livestatus using icinga2 feature enable: - -``` -# icinga2 feature enable livestatus -``` - -After that you will have to restart Icinga 2: - -``` -# systemctl restart icinga2 -``` - -By default the Livestatus socket is available in `/var/run/icinga2/cmd/livestatus`. - -In order for queries and commands to work you will need to add your query user -(e.g. your web server) to the `icingacmd` group: - -``` -# usermod -a -G icingacmd www-data -``` - -The Debian packages use `nagios` as the user and group name. Make sure to change `icingacmd` to -`nagios` if you're using Debian. - -Change `www-data` to the user you're using to run queries. - -In order to use the historical tables provided by the livestatus feature (for example, the -`log` table) you need to have the `CompatLogger` feature enabled. By default these logs -are expected to be in `/var/log/icinga2/compat`. A different path can be set using the -`compat_log_path` configuration attribute. - -``` -# icinga2 feature enable compatlog -``` - -### Livestatus Sockets - -Other to the Icinga 1.x Addon, Icinga 2 supports two socket types - -* Unix socket (default) -* TCP socket - -Details on the configuration can be found in the [LivestatusListener](09-object-types.md#objecttype-livestatuslistener) -object configuration. - -### Livestatus GET Queries - -> **Note** -> -> All Livestatus queries require an additional empty line as query end identifier. -> The `nc` tool (`netcat`) provides the `-U` parameter to communicate using -> a unix socket. - -There also is a Perl module available in CPAN for accessing the Livestatus socket -programmatically: [Monitoring::Livestatus](http://search.cpan.org/~nierlein/Monitoring-Livestatus-0.74/) - - -Example using the unix socket: - -``` -# echo -e "GET services\n" | /usr/bin/nc -U /var/run/icinga2/cmd/livestatus - -Example using the tcp socket listening on port `6558`: - -# echo -e 'GET services\n' | netcat 127.0.0.1 6558 - -# cat servicegroups < - -A list of available external commands and their parameters can be found [here](24-appendix.md#external-commands-list-detail) - -``` -$ echo -e 'COMMAND ' | netcat 127.0.0.1 6558 -``` - -### Livestatus Filters - -and, or, negate - - Operator | Negate | Description - ----------|----------|------------- - = | != | Equality - ~ | !~ | Regex match - =~ | !=~ | Equality ignoring case - ~~ | !~~ | Regex ignoring case - < | | Less than - > | | Greater than - <= | | Less than or equal - >= | | Greater than or equal - - -### Livestatus Stats - -Schema: "Stats: aggregatefunction aggregateattribute" - - Aggregate Function | Description - -------------------|-------------- - sum |   - min |   - max |   - avg | sum / count - std | standard deviation - suminv | sum (1 / value) - avginv | suminv / count - count | ordinary default for any stats query if not aggregate function defined - -Example: - -``` -GET hosts -Filter: has_been_checked = 1 -Filter: check_type = 0 -Stats: sum execution_time -Stats: sum latency -Stats: sum percent_state_change -Stats: min execution_time -Stats: min latency -Stats: min percent_state_change -Stats: max execution_time -Stats: max latency -Stats: max percent_state_change -OutputFormat: json -ResponseHeader: fixed16 -``` - -### Livestatus Output - -* CSV - -CSV output uses two levels of array separators: The members array separator -is a comma (1st level) while extra info and host|service relation separator -is a pipe (2nd level). - -Separators can be set using ASCII codes like: - -``` -Separators: 10 59 44 124 -``` - -* JSON - -Default separators. - -### Livestatus Error Codes - - Code | Description - ----------|-------------- - 200 | OK - 404 | Table does not exist - 452 | Exception on query - -### Livestatus Tables - - Table | Join |Description - --------------|-----------|---------------------------- - hosts |   | host config and status attributes, services counter - hostgroups |   | hostgroup config, status attributes and host/service counters - services | hosts | service config and status attributes - servicegroups |   | servicegroup config, status attributes and service counters - contacts |   | contact config and status attributes - contactgroups |   | contact config, members - commands |   | command name and line - status |   | programstatus, config and stats - comments | services | status attributes - downtimes | services | status attributes - timeperiods |   | name and is inside flag - endpoints |   | config and status attributes - log | services, hosts, contacts, commands | parses [compatlog](09-object-types.md#objecttype-compatlogger) and shows log attributes - statehist | hosts, services | parses [compatlog](09-object-types.md#objecttype-compatlogger) and aggregates state change attributes - hostsbygroup | hostgroups | host attributes grouped by hostgroup and its attributes - servicesbygroup | servicegroups | service attributes grouped by servicegroup and its attributes - servicesbyhostgroup | hostgroups | service attributes grouped by hostgroup and its attributes - -The `commands` table is populated with `CheckCommand`, `EventCommand` and `NotificationCommand` objects. - -A detailed list on the available table attributes can be found in the [Livestatus Schema documentation](24-appendix.md#schema-livestatus). - ## Deprecated Features @@ -1237,3 +1033,212 @@ object CheckResultReader "reader" { spool_dir = "/data/check-results" } ``` + +### Livestatus + +> **Note** +> +> This feature is DEPRECATED and will be removed in future releases. +> Check the [roadmap](https://github.com/Icinga/icinga2/milestones). + +The [MK Livestatus](https://mathias-kettner.de/checkmk_livestatus.html) project +implements a query protocol that lets users query their Icinga instance for +status information. It can also be used to send commands. + +The Livestatus component that is distributed as part of Icinga 2 is a +re-implementation of the Livestatus protocol which is compatible with MK +Livestatus. + +> **Tip** +> +> Only install the Livestatus feature if your web interface or addon requires +> you to do so. +> [Icinga Web 2](02-installation.md#setting-up-icingaweb2) does not need +> Livestatus. + +Details on the available tables and attributes with Icinga 2 can be found +in the [Livestatus Schema](24-appendix.md#schema-livestatus) section. + +You can enable Livestatus using icinga2 feature enable: + +``` +# icinga2 feature enable livestatus +``` + +After that you will have to restart Icinga 2: + +``` +# systemctl restart icinga2 +``` + +By default the Livestatus socket is available in `/var/run/icinga2/cmd/livestatus`. + +In order for queries and commands to work you will need to add your query user +(e.g. your web server) to the `icingacmd` group: + +``` +# usermod -a -G icingacmd www-data +``` + +The Debian packages use `nagios` as the user and group name. Make sure to change `icingacmd` to +`nagios` if you're using Debian. + +Change `www-data` to the user you're using to run queries. + +In order to use the historical tables provided by the livestatus feature (for example, the +`log` table) you need to have the `CompatLogger` feature enabled. By default these logs +are expected to be in `/var/log/icinga2/compat`. A different path can be set using the +`compat_log_path` configuration attribute. + +``` +# icinga2 feature enable compatlog +``` + +#### Livestatus Sockets + +Other to the Icinga 1.x Addon, Icinga 2 supports two socket types + +* Unix socket (default) +* TCP socket + +Details on the configuration can be found in the [LivestatusListener](09-object-types.md#objecttype-livestatuslistener) +object configuration. + +#### Livestatus GET Queries + +> **Note** +> +> All Livestatus queries require an additional empty line as query end identifier. +> The `nc` tool (`netcat`) provides the `-U` parameter to communicate using +> a unix socket. + +There also is a Perl module available in CPAN for accessing the Livestatus socket +programmatically: [Monitoring::Livestatus](http://search.cpan.org/~nierlein/Monitoring-Livestatus-0.74/) + + +Example using the unix socket: + +``` +# echo -e "GET services\n" | /usr/bin/nc -U /var/run/icinga2/cmd/livestatus + +Example using the tcp socket listening on port `6558`: + +# echo -e 'GET services\n' | netcat 127.0.0.1 6558 + +# cat servicegroups < + +A list of available external commands and their parameters can be found [here](24-appendix.md#external-commands-list-detail) + +``` +$ echo -e 'COMMAND ' | netcat 127.0.0.1 6558 +``` + +#### Livestatus Filters + +and, or, negate + + Operator | Negate | Description + ----------|----------|------------- + = | != | Equality + ~ | !~ | Regex match + =~ | !=~ | Equality ignoring case + ~~ | !~~ | Regex ignoring case + < | | Less than + > | | Greater than + <= | | Less than or equal + >= | | Greater than or equal + + +#### Livestatus Stats + +Schema: "Stats: aggregatefunction aggregateattribute" + + Aggregate Function | Description + -------------------|-------------- + sum |   + min |   + max |   + avg | sum / count + std | standard deviation + suminv | sum (1 / value) + avginv | suminv / count + count | ordinary default for any stats query if not aggregate function defined + +Example: + +``` +GET hosts +Filter: has_been_checked = 1 +Filter: check_type = 0 +Stats: sum execution_time +Stats: sum latency +Stats: sum percent_state_change +Stats: min execution_time +Stats: min latency +Stats: min percent_state_change +Stats: max execution_time +Stats: max latency +Stats: max percent_state_change +OutputFormat: json +ResponseHeader: fixed16 +``` + +#### Livestatus Output + +* CSV + +CSV output uses two levels of array separators: The members array separator +is a comma (1st level) while extra info and host|service relation separator +is a pipe (2nd level). + +Separators can be set using ASCII codes like: + +``` +Separators: 10 59 44 124 +``` + +* JSON + +Default separators. + +#### Livestatus Error Codes + + Code | Description + ----------|-------------- + 200 | OK + 404 | Table does not exist + 452 | Exception on query + +#### Livestatus Tables + + Table | Join |Description + --------------|-----------|---------------------------- + hosts |   | host config and status attributes, services counter + hostgroups |   | hostgroup config, status attributes and host/service counters + services | hosts | service config and status attributes + servicegroups |   | servicegroup config, status attributes and service counters + contacts |   | contact config and status attributes + contactgroups |   | contact config, members + commands |   | command name and line + status |   | programstatus, config and stats + comments | services | status attributes + downtimes | services | status attributes + timeperiods |   | name and is inside flag + endpoints |   | config and status attributes + log | services, hosts, contacts, commands | parses [compatlog](09-object-types.md#objecttype-compatlogger) and shows log attributes + statehist | hosts, services | parses [compatlog](09-object-types.md#objecttype-compatlogger) and aggregates state change attributes + hostsbygroup | hostgroups | host attributes grouped by hostgroup and its attributes + servicesbygroup | servicegroups | service attributes grouped by servicegroup and its attributes + servicesbyhostgroup | hostgroups | service attributes grouped by hostgroup and its attributes + +The `commands` table is populated with `CheckCommand`, `EventCommand` and `NotificationCommand` objects. + +A detailed list on the available table attributes can be found in the [Livestatus Schema documentation](24-appendix.md#schema-livestatus). From cd2ffd175ba107c8f4a56b9633e52ff881eb1f7d Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 5 Mar 2020 14:33:31 +0100 Subject: [PATCH 15/75] CheckerComponent#Stop(): don't wait for checks refs #7888 --- lib/checker/checkercomponent.cpp | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/lib/checker/checkercomponent.cpp b/lib/checker/checkercomponent.cpp index 20300b3ab..0b5980acd 100644 --- a/lib/checker/checkercomponent.cpp +++ b/lib/checker/checkercomponent.cpp @@ -74,30 +74,6 @@ void CheckerComponent::Stop(bool runtimeRemoved) m_CV.notify_all(); } - double wait = 0.0; - - while (Checkable::GetPendingChecks() > 0) { - Log(LogDebug, "CheckerComponent") - << "Waiting for running checks (" << Checkable::GetPendingChecks() - << ") to finish. Waited for " << wait << " seconds now."; - - Utility::Sleep(0.1); - wait += 0.1; - - /* Pick a timeout slightly shorther than the process reload timeout. */ - double reloadTimeout = Application::GetReloadTimeout(); - double waitMax = reloadTimeout - 30; - if (waitMax <= 0) - waitMax = 1; - - if (wait > waitMax) { - Log(LogWarning, "CheckerComponent") - << "Checks running too long for " << wait - << " seconds, hard shutdown before reload timeout: " << reloadTimeout << "."; - break; - } - } - m_ResultTimer->Stop(); m_Thread.join(); From 11c6c110765ca652852bcb872191af7e2e44de7a Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 5 Mar 2020 14:36:58 +0100 Subject: [PATCH 16/75] Add Checkable#last_check_started refs #7888 --- lib/icinga/checkable.ti | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/icinga/checkable.ti b/lib/icinga/checkable.ti index 98d18cd7f..ad34f6d42 100644 --- a/lib/icinga/checkable.ti +++ b/lib/icinga/checkable.ti @@ -90,6 +90,8 @@ abstract class Checkable : CustomVarObject [config] String icon_image_alt; [state] Timestamp next_check; + [state, no_user_view, no_user_modify] Timestamp last_check_started; + [state] int check_attempt { default {{{ return 1; }}} }; From 9c85401914ad495b4ef16af01b2b5cf93f625b0f Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 9 Jun 2020 12:26:16 +0200 Subject: [PATCH 17/75] Ensure the custom function is not null in Array#{sort,map,reduce,filter,any,all}() refs #8047 --- lib/base/array-script.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/base/array-script.cpp b/lib/base/array-script.cpp index 1c00f1106..80c075e17 100644 --- a/lib/base/array-script.cpp +++ b/lib/base/array-script.cpp @@ -83,6 +83,7 @@ static Array::Ptr ArraySort(const std::vector& args) std::sort(arr->Begin(), arr->End()); } else { Function::Ptr function = args[0]; + REQUIRE_NOT_NULL(function); if (vframe->Sandboxed && !function->IsSideEffectFree()) BOOST_THROW_EXCEPTION(ScriptError("Sort function must be side-effect free.")); @@ -123,6 +124,7 @@ static Array::Ptr ArrayMap(const Function::Ptr& function) ScriptFrame *vframe = ScriptFrame::GetCurrentFrame(); Array::Ptr self = static_cast(vframe->Self); REQUIRE_NOT_NULL(self); + REQUIRE_NOT_NULL(function); if (vframe->Sandboxed && !function->IsSideEffectFree()) BOOST_THROW_EXCEPTION(ScriptError("Map function must be side-effect free.")); @@ -142,6 +144,7 @@ static Value ArrayReduce(const Function::Ptr& function) ScriptFrame *vframe = ScriptFrame::GetCurrentFrame(); Array::Ptr self = static_cast(vframe->Self); REQUIRE_NOT_NULL(self); + REQUIRE_NOT_NULL(function); if (vframe->Sandboxed && !function->IsSideEffectFree()) BOOST_THROW_EXCEPTION(ScriptError("Reduce function must be side-effect free.")); @@ -164,6 +167,7 @@ static Array::Ptr ArrayFilter(const Function::Ptr& function) ScriptFrame *vframe = ScriptFrame::GetCurrentFrame(); Array::Ptr self = static_cast(vframe->Self); REQUIRE_NOT_NULL(self); + REQUIRE_NOT_NULL(function); if (vframe->Sandboxed && !function->IsSideEffectFree()) BOOST_THROW_EXCEPTION(ScriptError("Filter function must be side-effect free.")); @@ -184,6 +188,7 @@ static bool ArrayAny(const Function::Ptr& function) ScriptFrame *vframe = ScriptFrame::GetCurrentFrame(); Array::Ptr self = static_cast(vframe->Self); REQUIRE_NOT_NULL(self); + REQUIRE_NOT_NULL(function); if (vframe->Sandboxed && !function->IsSideEffectFree()) BOOST_THROW_EXCEPTION(ScriptError("Filter function must be side-effect free.")); @@ -202,6 +207,7 @@ static bool ArrayAll(const Function::Ptr& function) ScriptFrame *vframe = ScriptFrame::GetCurrentFrame(); Array::Ptr self = static_cast(vframe->Self); REQUIRE_NOT_NULL(self); + REQUIRE_NOT_NULL(function); if (vframe->Sandboxed && !function->IsSideEffectFree()) BOOST_THROW_EXCEPTION(ScriptError("Filter function must be side-effect free.")); From 647f1547a9c2f94d6f1432c2e4c68bac1ad2944f Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 5 Feb 2020 17:17:41 +0100 Subject: [PATCH 18/75] Generalize I/O timeout emulation --- lib/base/io-engine.cpp | 8 ++++++ lib/base/io-engine.hpp | 52 ++++++++++++++++++++++++++++++++++++++ lib/remote/apilistener.cpp | 31 +++++++---------------- 3 files changed, 69 insertions(+), 22 deletions(-) diff --git a/lib/base/io-engine.cpp b/lib/base/io-engine.cpp index 5dd3ee59c..d3197790e 100644 --- a/lib/base/io-engine.cpp +++ b/lib/base/io-engine.cpp @@ -144,3 +144,11 @@ void AsioConditionVariable::Wait(boost::asio::yield_context yc) boost::system::error_code ec; m_Timer.async_wait(yc[ec]); } + +void Timeout::Cancel() +{ + m_Cancelled.store(true); + + boost::system::error_code ec; + m_Timer.cancel(ec); +} diff --git a/lib/base/io-engine.hpp b/lib/base/io-engine.hpp index ba4ebcfc5..dabd6730b 100644 --- a/lib/base/io-engine.hpp +++ b/lib/base/io-engine.hpp @@ -6,10 +6,12 @@ #include "base/exception.hpp" #include "base/lazy-init.hpp" #include "base/logger.hpp" +#include "base/shared-object.hpp" #include #include #include #include +#include #include #include #include @@ -153,6 +155,56 @@ private: boost::asio::deadline_timer m_Timer; }; +/** + * I/O timeout emulator + * + * @ingroup base + */ +class Timeout : public SharedObject +{ +public: + DECLARE_PTR_TYPEDEFS(Timeout); + + template + Timeout(boost::asio::io_context& io, Executor& executor, TimeoutFromNow timeoutFromNow, OnTimeout onTimeout) + : m_Timer(io) + { + Ptr keepAlive (this); + + m_Cancelled.store(false); + m_Timer.expires_from_now(std::move(timeoutFromNow)); + + IoEngine::SpawnCoroutine(executor, [this, keepAlive, onTimeout](boost::asio::yield_context yc) { + if (m_Cancelled.load()) { + return; + } + + { + boost::system::error_code ec; + + m_Timer.async_wait(yc[ec]); + + if (ec) { + return; + } + } + + if (m_Cancelled.load()) { + return; + } + + auto f (onTimeout); + f(std::move(yc)); + }); + } + + void Cancel(); + +private: + boost::asio::deadline_timer m_Timer; + std::atomic m_Cancelled; +}; + } #endif /* IO_ENGINE_H */ diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index cb024e1fd..0eb9c248b 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -542,32 +542,19 @@ void ApiListener::NewClientHandlerInternal( boost::system::error_code ec; { - struct DoneHandshake - { - bool Done = false; - }; - - auto doneHandshake (Shared::Make()); - - IoEngine::SpawnCoroutine(*strand, [strand, client, doneHandshake](asio::yield_context yc) { - namespace sys = boost::system; - - { - boost::asio::deadline_timer timer (strand->context()); - timer.expires_from_now(boost::posix_time::microseconds(intmax_t(Configuration::TlsHandshakeTimeout * 1000000))); - - sys::error_code ec; - timer.async_wait(yc[ec]); - } - - if (!doneHandshake->Done) { - sys::error_code ec; + Timeout::Ptr handshakeTimeout (new Timeout( + strand->context(), + *strand, + boost::posix_time::microseconds(intmax_t(Configuration::TlsHandshakeTimeout * 1000000)), + [strand, client](asio::yield_context yc) { + boost::system::error_code ec; client->lowest_layer().cancel(ec); } - }); + )); sslConn.async_handshake(role == RoleClient ? sslConn.client : sslConn.server, yc[ec]); - doneHandshake->Done = true; + + handshakeTimeout->Cancel(); } if (ec) { From 19c632e44bc793f8240e2a71d1f29f4eb39333f0 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 27 Feb 2020 13:24:57 +0100 Subject: [PATCH 19/75] Add timeout for boost::asio::ssl::stream#async_shutdown() refs #7203 --- lib/remote/jsonrpcconnection.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/remote/jsonrpcconnection.cpp b/lib/remote/jsonrpcconnection.cpp index e4337f0f0..e489a3781 100644 --- a/lib/remote/jsonrpcconnection.cpp +++ b/lib/remote/jsonrpcconnection.cpp @@ -233,8 +233,20 @@ void JsonRpcConnection::Disconnect() m_Stream->lowest_layer().cancel(ec); + Timeout::Ptr shutdownTimeout (new Timeout( + m_IoStrand.context(), + m_IoStrand, + boost::posix_time::seconds(10), + [this, keepAlive](asio::yield_context yc) { + boost::system::error_code ec; + m_Stream->lowest_layer().cancel(ec); + } + )); + m_Stream->next_layer().async_shutdown(yc[ec]); + shutdownTimeout->Cancel(); + m_Stream->lowest_layer().shutdown(m_Stream->lowest_layer().shutdown_both, ec); } }); From 58f3343162d4bba7a4bbcb51650e6176250c710d Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 19 Jun 2020 14:22:06 +0200 Subject: [PATCH 20/75] GitHub actions: build Docker images --- .github/workflows/docker.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 000000000..d8016b7d8 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,21 @@ +name: Docker image + +on: + pull_request: {} + push: + branches: + - master + release: + types: + - published + +jobs: + docker: + runs-on: ubuntu-latest + + steps: + - name: Docker image + uses: Icinga/docker-icinga2@master + env: + INPUT_TOKEN: '${{ github.token }}' + DOCKER_HUB_PASSWORD: '${{ secrets.DOCKER_HUB_PERSONAL_TOKEN }}' From 535f97551f6498f314440484574e4d52a26f4d2f Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 22 Jun 2020 14:20:08 +0200 Subject: [PATCH 21/75] Docs: Add missing file prefix to link --- doc/06-distributed-monitoring.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/06-distributed-monitoring.md b/doc/06-distributed-monitoring.md index 762fbeb65..c98874ac3 100644 --- a/doc/06-distributed-monitoring.md +++ b/doc/06-distributed-monitoring.md @@ -791,7 +791,7 @@ after the installation. Select the check box to proceed. #### Agent Setup on Windows: Configuration Wizard On a fresh installation the setup wizard guides you through the initial configuration. -It also provides a mechanism to send a certificate request to the [CSR signing master](distributed-monitoring-setup-sign-certificates-master). +It also provides a mechanism to send a certificate request to the [CSR signing master](06-distributed-monitoring.md#distributed-monitoring-setup-sign-certificates-master). The following configuration details are required: From af2e1a37db8167c9489b23f724d3c51d77e53dda Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 26 Jun 2020 15:38:02 +0200 Subject: [PATCH 22/75] Deprecate Windows plugins --- doc/10-icinga-template-library.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/10-icinga-template-library.md b/doc/10-icinga-template-library.md index 83377f5c6..644d709d6 100644 --- a/doc/10-icinga-template-library.md +++ b/doc/10-icinga-template-library.md @@ -1503,6 +1503,13 @@ uptime_since | **Optional.** Show last boot in yyyy-mm-dd HH:MM:SS format (ou ## Windows Plugins for Icinga 2 +> **Note** +> +> These plugins are DEPRECATED in favor of our +> [PowerShell Plugins](https://github.com/Icinga/icinga-powershell-plugins) +> and will be removed in a future release. +> Check the [roadmap](https://github.com/Icinga/icinga2/milestones). + To allow a basic monitoring of Windows clients Icinga 2 comes with a set of Windows only plugins. While trying to mirror the functionalities of their linux cousins from the monitoring-plugins package, the differences between Windows and Linux are too big to be able use the same CheckCommands for both systems. A check-commands-windows.conf comes with Icinga 2, it assumes that the Windows Plugins are installed in the PluginDir set in your constants.conf. To enable them the following include directive is needed in you icinga2.conf: From cf5ec5e341e7c429063d7f988f410d8e87c034f8 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Thu, 9 Jul 2020 13:16:48 +0200 Subject: [PATCH 23/75] Send heartbeat every 20s and not 10s --- lib/remote/jsonrpcconnection-heartbeat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/remote/jsonrpcconnection-heartbeat.cpp b/lib/remote/jsonrpcconnection-heartbeat.cpp index 993877e4e..0644ced85 100644 --- a/lib/remote/jsonrpcconnection-heartbeat.cpp +++ b/lib/remote/jsonrpcconnection-heartbeat.cpp @@ -20,7 +20,7 @@ void JsonRpcConnection::HandleAndWriteHeartbeats(boost::asio::yield_context yc) boost::system::error_code ec; for (;;) { - m_HeartbeatTimer.expires_from_now(boost::posix_time::seconds(10)); + m_HeartbeatTimer.expires_from_now(boost::posix_time::seconds(20)); m_HeartbeatTimer.async_wait(yc[ec]); if (m_ShuttingDown) { From d73cc5d08ff2b3ba4fb26c8b3cfdda1caf7a80fc Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 9 Jul 2020 17:06:51 +0200 Subject: [PATCH 24/75] GitHub actions: don't cancel not yet failed jobs on a failure --- .github/workflows/packages.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 19503abe3..4d4853675 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -11,6 +11,7 @@ jobs: name: .deb strategy: + fail-fast: false matrix: distro: - name: debian From 06bbf8f7431e51576c12b1479305d66d0cdf7de8 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 14 Jul 2020 10:36:14 +0200 Subject: [PATCH 25/75] GitHub actions: make ccache actually working --- .github/workflows/packages.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 19503abe3..acbcca584 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -63,7 +63,8 @@ jobs: uses: actions/cache@v1 with: path: deb-icinga2/ccache - key: '${{ matrix.distro.name }}/${{ matrix.distro.codename }}-ccache' + key: |- + ${{ matrix.distro.name }}/${{ matrix.distro.codename }}-ccache-${{ hashFiles('deb-icinga2/ccache') }} - name: Binary x64 run: | @@ -197,7 +198,8 @@ jobs: uses: actions/cache@v1 with: path: rpm-icinga2/ccache - key: '${{ matrix.distro.name }}/${{ matrix.distro.release }}-ccache' + key: |- + ${{ matrix.distro.name }}/${{ matrix.distro.release }}-ccache-${{ hashFiles('rpm-icinga2/ccache') }} - name: Binary if: "steps.vars.outputs.CAN_BUILD == 'true'" @@ -266,7 +268,8 @@ jobs: uses: actions/cache@v1 with: path: raspbian-icinga2/ccache - key: 'raspbian/${{ matrix.codename }}-ccache' + key: |- + raspbian/${{ matrix.codename }}-ccache-${{ hashFiles('raspbian-icinga2/ccache') }} - name: Binary run: | From 3cb073d6d78dde3e3dde0f5aca191f861395f00e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 16 Jul 2020 10:07:15 +0200 Subject: [PATCH 26/75] GitHub actions: drop Debian "jessie" --- .github/workflows/packages.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 19503abe3..681998c22 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -19,9 +19,6 @@ jobs: - name: debian codename: stretch has32bit: true - - name: debian - codename: jessie - has32bit: true - name: ubuntu codename: focal has32bit: false From 9dc297987ea5155b75a98beeef17573b0289f1cf Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 5 Mar 2020 15:34:12 +0100 Subject: [PATCH 27/75] Sync Checkable#last_check_started refs #7888 --- doc/19-technical-concepts.md | 34 +++++++++++++++++++ lib/icinga/clusterevents.cpp | 64 ++++++++++++++++++++++++++++++++++++ lib/icinga/clusterevents.hpp | 3 ++ 3 files changed, 101 insertions(+) diff --git a/doc/19-technical-concepts.md b/doc/19-technical-concepts.md index 4586410f6..9fc0edeed 100644 --- a/doc/19-technical-concepts.md +++ b/doc/19-technical-concepts.md @@ -1397,6 +1397,40 @@ Message updates will be dropped when: * Checkable does not exist. * Origin endpoint's zone is not allowed to access this checkable. +#### event::SetLastCheckStarted + +> Location: `clusterevents.cpp` + +##### Message Body + +Key | Value +----------|--------- +jsonrpc | 2.0 +method | event::SetLastCheckStarted +params | Dictionary + +##### Params + +Key | Type | Description +---------------------|-----------|------------------ +host | String | Host name +service | String | Service name +last\_check\_started | Timestamp | Last check's start time as UNIX timestamp. + +##### Functions + +Event Sender: `Checkable::OnLastCheckStartedChanged` +Event Receiver: `LastCheckStartedChangedAPIHandler` + +##### Permissions + +The receiver will not process messages from not configured endpoints. + +Message updates will be dropped when: + +* Checkable does not exist. +* Origin endpoint's zone is not allowed to access this checkable. + #### event::SuppressedNotifications > Location: `clusterevents.cpp` diff --git a/lib/icinga/clusterevents.cpp b/lib/icinga/clusterevents.cpp index bc61a58cb..37809a8c6 100644 --- a/lib/icinga/clusterevents.cpp +++ b/lib/icinga/clusterevents.cpp @@ -24,6 +24,7 @@ INITIALIZE_ONCE(&ClusterEvents::StaticInitialize); REGISTER_APIFUNCTION(CheckResult, event, &ClusterEvents::CheckResultAPIHandler); REGISTER_APIFUNCTION(SetNextCheck, event, &ClusterEvents::NextCheckChangedAPIHandler); +REGISTER_APIFUNCTION(SetLastCheckStarted, event, &ClusterEvents::LastCheckStartedChangedAPIHandler); REGISTER_APIFUNCTION(SetSuppressedNotifications, event, &ClusterEvents::SuppressedNotificationsChangedAPIHandler); REGISTER_APIFUNCTION(SetNextNotification, event, &ClusterEvents::NextNotificationChangedAPIHandler); REGISTER_APIFUNCTION(SetForceNextCheck, event, &ClusterEvents::ForceNextCheckChangedAPIHandler); @@ -39,6 +40,7 @@ void ClusterEvents::StaticInitialize() { Checkable::OnNewCheckResult.connect(&ClusterEvents::CheckResultHandler); Checkable::OnNextCheckChanged.connect(&ClusterEvents::NextCheckChangedHandler); + Checkable::OnLastCheckStartedChanged.connect(&ClusterEvents::LastCheckStartedChangedHandler); Checkable::OnSuppressedNotificationsChanged.connect(&ClusterEvents::SuppressedNotificationsChangedHandler); Notification::OnNextNotificationChanged.connect(&ClusterEvents::NextNotificationChangedHandler); Checkable::OnForceNextCheckChanged.connect(&ClusterEvents::ForceNextCheckChangedHandler); @@ -234,6 +236,68 @@ Value ClusterEvents::NextCheckChangedAPIHandler(const MessageOrigin::Ptr& origin return Empty; } +void ClusterEvents::LastCheckStartedChangedHandler(const Checkable::Ptr& checkable, const MessageOrigin::Ptr& origin) +{ + ApiListener::Ptr listener = ApiListener::GetInstance(); + + if (!listener) + return; + + Host::Ptr host; + Service::Ptr service; + tie(host, service) = GetHostService(checkable); + + Dictionary::Ptr params = new Dictionary(); + params->Set("host", host->GetName()); + if (service) + params->Set("service", service->GetShortName()); + params->Set("last_check_started", checkable->GetLastCheckStarted()); + + Dictionary::Ptr message = new Dictionary(); + message->Set("jsonrpc", "2.0"); + message->Set("method", "event::SetLastCheckStarted"); + message->Set("params", params); + + listener->RelayMessage(origin, checkable, message, true); +} + +Value ClusterEvents::LastCheckStartedChangedAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params) +{ + Endpoint::Ptr endpoint = origin->FromClient->GetEndpoint(); + + if (!endpoint) { + Log(LogNotice, "ClusterEvents") + << "Discarding 'last_check_started changed' message from '" << origin->FromClient->GetIdentity() << "': Invalid endpoint origin (client not allowed)."; + return Empty; + } + + Host::Ptr host = Host::GetByName(params->Get("host")); + + if (!host) + return Empty; + + Checkable::Ptr checkable; + + if (params->Contains("service")) + checkable = host->GetServiceByShortName(params->Get("service")); + else + checkable = host; + + if (!checkable) + return Empty; + + if (origin->FromZone && !origin->FromZone->CanAccessObject(checkable)) { + Log(LogNotice, "ClusterEvents") + << "Discarding 'last_check_started changed' message for checkable '" << checkable->GetName() + << "' from '" << origin->FromClient->GetIdentity() << "': Unauthorized access."; + return Empty; + } + + checkable->SetLastCheckStarted(params->Get("last_check_started"), false, origin); + + return Empty; +} + void ClusterEvents::SuppressedNotificationsChangedHandler(const Checkable::Ptr& checkable, const MessageOrigin::Ptr& origin) { ApiListener::Ptr listener = ApiListener::GetInstance(); diff --git a/lib/icinga/clusterevents.hpp b/lib/icinga/clusterevents.hpp index a51bc60db..539dd961f 100644 --- a/lib/icinga/clusterevents.hpp +++ b/lib/icinga/clusterevents.hpp @@ -26,6 +26,9 @@ public: static void NextCheckChangedHandler(const Checkable::Ptr& checkable, const MessageOrigin::Ptr& origin); static Value NextCheckChangedAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params); + static void LastCheckStartedChangedHandler(const Checkable::Ptr& checkable, const MessageOrigin::Ptr& origin); + static Value LastCheckStartedChangedAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params); + static void SuppressedNotificationsChangedHandler(const Checkable::Ptr& checkable, const MessageOrigin::Ptr& origin); static Value SuppressedNotificationsChangedAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params); From 4585a404d6d5793f2695f487e910d08cede9e98e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 5 Mar 2020 15:42:07 +0100 Subject: [PATCH 28/75] Checkable#ExecuteCheck(): set #last_check_started to now before #UpdateNextCheck() refs #7888 --- lib/icinga/checkable-check.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/icinga/checkable-check.cpp b/lib/icinga/checkable-check.cpp index 68361fe17..ca5b7ba71 100644 --- a/lib/icinga/checkable-check.cpp +++ b/lib/icinga/checkable-check.cpp @@ -514,6 +514,8 @@ void Checkable::ExecuteCheck() double scheduled_start = GetNextCheck(); double before_check = Utility::GetTime(); + SetLastCheckStarted(Utility::GetTime()); + /* This calls SetNextCheck() which updates the CheckerComponent's idle/pending * queues and ensures that checks are not fired multiple times. ProcessCheckResult() * is called too late. See #6421. From 37c2c7ba901144aa18ca64dff0b73d8ea61921f2 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 5 Mar 2020 15:54:17 +0100 Subject: [PATCH 29/75] Checkable#Start(): if #last_check_started > last check, set #next_check to #last_check_started refs #7888 --- lib/icinga/checkable.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/icinga/checkable.cpp b/lib/icinga/checkable.cpp index 0f4d1399b..8f21e71f1 100644 --- a/lib/icinga/checkable.cpp +++ b/lib/icinga/checkable.cpp @@ -63,6 +63,14 @@ void Checkable::Start(bool runtimeCreated) { double now = Utility::GetTime(); + { + auto cr (GetLastCheckResult()); + + if (GetLastCheckStarted() > (cr ? cr->GetExecutionEnd() : 0.0)) { + SetNextCheck(GetLastCheckStarted()); + } + } + if (GetNextCheck() < now + 60) { double delta = std::min(GetCheckInterval(), 60.0); delta *= (double)std::rand() / RAND_MAX; From 964a90fa4b244b9d0a2e85e16bc3ef8b8baaa756 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Wed, 8 Jul 2020 14:14:07 +0200 Subject: [PATCH 30/75] Remove all codes related to the heartbeat timeout until now, if the timeout is exceeded, the connection is immediately terminated. But since we do not want to disconnect even if the timeout is exceeded, it is better to send the messages without timeout and have deleted everything that related to the heartbeat timeout. We also have another mechanism in JRPC::CheckLiveness that does the disconnect. --- lib/remote/jsonrpcconnection-heartbeat.cpp | 45 ++++++---------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/lib/remote/jsonrpcconnection-heartbeat.cpp b/lib/remote/jsonrpcconnection-heartbeat.cpp index 4d023958f..2474688e7 100644 --- a/lib/remote/jsonrpcconnection-heartbeat.cpp +++ b/lib/remote/jsonrpcconnection-heartbeat.cpp @@ -15,6 +15,12 @@ using namespace icinga; REGISTER_APIFUNCTION(Heartbeat, event, &JsonRpcConnection::HeartbeatAPIHandler); +/** + * We still send a heartbeat without timeout here + * to keep the m_Seen variable up to date. This is to keep the + * cluster connection alive when there isn't much going on. + */ + void JsonRpcConnection::HandleAndWriteHeartbeats(boost::asio::yield_context yc) { boost::system::error_code ec; @@ -26,44 +32,17 @@ void JsonRpcConnection::HandleAndWriteHeartbeats(boost::asio::yield_context yc) if (m_ShuttingDown) { break; } - - if (m_NextHeartbeat != 0 && m_NextHeartbeat < Utility::GetTime()) { - { - Log logMsg (LogWarning, "JsonRpcConnection"); - - if (m_Endpoint) { - logMsg << "Client for endpoint '" << m_Endpoint->GetName() << "'"; - } else { - logMsg << "Anonymous client"; - } - - logMsg << " has requested heartbeat message but hasn't responded in time. Closing connection."; - } - - Disconnect(); - break; - } - - if (m_Endpoint) { - SendMessageInternal(new Dictionary({ - { "jsonrpc", "2.0" }, - { "method", "event::Heartbeat" }, - { "params", new Dictionary({ - { "timeout", 120 } - }) } - })); - } + + SendMessageInternal(new Dictionary({ + { "jsonrpc", "2.0" }, + { "method", "event::Heartbeat" }, + { "params", new Dictionary() } + })); } } Value JsonRpcConnection::HeartbeatAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params) { - Value vtimeout = params->Get("timeout"); - - if (!vtimeout.IsEmpty()) { - origin->FromClient->m_NextHeartbeat = Utility::GetTime() + vtimeout; - } - return Empty; } From c5158226492c34e3688446514db4cff11f10154a Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 29 Jul 2020 17:13:41 +0200 Subject: [PATCH 31/75] Re-send notifications previously suppressed by their time periods refs #6167 --- doc/19-technical-concepts.md | 33 ++++++ lib/icinga/checkable-notification.cpp | 103 ++++++++++-------- lib/icinga/checkable.hpp | 3 + lib/icinga/clusterevents.cpp | 48 +++++++++ lib/icinga/clusterevents.hpp | 3 + lib/icinga/notification.cpp | 33 ++++++ lib/icinga/notification.ti | 4 + lib/notification/notificationcomponent.cpp | 115 ++++++++++++++++----- 8 files changed, 277 insertions(+), 65 deletions(-) diff --git a/doc/19-technical-concepts.md b/doc/19-technical-concepts.md index 4586410f6..e29e26da9 100644 --- a/doc/19-technical-concepts.md +++ b/doc/19-technical-concepts.md @@ -1431,6 +1431,39 @@ Message updates will be dropped when: * Checkable does not exist. * Origin endpoint's zone is not allowed to access this checkable. +#### event::SetSuppressedNotificationTypes + +> Location: `clusterevents.cpp` + +##### Message Body + +Key | Value +----------|--------- +jsonrpc | 2.0 +method | event::SetSuppressedNotificationTypes +params | Dictionary + +##### Params + +Key | Type | Description +-------------------------|--------|------------------ +notification | String | Notification name +supressed\_notifications | Number | Bitmask for suppressed notifications. + +##### Functions + +Event Sender: `Notification::OnSuppressedNotificationsChanged` +Event Receiver: `SuppressedNotificationTypesChangedAPIHandler` + +##### Permissions + +The receiver will not process messages from not configured endpoints. + +Message updates will be dropped when: + +* Notification does not exist. +* Origin endpoint's zone is not allowed to access this notification. + #### event::SetNextNotification diff --git a/lib/icinga/checkable-notification.cpp b/lib/icinga/checkable-notification.cpp index 43b4589fa..8dc0dfd58 100644 --- a/lib/icinga/checkable-notification.cpp +++ b/lib/icinga/checkable-notification.cpp @@ -147,25 +147,7 @@ static void FireSuppressedNotifications(Checkable* checkable) for (auto type : {NotificationProblem, NotificationRecovery, NotificationFlappingStart, NotificationFlappingEnd}) { if (suppressed_types & type) { - bool still_applies; - auto cr (checkable->GetLastCheckResult()); - - switch (type) { - case NotificationProblem: - still_applies = cr && !checkable->IsStateOK(cr->GetState()) && checkable->GetStateType() == StateTypeHard; - break; - case NotificationRecovery: - still_applies = cr && checkable->IsStateOK(cr->GetState()); - break; - case NotificationFlappingStart: - still_applies = checkable->IsFlapping(); - break; - case NotificationFlappingEnd: - still_applies = !checkable->IsFlapping(); - break; - default: - break; - } + bool still_applies = checkable->NotificationReasonApplies(type); if (still_applies) { bool still_suppressed; @@ -185,28 +167,8 @@ static void FireSuppressedNotifications(Checkable* checkable) break; } - if (!still_suppressed && checkable->GetEnableActiveChecks()) { - /* If e.g. the downtime just ended, but the service is still not ok, we would re-send the stashed problem notification. - * But if the next check result recovers the service soon, we would send a recovery notification soon after the problem one. - * This is not desired, especially for lots of services at once. - * Because of that if there's likely to be a check result soon, - * we delay the re-sending of the stashed notification until the next check. - * That check either doesn't change anything and we finally re-send the stashed problem notification - * or recovers the service and we drop the stashed notification. */ - - /* One minute unless the check interval is too short so the next check will always run during the next minute. */ - auto threshold (checkable->GetCheckInterval() - 10); - - if (threshold > 60) - threshold = 60; - else if (threshold < 0) - threshold = 0; - - still_suppressed = checkable->GetNextCheck() <= Utility::GetTime() + threshold; - } - - if (!still_suppressed) { - Checkable::OnNotificationsRequested(checkable, type, cr, "", "", nullptr); + if (!still_suppressed && !checkable->IsLikelyToBeCheckedSoon()) { + Checkable::OnNotificationsRequested(checkable, type, checkable->GetLastCheckResult(), "", "", nullptr); subtract |= type; } @@ -241,3 +203,62 @@ void Checkable::FireSuppressedNotifications(const Timer * const&) ::FireSuppressedNotifications(service.get()); } } + +/** + * Returns whether sending a notification of type type right now would represent *this' current state correctly. + * + * @param type The type of notification to send (or not to send). + * + * @return Whether to send the notification. + */ +bool Checkable::NotificationReasonApplies(NotificationType type) +{ + switch (type) { + case NotificationProblem: + { + auto cr (GetLastCheckResult()); + return cr && !IsStateOK(cr->GetState()) && GetStateType() == StateTypeHard; + } + case NotificationRecovery: + { + auto cr (GetLastCheckResult()); + return cr && IsStateOK(cr->GetState()); + } + case NotificationFlappingStart: + return IsFlapping(); + case NotificationFlappingEnd: + return !IsFlapping(); + default: + VERIFY(!"Checkable#NotificationReasonStillApplies(): given type not implemented"); + return false; + } +} + +/** + * E.g. we're going to re-send a stashed problem notification as *this is still not ok. + * But if the next check result recovers *this soon, we would send a recovery notification soon after the problem one. + * This is not desired, especially for lots of checkables at once. + * Because of that if there's likely to be a check result soon, + * we delay the re-sending of the stashed notification until the next check. + * That check either doesn't change anything and we finally re-send the stashed problem notification + * or recovers *this and we drop the stashed notification. + * + * @return Whether *this is likely to be checked soon + */ +bool Checkable::IsLikelyToBeCheckedSoon() +{ + if (!GetEnableActiveChecks()) { + return false; + } + + // One minute unless the check interval is too short so the next check will always run during the next minute. + auto threshold (GetCheckInterval() - 10); + + if (threshold > 60) { + threshold = 60; + } else if (threshold < 0) { + threshold = 0; + } + + return GetNextCheck() <= Utility::GetTime() + threshold; +} diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index 0eb1c5950..dc782aca5 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -171,6 +171,9 @@ public: void ValidateRetryInterval(const Lazy& lvalue, const ValidationUtils& value) final; void ValidateMaxCheckAttempts(const Lazy& lvalue, const ValidationUtils& value) final; + bool NotificationReasonApplies(NotificationType type); + bool IsLikelyToBeCheckedSoon(); + static void IncreasePendingChecks(); static void DecreasePendingChecks(); static int GetPendingChecks(); diff --git a/lib/icinga/clusterevents.cpp b/lib/icinga/clusterevents.cpp index bc61a58cb..e8841397a 100644 --- a/lib/icinga/clusterevents.cpp +++ b/lib/icinga/clusterevents.cpp @@ -25,6 +25,7 @@ INITIALIZE_ONCE(&ClusterEvents::StaticInitialize); REGISTER_APIFUNCTION(CheckResult, event, &ClusterEvents::CheckResultAPIHandler); REGISTER_APIFUNCTION(SetNextCheck, event, &ClusterEvents::NextCheckChangedAPIHandler); REGISTER_APIFUNCTION(SetSuppressedNotifications, event, &ClusterEvents::SuppressedNotificationsChangedAPIHandler); +REGISTER_APIFUNCTION(SetSuppressedNotificationTypes, event, &ClusterEvents::SuppressedNotificationTypesChangedAPIHandler); REGISTER_APIFUNCTION(SetNextNotification, event, &ClusterEvents::NextNotificationChangedAPIHandler); REGISTER_APIFUNCTION(SetForceNextCheck, event, &ClusterEvents::ForceNextCheckChangedAPIHandler); REGISTER_APIFUNCTION(SetForceNextNotification, event, &ClusterEvents::ForceNextNotificationChangedAPIHandler); @@ -40,6 +41,7 @@ void ClusterEvents::StaticInitialize() Checkable::OnNewCheckResult.connect(&ClusterEvents::CheckResultHandler); Checkable::OnNextCheckChanged.connect(&ClusterEvents::NextCheckChangedHandler); Checkable::OnSuppressedNotificationsChanged.connect(&ClusterEvents::SuppressedNotificationsChangedHandler); + Notification::OnSuppressedNotificationsChanged.connect(&ClusterEvents::SuppressedNotificationTypesChangedHandler); Notification::OnNextNotificationChanged.connect(&ClusterEvents::NextNotificationChangedHandler); Checkable::OnForceNextCheckChanged.connect(&ClusterEvents::ForceNextCheckChangedHandler); Checkable::OnForceNextNotificationChanged.connect(&ClusterEvents::ForceNextNotificationChangedHandler); @@ -296,6 +298,52 @@ Value ClusterEvents::SuppressedNotificationsChangedAPIHandler(const MessageOrigi return Empty; } +void ClusterEvents::SuppressedNotificationTypesChangedHandler(const Notification::Ptr& notification, const MessageOrigin::Ptr& origin) +{ + ApiListener::Ptr listener = ApiListener::GetInstance(); + + if (!listener) + return; + + Dictionary::Ptr params = new Dictionary(); + params->Set("notification", notification->GetName()); + params->Set("suppressed_notifications", notification->GetSuppressedNotifications()); + + Dictionary::Ptr message = new Dictionary(); + message->Set("jsonrpc", "2.0"); + message->Set("method", "event::SetSuppressedNotificationTypes"); + message->Set("params", params); + + listener->RelayMessage(origin, notification, message, true); +} + +Value ClusterEvents::SuppressedNotificationTypesChangedAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params) +{ + Endpoint::Ptr endpoint = origin->FromClient->GetEndpoint(); + + if (!endpoint) { + Log(LogNotice, "ClusterEvents") + << "Discarding 'suppressed notifications changed' message from '" << origin->FromClient->GetIdentity() << "': Invalid endpoint origin (client not allowed)."; + return Empty; + } + + auto notification (Notification::GetByName(params->Get("notification"))); + + if (!notification) + return Empty; + + if (origin->FromZone && !origin->FromZone->CanAccessObject(notification)) { + Log(LogNotice, "ClusterEvents") + << "Discarding 'suppressed notification types changed' message for notification '" << notification->GetName() + << "' from '" << origin->FromClient->GetIdentity() << "': Unauthorized access."; + return Empty; + } + + notification->SetSuppressedNotifications(params->Get("suppressed_notifications"), false, origin); + + return Empty; +} + void ClusterEvents::NextNotificationChangedHandler(const Notification::Ptr& notification, const MessageOrigin::Ptr& origin) { ApiListener::Ptr listener = ApiListener::GetInstance(); diff --git a/lib/icinga/clusterevents.hpp b/lib/icinga/clusterevents.hpp index a51bc60db..04110577f 100644 --- a/lib/icinga/clusterevents.hpp +++ b/lib/icinga/clusterevents.hpp @@ -29,6 +29,9 @@ public: static void SuppressedNotificationsChangedHandler(const Checkable::Ptr& checkable, const MessageOrigin::Ptr& origin); static Value SuppressedNotificationsChangedAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params); + static void SuppressedNotificationTypesChangedHandler(const Notification::Ptr& notification, const MessageOrigin::Ptr& origin); + static Value SuppressedNotificationTypesChangedAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params); + static void NextNotificationChangedHandler(const Notification::Ptr& notification, const MessageOrigin::Ptr& origin); static Value NextNotificationChangedAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params); diff --git a/lib/icinga/notification.cpp b/lib/icinga/notification.cpp index 9c2ae7ec8..7555b457c 100644 --- a/lib/icinga/notification.cpp +++ b/lib/icinga/notification.cpp @@ -234,6 +234,39 @@ void Notification::BeginExecuteNotification(NotificationType type, const CheckRe Log(LogNotice, "Notification") << "Not sending " << (reminder ? "reminder " : "") << "notifications for notification object '" << notificationName << "': not in timeperiod '" << tp->GetName() << "'"; + + if (!reminder) { + switch (type) { + case NotificationProblem: + case NotificationRecovery: + case NotificationFlappingStart: + case NotificationFlappingEnd: + { + /* If a non-reminder notification was suppressed, but just because of its time period, + * stash it into a notification types bitmask for maybe re-sending later. + */ + + ObjectLock olock (this); + int suppressedTypesBefore (GetSuppressedNotifications()); + int suppressedTypesAfter (suppressedTypesBefore | type); + + for (int conflict : {NotificationProblem | NotificationRecovery, NotificationFlappingStart | NotificationFlappingEnd}) { + /* E.g. problem and recovery notifications neutralize each other. */ + + if ((suppressedTypesAfter & conflict) == conflict) { + suppressedTypesAfter &= ~conflict; + } + } + + if (suppressedTypesAfter != suppressedTypesBefore) { + SetSuppressedNotifications(suppressedTypesAfter); + } + } + default: + ; // Cheating the compiler on "5 enumeration values not handled in switch" + } + } + return; } diff --git a/lib/icinga/notification.ti b/lib/icinga/notification.ti index a283bbb84..e76a4f775 100644 --- a/lib/icinga/notification.ti +++ b/lib/icinga/notification.ti @@ -86,6 +86,10 @@ class Notification : CustomVarObject < NotificationNameComposer [state] int notification_number; [state] Timestamp last_problem_notification; + [state, no_user_view, no_user_modify] int suppressed_notifications { + default {{{ return 0; }}} + }; + [config, navigation] name(Endpoint) command_endpoint (CommandEndpointRaw) { navigate {{{ return Endpoint::GetByName(GetCommandEndpointRaw()); diff --git a/lib/notification/notificationcomponent.cpp b/lib/notification/notificationcomponent.cpp index aa9601201..3bc4b5a63 100644 --- a/lib/notification/notificationcomponent.cpp +++ b/lib/notification/notificationcomponent.cpp @@ -56,6 +56,69 @@ void NotificationComponent::Stop(bool runtimeRemoved) ObjectImpl::Stop(runtimeRemoved); } +static inline +void SubtractSuppressedNotificationTypes(const Notification::Ptr& notification, int types) +{ + ObjectLock olock (notification); + + int suppressedTypesBefore (notification->GetSuppressedNotifications()); + int suppressedTypesAfter (suppressedTypesBefore & ~types); + + if (suppressedTypesAfter != suppressedTypesBefore) { + notification->SetSuppressedNotifications(suppressedTypesAfter); + } +} + +static inline +void FireSuppressedNotifications(const Notification::Ptr& notification) +{ + int suppressedTypes (notification->GetSuppressedNotifications()); + if (!suppressedTypes) + return; + + int subtract = 0; + auto checkable (notification->GetCheckable()); + + for (auto type : {NotificationProblem, NotificationRecovery, NotificationFlappingStart, NotificationFlappingEnd}) { + if ((suppressedTypes & type) && !checkable->NotificationReasonApplies(type)) { + subtract |= type; + suppressedTypes &= ~type; + } + } + + if (suppressedTypes) { + auto tp (notification->GetPeriod()); + + if ((!tp || tp->IsInside(Utility::GetTime())) && !checkable->IsLikelyToBeCheckedSoon()) { + for (auto type : {NotificationProblem, NotificationRecovery, NotificationFlappingStart, NotificationFlappingEnd}) { + if (!(suppressedTypes & type)) + continue; + + auto notificationName (notification->GetName()); + + Log(LogNotice, "NotificationComponent") + << "Attempting to re-send previously suppressed notification '" << notificationName << "'."; + + subtract |= type; + SubtractSuppressedNotificationTypes(notification, subtract); + subtract = 0; + + try { + notification->BeginExecuteNotification(type, checkable->GetLastCheckResult(), false, false); + } catch (const std::exception& ex) { + Log(LogWarning, "NotificationComponent") + << "Exception occurred during notification for object '" + << notificationName << "': " << DiagnosticInformation(ex, false); + } + } + } + } + + if (subtract) { + SubtractSuppressedNotificationTypes(notification, subtract); + } +} + /** * Periodically sends notifications. * @@ -104,37 +167,41 @@ void NotificationComponent::NotificationTimerHandler() bool reachable = checkable->IsReachable(DependencyNotification); if (reachable) { - Array::Ptr unstashedNotifications = new Array(); - { - auto stashedNotifications (notification->GetStashedNotifications()); - ObjectLock olock(stashedNotifications); + Array::Ptr unstashedNotifications = new Array(); - stashedNotifications->CopyTo(unstashedNotifications); - stashedNotifications->Clear(); - } + { + auto stashedNotifications (notification->GetStashedNotifications()); + ObjectLock olock(stashedNotifications); - ObjectLock olock(unstashedNotifications); + stashedNotifications->CopyTo(unstashedNotifications); + stashedNotifications->Clear(); + } - for (Dictionary::Ptr unstashedNotification : unstashedNotifications) { - try { - Log(LogNotice, "NotificationComponent") - << "Attempting to send stashed notification '" << notificationName << "'."; + ObjectLock olock(unstashedNotifications); - notification->BeginExecuteNotification( - (NotificationType)(int)unstashedNotification->Get("type"), - (CheckResult::Ptr)unstashedNotification->Get("cr"), - (bool)unstashedNotification->Get("force"), - (bool)unstashedNotification->Get("reminder"), - (String)unstashedNotification->Get("author"), - (String)unstashedNotification->Get("text") - ); - } catch (const std::exception& ex) { - Log(LogWarning, "NotificationComponent") - << "Exception occurred during notification for object '" - << notificationName << "': " << DiagnosticInformation(ex, false); + for (Dictionary::Ptr unstashedNotification : unstashedNotifications) { + try { + Log(LogNotice, "NotificationComponent") + << "Attempting to send stashed notification '" << notificationName << "'."; + + notification->BeginExecuteNotification( + (NotificationType)(int)unstashedNotification->Get("type"), + (CheckResult::Ptr)unstashedNotification->Get("cr"), + (bool)unstashedNotification->Get("force"), + (bool)unstashedNotification->Get("reminder"), + (String)unstashedNotification->Get("author"), + (String)unstashedNotification->Get("text") + ); + } catch (const std::exception& ex) { + Log(LogWarning, "NotificationComponent") + << "Exception occurred during notification for object '" + << notificationName << "': " << DiagnosticInformation(ex, false); + } } } + + FireSuppressedNotifications(notification); } if (notification->GetInterval() <= 0 && notification->GetNoMoreNotifications()) { From 6d7b3725e8309f89287da0c563151b60b5adbd5b Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 3 Aug 2020 10:39:37 +0200 Subject: [PATCH 32/75] RELEASE.md: replace "git push --tags" with "git push origin v$VERSION" --- RELEASE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index 98cf84ade..1551f5314 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -88,7 +88,7 @@ git tag -s -m "Version $VERSION" v$VERSION Push the tag: ``` -git push --tags +git push origin v$VERSION ``` **For major releases:** Create a new `support` branch: From ac87b2369569f70bdce8386eb0cb96d5adc0370e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 3 Aug 2020 11:22:07 +0200 Subject: [PATCH 33/75] RELEASE.md: add Docker section --- RELEASE.md | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 98cf84ade..dfae94faf 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -16,10 +16,11 @@ - [7. Release Tests](#release-tests) - [8. GitHub Release](#github-release) - [9. Chocolatey](#chocolatey) -- [10. Post Release](#post-release) - - [10.1. Online Documentation](#online-documentation) - - [10.2. Announcement](#announcement) - - [10.3. Project Management](#project-management) +- [10. Docker](#docker) +- [11. Post Release](#post-release) + - [11.1. Online Documentation](#online-documentation) + - [11.2. Announcement](#announcement) + - [11.3. Project Management](#project-management) ## Preparations @@ -321,6 +322,31 @@ choco push Icinga2-v2.11.0.nupkg --source https://push.chocolatey.org/ ``` +## Docker + +> Only for final versions (not for RCs). + +Once the release has been published on GitHub, wait for its +[GitHub actions](https://github.com/Icinga/icinga2/actions) to complete. + +```bash +VERSION=2.12.1 + +TAGS=(2.12) +#TAGS=(2.12 2 latest) + +docker pull icinga/icinga2:$VERSION + +for t in "${TAGS[@]}"; do + docker tag icinga/icinga2:$VERSION icinga/icinga2:$t +done + +for t in "${TAGS[@]}"; do + docker push icinga/icinga2:$t +done +``` + + ## Post Release ### Online Documentation From c8c374331474d0c040ca2b43dae7b74e80f5fd1d Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 3 Aug 2020 15:42:16 +0200 Subject: [PATCH 34/75] Icinga 2.12.0 --- AUTHORS | 1 + CHANGELOG.md | 116 +++++++++++++++++++++++++++++++++++++++++++++++++++ VERSION | 2 +- 3 files changed, 118 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 3ea93384e..9fd4ddcd1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -260,6 +260,7 @@ Winfried Angele Wolfgang Nieder Yannick Charton Yohan Jarosz +Yonas Habteab Zachary McGibbon Zoltan Nagy Élie Bouttier diff --git a/CHANGELOG.md b/CHANGELOG.md index 051f57b44..1cc214bd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,122 @@ documentation before upgrading to a new release. Released closed milestones can be found on [GitHub](https://github.com/Icinga/icinga2/milestones?state=closed). +## 2.12.0 (2020-08-05) + +[Issue and PRs](https://github.com/Icinga/icinga2/issues?utf8=%E2%9C%93&q=milestone%3A2.12.0) + +### Notes + +Upgrading docs: https://icinga.com/docs/icinga2/snapshot/doc/16-upgrading-icinga-2/#upgrading-to-v212 + +Thanks to all contributors: +[Ant1x](https://github.com/Icinga/icinga2/pulls?q=is%3Apr+author%3AAnt1x+milestone%3A2.12.0), +[azthec](https://github.com/Icinga/icinga2/pulls?q=is%3Apr+author%3Aazthec+milestone%3A2.12.0), +[baurmatt](https://github.com/Icinga/icinga2/pulls?q=is%3Apr+author%3Abaurmatt+milestone%3A2.12.0), +[bootc](https://github.com/Icinga/icinga2/pulls?q=is%3Apr+author%3Abootc+milestone%3A2.12.0), +[Foxeronie](https://github.com/Icinga/icinga2/pulls?q=is%3Apr+author%3AFoxeronie+milestone%3A2.12.0), +[ggzengel](https://github.com/Icinga/icinga2/pulls?q=is%3Apr+author%3Aggzengel+milestone%3A2.12.0), +[islander](https://github.com/Icinga/icinga2/pulls?q=is%3Apr+author%3Aislander+milestone%3A2.12.0), +[joni1993](https://github.com/Icinga/icinga2/pulls?q=is%3Apr+author%3Ajoni1993+milestone%3A2.12.0), +[KAMI911](https://github.com/Icinga/icinga2/pulls?q=is%3Apr+author%3AKAMI911+milestone%3A2.12.0), +[mcktr](https://github.com/Icinga/icinga2/pulls?q=is%3Apr+author%3Amcktr+milestone%3A2.12.0), +[MichalMMac](https://github.com/Icinga/icinga2/pulls?q=is%3Apr+author%3AMichalMMac+milestone%3A2.12.0), +[sebastic](https://github.com/Icinga/icinga2/pulls?q=is%3Apr+author%3Asebastic+milestone%3A2.12.0), +[sthen](https://github.com/Icinga/icinga2/pulls?q=is%3Apr+author%3Asthen+milestone%3A2.12.0), +[unki](https://github.com/Icinga/icinga2/pulls?q=is%3Apr+author%3Aunki+milestone%3A2.12.0), +[vigiroux](https://github.com/Icinga/icinga2/pulls?q=is%3Apr+author%3Avigiroux+milestone%3A2.12.0), +[wopfel](https://github.com/Icinga/icinga2/pulls?q=is%3Apr+author%3Awopfel+milestone%3A2.12.0) + +### Breaking changes + +* Deprecate Windows plugins in favor of our + [PowerShell plugins](https://github.com/Icinga/icinga-powershell-plugins) #8071 +* Deprecate Livestatus #8051 +* Refuse acknowledging an already acknowledged checkable #7695 +* Config lexer: complain on EOF in heredocs, i.e. `{{{abc` #7541 + +### Enhancements + +* Core + * Implement new database backend: Icinga DB #7571 + * Re-send notifications previously suppressed by their time periods #7816 +* API + * Host/Service: Add `acknowledgement_last_change` and `next_update` attributes #7881 #7534 + * Improve error message for POST queries #7681 + * /v1/actions/remove-comment: let users specify themselves #7646 + * /v1/actions/remove-downtime: let users specify themselves #7645 + * /v1/config/stages: Add 'activate' parameter #7535 +* CLI + * Add `pki verify` command for better TLS certificate troubleshooting #7843 + * Add OpenSSL version to 'Build' section in --version #7833 + * Improve experience with 'Node Setup for Agents/Satellite' #7835 +* DSL + * Add `get_template()` and `get_templates()` #7632 + * `MacroProcessor::ResolveArguments()`: skip null argument values #7567 + * Fix crash due to dependency apply rule with `ignore_on_error` and non-existing parent #7538 + * Introduce ternary operator (`x ? y : z`) #7442 + * LegacyTimePeriod: support specifying seconds #7439 + * Add support for Lambda Closures (`() use(x) => x and () use(x) => { return x }`) #7417 +* ITL + * Add notemp parameter to oracle health #7748 + * Add extended checks options to snmp-interface command template #7602 + * Add file age check for Windows command definition #7540 +* Docs + * Development: Update debugging instructions #7867 + * Add new API clients #7859 + * Clarify CRITICAL vs. UNKNOWN #7665 + * Explicitly explain how to disable freshness checks #7664 + * Update installation for RHEL/CentOS 8 and SLES 15 #7640 + * Add Powershell example to validate the certificate #7603 +* Misc + * Don't send `event::Heartbeat` to unauthenticated peers #7747 + * OpenTsdbWriter: Add custom tag support #7357 + +### Bugfixes + +* Core + * Fix JSON-RPC crashes #7532 #7737 + * Fix zone definitions in zones #7546 + * Fix deadlock during start on OpenBSD #7739 + * Consider PENDING not a problem #7685 + * Fix zombie processes after reload #7606 + * Don't wait for checks to finish during reload #7894 +* Cluster + * Fix segfault during heartbeat timeout with clients not yet signed #7970 + * Make the config update process mutually exclusive (Prevents file system race conditions) #7936 + * Fix `check_timeout` not being forwarded to agent command endpoints #7861 + * Config sync: Use a more friendly message when configs are equal and don't need a reload #7811 + * Fix open connections when agent waits for CA approval #7686 + * Consider a JsonRpcConnection alive on a single byte of TLS payload, not only on a whole message #7836 + * Send JsonRpcConnection heartbeat every 20s instead of 10s #8102 + * Use JsonRpcConnection heartbeat only to update connection liveness (m\_Seen) #8142 + * Fix TLS context not being updated on signed certificate messages on agents #7654 +* API + * Close connections w/o successful TLS handshakes after 10s #7809 + * Handle permission exceptions soon enough, returning 404 #7528 +* SELinux + * Fix safe-reload #7858 + * Allow direct SMTP notifications #7749 +* Windows + * Terminate check processes with UNKNOWN state on timeout #7788 + * Ensure that log replay files are properly renamed #7767 +* Metrics + * Graphite/OpenTSDB: Ensure that reconnect failure is detected #7765 + * Always send 0 as value for thresholds #7696 +* Scripts + * Fix notification scripts to stay compatible with Dash #7706 + * Fix bash line continuation in mail-host-notification.sh #7701 + * Fix notification scripts string comparison #7647 + * Service and host mail-notifications: Add line-breaks to very long output #6822 + * Set correct UTF-8 email subject header (RFC1342) #6369 +* Misc + * DSL: Fix segfault due to passing null as custom function to `Array#{sort,map,reduce,filter,any,all}()` #8053 + * CLI: `pki save-cert`: allow to specify --key and --cert for backwards compatibility #7995 + * Catch exception when trusted cert is not readable during node setup on agent/satellite #7838 + * CheckCommand ssl: Fix wrong parameter `-N` #7741 + * Code quality fixes + * Small documentation fixes + ## 2.12.0 RC1 (2020-03-13) [Issue and PRs](https://github.com/Icinga/icinga2/issues?utf8=%E2%9C%93&q=milestone%3A2.12.0) diff --git a/VERSION b/VERSION index 168b12744..ca6922f5c 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -Version: 2.12.0-rc1 +Version: 2.12.0 Revision: 1 From ddf1e50d93280059348da967a0af1d0a2a5dcbe0 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 11 Aug 2020 15:24:54 +0200 Subject: [PATCH 35/75] ProcessCheckResult(): Make sure hosts aren't locked during Service::GetSeverity() --- lib/icinga/checkable-check.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/icinga/checkable-check.cpp b/lib/icinga/checkable-check.cpp index ca5b7ba71..9be429ec0 100644 --- a/lib/icinga/checkable-check.cpp +++ b/lib/icinga/checkable-check.cpp @@ -358,9 +358,12 @@ void Checkable::ProcessCheckResult(const CheckResult::Ptr& cr, const MessageOrig SetLastCheckResult(cr); if (GetProblem() != wasProblem) { - for (auto& service : host->GetServices()) { + auto services = host->GetServices(); + olock.Unlock(); + for (auto& service : services) { Service::OnHostProblemChanged(service, cr, origin); } + olock.Lock(); } } From ade891bbf5c1e1740f119599c4898a5ca8daf11d Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 13 Aug 2020 10:39:55 +0200 Subject: [PATCH 36/75] Revert "MacroProcessor::ResolveArguments(): skip null argument values" This reverts commit e4bdcedbca069cb9d42c40ce6ce8054ed3ee1b58. --- lib/icinga/macroprocessor.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/icinga/macroprocessor.cpp b/lib/icinga/macroprocessor.cpp index c92ab19d4..df1e41d17 100644 --- a/lib/icinga/macroprocessor.cpp +++ b/lib/icinga/macroprocessor.cpp @@ -511,8 +511,6 @@ Value MacroProcessor::ResolveArguments(const Value& command, const Dictionary::P continue; } - arg.SkipValue = arg.SkipValue || arg.AValue.GetType() == ValueEmpty; - args.emplace_back(std::move(arg)); } From d69c6879fa7b52b93ecd163a13cfa2cc47b00604 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 3 Sep 2020 14:04:42 +0200 Subject: [PATCH 37/75] Application#RunEventLoop(): don't wait for the thread pool to stop on shutdown refs #8173 --- lib/base/application.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/base/application.cpp b/lib/base/application.cpp index 9ef9839a1..2339250ec 100644 --- a/lib/base/application.cpp +++ b/lib/base/application.cpp @@ -342,7 +342,9 @@ void Application::RunEventLoop() ConfigObject::StopObjects(); Application::GetInstance()->OnShutdown(); - UninitializeBase(); +#ifdef I2_DEBUG + UninitializeBase(); // Inspired from Exit() +#endif /* I2_DEBUG */ } bool Application::IsShuttingDown() From ea630472a77d7e5b883b18ae2e500ca3da62f235 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 8 Sep 2020 17:11:47 +0200 Subject: [PATCH 38/75] GitHub actions: don't upload any artifacts --- .github/workflows/packages.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index a56443175..34e319bfd 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -112,12 +112,6 @@ jobs: -e ICINGA_BUILD_TYPE=snapshot \ registry.icinga.com/build-docker/${{ matrix.distro.name }}/${{ matrix.distro.codename }}:x86 \ icinga-build-test - - - name: Artifacts - uses: actions/upload-artifact@v1 - with: - name: '${{ matrix.distro.name }}-${{ matrix.distro.codename }}-packages' - path: deb-icinga2/build rpm: name: .rpm @@ -228,13 +222,6 @@ jobs: -e ICINGA_BUILD_TYPE=snapshot \ registry.icinga.com/build-docker/${{ matrix.distro.name }}/${{ matrix.distro.release }} \ icinga-build-test - - - name: Artifacts - if: "steps.vars.outputs.CAN_BUILD == 'true'" - uses: actions/upload-artifact@v1 - with: - name: '${{ matrix.distro.name }}-${{ matrix.distro.release }}-packages' - path: rpm-icinga2/build raspbian: name: Raspbian @@ -306,9 +293,3 @@ jobs: # -e ICINGA_BUILD_DEB_DEFAULT_ARCH=armhf \ # registry.icinga.com/build-docker/raspbian/${{ matrix.codename }} \ # icinga-build-test - - - name: Artifacts - uses: actions/upload-artifact@v1 - with: - name: 'raspbian-${{ matrix.codename }}-packages' - path: raspbian-icinga2/build From 377a9e50481bcaebccf5aa6d17aadaeed4329147 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 9 Sep 2020 11:36:29 +0200 Subject: [PATCH 39/75] GitHub actions: don't cancel not yet failed jobs on a failure --- .github/workflows/packages.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index a56443175..7ba01c5c9 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -122,6 +122,7 @@ jobs: name: .rpm strategy: + fail-fast: false matrix: distro: - name: centos @@ -239,6 +240,7 @@ jobs: name: Raspbian strategy: + fail-fast: false matrix: codename: - buster From 8b0ba2275a3958a63d165d8fb5ff4c0e3be7a2bd Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 11 Sep 2020 14:31:31 +0200 Subject: [PATCH 40/75] Check !!downtime->GetCheckable() before downtime->GetCheckable()->GetName() ... not to crash while removing a downtime from a disappeared checkable. --- lib/icinga/downtime.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/icinga/downtime.cpp b/lib/icinga/downtime.cpp index f003bc746..1364f722c 100644 --- a/lib/icinga/downtime.cpp +++ b/lib/icinga/downtime.cpp @@ -345,10 +345,19 @@ void Downtime::RemoveDowntime(const String& id, bool cancelled, bool expired, co reason = ""; } - Log(LogInformation, "Downtime") - << "Removed downtime '" << downtime->GetName() << "' from checkable '" - << downtime->GetCheckable()->GetName() << "' (Reason: " << reason << ")."; + Log msg (LogInformation, "Downtime"); + msg << "Removed downtime '" << downtime->GetName() << "' from checkable"; + + { + auto checkable (downtime->GetCheckable()); + + if (checkable) { + msg << " '" << checkable->GetName() << "'"; + } + } + + msg << " (Reason: " << reason << ")."; } bool Downtime::CanBeTriggered() From 4768874ed1e04e768b61f51e257645a6a411e04b Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 15 Sep 2020 17:39:25 +0200 Subject: [PATCH 41/75] RELEASE.md: remove unneccessary "Authors" section --- RELEASE.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index dfae94faf..818202e57 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -5,7 +5,6 @@ - [1. Preparations](#preparations) - [1.1. Issues](#issues) - [1.2. Backport Commits](#backport-commits) - - [1.3. Authors](#authors) - [2. Version](#version) - [3. Changelog](#changelog) - [4. Git Tag](#git-tag) @@ -50,14 +49,6 @@ Check issues at https://github.com/Icinga/icinga2 For minor versions you need to manually backports any and all commits from the master branch which should be part of this release. -### Authors - -Update the [.mailmap](.mailmap) and [AUTHORS](AUTHORS) files: - -``` -git checkout master -git log --use-mailmap | grep '^Author:' | cut -f2- -d' ' | sort -f | uniq > AUTHORS -``` ## Version From 3c556350c802860904b22096e13757b64af41275 Mon Sep 17 00:00:00 2001 From: Henrik Triem Date: Mon, 14 Sep 2020 17:09:11 +0200 Subject: [PATCH 42/75] WorkQueue: Allow choosing stats log level --- lib/base/workqueue.cpp | 6 +++--- lib/base/workqueue.hpp | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/base/workqueue.cpp b/lib/base/workqueue.cpp index cacd17b76..4593b34ba 100644 --- a/lib/base/workqueue.cpp +++ b/lib/base/workqueue.cpp @@ -14,9 +14,9 @@ using namespace icinga; std::atomic WorkQueue::m_NextID(1); boost::thread_specific_ptr l_ThreadWorkQueue; -WorkQueue::WorkQueue(size_t maxItems, int threadCount) +WorkQueue::WorkQueue(size_t maxItems, int threadCount, LogSeverity statsLogLevel) : m_ID(m_NextID++), m_ThreadCount(threadCount), m_MaxItems(maxItems), - m_TaskStats(15 * 60) + m_TaskStats(15 * 60), m_StatsLogLevel(statsLogLevel) { /* Initialize logger. */ m_StatusTimerTimeout = Utility::GetTime(); @@ -216,7 +216,7 @@ void WorkQueue::StatusTimerHandler() /* Log if there are pending items, or 5 minute timeout is reached. */ if (pending > 0 || m_StatusTimerTimeout < now) { - Log(LogInformation, "WorkQueue") + Log(m_StatsLogLevel, "WorkQueue") << "#" << m_ID << " (" << m_Name << ") " << "items: " << pending << ", " << "rate: " << std::setw(2) << GetTaskCount(60) / 60.0 << "/s " diff --git a/lib/base/workqueue.hpp b/lib/base/workqueue.hpp index bc84d9176..1c3df57b2 100644 --- a/lib/base/workqueue.hpp +++ b/lib/base/workqueue.hpp @@ -6,6 +6,7 @@ #include "base/i2-base.hpp" #include "base/timer.hpp" #include "base/ringbuffer.hpp" +#include "base/logger.hpp" #include #include #include @@ -52,7 +53,7 @@ class WorkQueue public: typedef std::function ExceptionCallback; - WorkQueue(size_t maxItems = 0, int threadCount = 1); + WorkQueue(size_t maxItems = 0, int threadCount = 1, LogSeverity statsLogLevel = LogInformation); ~WorkQueue(); void SetName(const String& name); @@ -129,6 +130,7 @@ private: std::vector m_Exceptions; Timer::Ptr m_StatusTimer; double m_StatusTimerTimeout; + LogSeverity m_StatsLogLevel; RingBuffer m_TaskStats; size_t m_PendingTasks{0}; From 7a759a6427533c4dd6a8abb65b31145594bd7b8f Mon Sep 17 00:00:00 2001 From: Henrik Triem Date: Wed, 9 Sep 2020 18:17:37 +0200 Subject: [PATCH 43/75] IDO: Implement more accurate logging for query stats --- lib/db_ido/dbconnection.cpp | 50 +++++++++++++++++++ lib/db_ido/dbconnection.hpp | 9 ++++ lib/db_ido_mysql/idomysqlconnection.cpp | 65 +++++++++++++++++++++---- lib/db_ido_mysql/idomysqlconnection.hpp | 2 +- lib/db_ido_pgsql/idopgsqlconnection.cpp | 58 +++++++++++++++++++--- lib/db_ido_pgsql/idopgsqlconnection.hpp | 2 +- 6 files changed, 166 insertions(+), 20 deletions(-) diff --git a/lib/db_ido/dbconnection.cpp b/lib/db_ido/dbconnection.cpp index 0ae9a80ca..6ab9b4e10 100644 --- a/lib/db_ido/dbconnection.cpp +++ b/lib/db_ido/dbconnection.cpp @@ -76,6 +76,11 @@ void DbConnection::Resume() m_CleanUpTimer->SetInterval(60); m_CleanUpTimer->OnTimerExpired.connect(std::bind(&DbConnection::CleanUpHandler, this)); m_CleanUpTimer->Start(); + + m_LogStatsTimer = new Timer(); + m_LogStatsTimer->SetInterval(10); + m_LogStatsTimer->OnTimerExpired.connect([this](const Timer * const&) { LogStatsHandler(); }); + m_LogStatsTimer->Start(); } void DbConnection::Pause() @@ -236,6 +241,39 @@ void DbConnection::CleanUpHandler() } +void DbConnection::LogStatsHandler() +{ + auto pending = m_PendingQueries.load(); + + if (pending == 0u) { + return; + } + + auto now = Utility::GetTime(); + auto output = round(m_OutputQueries.CalculateRate(now, 10)); + + if (pending < output * 2) { + return; + } + + auto input = round(m_InputQueries.CalculateRate(now, 10)); + + String timeInfo = " empty in "; + + { + auto rate = output - input; + + if (rate <= 0) + timeInfo += "infinite time, your task handler isn't able to keep up"; + else + timeInfo += Utility::FormatDuration(pending / rate); + } + + Log(LogInformation, GetReflectionType()->GetName()) + << "Pending queries: " << pending << " (Input: " << input + << "/s; Output: " << output << "/s)" << timeInfo; +} + void DbConnection::CleanUpExecuteQuery(const String&, const String&, double) { /* Default handler does nothing. */ @@ -507,3 +545,15 @@ int DbConnection::GetSessionToken() { return Application::GetStartTime(); } + +void DbConnection::IncreasePendingQueries(int count) +{ + m_PendingQueries.fetch_add(count); + m_InputQueries.InsertValue(Utility::GetTime(), count); +} + +void DbConnection::DecreasePendingQueries(int count) +{ + m_PendingQueries.fetch_sub(count); + m_OutputQueries.InsertValue(Utility::GetTime(), count); +} diff --git a/lib/db_ido/dbconnection.hpp b/lib/db_ido/dbconnection.hpp index 3cb049f64..84e28d4c6 100644 --- a/lib/db_ido/dbconnection.hpp +++ b/lib/db_ido/dbconnection.hpp @@ -92,6 +92,9 @@ protected: static int GetSessionToken(); + void IncreasePendingQueries(int count); + void DecreasePendingQueries(int count); + private: bool m_IDCacheValid{false}; std::map, String> m_ConfigHashes; @@ -101,8 +104,10 @@ private: std::set m_ConfigUpdates; std::set m_StatusUpdates; Timer::Ptr m_CleanUpTimer; + Timer::Ptr m_LogStatsTimer; void CleanUpHandler(); + void LogStatsHandler(); static Timer::Ptr m_ProgramStatusTimer; static boost::once_flag m_OnceFlag; @@ -112,6 +117,10 @@ private: mutable boost::mutex m_StatsMutex; RingBuffer m_QueryStats{15 * 60}; bool m_ActiveChangedHandler{false}; + + RingBuffer m_InputQueries{10}; + RingBuffer m_OutputQueries{10}; + Atomic m_PendingQueries{0}; }; struct database_error : virtual std::exception, virtual boost::exception { }; diff --git a/lib/db_ido_mysql/idomysqlconnection.cpp b/lib/db_ido_mysql/idomysqlconnection.cpp index 5dbaba071..915a7b3b8 100644 --- a/lib/db_ido_mysql/idomysqlconnection.cpp +++ b/lib/db_ido_mysql/idomysqlconnection.cpp @@ -13,6 +13,7 @@ #include "base/configtype.hpp" #include "base/exception.hpp" #include "base/statsfunction.hpp" +#include "base/defer.hpp" #include using namespace icinga; @@ -175,6 +176,8 @@ void IdoMysqlConnection::InternalNewTransaction() if (!GetConnected()) return; + IncreasePendingQueries(2); + AsyncQuery("COMMIT"); AsyncQuery("BEGIN"); } @@ -524,12 +527,28 @@ void IdoMysqlConnection::FinishAsyncQueries() std::vector::size_type offset = 0; + // This will be executed if there is a problem with executing the queries, + // at which point this function throws an exception and the queries should + // not be listed as still pending in the queue. + Defer decreaseQueries ([this, &offset, &queries]() { + auto lostQueries = queries.size() - offset; + + if (lostQueries > 0) { + DecreasePendingQueries(lostQueries); + } + }); + while (offset < queries.size()) { std::ostringstream querybuf; std::vector::size_type count = 0; size_t num_bytes = 0; + Defer decreaseQueries ([this, &offset, &count]() { + offset += count; + DecreasePendingQueries(count); + }); + for (std::vector::size_type i = offset; i < queries.size(); i++) { const IdoAsyncQuery& aq = queries[i]; @@ -608,8 +627,6 @@ void IdoMysqlConnection::FinishAsyncQueries() ); } } - - offset += count; } } @@ -617,6 +634,9 @@ IdoMysqlResult IdoMysqlConnection::Query(const String& query) { AssertOnWorkQueue(); + IncreasePendingQueries(1); + Defer decreaseQueries ([this]() { DecreasePendingQueries(1); }); + /* finish all async queries to maintain the right order for queries */ FinishAsyncQueries(); @@ -770,6 +790,7 @@ void IdoMysqlConnection::InternalActivateObject(const DbObject::Ptr& dbobj) SetObjectID(dbobj, GetLastInsertID()); } else { qbuf << "UPDATE " + GetTablePrefix() + "objects SET is_active = 1 WHERE object_id = " << static_cast(dbref); + IncreasePendingQueries(1); AsyncQuery(qbuf.str()); } } @@ -804,6 +825,7 @@ void IdoMysqlConnection::InternalDeactivateObject(const DbObject::Ptr& dbobj) std::ostringstream qbuf; qbuf << "UPDATE " + GetTablePrefix() + "objects SET is_active = 0 WHERE object_id = " << static_cast(dbref); + IncreasePendingQueries(1); AsyncQuery(qbuf.str()); /* Note that we're _NOT_ clearing the db refs via SetReference/SetConfigUpdate/SetStatusUpdate @@ -893,6 +915,7 @@ void IdoMysqlConnection::ExecuteQuery(const DbQuery& query) << "Scheduling execute query task, type " << query.Type << ", table '" << query.Table << "'."; #endif /* I2_DEBUG */ + IncreasePendingQueries(1); m_QueryQueue.Enqueue(std::bind(&IdoMysqlConnection::InternalExecuteQuery, this, query, -1), query.Priority, true); } @@ -909,6 +932,7 @@ void IdoMysqlConnection::ExecuteMultipleQueries(const std::vector& quer << "Scheduling multiple execute query task, type " << queries[0].Type << ", table '" << queries[0].Table << "'."; #endif /* I2_DEBUG */ + IncreasePendingQueries(queries.size()); m_QueryQueue.Enqueue(std::bind(&IdoMysqlConnection::InternalExecuteMultipleQueries, this, queries), queries[0].Priority, true); } @@ -948,11 +972,16 @@ void IdoMysqlConnection::InternalExecuteMultipleQueries(const std::vectorGetObject()->GetExtension("agent_check").ToBool()) + if (query.Object && query.Object->GetObject()->GetExtension("agent_check").ToBool()) { + DecreasePendingQueries(1); return; + } /* check if there are missing object/insert ids and re-enqueue the query */ if (!CanExecuteQuery(query)) { @@ -1066,6 +1104,7 @@ void IdoMysqlConnection::InternalExecuteQuery(const DbQuery& query, int typeOver if ((type & DbQueryInsert) && (type & DbQueryDelete)) { std::ostringstream qdel; qdel << "DELETE FROM " << GetTablePrefix() << query.Table << where.str(); + IncreasePendingQueries(1); AsyncQuery(qdel.str()); type = DbQueryInsert; @@ -1150,6 +1189,7 @@ void IdoMysqlConnection::FinishExecuteQuery(const DbQuery& query, int type, bool << "Rescheduling DELETE/INSERT query: Upsert UPDATE did not affect rows, type " << type << ", table '" << query.Table << "'."; #endif /* I2_DEBUG */ + IncreasePendingQueries(1); m_QueryQueue.Enqueue(std::bind(&IdoMysqlConnection::InternalExecuteQuery, this, query, DbQueryDelete | DbQueryInsert), query.Priority); return; @@ -1178,6 +1218,7 @@ void IdoMysqlConnection::CleanUpExecuteQuery(const String& table, const String& << time_column << "'. max_age is set to '" << max_age << "'."; #endif /* I2_DEBUG */ + IncreasePendingQueries(1); m_QueryQueue.Enqueue(std::bind(&IdoMysqlConnection::InternalCleanUpExecuteQuery, this, table, time_column, max_age), PriorityLow, true); } @@ -1185,11 +1226,15 @@ void IdoMysqlConnection::InternalCleanUpExecuteQuery(const String& table, const { AssertOnWorkQueue(); - if (IsPaused()) + if (IsPaused()) { + DecreasePendingQueries(1); return; + } - if (!GetConnected()) + if (!GetConnected()) { + DecreasePendingQueries(1); return; + } AsyncQuery("DELETE FROM " + GetTablePrefix() + table + " WHERE instance_id = " + Convert::ToString(static_cast(m_InstanceID)) + " AND " + time_column + diff --git a/lib/db_ido_mysql/idomysqlconnection.hpp b/lib/db_ido_mysql/idomysqlconnection.hpp index 748243a4c..3c1bf916b 100644 --- a/lib/db_ido_mysql/idomysqlconnection.hpp +++ b/lib/db_ido_mysql/idomysqlconnection.hpp @@ -54,7 +54,7 @@ protected: private: DbReference m_InstanceID; - WorkQueue m_QueryQueue{10000000}; + WorkQueue m_QueryQueue{10000000, 1, LogNotice}; Library m_Library; std::unique_ptr m_Mysql; diff --git a/lib/db_ido_pgsql/idopgsqlconnection.cpp b/lib/db_ido_pgsql/idopgsqlconnection.cpp index e458e0d63..a64da281f 100644 --- a/lib/db_ido_pgsql/idopgsqlconnection.cpp +++ b/lib/db_ido_pgsql/idopgsqlconnection.cpp @@ -14,6 +14,7 @@ #include "base/exception.hpp" #include "base/context.hpp" #include "base/statsfunction.hpp" +#include "base/defer.hpp" #include using namespace icinga; @@ -137,6 +138,7 @@ void IdoPgsqlConnection::Disconnect() if (!GetConnected()) return; + IncreasePendingQueries(1); Query("COMMIT"); m_Pgsql->finish(m_Connection); @@ -166,6 +168,7 @@ void IdoPgsqlConnection::InternalNewTransaction() if (!GetConnected()) return; + IncreasePendingQueries(2); Query("COMMIT"); Query("BEGIN"); } @@ -191,6 +194,7 @@ void IdoPgsqlConnection::Reconnect() if (GetConnected()) { /* Check if we're really still connected */ try { + IncreasePendingQueries(1); Query("SELECT 1"); return; } catch (const std::exception&) { @@ -260,10 +264,13 @@ void IdoPgsqlConnection::Reconnect() /* explicitely require legacy mode for string escaping in PostgreSQL >= 9.1 * changing standard_conforming_strings to on by default */ - if (m_Pgsql->serverVersion(m_Connection) >= 90100) + if (m_Pgsql->serverVersion(m_Connection) >= 90100) { + IncreasePendingQueries(1); result = Query("SET standard_conforming_strings TO off"); + } String dbVersionName = "idoutils"; + IncreasePendingQueries(1); result = Query("SELECT version FROM " + GetTablePrefix() + "dbversion WHERE name=E'" + Escape(dbVersionName) + "'"); Dictionary::Ptr row = FetchRow(result, 0); @@ -295,10 +302,12 @@ void IdoPgsqlConnection::Reconnect() String instanceName = GetInstanceName(); + IncreasePendingQueries(1); result = Query("SELECT instance_id FROM " + GetTablePrefix() + "instances WHERE instance_name = E'" + Escape(instanceName) + "'"); row = FetchRow(result, 0); if (!row) { + IncreasePendingQueries(1); Query("INSERT INTO " + GetTablePrefix() + "instances (instance_name, instance_description) VALUES (E'" + Escape(instanceName) + "', E'" + Escape(GetInstanceDescription()) + "')"); m_InstanceID = GetSequenceValue(GetTablePrefix() + "instances", "instance_id"); } else { @@ -310,6 +319,7 @@ void IdoPgsqlConnection::Reconnect() /* we have an endpoint in a cluster setup, so decide if we can proceed here */ if (my_endpoint && GetHAMode() == HARunOnce) { /* get the current endpoint writing to programstatus table */ + IncreasePendingQueries(1); result = Query("SELECT UNIX_TIMESTAMP(status_update_time) AS status_update_time, endpoint_name FROM " + GetTablePrefix() + "programstatus WHERE instance_id = " + Convert::ToString(m_InstanceID)); row = FetchRow(result, 0); @@ -372,12 +382,14 @@ void IdoPgsqlConnection::Reconnect() << "PGSQL IDO instance id: " << static_cast(m_InstanceID) << " (schema version: '" + version + "')" << (!sslMode.IsEmpty() ? ", sslmode='" + sslMode + "'" : ""); + IncreasePendingQueries(1); Query("BEGIN"); /* update programstatus table */ UpdateProgramStatus(); /* record connection */ + IncreasePendingQueries(1); Query("INSERT INTO " + GetTablePrefix() + "conninfo " + "(instance_id, connect_time, last_checkin_time, agent_name, agent_version, connect_type, data_start_time) VALUES (" + Convert::ToString(static_cast(m_InstanceID)) + ", NOW(), NOW(), E'icinga2 db_ido_pgsql', E'" + Escape(Application::GetAppVersion()) @@ -388,6 +400,7 @@ void IdoPgsqlConnection::Reconnect() std::ostringstream q1buf; q1buf << "SELECT object_id, objecttype_id, name1, name2, is_active FROM " + GetTablePrefix() + "objects WHERE instance_id = " << static_cast(m_InstanceID); + IncreasePendingQueries(1); result = Query(q1buf.str()); std::vector activeDbObjs; @@ -442,6 +455,7 @@ void IdoPgsqlConnection::FinishConnect(double startTime) << "Finished reconnecting to '" << GetName() << "' database '" << GetDatabase() << "' in " << std::setw(2) << Utility::GetTime() - startTime << " second(s)."; + IncreasePendingQueries(2); Query("COMMIT"); Query("BEGIN"); } @@ -455,6 +469,7 @@ void IdoPgsqlConnection::ClearTablesBySession() void IdoPgsqlConnection::ClearTableBySession(const String& table) { + IncreasePendingQueries(1); Query("DELETE FROM " + GetTablePrefix() + table + " WHERE instance_id = " + Convert::ToString(static_cast(m_InstanceID)) + " AND session_token <> " + Convert::ToString(GetSessionToken())); @@ -464,6 +479,8 @@ IdoPgsqlResult IdoPgsqlConnection::Query(const String& query) { AssertOnWorkQueue(); + Defer decreaseQueries ([this]() { DecreasePendingQueries(1); }); + Log(LogDebug, "IdoPgsqlConnection") << "Query: " << query; @@ -512,6 +529,7 @@ DbReference IdoPgsqlConnection::GetSequenceValue(const String& table, const Stri { AssertOnWorkQueue(); + IncreasePendingQueries(1); IdoPgsqlResult result = Query("SELECT CURRVAL(pg_get_serial_sequence(E'" + Escape(table) + "', E'" + Escape(column) + "')) AS id"); Dictionary::Ptr row = FetchRow(result, 0); @@ -601,10 +619,12 @@ void IdoPgsqlConnection::InternalActivateObject(const DbObject::Ptr& dbobj) << "E'" << Escape(dbobj->GetName1()) << "', 1)"; } + IncreasePendingQueries(1); Query(qbuf.str()); SetObjectID(dbobj, GetSequenceValue(GetTablePrefix() + "objects", "object_id")); } else { qbuf << "UPDATE " + GetTablePrefix() + "objects SET is_active = 1 WHERE object_id = " << static_cast(dbref); + IncreasePendingQueries(1); Query(qbuf.str()); } } @@ -631,6 +651,7 @@ void IdoPgsqlConnection::InternalDeactivateObject(const DbObject::Ptr& dbobj) std::ostringstream qbuf; qbuf << "UPDATE " + GetTablePrefix() + "objects SET is_active = 0 WHERE object_id = " << static_cast(dbref); + IncreasePendingQueries(1); Query(qbuf.str()); /* Note that we're _NOT_ clearing the db refs via SetReference/SetConfigUpdate/SetStatusUpdate @@ -715,6 +736,7 @@ void IdoPgsqlConnection::ExecuteQuery(const DbQuery& query) ASSERT(query.Category != DbCatInvalid); + IncreasePendingQueries(1); m_QueryQueue.Enqueue(std::bind(&IdoPgsqlConnection::InternalExecuteQuery, this, query, -1), query.Priority, true); } @@ -726,6 +748,7 @@ void IdoPgsqlConnection::ExecuteMultipleQueries(const std::vector& quer if (queries.empty()) return; + IncreasePendingQueries(queries.size()); m_QueryQueue.Enqueue(std::bind(&IdoPgsqlConnection::InternalExecuteMultipleQueries, this, queries), queries[0].Priority, true); } @@ -765,11 +788,15 @@ void IdoPgsqlConnection::InternalExecuteMultipleQueries(const std::vectorGetObject()->GetExtension("agent_check").ToBool()) + if (query.Object && query.Object->GetObject()->GetExtension("agent_check").ToBool()) { + DecreasePendingQueries(1); return; + } /* check if there are missing object/insert ids and re-enqueue the query */ if (!CanExecuteQuery(query)) { @@ -862,6 +898,7 @@ void IdoPgsqlConnection::InternalExecuteQuery(const DbQuery& query, int typeOver if ((type & DbQueryInsert) && (type & DbQueryDelete)) { std::ostringstream qdel; qdel << "DELETE FROM " << GetTablePrefix() << query.Table << where.str(); + IncreasePendingQueries(1); Query(qdel.str()); type = DbQueryInsert; @@ -929,6 +966,7 @@ void IdoPgsqlConnection::InternalExecuteQuery(const DbQuery& query, int typeOver Query(qbuf.str()); if (upsert && GetAffectedRows() == 0) { + IncreasePendingQueries(1); InternalExecuteQuery(query, DbQueryDelete | DbQueryInsert); return; @@ -959,6 +997,7 @@ void IdoPgsqlConnection::CleanUpExecuteQuery(const String& table, const String& if (IsPaused()) return; + IncreasePendingQueries(1); m_QueryQueue.Enqueue(std::bind(&IdoPgsqlConnection::InternalCleanUpExecuteQuery, this, table, time_column, max_age), PriorityLow, true); } @@ -966,8 +1005,10 @@ void IdoPgsqlConnection::InternalCleanUpExecuteQuery(const String& table, const { AssertOnWorkQueue(); - if (!GetConnected()) + if (!GetConnected()) { + DecreasePendingQueries(1); return; + } Query("DELETE FROM " + GetTablePrefix() + table + " WHERE instance_id = " + Convert::ToString(static_cast(m_InstanceID)) + " AND " + time_column + @@ -977,6 +1018,7 @@ void IdoPgsqlConnection::InternalCleanUpExecuteQuery(const String& table, const void IdoPgsqlConnection::FillIDCache(const DbType::Ptr& type) { String query = "SELECT " + type->GetIDColumn() + " AS object_id, " + type->GetTable() + "_id, config_hash FROM " + GetTablePrefix() + type->GetTable() + "s"; + IncreasePendingQueries(1); IdoPgsqlResult result = Query(query); Dictionary::Ptr row; diff --git a/lib/db_ido_pgsql/idopgsqlconnection.hpp b/lib/db_ido_pgsql/idopgsqlconnection.hpp index 83e4d3f9f..e355116d4 100644 --- a/lib/db_ido_pgsql/idopgsqlconnection.hpp +++ b/lib/db_ido_pgsql/idopgsqlconnection.hpp @@ -48,7 +48,7 @@ protected: private: DbReference m_InstanceID; - WorkQueue m_QueryQueue{1000000}; + WorkQueue m_QueryQueue{1000000, 1, LogNotice}; Library m_Library; std::unique_ptr m_Pgsql; From 177f930375330ad762070c6da084ca19d53946b2 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 23 Sep 2020 11:49:13 +0200 Subject: [PATCH 44/75] Call Process::InitializeSpawnHelper() ASAP refs #8196 --- lib/cli/daemoncommand.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/cli/daemoncommand.cpp b/lib/cli/daemoncommand.cpp index 090bb0257..29a73c89c 100644 --- a/lib/cli/daemoncommand.cpp +++ b/lib/cli/daemoncommand.cpp @@ -11,6 +11,7 @@ #include "base/defer.hpp" #include "base/logger.hpp" #include "base/application.hpp" +#include "base/process.hpp" #include "base/timer.hpp" #include "base/utility.hpp" #include "base/exception.hpp" @@ -504,6 +505,14 @@ static pid_t StartUnixWorker(const std::vector& configs, bool close _exit(EXIT_FAILURE); } + try { + Process::InitializeSpawnHelper(); + } catch (const std::exception& ex) { + Log(LogCritical, "cli") + << "Failed to initialize process spawn helper after forking (child): " << DiagnosticInformation(ex); + _exit(EXIT_FAILURE); + } + _exit(RunWorker(configs, closeConsoleLog, stderrFile)); } catch (...) { _exit(EXIT_FAILURE); From f4a951139d1daee25285e871a944f1e79d8b3c77 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 25 Sep 2020 10:10:14 +0200 Subject: [PATCH 45/75] Update .mailmap --- .mailmap | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.mailmap b/.mailmap index 918d8db55..bcaffb34d 100644 --- a/.mailmap +++ b/.mailmap @@ -24,6 +24,7 @@ Alexander A. Klimov +Alex Carsten Köbke Carsten Koebke Claudio Kuenzler Diana Flach @@ -36,11 +37,15 @@ Henrik Triem Henrik Triem <43344334+htriem@users.norep Jens Schanz Jens Schanz Schanz, Jens +Kálmán „KAMI” Szalai Marianne Spiller Markus Waldmüller Michael Insel Michael Insel Michael Insel nemtrif +nemtrif Robin O'Brien +Roman Gerhardt +Sebastian Chrostek Thomas Gelf From 90371b4836b1f50bb734805d1cc2d0d809d02f6b Mon Sep 17 00:00:00 2001 From: "icinga-probot[bot]" <70573609+icinga-probot[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2020 08:13:29 +0000 Subject: [PATCH 46/75] Update AUTHORS --- AUTHORS | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/AUTHORS b/AUTHORS index 9fd4ddcd1..b3419bf4c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -23,6 +23,7 @@ Arnd Hannemann Assaf Flatto azthec BarbUk +Bård Dahlmo-Lerbæk Bas Couwenberg bascarsija Bastian Guse @@ -36,7 +37,6 @@ Brendan Jurd Brian De Wolf Brian Dockter Bruno Lingner -Bård Dahlmo-Lerbæk Carlos Cesario Carsten Köbke Chris Boot @@ -48,7 +48,6 @@ Christian Lehmann Christian Loos Christian Schmidt Christopher Schirner -chrostek Claudio Bilotta Claudio Kuenzler Conrad Clement @@ -72,6 +71,7 @@ Edgar Fuß Eduard Güldner Edvin Seferovic Elias Ohm +Élie Bouttier Eric Lippmann Evgeni Golov Ewoud Kohl van Wijngaarden @@ -80,13 +80,11 @@ fbachmann Federico Cuello Federico Pires Ferdi Gueran -fluxX04 Francesco Colista Gaël Beaudoin Georg Faerber Georg Haas Gerd von Egidy -Gerhardt Roman gitmopp Glauco Vinicius Greg Hewgill @@ -115,6 +113,7 @@ Jens Link Jens Schanz Jeon Sang Wan Jeremy Armstrong +Jérôme Drouet Jesse Morgan Jo Goossens Johannes Meyer @@ -123,14 +122,12 @@ Jordi van Scheijen Joseph L. Casale jre3brg Julian Brost -Jérôme Drouet K0nne <34264690+K0nne@users.noreply.github.com> Kai Goller +Kálmán „KAMI” Szalai kiba Konstantin Kelemen krishna -Kálmán Szalai - KAMI -Kálmán „KAMI” Szalai Lars Engels Lars Krüger Leah Oswald @@ -182,7 +179,6 @@ Mirco Bauer Mirko Nardin mocruz Muhammad Mominul Huque -Nemanja Trifunovic nemtrif Nicolai Nicolas Limage @@ -263,4 +259,3 @@ Yohan Jarosz Yonas Habteab Zachary McGibbon Zoltan Nagy -Élie Bouttier From 0a1a2869134f254b032f47e10ac1383e35feec2f Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 30 Sep 2020 10:30:21 +0200 Subject: [PATCH 47/75] Don't send reminder notifications before suppressed ones refs #8201 --- lib/notification/notificationcomponent.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/notification/notificationcomponent.cpp b/lib/notification/notificationcomponent.cpp index aa9601201..60574b9b1 100644 --- a/lib/notification/notificationcomponent.cpp +++ b/lib/notification/notificationcomponent.cpp @@ -165,6 +165,10 @@ void NotificationComponent::NotificationTimerHandler() if ((service && service->GetState() == ServiceOK) || (!service && host->GetState() == HostUp)) continue; + /* Don't send reminder notifications before initial ones. */ + if (checkable->GetSuppressedNotifications() & NotificationProblem) + continue; + /* Skip in runtime filters. */ if (!reachable || checkable->IsInDowntime() || checkable->IsAcknowledged() || checkable->IsFlapping()) continue; From 22c14fd964638bb4ce75863e495c4b942c63a110 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 30 Sep 2020 14:35:34 +0200 Subject: [PATCH 48/75] IDO: Always insert customvar status on heavy config update --- lib/db_ido/dbobject.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/db_ido/dbobject.cpp b/lib/db_ido/dbobject.cpp index c48739ede..180abe844 100644 --- a/lib/db_ido/dbobject.cpp +++ b/lib/db_ido/dbobject.cpp @@ -111,7 +111,6 @@ void DbObject::SendConfigUpdateHeavy(const Dictionary::Ptr& configFields) { /* update custom var config and status */ SendVarsConfigUpdateHeavy(); - SendVarsStatusUpdate(); /* config attributes */ if (!configFields) @@ -245,6 +244,22 @@ void DbObject::SendVarsConfigUpdateHeavy() { "instance_id", 0 } /* DbConnection class fills in real ID */ }); queries.emplace_back(std::move(query3)); + + DbQuery query4; + query4.Table = "customvariablestatus"; + query4.Type = DbQueryInsert; + query4.Category = DbCatState; + + query4.Fields = new Dictionary({ + { "varname", kv.first }, + { "varvalue", value }, + { "is_json", is_json }, + { "status_update_time", DbValue::FromTimestamp(Utility::GetTime()) }, + { "object_id", obj }, + { "instance_id", 0 } /* DbConnection class fills in real ID */ + }); + + queries.emplace_back(std::move(query4)); } } From be3eb0821af06f8bcbe37ae415a97a7e092a1001 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Thu, 8 Oct 2020 11:35:59 +0200 Subject: [PATCH 49/75] IDO-Logging: Add log timeout to log every 5 minutes --- lib/db_ido/dbconnection.cpp | 22 +++++++++++++++++----- lib/db_ido/dbconnection.hpp | 2 ++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/lib/db_ido/dbconnection.cpp b/lib/db_ido/dbconnection.cpp index 6ab9b4e10..9dd7301ff 100644 --- a/lib/db_ido/dbconnection.cpp +++ b/lib/db_ido/dbconnection.cpp @@ -77,6 +77,8 @@ void DbConnection::Resume() m_CleanUpTimer->OnTimerExpired.connect(std::bind(&DbConnection::CleanUpHandler, this)); m_CleanUpTimer->Start(); + m_LogStatsTimeout = 0; + m_LogStatsTimer = new Timer(); m_LogStatsTimer->SetInterval(10); m_LogStatsTimer->OnTimerExpired.connect([this](const Timer * const&) { LogStatsHandler(); }); @@ -245,22 +247,27 @@ void DbConnection::LogStatsHandler() { auto pending = m_PendingQueries.load(); - if (pending == 0u) { + auto now = Utility::GetTime(); + bool timeoutReached = m_LogStatsTimeout < now; + + if (pending == 0u && !timeoutReached) { return; } - auto now = Utility::GetTime(); auto output = round(m_OutputQueries.CalculateRate(now, 10)); - if (pending < output * 2) { + if (pending < output * 2 && !timeoutReached) { return; } auto input = round(m_InputQueries.CalculateRate(now, 10)); - String timeInfo = " empty in "; + String timeInfo = ""; - { + // If we run into our logging timeout, we don't want to display our calculations + // because it should already be basically empty for over 5 minutes. + if (!timeoutReached) { + timeInfo = " empty in "; auto rate = output - input; if (rate <= 0) @@ -272,6 +279,11 @@ void DbConnection::LogStatsHandler() Log(LogInformation, GetReflectionType()->GetName()) << "Pending queries: " << pending << " (Input: " << input << "/s; Output: " << output << "/s)" << timeInfo; + + /* Reschedule next log entry in 5 minutes. */ + if (timeoutReached) { + m_LogStatsTimeout = now + 60 * 5; + } } void DbConnection::CleanUpExecuteQuery(const String&, const String&, double) diff --git a/lib/db_ido/dbconnection.hpp b/lib/db_ido/dbconnection.hpp index 84e28d4c6..6af4fd908 100644 --- a/lib/db_ido/dbconnection.hpp +++ b/lib/db_ido/dbconnection.hpp @@ -106,6 +106,8 @@ private: Timer::Ptr m_CleanUpTimer; Timer::Ptr m_LogStatsTimer; + double m_LogStatsTimeout; + void CleanUpHandler(); void LogStatsHandler(); From 82c4d58d142143de14fb08a8e113a483a5caa0a0 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 9 Oct 2020 14:56:17 +0200 Subject: [PATCH 50/75] IDO-Logging: Remove useless ETA --- lib/db_ido/dbconnection.cpp | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/lib/db_ido/dbconnection.cpp b/lib/db_ido/dbconnection.cpp index 9dd7301ff..ed78da2e4 100644 --- a/lib/db_ido/dbconnection.cpp +++ b/lib/db_ido/dbconnection.cpp @@ -262,23 +262,9 @@ void DbConnection::LogStatsHandler() auto input = round(m_InputQueries.CalculateRate(now, 10)); - String timeInfo = ""; - - // If we run into our logging timeout, we don't want to display our calculations - // because it should already be basically empty for over 5 minutes. - if (!timeoutReached) { - timeInfo = " empty in "; - auto rate = output - input; - - if (rate <= 0) - timeInfo += "infinite time, your task handler isn't able to keep up"; - else - timeInfo += Utility::FormatDuration(pending / rate); - } - Log(LogInformation, GetReflectionType()->GetName()) << "Pending queries: " << pending << " (Input: " << input - << "/s; Output: " << output << "/s)" << timeInfo; + << "/s; Output: " << output << "/s)"; /* Reschedule next log entry in 5 minutes. */ if (timeoutReached) { From d3779961764c0c64bbd8e07d50d9caacd1dd7099 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 9 Oct 2020 15:27:05 +0200 Subject: [PATCH 51/75] IDO-Logging: Don't log when not connected --- lib/db_ido/dbconnection.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/db_ido/dbconnection.cpp b/lib/db_ido/dbconnection.cpp index ed78da2e4..83cf6517e 100644 --- a/lib/db_ido/dbconnection.cpp +++ b/lib/db_ido/dbconnection.cpp @@ -245,6 +245,9 @@ void DbConnection::CleanUpHandler() void DbConnection::LogStatsHandler() { + if (!GetConnected()) + return; + auto pending = m_PendingQueries.load(); auto now = Utility::GetTime(); From 04fc0014ce823dec5a8e4781589896f465cffad4 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 9 Oct 2020 15:40:13 +0200 Subject: [PATCH 52/75] IDO-Logging: Increase logging threshold for nearly empty queue --- lib/db_ido/dbconnection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/db_ido/dbconnection.cpp b/lib/db_ido/dbconnection.cpp index 83cf6517e..62a9c5c3d 100644 --- a/lib/db_ido/dbconnection.cpp +++ b/lib/db_ido/dbconnection.cpp @@ -259,7 +259,7 @@ void DbConnection::LogStatsHandler() auto output = round(m_OutputQueries.CalculateRate(now, 10)); - if (pending < output * 2 && !timeoutReached) { + if (pending < output * 5 && !timeoutReached) { return; } From cb00a7fd6a8d73fe9ce522eca5e84a039ffa1793 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 13 Oct 2020 13:47:49 +0200 Subject: [PATCH 53/75] *DbObject#CalculateConfigHash(): sort groups to be hashed ... to ensure consistent hashes across config reloads. This will likely cause a heavy update once for all objects in >1 group, but it will ensure that this happens the last time. --- lib/db_ido/hostdbobject.cpp | 6 +++++- lib/db_ido/servicedbobject.cpp | 6 +++++- lib/db_ido/userdbobject.cpp | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/db_ido/hostdbobject.cpp b/lib/db_ido/hostdbobject.cpp index 18be0bd52..60d1a99d1 100644 --- a/lib/db_ido/hostdbobject.cpp +++ b/lib/db_ido/hostdbobject.cpp @@ -356,8 +356,12 @@ String HostDbObject::CalculateConfigHash(const Dictionary::Ptr& configFields) co Array::Ptr groups = host->GetGroups(); - if (groups) + if (groups) { + groups = groups->ShallowClone(); + ObjectLock oLock (groups); + std::sort(groups->Begin(), groups->End()); hashData += DbObject::HashValue(groups); + } ArrayData parents; diff --git a/lib/db_ido/servicedbobject.cpp b/lib/db_ido/servicedbobject.cpp index ac1f78897..7f711df5e 100644 --- a/lib/db_ido/servicedbobject.cpp +++ b/lib/db_ido/servicedbobject.cpp @@ -308,8 +308,12 @@ String ServiceDbObject::CalculateConfigHash(const Dictionary::Ptr& configFields) Array::Ptr groups = service->GetGroups(); - if (groups) + if (groups) { + groups = groups->ShallowClone(); + ObjectLock oLock (groups); + std::sort(groups->Begin(), groups->End()); hashData += DbObject::HashValue(groups); + } ArrayData dependencies; diff --git a/lib/db_ido/userdbobject.cpp b/lib/db_ido/userdbobject.cpp index f0d80b604..439b8fb05 100644 --- a/lib/db_ido/userdbobject.cpp +++ b/lib/db_ido/userdbobject.cpp @@ -150,8 +150,12 @@ String UserDbObject::CalculateConfigHash(const Dictionary::Ptr& configFields) co Array::Ptr groups = user->GetGroups(); - if (groups) + if (groups) { + groups = groups->ShallowClone(); + ObjectLock oLock (groups); + std::sort(groups->Begin(), groups->End()); hashData += DbObject::HashValue(groups); + } return SHA256(hashData); } From e04d618edeeb06dcddb7c3833c6ebc3aaaebfcb7 Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Thu, 8 Oct 2020 12:47:18 +0200 Subject: [PATCH 54/75] Catch exceptions in the thread running HandleConfigUpdate With dc3062a9b06fed69cdbb1508ace6eb2f77f87553, exceptions in this code path were no longer caught properly. This commit restores exception handling for this function. --- lib/remote/apilistener-filesync.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 1feb13097..468c0038f 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -312,7 +312,16 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D return Empty; } - std::thread([origin, params]() { HandleConfigUpdate(origin, params); }).detach(); + std::thread([origin, params, listener]() { + try { + HandleConfigUpdate(origin, params); + } catch (const std::exception& ex) { + auto msg ("Exception during config sync: " + DiagnosticInformation(ex)); + + Log(LogCritical, "ApiListener") << msg; + listener->UpdateLastFailedZonesStageValidation(msg); + } + }).detach(); return Empty; } From 132f501148b019d3135d50ec1428acb2e7459369 Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Tue, 13 Oct 2020 17:18:31 +0200 Subject: [PATCH 55/75] RELEASE.md: fix packaging repo URLs --- RELEASE.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 818202e57..e5517736b 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -104,7 +104,7 @@ cd $HOME/dev/icinga/packaging ### RPM Packages ``` -git clone git@git.icinga.com:icinga/rpm-icinga2.git && cd rpm-icinga2 +git clone git@git.icinga.com:packaging/rpm-icinga2.git && cd rpm-icinga2 ``` ### DEB Packages @@ -116,13 +116,13 @@ git clone git@git.icinga.com:packaging/deb-icinga2.git && cd deb-icinga2 #### Raspbian Packages ``` -git clone git@git.icinga.com:icinga/raspbian-icinga2.git && cd raspbian-icinga2 +git clone git@git.icinga.com:packaging/raspbian-icinga2.git && cd raspbian-icinga2 ``` ### Windows Packages ``` -git clone git@git.icinga.com:icinga/windows-icinga2.git && cd windows-icinga2 +git clone git@git.icinga.com:packaging/windows-icinga2.git && cd windows-icinga2 ``` From bc7f1320e6c75abdf9d05dc150624cfb36e50609 Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Tue, 13 Oct 2020 17:39:25 +0200 Subject: [PATCH 56/75] RELEASE.md: Add instructions how to prepare releases for Windows --- RELEASE.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index 818202e57..cf35057cd 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -178,6 +178,16 @@ icinga2 (2.11.0-1) icinga; urgency=medium ``` +#### Windows Release Preparations + +Update the file `.gitlab-ci.yml`: + +``` +sed -i "s/^ UPSTREAM_GIT_BRANCH: .*/ UPSTREAM_GIT_BRANCH: v$VERSION/g" .gitlab-ci.yml +sed -i "s/^ ICINGA_FORCE_VERSION: .*/ ICINGA_FORCE_VERSION: v$VERSION/g" .gitlab-ci.yml +``` + + ### Release Commit Commit the changes and push the branch. From e09dc838d5287cf65f726352e8a6b082d5fd5521 Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Tue, 13 Oct 2020 18:01:04 +0200 Subject: [PATCH 57/75] Add myself to mailmap --- .mailmap | 1 + 1 file changed, 1 insertion(+) diff --git a/.mailmap b/.mailmap index bcaffb34d..7f8682886 100644 --- a/.mailmap +++ b/.mailmap @@ -12,6 +12,7 @@ Alexander A. Klimov + From a2c3c33746d8ca0886b84d7b115ee4eee814a5c7 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 14 Oct 2020 09:27:29 +0200 Subject: [PATCH 58/75] RELEASE.md: remove Chocolatey section --- RELEASE.md | 32 +++++--------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index dfae94faf..d289570fe 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -15,12 +15,11 @@ - [6. Build Server](#build-infrastructure) - [7. Release Tests](#release-tests) - [8. GitHub Release](#github-release) -- [9. Chocolatey](#chocolatey) -- [10. Docker](#docker) -- [11. Post Release](#post-release) - - [11.1. Online Documentation](#online-documentation) - - [11.2. Announcement](#announcement) - - [11.3. Project Management](#project-management) +- [9. Docker](#docker) +- [10. Post Release](#post-release) + - [10.1. Online Documentation](#online-documentation) + - [10.2. Announcement](#announcement) + - [10.3. Project Management](#project-management) ## Preparations @@ -301,27 +300,6 @@ The release body should contain a short changelog, with links into the roadmap, changelog and blogpost. -## Chocolatey - -Navigate to the git repository on your Windows box which -already has chocolatey installed. Pull/checkout the release. - -Create the nupkg package (or use the one generated on https://packages.icinga.com/windows): - -``` -cpack -``` - -Fetch the API key from https://chocolatey.org/account and use the `choco push` -command line. - -``` -choco apikey --key xxx --source https://push.chocolatey.org/ - -choco push Icinga2-v2.11.0.nupkg --source https://push.chocolatey.org/ -``` - - ## Docker > Only for final versions (not for RCs). From 0253c9f80e6b2e40ecb3c142b76be278beaa5c93 Mon Sep 17 00:00:00 2001 From: "icinga-probot[bot]" <70573609+icinga-probot[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2020 07:38:00 +0000 Subject: [PATCH 59/75] Update AUTHORS --- AUTHORS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index b3419bf4c..f41372262 100644 --- a/AUTHORS +++ b/AUTHORS @@ -121,7 +121,7 @@ Jonas Meurer Jordi van Scheijen Joseph L. Casale jre3brg -Julian Brost +Julian Brost K0nne <34264690+K0nne@users.noreply.github.com> Kai Goller Kálmán „KAMI” Szalai From cd72eaf2b9c512e17131ed53e58f378ef23dcff5 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 29 Sep 2020 16:34:26 +0200 Subject: [PATCH 60/75] IDO MySQL: actually COMMIT after 25000 async queries --- lib/db_ido_mysql/idomysqlconnection.cpp | 13 ++++++++----- lib/db_ido_mysql/idomysqlconnection.hpp | 2 ++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/db_ido_mysql/idomysqlconnection.cpp b/lib/db_ido_mysql/idomysqlconnection.cpp index 915a7b3b8..f4fa465e9 100644 --- a/lib/db_ido_mysql/idomysqlconnection.cpp +++ b/lib/db_ido_mysql/idomysqlconnection.cpp @@ -513,11 +513,6 @@ void IdoMysqlConnection::AsyncQuery(const String& query, const std::function 25000) { - FinishAsyncQueries(); - InternalNewTransaction(); - } } void IdoMysqlConnection::FinishAsyncQueries() @@ -547,6 +542,7 @@ void IdoMysqlConnection::FinishAsyncQueries() Defer decreaseQueries ([this, &offset, &count]() { offset += count; DecreasePendingQueries(count); + m_UncommittedAsyncQueries += count; }); for (std::vector::size_type i = offset; i < queries.size(); i++) { @@ -628,6 +624,13 @@ void IdoMysqlConnection::FinishAsyncQueries() } } } + + if (m_UncommittedAsyncQueries > 25000) { + m_UncommittedAsyncQueries = 0; + + Query("COMMIT"); + Query("BEGIN"); + } } IdoMysqlResult IdoMysqlConnection::Query(const String& query) diff --git a/lib/db_ido_mysql/idomysqlconnection.hpp b/lib/db_ido_mysql/idomysqlconnection.hpp index 3c1bf916b..d1003c286 100644 --- a/lib/db_ido_mysql/idomysqlconnection.hpp +++ b/lib/db_ido_mysql/idomysqlconnection.hpp @@ -9,6 +9,7 @@ #include "base/timer.hpp" #include "base/workqueue.hpp" #include "base/library.hpp" +#include namespace icinga { @@ -64,6 +65,7 @@ private: unsigned int m_MaxPacketSize; std::vector m_AsyncQueries; + uint_fast32_t m_UncommittedAsyncQueries = 0; Timer::Ptr m_ReconnectTimer; Timer::Ptr m_TxTimer; From 4af450141b960666d2aebf164c51170ea9a074df Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 2 Oct 2020 17:41:12 +0200 Subject: [PATCH 61/75] Introduce SpinLock --- lib/base/CMakeLists.txt | 1 + lib/base/spinlock.cpp | 22 ++++++++++++++++++++++ lib/base/spinlock.hpp | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 lib/base/spinlock.cpp create mode 100644 lib/base/spinlock.hpp diff --git a/lib/base/CMakeLists.txt b/lib/base/CMakeLists.txt index 108ca27c1..c1f28a06d 100644 --- a/lib/base/CMakeLists.txt +++ b/lib/base/CMakeLists.txt @@ -64,6 +64,7 @@ set(base_SOURCES shared-object.hpp singleton.hpp socket.cpp socket.hpp + spinlock.cpp spinlock.hpp stacktrace.cpp stacktrace.hpp statsfunction.hpp stdiostream.cpp stdiostream.hpp diff --git a/lib/base/spinlock.cpp b/lib/base/spinlock.cpp new file mode 100644 index 000000000..03de2e6be --- /dev/null +++ b/lib/base/spinlock.cpp @@ -0,0 +1,22 @@ +/* Icinga 2 | (c) 2020 Icinga GmbH | GPLv2+ */ + +#include "base/spinlock.hpp" +#include + +using namespace icinga; + +void SpinLock::lock() +{ + while (m_Locked.test_and_set(std::memory_order_acquire)) { + } +} + +bool SpinLock::try_lock() +{ + return !m_Locked.test_and_set(std::memory_order_acquire); +} + +void SpinLock::unlock() +{ + m_Locked.clear(std::memory_order_release); +} diff --git a/lib/base/spinlock.hpp b/lib/base/spinlock.hpp new file mode 100644 index 000000000..d6da5876e --- /dev/null +++ b/lib/base/spinlock.hpp @@ -0,0 +1,35 @@ +/* Icinga 2 | (c) 2020 Icinga GmbH | GPLv2+ */ + +#ifndef SPINLOCK_H +#define SPINLOCK_H + +#include + +namespace icinga +{ + +/** + * A spin lock. + * + * @ingroup base + */ +class SpinLock +{ +public: + SpinLock() = default; + SpinLock(const SpinLock&) = delete; + SpinLock& operator=(const SpinLock&) = delete; + SpinLock(SpinLock&&) = delete; + SpinLock& operator=(SpinLock&&) = delete; + + void lock(); + bool try_lock(); + void unlock(); + +private: + std::atomic_flag m_Locked = ATOMIC_FLAG_INIT; +}; + +} + +#endif /* SPINLOCK_H */ From a083635de41e8338a5379b5f1f53a3a4a3be92d3 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 2 Oct 2020 17:53:17 +0200 Subject: [PATCH 62/75] Make ApiListener::m_ConfigSyncStageLock a SpinLock --- lib/remote/apilistener-filesync.cpp | 6 +++--- lib/remote/apilistener.hpp | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 1feb13097..890e3eb5a 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -20,7 +20,7 @@ using namespace icinga; REGISTER_APIFUNCTION(Update, config, &ApiListener::ConfigUpdateHandler); -boost::mutex ApiListener::m_ConfigSyncStageLock; +SpinLock ApiListener::m_ConfigSyncStageLock; /** * Entrypoint for updating all authoritative configs from /etc/zones.d, packages, etc. @@ -321,7 +321,7 @@ void ApiListener::HandleConfigUpdate(const MessageOrigin::Ptr& origin, const Dic /* Only one transaction is allowed, concurrent message handlers need to wait. * This affects two parent endpoints sending the config in the same moment. */ - auto lock (Shared::Make(m_ConfigSyncStageLock)); + auto lock (Shared>::Make(m_ConfigSyncStageLock)); String apiZonesStageDir = GetApiZonesStageDir(); String fromEndpointName = origin->FromClient->GetEndpoint()->GetName(); @@ -618,7 +618,7 @@ void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, * * @param relativePaths Required for later file operations in the callback. Provides the zone name plus path in a list. */ -void ApiListener::AsyncTryActivateZonesStage(const std::vector& relativePaths, const Shared::Ptr& lock) +void ApiListener::AsyncTryActivateZonesStage(const std::vector& relativePaths, const Shared>::Ptr& lock) { VERIFY(Application::GetArgC() >= 1); diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp index 4a3623a68..3205fa265 100644 --- a/lib/remote/apilistener.hpp +++ b/lib/remote/apilistener.hpp @@ -11,6 +11,7 @@ #include "base/configobject.hpp" #include "base/process.hpp" #include "base/shared.hpp" +#include "base/spinlock.hpp" #include "base/timer.hpp" #include "base/workqueue.hpp" #include "base/tcpsocket.hpp" @@ -21,6 +22,7 @@ #include #include #include +#include #include namespace icinga @@ -186,7 +188,7 @@ private: void RemoveStatusFile(); /* filesync */ - static boost::mutex m_ConfigSyncStageLock; + static SpinLock m_ConfigSyncStageLock; void SyncLocalZoneDirs() const; void SyncLocalZoneDir(const Zone::Ptr& zone) const; @@ -200,7 +202,7 @@ private: static void TryActivateZonesStageCallback(const ProcessResult& pr, const std::vector& relativePaths); - static void AsyncTryActivateZonesStage(const std::vector& relativePaths, const Shared::Ptr& lock); + static void AsyncTryActivateZonesStage(const std::vector& relativePaths, const Shared>::Ptr& lock); static String GetChecksum(const String& content); static bool CheckConfigChange(const ConfigDirInformation& oldConfig, const ConfigDirInformation& newConfig); From 291823f1a9101c74f94a43b8bb88c9a402eef580 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 14 Oct 2020 11:09:25 +0200 Subject: [PATCH 63/75] Update .mailmap --- .mailmap | 1 + 1 file changed, 1 insertion(+) diff --git a/.mailmap b/.mailmap index 7f8682886..cdbec5b2f 100644 --- a/.mailmap +++ b/.mailmap @@ -34,6 +34,7 @@ Diana Flach Diana Flach Jean Flach Dolf Schimmel Gunnar Beutner +Henrik Triem Henrik Triem Henrik Triem <43344334+htriem@users.noreply.github.com> Jens Schanz From fbfa931b31d664e02a03a6f14730c3db8e0cb5b4 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 11 Sep 2020 13:38:35 +0200 Subject: [PATCH 64/75] Clear ApiListener#last_failed_zones_stage_validation on config::Update if config not changed refs #7642 --- lib/remote/apilistener-filesync.cpp | 3 ++- lib/remote/apilistener.hpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 468c0038f..738dad79c 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -314,7 +314,7 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D std::thread([origin, params, listener]() { try { - HandleConfigUpdate(origin, params); + listener->HandleConfigUpdate(origin, params); } catch (const std::exception& ex) { auto msg ("Exception during config sync: " + DiagnosticInformation(ex)); @@ -543,6 +543,7 @@ void ApiListener::HandleConfigUpdate(const MessageOrigin::Ptr& origin, const Dic Log(LogInformation, "ApiListener") << "Received configuration updates (" << count << ") from endpoint '" << fromEndpointName << "' are equal to production, skipping validation and reload."; + ClearLastFailedZonesStageValidation(); } } diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp index 4a3623a68..d3ed17d23 100644 --- a/lib/remote/apilistener.hpp +++ b/lib/remote/apilistener.hpp @@ -85,7 +85,7 @@ public: /* filesync */ static Value ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params); - static void HandleConfigUpdate(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params); + void HandleConfigUpdate(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params); /* configsync */ static void ConfigUpdateObjectHandler(const ConfigObject::Ptr& object, const Value& cookie); From 88dc5fea98d22d8786be6be7e5306cc6cf803a08 Mon Sep 17 00:00:00 2001 From: "icinga-probot[bot]" <70573609+icinga-probot[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2020 11:40:19 +0000 Subject: [PATCH 65/75] Update AUTHORS --- AUTHORS | 1 - 1 file changed, 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index f41372262..a9d5db2b4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -96,7 +96,6 @@ Harald Laabs Heike Jurzik Hendrik Röder Henrik Triem -htriem Ian Kelling Ildar Hizbulin Irina Kaprizkina From d3f6a97a7ede7c049a12b71a6ea473f2180252d6 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 14 Oct 2020 14:02:44 +0200 Subject: [PATCH 66/75] Ensure the daemon doesn't get killed by logrotate refs #8050 --- lib/cli/daemoncommand.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/cli/daemoncommand.cpp b/lib/cli/daemoncommand.cpp index 28fce401d..7f0c31b3f 100644 --- a/lib/cli/daemoncommand.cpp +++ b/lib/cli/daemoncommand.cpp @@ -367,6 +367,10 @@ static void UmbrellaSignalHandler(int num, siginfo_t *info, void*) static void WorkerSignalHandler(int num, siginfo_t *info, void*) { switch (num) { + case SIGUSR1: + // Catches SIGUSR1 as long as the actual handler (logrotate) + // has not been installed not to let SIGUSR1 terminate the process + break; case SIGUSR2: if (info->si_pid == l_UmbrellaPid) { // The umbrella process allowed us to continue working beyond config validation @@ -459,6 +463,7 @@ static pid_t StartUnixWorker(const std::vector& configs, bool close sa.sa_sigaction = &WorkerSignalHandler; sa.sa_flags = SA_RESTART | SA_SIGINFO; + (void)sigaction(SIGUSR1, &sa, nullptr); (void)sigaction(SIGUSR2, &sa, nullptr); (void)sigaction(SIGINT, &sa, nullptr); (void)sigaction(SIGTERM, &sa, nullptr); From 91265a5b0e134006893576a465c4c84906f1eb82 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 14 Oct 2020 18:00:16 +0200 Subject: [PATCH 67/75] icinga2 daemon: reap remaining child processes after reload ... as we may be PID 1. --- lib/cli/daemoncommand.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/cli/daemoncommand.cpp b/lib/cli/daemoncommand.cpp index 28fce401d..4c33c6dfc 100644 --- a/lib/cli/daemoncommand.cpp +++ b/lib/cli/daemoncommand.cpp @@ -746,6 +746,17 @@ int DaemonCommand::Run(const po::variables_map& vm, const std::vector Date: Thu, 15 Oct 2020 09:58:39 +0200 Subject: [PATCH 68/75] IDO: Do not log stats, if paused --- lib/db_ido/dbconnection.cpp | 2 +- lib/db_ido_mysql/idomysqlconnection.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/db_ido/dbconnection.cpp b/lib/db_ido/dbconnection.cpp index 62a9c5c3d..3650b73ac 100644 --- a/lib/db_ido/dbconnection.cpp +++ b/lib/db_ido/dbconnection.cpp @@ -245,7 +245,7 @@ void DbConnection::CleanUpHandler() void DbConnection::LogStatsHandler() { - if (!GetConnected()) + if (!GetConnected() || IsPaused()) return; auto pending = m_PendingQueries.load(); diff --git a/lib/db_ido_mysql/idomysqlconnection.cpp b/lib/db_ido_mysql/idomysqlconnection.cpp index f4fa465e9..83919a330 100644 --- a/lib/db_ido_mysql/idomysqlconnection.cpp +++ b/lib/db_ido_mysql/idomysqlconnection.cpp @@ -475,7 +475,7 @@ void IdoMysqlConnection::FinishConnect(double startTime) { AssertOnWorkQueue(); - if (!GetConnected()) + if (!GetConnected() || IsPaused()) return; FinishAsyncQueries(); From 03af068302070f5fba3a63ccc7db27fb508374d2 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Thu, 24 Sep 2020 14:18:49 +0200 Subject: [PATCH 69/75] IDO: Make sure to insert program status during reconnect() --- lib/db_ido/dbconnection.cpp | 6 +++--- lib/db_ido/dbconnection.hpp | 2 ++ lib/db_ido_mysql/idomysqlconnection.hpp | 2 -- lib/db_ido_pgsql/idopgsqlconnection.hpp | 2 -- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/db_ido/dbconnection.cpp b/lib/db_ido/dbconnection.cpp index 3650b73ac..4afe82583 100644 --- a/lib/db_ido/dbconnection.cpp +++ b/lib/db_ido/dbconnection.cpp @@ -152,7 +152,7 @@ void DbConnection::UpdateProgramStatus() DbQuery query1; query1.Table = "programstatus"; query1.IdColumn = "programstatus_id"; - query1.Type = DbQueryInsert | DbQueryUpdate; + query1.Type = DbQueryInsert | DbQueryDelete; query1.Category = DbCatProgramStatus; query1.Fields = new Dictionary({ @@ -179,7 +179,7 @@ void DbConnection::UpdateProgramStatus() { "instance_id", 0 } /* DbConnection class fills in real ID */ }); - query1.Priority = PriorityHigh; + query1.Priority = PriorityImmediate; queries.emplace_back(std::move(query1)); DbQuery query2; @@ -485,7 +485,7 @@ void DbConnection::UpdateAllObjects() continue; for (const ConfigObject::Ptr& object : dtype->GetObjects()) { - UpdateObject(object); + m_QueryQueue.Enqueue([this, object](){ UpdateObject(object); }, PriorityHigh); } } } diff --git a/lib/db_ido/dbconnection.hpp b/lib/db_ido/dbconnection.hpp index 6af4fd908..a7749d7fa 100644 --- a/lib/db_ido/dbconnection.hpp +++ b/lib/db_ido/dbconnection.hpp @@ -95,6 +95,8 @@ protected: void IncreasePendingQueries(int count); void DecreasePendingQueries(int count); + WorkQueue m_QueryQueue{10000000, 1, LogNotice}; + private: bool m_IDCacheValid{false}; std::map, String> m_ConfigHashes; diff --git a/lib/db_ido_mysql/idomysqlconnection.hpp b/lib/db_ido_mysql/idomysqlconnection.hpp index d1003c286..4cd9cac0d 100644 --- a/lib/db_ido_mysql/idomysqlconnection.hpp +++ b/lib/db_ido_mysql/idomysqlconnection.hpp @@ -55,8 +55,6 @@ protected: private: DbReference m_InstanceID; - WorkQueue m_QueryQueue{10000000, 1, LogNotice}; - Library m_Library; std::unique_ptr m_Mysql; diff --git a/lib/db_ido_pgsql/idopgsqlconnection.hpp b/lib/db_ido_pgsql/idopgsqlconnection.hpp index e355116d4..70726357d 100644 --- a/lib/db_ido_pgsql/idopgsqlconnection.hpp +++ b/lib/db_ido_pgsql/idopgsqlconnection.hpp @@ -48,8 +48,6 @@ protected: private: DbReference m_InstanceID; - WorkQueue m_QueryQueue{1000000, 1, LogNotice}; - Library m_Library; std::unique_ptr m_Pgsql; From c30d8ff3731efecb12de49a48e504c15c146ae0f Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Tue, 20 Oct 2020 12:19:15 +0200 Subject: [PATCH 70/75] Remove Utility::GetEnv from header It is never used and not even implemented anywhere, probably a leftover. --- lib/base/utility.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/base/utility.hpp b/lib/base/utility.hpp index 4505dc918..676016089 100644 --- a/lib/base/utility.hpp +++ b/lib/base/utility.hpp @@ -52,8 +52,6 @@ public: static String DirName(const String& path); static String BaseName(const String& path); - static String GetEnv(const String& key); - static void NullDeleter(void *); static double GetTime(); From 4ede8e484be9a3d117b7062bd90ea35fd4febaa9 Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Tue, 20 Oct 2020 16:39:19 +0200 Subject: [PATCH 71/75] Update Windows build scripts for Visual Studio 2019 $env:CMAKE_GENERATOR_PLATFORM is only used in configure-dev.ps1 but now is also required in configure.ps1 to allow the build pipeline to be upgraded to Visual Studio 2019. Additionally bump the versions in paths for Boost and OpenSSL. --- tools/win32/configure.ps1 | 13 ++++++++----- tools/win32/load-vsenv.ps1 | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tools/win32/configure.ps1 b/tools/win32/configure.ps1 index 63305e5dc..7afd922ae 100644 --- a/tools/win32/configure.ps1 +++ b/tools/win32/configure.ps1 @@ -17,16 +17,19 @@ if (-not ($env:PATH -contains $env:CMAKE_PATH)) { $env:PATH = $env:CMAKE_PATH + ';' + $env:PATH } if (-not (Test-Path env:CMAKE_GENERATOR)) { - $env:CMAKE_GENERATOR = 'Visual Studio 15 2017 Win64' + $env:CMAKE_GENERATOR = 'Visual Studio 16 2019' +} +if (-not (Test-Path env:CMAKE_GENERATOR_PLATFORM)) { + $env:CMAKE_GENERATOR_PLATFORM = 'x64' } if (-not (Test-Path env:OPENSSL_ROOT_DIR)) { - $env:OPENSSL_ROOT_DIR = 'c:\local\OpenSSL_1_1_1b-Win64' + $env:OPENSSL_ROOT_DIR = 'c:\local\OpenSSL_1_1_1h-Win64' } if (-not (Test-Path env:BOOST_ROOT)) { - $env:BOOST_ROOT = 'c:\local\boost_1_69_0-Win64' + $env:BOOST_ROOT = 'c:\local\boost_1_71_0-Win64' } if (-not (Test-Path env:BOOST_LIBRARYDIR)) { - $env:BOOST_LIBRARYDIR = 'c:\local\boost_1_69_0-Win64\lib64-msvc-14.1' + $env:BOOST_LIBRARYDIR = 'c:\local\boost_1_71_0-Win64\lib64-msvc-14.2' } if (-not (Test-Path env:FLEX_BINARY)) { $env:FLEX_BINARY = 'C:\ProgramData\chocolatey\bin\win_flex.exe' @@ -48,7 +51,7 @@ if (Test-Path CMakeCache.txt) { & cmake.exe "$sourcePath" ` -DCMAKE_BUILD_TYPE="$env:CMAKE_BUILD_TYPE" ` - -G "$env:CMAKE_GENERATOR" -DCPACK_GENERATOR=WIX ` + -G "$env:CMAKE_GENERATOR" -A "$env:CMAKE_GENERATOR_PLATFORM" -DCPACK_GENERATOR=WIX ` -DICINGA2_WITH_MYSQL=OFF -DICINGA2_WITH_PGSQL=OFF ` -DICINGA2_WITH_LIVESTATUS=OFF -DICINGA2_WITH_COMPAT=OFF ` -DOPENSSL_ROOT_DIR="$env:OPENSSL_ROOT_DIR" ` diff --git a/tools/win32/load-vsenv.ps1 b/tools/win32/load-vsenv.ps1 index 86fcf4fba..984a4f133 100644 --- a/tools/win32/load-vsenv.ps1 +++ b/tools/win32/load-vsenv.ps1 @@ -18,7 +18,7 @@ if (-not (Test-Path $BUILD)) { if (Test-Path env:VS_INSTALL_PATH) { $VSBASE = $env:VS_INSTALL_PATH } else { - $VSBASE = "C:\Program Files (x86)\Microsoft Visual Studio\2017" + $VSBASE = "C:\Program Files (x86)\Microsoft Visual Studio\2019" } if (Test-Path env:BITS) { From 025d38bb76a48f822eb9b2a035477da3a513312c Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Wed, 21 Oct 2020 10:53:59 +0200 Subject: [PATCH 72/75] Update AppVeyor Config to use Visual Studio 2019 --- appveyor.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 92dfe7bef..cd349bb65 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,16 +1,17 @@ --- version: 2.11.0.dev.{build} -os: Visual Studio 2017 +os: Visual Studio 2019 platform: x64 environment: BITS: 64 CMAKE_BUILD_TYPE: Debug - CMAKE_GENERATOR: "Visual Studio 15 2017 Win64" + CMAKE_GENERATOR: "Visual Studio 16 2019" + CMAKE_GENERATOR_PLATFORM: x64 # https://www.appveyor.com/docs/windows-images-software/#boost - BOOST_ROOT: 'C:\Libraries\boost_1_67_0' - BOOST_LIBRARYDIR: 'C:\Libraries\boost_1_67_0\lib64-msvc-14.1' + BOOST_ROOT: 'C:\Libraries\boost_1_71_0' + BOOST_LIBRARYDIR: 'C:\Libraries\boost_1_71_0\lib64-msvc-14.2' # https://www.appveyor.com/docs/windows-images-software/#tools OPENSSL_ROOT_DIR: 'C:\OpenSSL-v111-Win64' BISON_BINARY: 'C:\ProgramData\chocolatey\lib\winflexbison3\tools\win_bison.exe' From f3b72351a4b28673b532f00bb05a9717384b4d05 Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Wed, 21 Oct 2020 15:01:16 +0200 Subject: [PATCH 73/75] docs: remove references to Visual Studio 2017 PR #8390 changed the default version of Visual Studio to 2019 because PR #8373 introduces behavior that is incompatible with MSVC versions shipped with older versions of Visual Studio. --- doc/21-development.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/doc/21-development.md b/doc/21-development.md index 35ed4eecb..ad1515e7b 100644 --- a/doc/21-development.md +++ b/doc/21-development.md @@ -1789,8 +1789,7 @@ as community version, free for use for open source projects such as Icinga. The installation requires ~9GB disk space. [Download](https://www.visualstudio.com/downloads/) the web installer and start the installation. -Note: Both Visual Studio 2017 and 2019 are covered here. Older versions -are not supported. +Note: Only Visual Studio 2019 is covered here. Older versions are not supported. You need a free Microsoft account to download and also store your preferences. @@ -1883,7 +1882,6 @@ Icinga needs the development header and library files from the Boost library. Visual Studio translates into the following compiler versions: -- `msvc-14.1` = Visual Studio 2017 - `msvc-14.2` = Visual Studio 2019 ##### Pre-built Binaries @@ -1993,7 +1991,6 @@ Build Icinga with specific CMake variables. This generates a new Visual Studio p Visual Studio translates into the following: -- `msvc-14.1` = Visual Studio 2017 - `msvc-14.2` = Visual Studio 2019 You need to specify the previously installed component paths. @@ -2001,7 +1998,7 @@ You need to specify the previously installed component paths. Variable | Value | Description ----------------------|----------------------------------------------------------------------|------------------------------------------------------- `BOOST_ROOT` | `C:\local\boost_1_71_0` | Root path where you've extracted and compiled Boost. -`BOOST_LIBRARYDIR` | Binary: `C:\local\boost_1_71_0\lib64-msvc-14.1`, Source: `C:\local\boost_1_71_0\stage` | Path to the static compiled Boost libraries, directory must contain `lib`. +`BOOST_LIBRARYDIR` | Binary: `C:\local\boost_1_71_0\lib64-msvc-14.2`, Source: `C:\local\boost_1_71_0\stage` | Path to the static compiled Boost libraries, directory must contain `lib`. `BISON_EXECUTABLE` | `C:\ProgramData\chocolatey\lib\winflexbison\tools\win_bison.exe` | Path to the Bison executable. `FLEX_EXECUTABLE` | `C:\ProgramData\chocolatey\lib\winflexbison\tools\win_flex.exe` | Path to the Flex executable. `ICINGA2_WITH_MYSQL` | OFF | Requires extra setup for MySQL if set to `ON`. Not supported for client setups. @@ -2027,9 +2024,8 @@ cd %HOMEPATH%\source\repos\icinga2 The debug MSI package is located in the `debug` directory. -If you did not follow the above steps with Boost binaries -and OpenSSL paths, or using VS 2017, you can still modify -the environment variables. +If you did not follow the above steps with Boost binaries and OpenSSL +paths, you can still modify the environment variables. ``` $env:CMAKE_GENERATOR='Visual Studio 16 2019' From ef2da99104e6bf5e88beeeb08447c89edbcde67f Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Thu, 22 Oct 2020 12:18:14 +0200 Subject: [PATCH 74/75] RELEASE.md: update branch workflow for packaging repos --- RELEASE.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 846c28ef2..deda4cfc3 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -127,27 +127,27 @@ git clone git@git.icinga.com:packaging/windows-icinga2.git && cd windows-icinga2 ### Branch Workflow -Checkout `master` and create a new branch. - -* For releases use x.x[.x] as branch name (e.g. 2.11 or 2.11.1) -* For releases with revision use x.x.x-n (e.g. 2.11.0-2) +For each support branch in this repo (e.g. support/2.12), there exists a corresponding branch in the packaging repos +(e.g. 2.12). Each package revision is a tagged commit on these branches. When doing a major release, create the new +branch, otherweise switch to the existing one. ### Switch Build Type -Edit file `.gitlab-ci.yml` and comment variable `ICINGA_BUILD_TYPE` out. +Ensure that `ICINGA_BUILD_TYPE` is set to `release` in `.gitlab-ci.yml`. This should only be necessary after creating a +new branch. ```yaml variables: ... - #ICINGA_BUILD_TYPE: snapshot + ICINGA_BUILD_TYPE: release ... ``` Commit the change. ``` -git commit -av -m "Switch build type for $VERSION-1" +git commit -av -m "Switch build type for 2.13" ``` #### RPM Release Preparations From cb218aee2063e4890e98047025e7d8ac0308ab33 Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Mon, 26 Oct 2020 09:06:06 +0100 Subject: [PATCH 75/75] docs: better instructions for installing wixtoolset on Windows If you try to follow the existing instructions, you might be greeted by a long and useless error message when installing this which boils down to the fact that the DotNet3.5 Chocolatey package actually requires a reboot but does not properly ask for it. Enable-WindowsOptionalFeature basically does the same as the Chocolatey package would do, but nicely asks for a reboot after which the Chocolatey package can be installed (which basically is a noop then) properly. --- doc/21-development.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/21-development.md b/doc/21-development.md index ad1515e7b..7bcf3bded 100644 --- a/doc/21-development.md +++ b/doc/21-development.md @@ -1961,9 +1961,11 @@ CMake uses CPack and NSIS to create the setup executable including all binaries in addition to setup dialogues and configuration. Therefore we’ll need to install [NSIS](http://nsis.sourceforge.net/Download) first. -We also need to install the Windows Installer XML (WIX) toolset. +We also need to install the Windows Installer XML (WIX) toolset. This has .NET 3.5 as a dependency which might need a +reboot of the system which is not handled properly by Chocolatey. Therefore install it first and reboot when asked. ``` +Enable-WindowsOptionalFeature -FeatureName "NetFx3" -Online choco install -y wixtoolset ```