From 157e3750e304913e1277ab8ebdd09ab99c2db0a4 Mon Sep 17 00:00:00 2001 From: Johannes Schmidt Date: Wed, 11 Jun 2025 09:54:53 +0200 Subject: [PATCH 1/4] Add IsLockable method to WaitGroup --- lib/base/wait-group.cpp | 7 ++++++- lib/base/wait-group.hpp | 8 +++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/base/wait-group.cpp b/lib/base/wait-group.cpp index 1e1ad00ee..6ad716605 100644 --- a/lib/base/wait-group.cpp +++ b/lib/base/wait-group.cpp @@ -26,6 +26,11 @@ void StoppableWaitGroup::unlock_shared() } } +bool StoppableWaitGroup::IsLockable() const +{ + return !m_Stopped.load(std::memory_order_relaxed); +} + /** * Disallow new shared locks, wait for all existing ones. */ @@ -33,6 +38,6 @@ void StoppableWaitGroup::Join() { std::unique_lock lock (m_Mutex); - m_Stopped = true; + m_Stopped.store(true, std::memory_order_relaxed); m_CV.wait(lock, [this] { return !m_SharedLocks; }); } diff --git a/lib/base/wait-group.hpp b/lib/base/wait-group.hpp index 5b4527011..618f07aec 100644 --- a/lib/base/wait-group.hpp +++ b/lib/base/wait-group.hpp @@ -3,6 +3,7 @@ #pragma once #include "base/object.hpp" +#include "base/atomic.hpp" #include #include #include @@ -22,6 +23,8 @@ public: virtual bool try_lock_shared() = 0; virtual void unlock_shared() = 0; + + virtual bool IsLockable() const = 0; }; /** @@ -42,13 +45,16 @@ public: bool try_lock_shared() override; void unlock_shared() override; + + bool IsLockable() const override; + void Join(); private: std::mutex m_Mutex; std::condition_variable m_CV; uint_fast32_t m_SharedLocks = 0; - bool m_Stopped = false; + Atomic m_Stopped = false; }; } From 00802ed9fadf626f95a0e1766b6919407db0ebfa Mon Sep 17 00:00:00 2001 From: Johannes Schmidt Date: Mon, 26 May 2025 10:01:05 +0200 Subject: [PATCH 2/4] Stop ApiListener::ListenerCoroutineProc() when Stop() is called --- lib/remote/apilistener.cpp | 49 ++++++++++++++++++++++++++++++++++++-- lib/remote/apilistener.hpp | 4 ++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index 5ba45b94f..d8f7b0888 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -368,6 +368,8 @@ void ApiListener::Stop(bool runtimeDeleted) m_Timer->Stop(true); m_RenewOwnCertTimer->Stop(true); + StopListener(); + m_WaitGroup->Join(); ObjectImpl::Stop(runtimeDeleted); @@ -486,13 +488,38 @@ bool ApiListener::AddListener(const String& node, const String& service) Log(LogInformation, "ApiListener") << "Started new listener on '[" << localEndpoint.address() << "]:" << localEndpoint.port() << "'"; - IoEngine::SpawnCoroutine(io, [this, acceptor](asio::yield_context yc) { ListenerCoroutineProc(yc, acceptor); }); + auto strand = Shared::Make(io); + + boost::signals2::scoped_connection closeSignal = m_OnListenerShutdown.connect([strand, acceptor]() { + boost::asio::post(*strand, [acceptor] { + try { + acceptor->close(); + } catch (const std::exception& ex) { + Log(LogCritical, "ApiListener") + << "Failed to close acceptor socket: " << ex.what(); + } + }); + }); + + IoEngine::SpawnCoroutine(*strand, [this, acceptor, closeSignal = std::move(closeSignal)](asio::yield_context yc) { + ListenerCoroutineProc(yc, acceptor); + }); UpdateStatusFile(localEndpoint); return true; } +/** + * Stops the listener(s). + */ +void ApiListener::StopListener() +{ + m_OnListenerShutdown(); + + m_ListenerWaitGroup->Join(); +} + void ApiListener::ListenerCoroutineProc(boost::asio::yield_context yc, const Shared::Ptr& server) { namespace asio = boost::asio; @@ -506,7 +533,14 @@ void ApiListener::ListenerCoroutineProc(boost::asio::yield_context yc, const Sha lastModified = Utility::GetFileCreationTime(crlPath); } - for (;;) { + std::shared_lock wgLock(*m_ListenerWaitGroup, std::try_to_lock); + if (!wgLock) { + Log(LogCritical, "ApiListener") + << "Could not lock the listener wait group."; + return; + } + + while (server->is_open()) { try { asio::ip::tcp::socket socket (io); @@ -546,6 +580,12 @@ void ApiListener::ListenerCoroutineProc(boost::asio::yield_context yc, const Sha NewClientHandler(yc, strand, sslConn, String(), RoleServer); }); } catch (const std::exception& ex) { + auto se (dynamic_cast(&ex)); + + if (se && se->code() == boost::asio::error::operation_aborted) { + return; + } + Log(LogCritical, "ApiListener") << "Cannot accept new connection: " << ex.what(); } @@ -828,6 +868,11 @@ void ApiListener::NewClientHandlerInternal( throw; } + std::shared_lock wgLock(*m_ListenerWaitGroup, std::try_to_lock); + if (!wgLock) { + return; + } + if (ctype == ClientJsonRpc) { Log(LogNotice, "ApiListener", "New JSON-RPC client"); diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp index 4fef5ba7d..82137fa32 100644 --- a/lib/remote/apilistener.hpp +++ b/lib/remote/apilistener.hpp @@ -191,12 +191,16 @@ private: static ApiListener::Ptr m_Instance; static std::atomic m_UpdatedObjectAuthority; + boost::signals2::signal m_OnListenerShutdown; + StoppableWaitGroup::Ptr m_ListenerWaitGroup = new StoppableWaitGroup(); + void ApiTimerHandler(); void ApiReconnectTimerHandler(); void CleanupCertificateRequestsTimerHandler(); void CheckApiPackageIntegrity(); bool AddListener(const String& node, const String& service); + void StopListener(); void AddConnection(const Endpoint::Ptr& endpoint); void NewClientHandler( From 33777f6f3f3a8fa94d462d4e4b561acc0360034a Mon Sep 17 00:00:00 2001 From: Johannes Schmidt Date: Fri, 6 Jun 2025 09:53:03 +0200 Subject: [PATCH 3/4] Disconnect JSON-RPC clients on ApiListner::Stop() --- lib/remote/apilistener.cpp | 19 ++++++++++++++++++- lib/remote/apilistener.hpp | 1 + lib/remote/endpoint.cpp | 2 +- lib/remote/jsonrpcconnection.cpp | 15 ++++++++++----- lib/remote/jsonrpcconnection.hpp | 8 ++++++-- 5 files changed, 36 insertions(+), 9 deletions(-) diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index d8f7b0888..9285f747f 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -370,7 +370,10 @@ void ApiListener::Stop(bool runtimeDeleted) StopListener(); + DisconnectJsonRpcConnections(); + m_WaitGroup->Join(); + ObjectImpl::Stop(runtimeDeleted); Log(LogInformation, "ApiListener") @@ -891,7 +894,7 @@ void ApiListener::NewClientHandlerInternal( return; } - JsonRpcConnection::Ptr aclient = new JsonRpcConnection(identity, verify_ok, client, role); + JsonRpcConnection::Ptr aclient = new JsonRpcConnection(m_WaitGroup, identity, verify_ok, client, role); if (endpoint) { endpoint->AddClient(aclient); @@ -1805,6 +1808,20 @@ std::set ApiListener::GetAnonymousClients() const return m_AnonymousClients; } +void ApiListener::DisconnectJsonRpcConnections() +{ + for (auto endpoint : ConfigType::GetObjectsByType()) { + for (const auto& client : endpoint->GetClients()) { + client->Disconnect(); + } + } + + std::unique_lock lock(m_AnonymousClientsLock); + for (const auto & client : m_AnonymousClients){ + client->Disconnect(); + } +} + void ApiListener::AddHttpClient(const HttpServerConnection::Ptr& aclient) { std::unique_lock lock(m_HttpClientsLock); diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp index 82137fa32..f278c2e9b 100644 --- a/lib/remote/apilistener.hpp +++ b/lib/remote/apilistener.hpp @@ -114,6 +114,7 @@ public: bool AddAnonymousClient(const JsonRpcConnection::Ptr& aclient); void RemoveAnonymousClient(const JsonRpcConnection::Ptr& aclient); std::set GetAnonymousClients() const; + void DisconnectJsonRpcConnections(); void AddHttpClient(const HttpServerConnection::Ptr& aclient); void RemoveHttpClient(const HttpServerConnection::Ptr& aclient); diff --git a/lib/remote/endpoint.cpp b/lib/remote/endpoint.cpp index e534fc178..55ab68f12 100644 --- a/lib/remote/endpoint.cpp +++ b/lib/remote/endpoint.cpp @@ -60,7 +60,7 @@ void Endpoint::RemoveClient(const JsonRpcConnection::Ptr& client) std::unique_lock lock(m_ClientsLock); m_Clients.erase(client); - Log(LogWarning, "ApiListener") + Log(LogInformation, "ApiListener") << "Removing API client for endpoint '" << GetName() << "'. " << m_Clients.size() << " API clients left."; SetConnecting(false); diff --git a/lib/remote/jsonrpcconnection.cpp b/lib/remote/jsonrpcconnection.cpp index 889d4452c..a84f98d9f 100644 --- a/lib/remote/jsonrpcconnection.cpp +++ b/lib/remote/jsonrpcconnection.cpp @@ -29,17 +29,17 @@ REGISTER_APIFUNCTION(SetLogPosition, log, &SetLogPositionHandler); static RingBuffer l_TaskStats (15 * 60); -JsonRpcConnection::JsonRpcConnection(const String& identity, bool authenticated, +JsonRpcConnection::JsonRpcConnection(const WaitGroup::Ptr& waitGroup, const String& identity, bool authenticated, const Shared::Ptr& stream, ConnectionRole role) - : JsonRpcConnection(identity, authenticated, stream, role, IoEngine::Get().GetIoContext()) + : JsonRpcConnection(waitGroup, identity, authenticated, stream, role, IoEngine::Get().GetIoContext()) { } -JsonRpcConnection::JsonRpcConnection(const String& identity, bool authenticated, +JsonRpcConnection::JsonRpcConnection(const WaitGroup::Ptr& waitGroup, const String& identity, bool authenticated, const Shared::Ptr& stream, ConnectionRole role, boost::asio::io_context& io) : m_Identity(identity), m_Authenticated(authenticated), m_Stream(stream), m_Role(role), m_Timestamp(Utility::GetTime()), m_Seen(Utility::GetTime()), m_IoStrand(io), - m_OutgoingMessagesQueued(io), m_WriterDone(io), m_ShuttingDown(false), + m_OutgoingMessagesQueued(io), m_WriterDone(io), m_ShuttingDown(false), m_WaitGroup(waitGroup), m_CheckLivenessTimer(io), m_HeartbeatTimer(io) { if (authenticated) @@ -284,7 +284,7 @@ void JsonRpcConnection::Disconnect() ApiListener::GetInstance()->RemoveAnonymousClient(this); } - Log(LogWarning, "JsonRpcConnection") + Log(LogInformation, "JsonRpcConnection") << "API client disconnected for identity '" << m_Identity << "'"; }); } @@ -303,6 +303,11 @@ void JsonRpcConnection::Disconnect() */ void JsonRpcConnection::MessageHandler(const Dictionary::Ptr& message) { + std::shared_lock wgLock(*m_WaitGroup, std::try_to_lock); + if (!wgLock) { + return; + } + if (m_Endpoint && message->Contains("ts")) { double ts = message->Get("ts"); diff --git a/lib/remote/jsonrpcconnection.hpp b/lib/remote/jsonrpcconnection.hpp index 826d3b46a..df846527a 100644 --- a/lib/remote/jsonrpcconnection.hpp +++ b/lib/remote/jsonrpcconnection.hpp @@ -8,6 +8,7 @@ #include "base/atomic.hpp" #include "base/io-engine.hpp" #include "base/tlsstream.hpp" +#include "base/wait-group.hpp" #include "base/timer.hpp" #include "base/workqueue.hpp" #include @@ -43,7 +44,8 @@ class JsonRpcConnection final : public Object public: DECLARE_PTR_TYPEDEFS(JsonRpcConnection); - JsonRpcConnection(const String& identity, bool authenticated, const Shared::Ptr& stream, ConnectionRole role); + JsonRpcConnection(const WaitGroup::Ptr& waitgroup, const String& identity, bool authenticated, + const Shared::Ptr& stream, ConnectionRole role); void Start(); @@ -78,9 +80,11 @@ private: AsioEvent m_OutgoingMessagesQueued; AsioEvent m_WriterDone; Atomic m_ShuttingDown; + WaitGroup::Ptr m_WaitGroup; boost::asio::deadline_timer m_CheckLivenessTimer, m_HeartbeatTimer; - JsonRpcConnection(const String& identity, bool authenticated, const Shared::Ptr& stream, ConnectionRole role, boost::asio::io_context& io); + JsonRpcConnection(const WaitGroup::Ptr& waitgroup, const String& identity, bool authenticated, + const Shared::Ptr& stream, ConnectionRole role, boost::asio::io_context& io); void HandleIncomingMessages(boost::asio::yield_context yc); void WriteOutgoingMessages(boost::asio::yield_context yc); From 82bb636d2b9f4cb92a871f45a1bc5225f7af8cc5 Mon Sep 17 00:00:00 2001 From: Johannes Schmidt Date: Wed, 11 Jun 2025 10:28:16 +0200 Subject: [PATCH 4/4] Use WaitGroup to wait for or abort HTTP requests The wait group gets passed to HttpServerConnection, then down to the HttpHandlers. For those handlers that modify the program state, the wait group is locked so ApiListener will wait on Stop() for the request to complete. If the request iterates over config objects, a further check on the state of the wait group is added to abort early and not delay program shutdown. In that case, 503 responses will be sent to the client. Additionally, in HttpServerConnection, no further requests than the one already started will be allowed once the wait group is joining. --- lib/remote/actionshandler.cpp | 22 ++++++++++++++++++++++ lib/remote/actionshandler.hpp | 1 + lib/remote/apilistener.cpp | 2 +- lib/remote/configfileshandler.cpp | 1 + lib/remote/configfileshandler.hpp | 1 + lib/remote/configpackageshandler.cpp | 1 + lib/remote/configpackageshandler.hpp | 1 + lib/remote/configstageshandler.cpp | 1 + lib/remote/configstageshandler.hpp | 1 + lib/remote/consolehandler.cpp | 1 + lib/remote/consolehandler.hpp | 1 + lib/remote/createobjecthandler.cpp | 7 +++++++ lib/remote/createobjecthandler.hpp | 1 + lib/remote/deleteobjecthandler.cpp | 24 ++++++++++++++++++++++++ lib/remote/deleteobjecthandler.hpp | 1 + lib/remote/eventshandler.cpp | 1 + lib/remote/eventshandler.hpp | 1 + lib/remote/httphandler.cpp | 3 ++- lib/remote/httphandler.hpp | 2 ++ lib/remote/httpserverconnection.cpp | 15 ++++++++------- lib/remote/httpserverconnection.hpp | 9 ++++++--- lib/remote/infohandler.cpp | 1 + lib/remote/infohandler.hpp | 1 + lib/remote/mallocinfohandler.cpp | 1 + lib/remote/mallocinfohandler.hpp | 1 + lib/remote/modifyobjecthandler.cpp | 20 ++++++++++++++++++++ lib/remote/modifyobjecthandler.hpp | 1 + lib/remote/objectqueryhandler.cpp | 1 + lib/remote/objectqueryhandler.hpp | 1 + lib/remote/statushandler.cpp | 1 + lib/remote/statushandler.hpp | 1 + lib/remote/templatequeryhandler.cpp | 1 + lib/remote/templatequeryhandler.hpp | 1 + lib/remote/typequeryhandler.cpp | 1 + lib/remote/typequeryhandler.hpp | 1 + lib/remote/variablequeryhandler.cpp | 1 + lib/remote/variablequeryhandler.hpp | 1 + 37 files changed, 120 insertions(+), 12 deletions(-) diff --git a/lib/remote/actionshandler.cpp b/lib/remote/actionshandler.cpp index cd16c2bc0..5ae5fdc80 100644 --- a/lib/remote/actionshandler.cpp +++ b/lib/remote/actionshandler.cpp @@ -16,6 +16,7 @@ thread_local ApiUser::Ptr ActionsHandler::AuthenticatedApiUser; REGISTER_URLHANDLER("/v1/actions", ActionsHandler); bool ActionsHandler::HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, @@ -88,7 +89,28 @@ bool ActionsHandler::HandleRequest( if (params) verbose = HttpUtility::GetLastParameter(params, "verbose"); + std::shared_lock wgLock{*waitGroup, std::try_to_lock}; + if (!wgLock) { + HttpUtility::SendJsonError(response, params, 503, "Shutting down."); + return true; + } + for (ConfigObject::Ptr obj : objs) { + if (!waitGroup->IsLockable()) { + if (wgLock) { + wgLock.unlock(); + } + + results.emplace_back(new Dictionary({ + { "type", obj->GetReflectionType()->GetName() }, + { "name", obj->GetName() }, + { "code", 503 }, + { "status", "Action skipped: Shutting down."} + })); + + continue; + } + try { results.emplace_back(action->Invoke(obj, params)); } catch (const std::exception& ex) { diff --git a/lib/remote/actionshandler.hpp b/lib/remote/actionshandler.hpp index ca662caba..fbf716797 100644 --- a/lib/remote/actionshandler.hpp +++ b/lib/remote/actionshandler.hpp @@ -16,6 +16,7 @@ public: static thread_local ApiUser::Ptr AuthenticatedApiUser; bool HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp index 9285f747f..547eadc7f 100644 --- a/lib/remote/apilistener.cpp +++ b/lib/remote/apilistener.cpp @@ -917,7 +917,7 @@ void ApiListener::NewClientHandlerInternal( } else { Log(LogNotice, "ApiListener", "New HTTP client"); - HttpServerConnection::Ptr aclient = new HttpServerConnection(identity, verify_ok, client); + HttpServerConnection::Ptr aclient = new HttpServerConnection(m_WaitGroup, identity, verify_ok, client); AddHttpClient(aclient); aclient->Start(); shutdownSslConn.Cancel(); diff --git a/lib/remote/configfileshandler.cpp b/lib/remote/configfileshandler.cpp index 779ecd198..6c390e804 100644 --- a/lib/remote/configfileshandler.cpp +++ b/lib/remote/configfileshandler.cpp @@ -14,6 +14,7 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/config/files", ConfigFilesHandler); bool ConfigFilesHandler::HandleRequest( + const WaitGroup::Ptr&, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/configfileshandler.hpp b/lib/remote/configfileshandler.hpp index ea48b1ef4..a8826d8c1 100644 --- a/lib/remote/configfileshandler.hpp +++ b/lib/remote/configfileshandler.hpp @@ -14,6 +14,7 @@ public: DECLARE_PTR_TYPEDEFS(ConfigFilesHandler); bool HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/configpackageshandler.cpp b/lib/remote/configpackageshandler.cpp index 98b326890..7987092bc 100644 --- a/lib/remote/configpackageshandler.cpp +++ b/lib/remote/configpackageshandler.cpp @@ -11,6 +11,7 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/config/packages", ConfigPackagesHandler); bool ConfigPackagesHandler::HandleRequest( + const WaitGroup::Ptr&, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/configpackageshandler.hpp b/lib/remote/configpackageshandler.hpp index 0a05ea10a..2bae0e265 100644 --- a/lib/remote/configpackageshandler.hpp +++ b/lib/remote/configpackageshandler.hpp @@ -14,6 +14,7 @@ public: DECLARE_PTR_TYPEDEFS(ConfigPackagesHandler); bool HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/configstageshandler.cpp b/lib/remote/configstageshandler.cpp index edbb767e5..0dee5f25f 100644 --- a/lib/remote/configstageshandler.cpp +++ b/lib/remote/configstageshandler.cpp @@ -15,6 +15,7 @@ REGISTER_URLHANDLER("/v1/config/stages", ConfigStagesHandler); std::atomic ConfigStagesHandler::m_RunningPackageUpdates (false); bool ConfigStagesHandler::HandleRequest( + const WaitGroup::Ptr&, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/configstageshandler.hpp b/lib/remote/configstageshandler.hpp index 88f248c8f..a26ddc49c 100644 --- a/lib/remote/configstageshandler.hpp +++ b/lib/remote/configstageshandler.hpp @@ -15,6 +15,7 @@ public: DECLARE_PTR_TYPEDEFS(ConfigStagesHandler); bool HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/consolehandler.cpp b/lib/remote/consolehandler.cpp index 8ed36311c..c48821aae 100644 --- a/lib/remote/consolehandler.cpp +++ b/lib/remote/consolehandler.cpp @@ -54,6 +54,7 @@ static void EnsureFrameCleanupTimer() } bool ConsoleHandler::HandleRequest( + const WaitGroup::Ptr&, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/consolehandler.hpp b/lib/remote/consolehandler.hpp index df0d77d01..ba93d0001 100644 --- a/lib/remote/consolehandler.hpp +++ b/lib/remote/consolehandler.hpp @@ -23,6 +23,7 @@ public: DECLARE_PTR_TYPEDEFS(ConsoleHandler); bool HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/createobjecthandler.cpp b/lib/remote/createobjecthandler.cpp index 89977a3d3..119be1cd9 100644 --- a/lib/remote/createobjecthandler.cpp +++ b/lib/remote/createobjecthandler.cpp @@ -16,6 +16,7 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/objects", CreateObjectHandler); bool CreateObjectHandler::HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, @@ -102,6 +103,12 @@ bool CreateObjectHandler::HandleRequest( return true; } + std::shared_lock wgLock{*waitGroup, std::try_to_lock}; + if (!wgLock) { + HttpUtility::SendJsonError(response, params, 503, "Shutting down."); + return true; + } + /* Object creation can cause multiple errors and optionally diagnostic information. * We can't use SendJsonError() here. */ diff --git a/lib/remote/createobjecthandler.hpp b/lib/remote/createobjecthandler.hpp index 4bcf21b55..3f6a705c2 100644 --- a/lib/remote/createobjecthandler.hpp +++ b/lib/remote/createobjecthandler.hpp @@ -14,6 +14,7 @@ public: DECLARE_PTR_TYPEDEFS(CreateObjectHandler); bool HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/deleteobjecthandler.cpp b/lib/remote/deleteobjecthandler.cpp index 150de99e0..54d31f13d 100644 --- a/lib/remote/deleteobjecthandler.cpp +++ b/lib/remote/deleteobjecthandler.cpp @@ -16,6 +16,7 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/objects", DeleteObjectHandler); bool DeleteObjectHandler::HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, @@ -78,7 +79,30 @@ bool DeleteObjectHandler::HandleRequest( bool success = true; + std::shared_lock wgLock{*waitGroup, std::try_to_lock}; + if (!wgLock) { + HttpUtility::SendJsonError(response, params, 503, "Shutting down."); + return true; + } + for (ConfigObject::Ptr obj : objs) { + if (!waitGroup->IsLockable()) { + if (wgLock) { + wgLock.unlock(); + } + + results.emplace_back(new Dictionary({ + { "type", type->GetName() }, + { "name", obj->GetName() }, + { "code", 503 }, + { "status", "Action skipped: Shutting down."} + })); + + success = false; + + continue; + } + int code; String status; Array::Ptr errors = new Array(); diff --git a/lib/remote/deleteobjecthandler.hpp b/lib/remote/deleteobjecthandler.hpp index 19a46e475..0f9643277 100644 --- a/lib/remote/deleteobjecthandler.hpp +++ b/lib/remote/deleteobjecthandler.hpp @@ -14,6 +14,7 @@ public: DECLARE_PTR_TYPEDEFS(DeleteObjectHandler); bool HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/eventshandler.cpp b/lib/remote/eventshandler.cpp index 897398d4a..2cbee92f3 100644 --- a/lib/remote/eventshandler.cpp +++ b/lib/remote/eventshandler.cpp @@ -40,6 +40,7 @@ const std::map l_EventTypes ({ const String l_ApiQuery (""); bool EventsHandler::HandleRequest( + const WaitGroup::Ptr&, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/eventshandler.hpp b/lib/remote/eventshandler.hpp index c823415d3..49229733a 100644 --- a/lib/remote/eventshandler.hpp +++ b/lib/remote/eventshandler.hpp @@ -15,6 +15,7 @@ public: DECLARE_PTR_TYPEDEFS(EventsHandler); bool HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/httphandler.cpp b/lib/remote/httphandler.cpp index f67df4c69..79571d760 100644 --- a/lib/remote/httphandler.cpp +++ b/lib/remote/httphandler.cpp @@ -47,6 +47,7 @@ void HttpHandler::Register(const Url::Ptr& url, const HttpHandler::Ptr& handler) } void HttpHandler::ProcessRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, @@ -108,7 +109,7 @@ void HttpHandler::ProcessRequest( */ try { for (const HttpHandler::Ptr& handler : handlers) { - if (handler->HandleRequest(stream, user, request, url, response, params, yc, server)) { + if (handler->HandleRequest(waitGroup, stream, user, request, url, response, params, yc, server)) { processed = true; break; } diff --git a/lib/remote/httphandler.hpp b/lib/remote/httphandler.hpp index a6a730255..ec67ae8a4 100644 --- a/lib/remote/httphandler.hpp +++ b/lib/remote/httphandler.hpp @@ -27,6 +27,7 @@ public: DECLARE_PTR_TYPEDEFS(HttpHandler); virtual bool HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, @@ -39,6 +40,7 @@ public: static void Register(const Url::Ptr& url, const HttpHandler::Ptr& handler); static void ProcessRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/httpserverconnection.cpp b/lib/remote/httpserverconnection.cpp index cc0ecc376..17e61f160 100644 --- a/lib/remote/httpserverconnection.cpp +++ b/lib/remote/httpserverconnection.cpp @@ -35,13 +35,13 @@ using namespace icinga; auto const l_ServerHeader ("Icinga/" + Application::GetAppVersion()); -HttpServerConnection::HttpServerConnection(const String& identity, bool authenticated, const Shared::Ptr& stream) - : HttpServerConnection(identity, authenticated, stream, IoEngine::Get().GetIoContext()) +HttpServerConnection::HttpServerConnection(const WaitGroup::Ptr& waitGroup, const String& identity, bool authenticated, const Shared::Ptr& stream) + : HttpServerConnection(waitGroup, identity, authenticated, stream, IoEngine::Get().GetIoContext()) { } -HttpServerConnection::HttpServerConnection(const String& identity, bool authenticated, const Shared::Ptr& stream, boost::asio::io_context& io) - : m_Stream(stream), m_Seen(Utility::GetTime()), m_IoStrand(io), m_ShuttingDown(false), m_HasStartedStreaming(false), +HttpServerConnection::HttpServerConnection(const WaitGroup::Ptr& waitGroup, const String& identity, bool authenticated, const Shared::Ptr& stream, boost::asio::io_context& io) + : m_WaitGroup(waitGroup), m_Stream(stream), m_Seen(Utility::GetTime()), m_IoStrand(io), m_ShuttingDown(false), m_HasStartedStreaming(false), m_CheckLivenessTimer(io) { if (authenticated) { @@ -419,6 +419,7 @@ bool ProcessRequest( boost::beast::http::response& response, HttpServerConnection& server, bool& hasStartedStreaming, + const WaitGroup::Ptr& waitGroup, std::chrono::steady_clock::duration& cpuBoundWorkTime, boost::asio::yield_context& yc ) @@ -431,7 +432,7 @@ bool ProcessRequest( CpuBoundWork handlingRequest (yc); cpuBoundWorkTime = std::chrono::steady_clock::now() - start; - HttpHandler::ProcessRequest(stream, authenticatedUser, request, response, yc, server); + HttpHandler::ProcessRequest(waitGroup, stream, authenticatedUser, request, response, yc, server); } catch (const std::exception& ex) { if (hasStartedStreaming) { return false; @@ -477,7 +478,7 @@ void HttpServerConnection::ProcessMessages(boost::asio::yield_context yc) */ beast::flat_buffer buf; - for (;;) { + while (m_WaitGroup->IsLockable()) { m_Seen = Utility::GetTime(); http::parser parser; @@ -548,7 +549,7 @@ void HttpServerConnection::ProcessMessages(boost::asio::yield_context yc) m_Seen = std::numeric_limits::max(); - if (!ProcessRequest(*m_Stream, request, authenticatedUser, response, *this, m_HasStartedStreaming, cpuBoundWorkTime, yc)) { + if (!ProcessRequest(*m_Stream, request, authenticatedUser, response, *this, m_HasStartedStreaming, m_WaitGroup, cpuBoundWorkTime, yc)) { break; } diff --git a/lib/remote/httpserverconnection.hpp b/lib/remote/httpserverconnection.hpp index 63f99e19c..e4f7d257e 100644 --- a/lib/remote/httpserverconnection.hpp +++ b/lib/remote/httpserverconnection.hpp @@ -6,6 +6,7 @@ #include "remote/apiuser.hpp" #include "base/string.hpp" #include "base/tlsstream.hpp" +#include "base/wait-group.hpp" #include #include #include @@ -25,14 +26,15 @@ class HttpServerConnection final : public Object public: DECLARE_PTR_TYPEDEFS(HttpServerConnection); - HttpServerConnection(const String& identity, bool authenticated, const Shared::Ptr& stream); + HttpServerConnection(const WaitGroup::Ptr& waitGroup, const String& identity, bool authenticated, + const Shared::Ptr& stream); void Start(); void StartStreaming(); - bool Disconnected(); private: + WaitGroup::Ptr m_WaitGroup; ApiUser::Ptr m_ApiUser; Shared::Ptr m_Stream; double m_Seen; @@ -42,7 +44,8 @@ private: bool m_HasStartedStreaming; boost::asio::deadline_timer m_CheckLivenessTimer; - HttpServerConnection(const String& identity, bool authenticated, const Shared::Ptr& stream, boost::asio::io_context& io); + HttpServerConnection(const WaitGroup::Ptr& waitGroup, const String& identity, bool authenticated, + const Shared::Ptr& stream, boost::asio::io_context& io); void Disconnect(boost::asio::yield_context yc); diff --git a/lib/remote/infohandler.cpp b/lib/remote/infohandler.cpp index 6a1360b40..5fc621cd8 100644 --- a/lib/remote/infohandler.cpp +++ b/lib/remote/infohandler.cpp @@ -9,6 +9,7 @@ using namespace icinga; REGISTER_URLHANDLER("/", InfoHandler); bool InfoHandler::HandleRequest( + const WaitGroup::Ptr&, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/infohandler.hpp b/lib/remote/infohandler.hpp index e1fe98314..7396f5ac9 100644 --- a/lib/remote/infohandler.hpp +++ b/lib/remote/infohandler.hpp @@ -14,6 +14,7 @@ public: DECLARE_PTR_TYPEDEFS(InfoHandler); bool HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/mallocinfohandler.cpp b/lib/remote/mallocinfohandler.cpp index ac73e3650..f4c27cac4 100644 --- a/lib/remote/mallocinfohandler.cpp +++ b/lib/remote/mallocinfohandler.cpp @@ -18,6 +18,7 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/debug/malloc_info", MallocInfoHandler); bool MallocInfoHandler::HandleRequest( + const WaitGroup::Ptr&, AsioTlsStream&, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/mallocinfohandler.hpp b/lib/remote/mallocinfohandler.hpp index 0e188f3eb..9648fac9f 100644 --- a/lib/remote/mallocinfohandler.hpp +++ b/lib/remote/mallocinfohandler.hpp @@ -13,6 +13,7 @@ public: DECLARE_PTR_TYPEDEFS(MallocInfoHandler); bool HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/modifyobjecthandler.cpp b/lib/remote/modifyobjecthandler.cpp index dabe69523..c71be6a9a 100644 --- a/lib/remote/modifyobjecthandler.cpp +++ b/lib/remote/modifyobjecthandler.cpp @@ -14,6 +14,7 @@ using namespace icinga; REGISTER_URLHANDLER("/v1/objects", ModifyObjectHandler); bool ModifyObjectHandler::HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, @@ -104,12 +105,31 @@ bool ModifyObjectHandler::HandleRequest( ArrayData results; + std::shared_lock wgLock{*waitGroup, std::try_to_lock}; + if (!wgLock) { + HttpUtility::SendJsonError(response, params, 503, "Shutting down."); + return true; + } + for (ConfigObject::Ptr obj : objs) { Dictionary::Ptr result1 = new Dictionary(); result1->Set("type", type->GetName()); result1->Set("name", obj->GetName()); + if (!waitGroup->IsLockable()) { + if (wgLock) { + wgLock.unlock(); + } + + result1->Set("code", 503); + result1->Set("status", "Action skipped: Shutting down."); + + results.emplace_back(std::move(result1)); + + continue; + } + String key; // Lock the object name of the given type to prevent from being modified/deleted concurrently. diff --git a/lib/remote/modifyobjecthandler.hpp b/lib/remote/modifyobjecthandler.hpp index f4693013f..f299acd6e 100644 --- a/lib/remote/modifyobjecthandler.hpp +++ b/lib/remote/modifyobjecthandler.hpp @@ -14,6 +14,7 @@ public: DECLARE_PTR_TYPEDEFS(ModifyObjectHandler); bool HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/objectqueryhandler.cpp b/lib/remote/objectqueryhandler.cpp index fbd5c7e70..f6f049e4e 100644 --- a/lib/remote/objectqueryhandler.cpp +++ b/lib/remote/objectqueryhandler.cpp @@ -89,6 +89,7 @@ Dictionary::Ptr ObjectQueryHandler::SerializeObjectAttrs(const Object::Ptr& obje } bool ObjectQueryHandler::HandleRequest( + const WaitGroup::Ptr&, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/objectqueryhandler.hpp b/lib/remote/objectqueryhandler.hpp index 691b2cfcf..376eb661e 100644 --- a/lib/remote/objectqueryhandler.hpp +++ b/lib/remote/objectqueryhandler.hpp @@ -14,6 +14,7 @@ public: DECLARE_PTR_TYPEDEFS(ObjectQueryHandler); bool HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/statushandler.cpp b/lib/remote/statushandler.cpp index 310ce0d87..bf14152f8 100644 --- a/lib/remote/statushandler.cpp +++ b/lib/remote/statushandler.cpp @@ -69,6 +69,7 @@ public: }; bool StatusHandler::HandleRequest( + const WaitGroup::Ptr&, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/statushandler.hpp b/lib/remote/statushandler.hpp index c722ab3e2..109fd4881 100644 --- a/lib/remote/statushandler.hpp +++ b/lib/remote/statushandler.hpp @@ -14,6 +14,7 @@ public: DECLARE_PTR_TYPEDEFS(StatusHandler); bool HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/templatequeryhandler.cpp b/lib/remote/templatequeryhandler.cpp index e70dafb65..a68ad6dad 100644 --- a/lib/remote/templatequeryhandler.cpp +++ b/lib/remote/templatequeryhandler.cpp @@ -76,6 +76,7 @@ public: }; bool TemplateQueryHandler::HandleRequest( + const WaitGroup::Ptr&, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/templatequeryhandler.hpp b/lib/remote/templatequeryhandler.hpp index 503bc8560..312cf4221 100644 --- a/lib/remote/templatequeryhandler.hpp +++ b/lib/remote/templatequeryhandler.hpp @@ -14,6 +14,7 @@ public: DECLARE_PTR_TYPEDEFS(TemplateQueryHandler); bool HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/typequeryhandler.cpp b/lib/remote/typequeryhandler.cpp index b30dbb14a..b2184344d 100644 --- a/lib/remote/typequeryhandler.cpp +++ b/lib/remote/typequeryhandler.cpp @@ -47,6 +47,7 @@ public: }; bool TypeQueryHandler::HandleRequest( + const WaitGroup::Ptr&, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/typequeryhandler.hpp b/lib/remote/typequeryhandler.hpp index 5489cb232..45cbc38ec 100644 --- a/lib/remote/typequeryhandler.hpp +++ b/lib/remote/typequeryhandler.hpp @@ -14,6 +14,7 @@ public: DECLARE_PTR_TYPEDEFS(TypeQueryHandler); bool HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/variablequeryhandler.cpp b/lib/remote/variablequeryhandler.cpp index 7264338cb..40552dd7d 100644 --- a/lib/remote/variablequeryhandler.cpp +++ b/lib/remote/variablequeryhandler.cpp @@ -57,6 +57,7 @@ public: }; bool VariableQueryHandler::HandleRequest( + const WaitGroup::Ptr&, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request, diff --git a/lib/remote/variablequeryhandler.hpp b/lib/remote/variablequeryhandler.hpp index 48e73be35..d145f5b59 100644 --- a/lib/remote/variablequeryhandler.hpp +++ b/lib/remote/variablequeryhandler.hpp @@ -14,6 +14,7 @@ public: DECLARE_PTR_TYPEDEFS(VariableQueryHandler); bool HandleRequest( + const WaitGroup::Ptr& waitGroup, AsioTlsStream& stream, const ApiUser::Ptr& user, boost::beast::http::request& request,