From b75e21a998fa63404e68a318d34cc71ca12b4141 Mon Sep 17 00:00:00 2001 From: Gunnar Beutner Date: Mon, 25 Sep 2017 14:41:43 +0200 Subject: [PATCH 001/219] Redis support (WIP) --- CMakeLists.txt | 1 + etc/icinga2/features-available/redis.conf | 14 + lib/CMakeLists.txt | 4 + lib/redis/CMakeLists.txt | 53 + lib/redis/rediswriter-status.cpp | 287 +++++ lib/redis/rediswriter-utility.cpp | 120 ++ lib/redis/rediswriter.cpp | 336 +++++ lib/redis/rediswriter.hpp | 107 ++ lib/redis/rediswriter.ti | 40 + third-party/CMakeLists.txt | 4 + third-party/hiredis/.gitignore | 7 + third-party/hiredis/.travis.yml | 24 + third-party/hiredis/CHANGELOG.md | 110 ++ third-party/hiredis/CMakeLists.txt | 42 + third-party/hiredis/COPYING | 29 + third-party/hiredis/Makefile | 217 ++++ third-party/hiredis/README.md | 392 ++++++ third-party/hiredis/adapters/ae.h | 127 ++ third-party/hiredis/adapters/glib.h | 153 +++ third-party/hiredis/adapters/ivykis.h | 81 ++ third-party/hiredis/adapters/libev.h | 147 +++ third-party/hiredis/adapters/libevent.h | 108 ++ third-party/hiredis/adapters/libuv.h | 121 ++ third-party/hiredis/adapters/macosx.h | 114 ++ third-party/hiredis/adapters/qt.h | 135 ++ third-party/hiredis/async.c | 687 +++++++++++ third-party/hiredis/async.h | 129 ++ third-party/hiredis/dict.c | 338 +++++ third-party/hiredis/dict.h | 126 ++ third-party/hiredis/examples/example-ae.c | 62 + third-party/hiredis/examples/example-glib.c | 73 ++ third-party/hiredis/examples/example-ivykis.c | 58 + third-party/hiredis/examples/example-libev.c | 52 + .../hiredis/examples/example-libevent.c | 53 + third-party/hiredis/examples/example-libuv.c | 53 + third-party/hiredis/examples/example-macosx.c | 66 + third-party/hiredis/examples/example-qt.cpp | 46 + third-party/hiredis/examples/example-qt.h | 32 + third-party/hiredis/examples/example.c | 78 ++ third-party/hiredis/fmacros.h | 21 + third-party/hiredis/hiredis.c | 1021 +++++++++++++++ third-party/hiredis/hiredis.h | 223 ++++ third-party/hiredis/net.c | 458 +++++++ third-party/hiredis/net.h | 53 + third-party/hiredis/read.c | 525 ++++++++ third-party/hiredis/read.h | 116 ++ third-party/hiredis/sds.c | 1095 +++++++++++++++++ third-party/hiredis/sds.h | 105 ++ third-party/hiredis/test.c | 807 ++++++++++++ third-party/hiredis/win32.h | 42 + 50 files changed, 9092 insertions(+) create mode 100644 etc/icinga2/features-available/redis.conf create mode 100644 lib/redis/CMakeLists.txt create mode 100644 lib/redis/rediswriter-status.cpp create mode 100644 lib/redis/rediswriter-utility.cpp create mode 100644 lib/redis/rediswriter.cpp create mode 100644 lib/redis/rediswriter.hpp create mode 100644 lib/redis/rediswriter.ti create mode 100644 third-party/hiredis/.gitignore create mode 100644 third-party/hiredis/.travis.yml create mode 100644 third-party/hiredis/CHANGELOG.md create mode 100644 third-party/hiredis/CMakeLists.txt create mode 100644 third-party/hiredis/COPYING create mode 100644 third-party/hiredis/Makefile create mode 100644 third-party/hiredis/README.md create mode 100644 third-party/hiredis/adapters/ae.h create mode 100644 third-party/hiredis/adapters/glib.h create mode 100644 third-party/hiredis/adapters/ivykis.h create mode 100644 third-party/hiredis/adapters/libev.h create mode 100644 third-party/hiredis/adapters/libevent.h create mode 100644 third-party/hiredis/adapters/libuv.h create mode 100644 third-party/hiredis/adapters/macosx.h create mode 100644 third-party/hiredis/adapters/qt.h create mode 100644 third-party/hiredis/async.c create mode 100644 third-party/hiredis/async.h create mode 100644 third-party/hiredis/dict.c create mode 100644 third-party/hiredis/dict.h create mode 100644 third-party/hiredis/examples/example-ae.c create mode 100644 third-party/hiredis/examples/example-glib.c create mode 100644 third-party/hiredis/examples/example-ivykis.c create mode 100644 third-party/hiredis/examples/example-libev.c create mode 100644 third-party/hiredis/examples/example-libevent.c create mode 100644 third-party/hiredis/examples/example-libuv.c create mode 100644 third-party/hiredis/examples/example-macosx.c create mode 100644 third-party/hiredis/examples/example-qt.cpp create mode 100644 third-party/hiredis/examples/example-qt.h create mode 100644 third-party/hiredis/examples/example.c create mode 100644 third-party/hiredis/fmacros.h create mode 100644 third-party/hiredis/hiredis.c create mode 100644 third-party/hiredis/hiredis.h create mode 100644 third-party/hiredis/net.c create mode 100644 third-party/hiredis/net.h create mode 100644 third-party/hiredis/read.c create mode 100644 third-party/hiredis/read.h create mode 100644 third-party/hiredis/sds.c create mode 100644 third-party/hiredis/sds.h create mode 100644 third-party/hiredis/test.c create mode 100644 third-party/hiredis/win32.h diff --git a/CMakeLists.txt b/CMakeLists.txt index bfc558769..04349e75a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,7 @@ option(ICINGA2_WITH_COMPAT "Build the compat module" ON) option(ICINGA2_WITH_LIVESTATUS "Build the Livestatus module" ON) option(ICINGA2_WITH_NOTIFICATION "Build the notification module" ON) option(ICINGA2_WITH_PERFDATA "Build the perfdata module" ON) +option(ICINGA2_WITH_REDIS "Build the redis module" ON) option(ICINGA2_WITH_TESTS "Run unit tests" ON) option (USE_SYSTEMD diff --git a/etc/icinga2/features-available/redis.conf b/etc/icinga2/features-available/redis.conf new file mode 100644 index 000000000..bcf781ad2 --- /dev/null +++ b/etc/icinga2/features-available/redis.conf @@ -0,0 +1,14 @@ +/** + * The redis library implements functionality for putting Icinga + * event data into a redis database. + * + * NOTE: This is experimental and may change without further notice. + */ + +library "redis" + +object RedisWriter "redis" { + //host = "127.0.0.1" + //port = 6379 + //password = "xxx" +} diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 004fc154c..a00cf4db8 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -53,4 +53,8 @@ if(ICINGA2_WITH_PERFDATA) add_subdirectory(perfdata) endif() +if(ICINGA2_WITH_REDIS) + add_subdirectory(redis) +endif() + set(CPACK_NSIS_EXTRA_INSTALL_COMMANDS "${CPACK_NSIS_EXTRA_INSTALL_COMMANDS}" PARENT_SCOPE) diff --git a/lib/redis/CMakeLists.txt b/lib/redis/CMakeLists.txt new file mode 100644 index 000000000..540fffa06 --- /dev/null +++ b/lib/redis/CMakeLists.txt @@ -0,0 +1,53 @@ +# Icinga 2 +# Copyright (C) 2012-2017 Icinga Development Team (https://www.icinga.com/) +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. + +mkclass_target(rediswriter.ti rediswriter.tcpp rediswriter.thpp) + +set(redis_SOURCES + rediswriter.cpp rediswriter-status.cpp rediswriter-utility.cpp rediswriter.thpp +) + +if(ICINGA2_UNITY_BUILD) + mkunity_target(redis redis redis_SOURCES) +endif() + +add_library(redis SHARED ${redis_SOURCES}) + +target_link_libraries(redis ${Boost_LIBRARIES} base config icinga remote hiredis) + +include_directories(${icinga2_SOURCE_DIR}/third-party) +link_directories(${icinga2_BINARY_DIR}/third-party/hiredis) + +set_target_properties ( + redis PROPERTIES + INSTALL_RPATH ${CMAKE_INSTALL_FULL_LIBDIR}/icinga2 + DEFINE_SYMBOL I2_REDIS_BUILD + FOLDER Components + VERSION ${SPEC_VERSION} +) + +install_if_not_exists( + ${PROJECT_SOURCE_DIR}/etc/icinga2/features-available/redis.conf + ${CMAKE_INSTALL_SYSCONFDIR}/icinga2/features-available +) + +install( + TARGETS redis + RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}/icinga2 +) + diff --git a/lib/redis/rediswriter-status.cpp b/lib/redis/rediswriter-status.cpp new file mode 100644 index 000000000..8c830fbc8 --- /dev/null +++ b/lib/redis/rediswriter-status.cpp @@ -0,0 +1,287 @@ +/****************************************************************************** + * Icinga 2 * + * Copyright (C) 2012-2017 Icinga Development Team (https://www.icinga.com/) * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License * + * as published by the Free Software Foundation; either version 2 * + * of the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software Foundation * + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * + ******************************************************************************/ + +#include "redis/rediswriter.hpp" +#include "icinga/customvarobject.hpp" +#include "icinga/host.hpp" +#include "icinga/service.hpp" +#include "base/json.hpp" +#include "base/logger.hpp" +#include "base/serializer.hpp" +#include "base/tlsutility.hpp" +#include "base/initialize.hpp" + +using namespace icinga; + +/* +- icinga:config: as hash +key: sha1 checksum(name) +value: JsonEncode(Serialize(object, FAConfig)) + config_checksum + +Diff between calculated config_checksum and Redis json config_checksum +Alternative: Replace into. + + +- icinga:status: as hash +key: sha1 checksum(name) +value: JsonEncode(Serialize(object, FAState)) +*/ + +INITIALIZE_ONCE(&RedisWriter::ConfigStaticInitialize); + +void RedisWriter::ConfigStaticInitialize(void) +{ + /* triggered in ProcessCheckResult(), requires UpdateNextCheck() to be called before */ + ConfigObject::OnStateChanged.connect(boost::bind(&RedisWriter::StateChangedHandler, _1)); + CustomVarObject::OnVarsChanged.connect(boost::bind(&RedisWriter::VarsChangedHandler, _1)); + + /* triggered on create, update and delete objects */ + ConfigObject::OnActiveChanged.connect(boost::bind(&RedisWriter::VersionChangedHandler, _1)); + ConfigObject::OnVersionChanged.connect(boost::bind(&RedisWriter::VersionChangedHandler, _1)); +} + +void RedisWriter::UpdateAllConfigObjects(void) +{ + AssertOnWorkQueue(); + + double startTime = Utility::GetTime(); + + //TODO: "Publish" the config dump by adding another event, globally or by object + ExecuteQuery({ "MULTI" }); + + for (const Type::Ptr& type : Type::GetAllTypes()) { + ConfigType *ctype = dynamic_cast(type.get()); + if (!ctype) + continue; + + String typeName = type->GetName(); + + /* replace into aka delete insert is faster than a full diff */ + ExecuteQuery({ "DEL", "icinga:config:" + typeName, "icinga:config:" + typeName + ":checksum", "icinga:status:" + typeName }); + + /* fetch all objects and dump them */ + for (const ConfigObject::Ptr& object : ctype->GetObjects()) { + SendConfigUpdate(object, typeName); + SendStatusUpdate(object, typeName); + } + + /* publish config type dump finished */ + ExecuteQuery({ "PUBLISH", "icinga:config:dump", typeName }); + } + + ExecuteQuery({ "EXEC" }); + + Log(LogInformation, "RedisWriter") + << "Initial config/status dump finished in " << Utility::GetTime() - startTime << " seconds."; +} + +void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, const String& typeName, bool runtimeUpdate) +{ + AssertOnWorkQueue(); + + /* during startup we might send duplicated object config, ignore them without any connection */ + if (!m_Context) + return; + + /* TODO: This isn't essentially correct as we don't keep track of config objects ourselves. This would avoid duplicated config updates at startup. + if (!runtimeUpdate && m_ConfigDumpInProgress) + return; + */ + + /* Serialize config object attributes */ + Dictionary::Ptr objectAttrs = SerializeObjectAttrs(object, FAConfig); + + String jsonBody = JsonEncode(objectAttrs); + + String objectName = object->GetName(); + + ExecuteQuery({ "HSET", "icinga:config:" + typeName, objectName, jsonBody }); + + /* check sums */ + /* hset icinga:config:Host:checksums localhost { "name_checksum": "...", "properties_checksum": "...", "groups_checksum": "...", "vars_checksum": null } */ + Dictionary::Ptr checkSum = new Dictionary(); + + checkSum->Set("name_checksum", CalculateCheckSumString(object->GetName())); + + // TODO: move this elsewhere + Checkable::Ptr checkable = dynamic_pointer_cast(object); + + if (checkable) { + Host::Ptr host; + Service::Ptr service; + + tie(host, service) = GetHostService(checkable); + + if (service) + checkSum->Set("groups_checksum", CalculateCheckSumGroups(service->GetGroups())); + else + checkSum->Set("groups_checksum", CalculateCheckSumGroups(host->GetGroups())); + } + + checkSum->Set("properties_checksum", CalculateCheckSumProperties(object)); + checkSum->Set("vars_checksum", CalculateCheckSumVars(object)); + + String checkSumBody = JsonEncode(checkSum); + + ExecuteQuery({ "HSET", "icinga:config:" + typeName + ":checksum", objectName, checkSumBody }); + + /* publish runtime updated objects immediately */ + if (!runtimeUpdate) + return; + + ExecuteQuery({ "PUBLISH", "icinga:config:update", typeName + ":" + objectName + "!" + checkSumBody }); +} + +void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object, const String& typeName) +{ + AssertOnWorkQueue(); + + /* during startup we might send duplicated object config, ignore them without any connection */ + if (!m_Context) + return; + + String objectName = object->GetName(); + + ExecuteQuery({ "HDEL", "icinga:config:" + typeName, objectName }); + ExecuteQuery({ "HDEL", "icinga:config:" + typeName + ":checksum", objectName }); + ExecuteQuery({ "HDEL", "icinga:status:" + typeName, objectName }); + + ExecuteQuery({ "PUBLISH", "icinga:config:delete", typeName + ":" + objectName }); +} + +void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, const String& typeName) +{ + AssertOnWorkQueue(); + + /* during startup we might receive check results, ignore them without any connection */ + if (!m_Context) + return; + + /* Serialize config object attributes */ + Dictionary::Ptr objectAttrs = SerializeObjectAttrs(object, FAState); + + String jsonBody = JsonEncode(objectAttrs); + + String objectName = object->GetName(); + + ExecuteQuery({ "HSET", "icinga:status:" + typeName, objectName, jsonBody }); + + /* Icinga DB part for Icinga Web 2 */ + Checkable::Ptr checkable = dynamic_pointer_cast(object); + + if (checkable) { + Dictionary::Ptr attrs = new Dictionary(); + String tableName; + String objectCheckSum = CalculateCheckSumString(objectName, true); //store binary checksum here + + Host::Ptr host; + Service::Ptr service; + + tie(host, service) = GetHostService(checkable); + + if (service) { + tableName = "servicestate"; + attrs->Set("service_checksum", objectCheckSum); + attrs->Set("host_checksum", CalculateCheckSumString(host->GetName(), true)); + } else { + tableName = "hoststate"; + attrs->Set("host_checksum", objectCheckSum); + } + + attrs->Set("last_check", checkable->GetLastCheck()); + attrs->Set("next_check", checkable->GetNextCheck()); + + attrs->Set("severity", checkable->GetSeverity()); + +/* + 'host_checksum' => null, + 'command' => null, // JSON, array + 'execution_start' => null, + 'execution_end' => null, + 'schedule_start' => null, + 'schedule_end' => null, + 'exit_status' => null, + 'output' => null, + 'performance_data' => null, // JSON, array + + +10.0.3.12:6379> keys icinga:hoststate.* +1) "icinga:hoststate.~\xf5a\x91+\x03\x97\x99\xb5(\x16 CYm\xb1\xdf\x85\xa2\xcb" +10.0.3.12:6379> get "icinga:hoststate.~\xf5a\x91+\x03\x97\x99\xb5(\x16 CYm\xb1\xdf\x85\xa2\xcb" +"{\"command\":[\"\\/usr\\/lib\\/nagios\\/plugins\\/check_ping\",\"-H\",\"127.0.0.1\",\"-c\",\"5000,100%\",\"-w\",\"3000,80%\"],\"execution_start\":1492007581.7624,\"execution_end\":1492007585.7654,\"schedule_start\":1492007581.7609,\"schedule_end\":1492007585.7655,\"exit_status\":0,\"output\":\"PING OK - Packet loss = 0%, RTA = 0.08 ms\",\"performance_data\":[\"rta=0.076000ms;3000.000000;5000.000000;0.000000\",\"pl=0%;80;100;0\"]}" + +*/ + + CheckResult::Ptr cr = checkable->GetLastCheckResult(); + + if (cr) { + attrs->Set("command", JsonEncode(cr->GetCommand())); + attrs->Set("execution_start", cr->GetExecutionStart()); + attrs->Set("execution_end", cr->GetExecutionEnd()); + attrs->Set("schedule_start", cr->GetScheduleStart()); + attrs->Set("schedule_end", cr->GetScheduleStart()); + attrs->Set("exit_status", cr->GetExitStatus()); + attrs->Set("output", cr->GetOutput()); + attrs->Set("performance_data", JsonEncode(cr->GetPerformanceData())); + } + + String jsonAttrs = JsonEncode(attrs); + String key = "icinga:" + tableName + "." + objectCheckSum; + ExecuteQuery({ "SET", key, jsonAttrs }); + + /* expire in check_interval * attempts + timeout + some more seconds */ + double expireTime = checkable->GetCheckInterval() * checkable->GetMaxCheckAttempts() + 60; + ExecuteQuery({ "EXPIRE", key, String(expireTime) }); + } +} + +void RedisWriter::StateChangedHandler(const ConfigObject::Ptr& object) +{ + Type::Ptr type = object->GetReflectionType(); + + for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { + rw->m_WorkQueue.Enqueue(boost::bind(&RedisWriter::SendStatusUpdate, rw, object, type->GetName())); + } +} + +void RedisWriter::VarsChangedHandler(const ConfigObject::Ptr& object) +{ + Type::Ptr type = object->GetReflectionType(); + + for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { + rw->m_WorkQueue.Enqueue(boost::bind(&RedisWriter::SendConfigUpdate, rw, object, type->GetName(), true)); + } +} + +void RedisWriter::VersionChangedHandler(const ConfigObject::Ptr& object) +{ + Type::Ptr type = object->GetReflectionType(); + + if (object->IsActive()) { + /* Create or update the object config */ + for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { + rw->m_WorkQueue.Enqueue(boost::bind(&RedisWriter::SendConfigUpdate, rw.get(), object, type->GetName(), true)); + } + } else if (!object->IsActive() && object->GetExtension("ConfigObjectDeleted")) { /* same as in apilistener-configsync.cpp */ + /* Delete object config */ + for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { + rw->m_WorkQueue.Enqueue(boost::bind(&RedisWriter::SendConfigDelete, rw.get(), object, type->GetName())); + } + } +} diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp new file mode 100644 index 000000000..e91edc800 --- /dev/null +++ b/lib/redis/rediswriter-utility.cpp @@ -0,0 +1,120 @@ +/****************************************************************************** + * Icinga 2 * + * Copyright (C) 2012-2017 Icinga Development Team (https://www.icinga.com/) * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License * + * as published by the Free Software Foundation; either version 2 * + * of the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software Foundation * + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * + ******************************************************************************/ + +#include "redis/rediswriter.hpp" +#include "icinga/customvarobject.hpp" +#include "base/json.hpp" +#include "base/logger.hpp" +#include "base/serializer.hpp" +#include "base/tlsutility.hpp" +#include "base/initialize.hpp" + +using namespace icinga; + +String RedisWriter::FormatCheckSumBinary(const String& str) +{ + char output[20*2+1]; + for (int i = 0; i < 20; i++) + sprintf(output + 2 * i, "%02x", str[i]); + + return output; +} + +String RedisWriter::CalculateCheckSumString(const String& str, bool binary) +{ + return SHA1(str, binary); +} + +String RedisWriter::CalculateCheckSumGroups(const Array::Ptr& groups, bool binary) +{ + String output; + + ObjectLock olock(groups); + + for (const String& group : groups) { + output += SHA1(group, true); //binary checksum required here + } + + return SHA1(output, binary); +} + +String RedisWriter::CalculateCheckSumProperties(const ConfigObject::Ptr& object, bool binary) +{ + //TODO: consider precision of 6 for double values; use specific config fields for hashing? + return HashValue(object, binary); +} + +String RedisWriter::CalculateCheckSumVars(const ConfigObject::Ptr& object, bool binary) +{ + CustomVarObject::Ptr customVarObject = dynamic_pointer_cast(object); + + if (!customVarObject) + return HashValue(Empty, binary); + + Dictionary::Ptr vars = customVarObject->GetVars(); + + if (!vars) + return HashValue(Empty, binary); + + return HashValue(vars, binary); +} + +String RedisWriter::HashValue(const Value& value, bool binary) +{ + Value temp; + + Type::Ptr type = value.GetReflectionType(); + + if (ConfigObject::TypeInstance->IsAssignableFrom(type)) + temp = Serialize(value, FAConfig); + else + temp = value; + + return SHA1(JsonEncode(temp), binary); +} + +Dictionary::Ptr RedisWriter::SerializeObjectAttrs(const Object::Ptr& object, int fieldType) +{ + Type::Ptr type = object->GetReflectionType(); + + Dictionary::Ptr resultAttrs = new Dictionary(); + + for (int fid = 0; fid < type->GetFieldCount(); fid++) { + Field field = type->GetFieldInfo(fid); + + if ((field.Attributes & fieldType) == 0) + continue; + + Value val = object->GetField(fid); + + /* hide attributes which shouldn't be user-visible */ + if (field.Attributes & FANoUserView) + continue; + + /* hide internal navigation fields */ + if (field.Attributes & FANavigation && !(field.Attributes & (FAConfig | FAState))) + continue; + + Value sval = Serialize(val); + resultAttrs->Set(field.Name, sval); + } + + return resultAttrs; +} + diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp new file mode 100644 index 000000000..70403068b --- /dev/null +++ b/lib/redis/rediswriter.cpp @@ -0,0 +1,336 @@ +/****************************************************************************** + * Icinga 2 * + * Copyright (C) 2012-2017 Icinga Development Team (https://www.icinga.com/) * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License * + * as published by the Free Software Foundation; either version 2 * + * of the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software Foundation * + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * + ******************************************************************************/ + +#include "redis/rediswriter.hpp" +#include "redis/rediswriter.tcpp" +#include "remote/eventqueue.hpp" +#include "base/json.hpp" +#include "base/statsfunction.hpp" + +using namespace icinga; + +REGISTER_TYPE(RedisWriter); + +RedisWriter::RedisWriter(void) + : m_Context(NULL) +{ } + +/** + * Starts the component. + */ +void RedisWriter::Start(bool runtimeCreated) +{ + ObjectImpl::Start(runtimeCreated); + + Log(LogInformation, "RedisWriter") + << "'" << GetName() << "' started."; + + m_ConfigDumpInProgress = false; + + m_WorkQueue.SetExceptionCallback(boost::bind(&RedisWriter::ExceptionHandler, this, _1)); + + m_ReconnectTimer = new Timer(); + m_ReconnectTimer->SetInterval(15); + m_ReconnectTimer->OnTimerExpired.connect(boost::bind(&RedisWriter::ReconnectTimerHandler, this)); + m_ReconnectTimer->Start(); + m_ReconnectTimer->Reschedule(0); + + m_SubscriptionTimer = new Timer(); + m_SubscriptionTimer->SetInterval(15); + m_SubscriptionTimer->OnTimerExpired.connect(boost::bind(&RedisWriter::UpdateSubscriptionsTimerHandler, this)); + m_SubscriptionTimer->Start(); + + m_StatsTimer = new Timer(); + m_StatsTimer->SetInterval(10); + m_StatsTimer->OnTimerExpired.connect(boost::bind(&RedisWriter::PublishStatsTimerHandler, this)); + m_StatsTimer->Start(); + + boost::thread thread(boost::bind(&RedisWriter::HandleEvents, this)); + thread.detach(); +} + +void RedisWriter::ExceptionHandler(boost::exception_ptr exp) +{ + Log(LogCritical, "RedisWriter", "Exception during redis query. Verify that Redis is operational."); + + Log(LogDebug, "RedisWriter") + << "Exception during redis operation: " << DiagnosticInformation(exp); + + if (m_Context) { + redisFree(m_Context); + m_Context = NULL; + } +} + +void RedisWriter::ReconnectTimerHandler(void) +{ + m_WorkQueue.Enqueue(boost::bind(&RedisWriter::TryToReconnect, this)); +} + +void RedisWriter::TryToReconnect(void) +{ + AssertOnWorkQueue(); + + if (m_Context) + return; + + String path = GetPath(); + String host = GetHost(); + + Log(LogInformation, "RedisWriter", "Trying to connect to redis server"); + + if (path.IsEmpty()) + m_Context = redisConnect(host.CStr(), GetPort()); + else + m_Context = redisConnectUnix(path.CStr()); + + if (!m_Context || m_Context->err) { + if (!m_Context) { + Log(LogWarning, "RedisWriter", "Cannot allocate redis context."); + } else { + Log(LogWarning, "RedisWriter", "Connection error: ") + << m_Context->errstr; + } + + if (m_Context) { + redisFree(m_Context); + m_Context = NULL; + } + + return; + } + + String password = GetPassword(); + + /* TODO: exception is fired but terminates reconnect silently. + * Error case: Password does not match, or even: "Client sent AUTH, but no password is set" which also results in an error. + */ + if (!password.IsEmpty()) + ExecuteQuery({ "AUTH", password }); + + int dbIndex = GetDbIndex(); + + if (dbIndex != 0) + ExecuteQuery({ "SELECT", Convert::ToString(dbIndex) }); + + /* Config dump */ + m_ConfigDumpInProgress = true; + + UpdateAllConfigObjects(); + + m_ConfigDumpInProgress = false; +} + +void RedisWriter::UpdateSubscriptionsTimerHandler(void) +{ + m_WorkQueue.Enqueue(boost::bind(&RedisWriter::UpdateSubscriptions, this)); +} + +void RedisWriter::UpdateSubscriptions(void) +{ + AssertOnWorkQueue(); + + if (!m_Context) + return; + + Log(LogInformation, "RedisWriter", "Updating Redis subscriptions"); + + std::map subscriptions; + long long cursor = 0; + + do { + boost::shared_ptr reply = ExecuteQuery({ "SCAN", Convert::ToString(cursor), "MATCH", "icinga:subscription:*", "COUNT", "1000" }); + + VERIFY(reply->type == REDIS_REPLY_ARRAY); + VERIFY(reply->elements % 2 == 0); + + redisReply *cursorReply = reply->element[0]; + cursor = Convert::ToLong(cursorReply->str); + + redisReply *keysReply = reply->element[1]; + + for (size_t i = 0; i < keysReply->elements; i++) { + redisReply *keyReply = keysReply->element[i]; + VERIFY(keyReply->type == REDIS_REPLY_STRING); + + boost::shared_ptr vreply = ExecuteQuery({ "GET", keyReply->str }); + + subscriptions[keyReply->str] = vreply->str; + } + } while (cursor != 0); + + m_Subscriptions.clear(); + + for (const std::pair& kv : subscriptions) { + const String& key = kv.first.SubStr(20); /* removes the "icinga:subscription: prefix */ + const String& value = kv.second; + + try { + Dictionary::Ptr subscriptionInfo = JsonDecode(value); + + Log(LogInformation, "RedisWriter") + << "Subscriber Info - Key: " << key << " Value: " << Value(subscriptionInfo); + + RedisSubscriptionInfo rsi; + + Array::Ptr types = subscriptionInfo->Get("types"); + + if (types) + rsi.EventTypes = types->ToSet(); + + m_Subscriptions[key] = rsi; + } catch (const std::exception& ex) { + Log(LogWarning, "RedisWriter") + << "Invalid Redis subscriber info for subscriber '" << key << "': " << DiagnosticInformation(ex); + + continue; + } + //TODO + } + + Log(LogInformation, "RedisWriter") + << "Current Redis event subscriptions: " << m_Subscriptions.size(); +} + +void RedisWriter::PublishStatsTimerHandler(void) +{ + m_WorkQueue.Enqueue(boost::bind(&RedisWriter::PublishStats, this)); +} + +void RedisWriter::PublishStats(void) +{ + AssertOnWorkQueue(); + + if (!m_Context) + return; + + //TODO: Figure out if more stats can be useful here. + StatsFunction::Ptr func = StatsFunctionRegistry::GetInstance()->GetItem("CIB"); + Dictionary::Ptr status = new Dictionary(); + Array::Ptr perfdata = new Array(); + func->Invoke(status, perfdata); + String jsonStats = JsonEncode(status); + + ExecuteQuery({ "PUBLISH", "icinga:stats", jsonStats }); +} + +void RedisWriter::HandleEvents(void) +{ + String queueName = Utility::NewUniqueID(); + EventQueue::Ptr queue = new EventQueue(queueName); + EventQueue::Register(queueName, queue); + + std::set types; + types.insert("CheckResult"); + types.insert("StateChange"); + types.insert("Notification"); + types.insert("AcknowledgementSet"); + types.insert("AcknowledgementCleared"); + types.insert("CommentAdded"); + types.insert("CommentRemoved"); + types.insert("DowntimeAdded"); + types.insert("DowntimeRemoved"); + types.insert("DowntimeStarted"); + types.insert("DowntimeTriggered"); + + queue->SetTypes(types); + + queue->AddClient(this); + + for (;;) { + Dictionary::Ptr event = queue->WaitForEvent(this); + + if (!event) + continue; + + m_WorkQueue.Enqueue(boost::bind(&RedisWriter::HandleEvent, this, event)); + } + + queue->RemoveClient(this); + EventQueue::UnregisterIfUnused(queueName, queue); +} + +void RedisWriter::HandleEvent(const Dictionary::Ptr& event) +{ + AssertOnWorkQueue(); + + if (!m_Context) + return; + + for (const std::pair& kv : m_Subscriptions) { + const auto& name = kv.first; + const auto& rsi = kv.second; + + if (rsi.EventTypes.find(event->Get("type")) == rsi.EventTypes.end()) + continue; + + String body = JsonEncode(event); + + ExecuteQuery({ "LPUSH", "icinga:event:" + name, body }); + } +} + +void RedisWriter::Stop(bool runtimeRemoved) +{ + Log(LogInformation, "RedisWriter") + << "'" << GetName() << "' stopped."; + + ObjectImpl::Stop(runtimeRemoved); +} + +void RedisWriter::AssertOnWorkQueue(void) +{ + ASSERT(m_WorkQueue.IsWorkerThread()); +} + +boost::shared_ptr RedisWriter::ExecuteQuery(const std::vector& query) +{ + const char **argv; + size_t *argvlen; + + argv = new const char *[query.size()]; + argvlen = new size_t[query.size()]; + + for (std::vector::size_type i = 0; i < query.size(); i++) { + argv[i] = query[i].CStr(); + argvlen[i] = query[i].GetLength(); + } + + redisReply *reply = reinterpret_cast(redisCommandArgv(m_Context, query.size(), argv, argvlen)); + + delete [] argv; + delete [] argvlen; + + if (reply->type == REDIS_REPLY_ERROR) { + Log(LogCritical, "RedisWriter") + << "Redis query failed: " << reply->str; + + String msg = reply->str; + + freeReplyObject(reply); + + BOOST_THROW_EXCEPTION( + redis_error() + << errinfo_message(msg) + << errinfo_redis_query(Utility::Join(Array::FromVector(query), ' ', false)) + ); + } + + return boost::shared_ptr(reply); +} diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp new file mode 100644 index 000000000..07f56a15f --- /dev/null +++ b/lib/redis/rediswriter.hpp @@ -0,0 +1,107 @@ +/****************************************************************************** + * Icinga 2 * + * Copyright (C) 2012-2017 Icinga Development Team (https://www.icinga.com/) * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License * + * as published by the Free Software Foundation; either version 2 * + * of the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software Foundation * + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * + ******************************************************************************/ + +#ifndef REDISWRITER_H +#define REDISWRITER_H + +#include "redis/rediswriter.thpp" +#include "remote/messageorigin.hpp" +#include "base/timer.hpp" +#include "base/workqueue.hpp" +#include + +namespace icinga +{ + +struct RedisSubscriptionInfo +{ + std::set EventTypes; +}; + +/** + * @ingroup redis + */ +class RedisWriter : public ObjectImpl +{ +public: + DECLARE_OBJECT(RedisWriter); + DECLARE_OBJECTNAME(RedisWriter); + + RedisWriter(void); + + static void ConfigStaticInitialize(void); + + virtual void Start(bool runtimeCreated) override; + virtual void Stop(bool runtimeRemoved) override; + +private: + void ReconnectTimerHandler(void); + void TryToReconnect(void); + void HandleEvents(void); + void HandleEvent(const Dictionary::Ptr& event); + + void UpdateSubscriptionsTimerHandler(void); + void UpdateSubscriptions(void); + void PublishStatsTimerHandler(void); + void PublishStats(void); + + /* config & status dump */ + void UpdateAllConfigObjects(void); + void SendConfigUpdate(const ConfigObject::Ptr& object, const String& typeName, bool runtimeUpdate = false); + void SendConfigDelete(const ConfigObject::Ptr& object, const String& typeName); + void SendStatusUpdate(const ConfigObject::Ptr& object, const String& typeName); + + /* utilities */ + static String FormatCheckSumBinary(const String& str); + + static String CalculateCheckSumString(const String& str, bool binary = false); + static String CalculateCheckSumGroups(const Array::Ptr& groups, bool binary = false); + static String CalculateCheckSumProperties(const ConfigObject::Ptr& object, bool binary = false); + static String CalculateCheckSumVars(const ConfigObject::Ptr& object, bool binary = false); + + static String HashValue(const Value& value, bool binary = false); + static Dictionary::Ptr SerializeObjectAttrs(const Object::Ptr& object, int fieldType); + + static void StateChangedHandler(const ConfigObject::Ptr& object); + static void VarsChangedHandler(const ConfigObject::Ptr& object); + static void VersionChangedHandler(const ConfigObject::Ptr& object); + + void AssertOnWorkQueue(void); + + void ExceptionHandler(boost::exception_ptr exp); + + boost::shared_ptr ExecuteQuery(const std::vector& query); + + Timer::Ptr m_StatsTimer; + Timer::Ptr m_ReconnectTimer; + Timer::Ptr m_SubscriptionTimer; + WorkQueue m_WorkQueue; + redisContext *m_Context; + std::map m_Subscriptions; + bool m_ConfigDumpInProgress; +}; + +struct redis_error : virtual std::exception, virtual boost::exception { }; + +struct errinfo_redis_query_; +typedef boost::error_info errinfo_redis_query; + +} + +#endif /* REDISWRITER_H */ diff --git a/lib/redis/rediswriter.ti b/lib/redis/rediswriter.ti new file mode 100644 index 000000000..0aa76c9c0 --- /dev/null +++ b/lib/redis/rediswriter.ti @@ -0,0 +1,40 @@ +/****************************************************************************** + * Icinga 2 * + * Copyright (C) 2012-2017 Icinga Development Team (https://www.icinga.com/) * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License * + * as published by the Free Software Foundation; either version 2 * + * of the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software Foundation * + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * + ******************************************************************************/ + +#include "base/configobject.hpp" + +library demo; + +namespace icinga +{ + +class RedisWriter : ConfigObject +{ + [config] String host { + default {{{ return "127.0.0.1"; }}} + }; + [config] int port { + default {{{ return 6379; }}} + }; + [config] String path; + [config] String password; + [config] int db_index; +}; + +} diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index fea750f1d..fa5f5bfce 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -6,4 +6,8 @@ if(UNIX OR CYGWIN) add_subdirectory(execvpe) endif() +if(ICINGA2_WITH_REDIS) + add_subdirectory(hiredis) +endif() + add_subdirectory(socketpair) diff --git a/third-party/hiredis/.gitignore b/third-party/hiredis/.gitignore new file mode 100644 index 000000000..c44b5c537 --- /dev/null +++ b/third-party/hiredis/.gitignore @@ -0,0 +1,7 @@ +/hiredis-test +/examples/hiredis-example* +/*.o +/*.so +/*.dylib +/*.a +/*.pc diff --git a/third-party/hiredis/.travis.yml b/third-party/hiredis/.travis.yml new file mode 100644 index 000000000..1e1ce3006 --- /dev/null +++ b/third-party/hiredis/.travis.yml @@ -0,0 +1,24 @@ +language: c +sudo: false +compiler: + - gcc + - clang + +addons: + apt: + packages: + - libc6-dbg + - libc6-dev + - libc6:i386 + - libc6-dev-i386 + - libc6-dbg:i386 + - gcc-multilib + - valgrind + +env: + - CFLAGS="-Werror" + - PRE="valgrind --track-origins=yes --leak-check=full" + - TARGET="32bit" TARGET_VARS="32bit-vars" CFLAGS="-Werror" + - TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" + +script: make $TARGET CFLAGS="$CFLAGS" && make check PRE="$PRE" && make $TARGET_VARS hiredis-example diff --git a/third-party/hiredis/CHANGELOG.md b/third-party/hiredis/CHANGELOG.md new file mode 100644 index 000000000..a5015c5da --- /dev/null +++ b/third-party/hiredis/CHANGELOG.md @@ -0,0 +1,110 @@ +### 0.13.3 (2015-09-16) + +* Revert "Clear `REDIS_CONNECTED` flag when connection is closed". +* Make tests pass on FreeBSD (Thanks, Giacomo Olgeni) + + +If the `REDIS_CONNECTED` flag is cleared, +the async onDisconnect callback function will never be called. +This causes problems as the disconnect is never reported back to the user. + +### 0.13.2 (2015-08-25) + +* Prevent crash on pending replies in async code (Thanks, @switch-st) +* Clear `REDIS_CONNECTED` flag when connection is closed (Thanks, Jerry Jacobs) +* Add MacOS X addapter (Thanks, @dizzus) +* Add Qt adapter (Thanks, Pietro Cerutti) +* Add Ivykis adapter (Thanks, Gergely Nagy) + +All adapters are provided as is and are only tested where possible. + +### 0.13.1 (2015-05-03) + +This is a bug fix release. +The new `reconnect` method introduced new struct members, which clashed with pre-defined names in pre-C99 code. +Another commit forced C99 compilation just to make it work, but of course this is not desirable for outside projects. +Other non-C99 code can now use hiredis as usual again. +Sorry for the inconvenience. + +* Fix memory leak in async reply handling (Salvatore Sanfilippo) +* Rename struct member to avoid name clash with pre-c99 code (Alex Balashov, ncopa) + +### 0.13.0 (2015-04-16) + +This release adds a minimal Windows compatibility layer. +The parser, standalone since v0.12.0, can now be compiled on Windows +(and thus used in other client libraries as well) + +* Windows compatibility layer for parser code (tzickel) +* Properly escape data printed to PKGCONF file (Dan Skorupski) +* Fix tests when assert() undefined (Keith Bennett, Matt Stancliff) +* Implement a reconnect method for the client context, this changes the structure of `redisContext` (Aaron Bedra) + +### 0.12.1 (2015-01-26) + +* Fix `make install`: DESTDIR support, install all required files, install PKGCONF in proper location +* Fix `make test` as 32 bit build on 64 bit platform + +### 0.12.0 (2015-01-22) + +* Add optional KeepAlive support + +* Try again on EINTR errors + +* Add libuv adapter + +* Add IPv6 support + +* Remove possiblity of multiple close on same fd + +* Add ability to bind source address on connect + +* Add redisConnectFd() and redisFreeKeepFd() + +* Fix getaddrinfo() memory leak + +* Free string if it is unused (fixes memory leak) + +* Improve redisAppendCommandArgv performance 2.5x + +* Add support for SO_REUSEADDR + +* Fix redisvFormatCommand format parsing + +* Add GLib 2.0 adapter + +* Refactor reading code into read.c + +* Fix errno error buffers to not clobber errors + +* Generate pkgconf during build + +* Silence _BSD_SOURCE warnings + +* Improve digit counting for multibulk creation + + +### 0.11.0 + +* Increase the maximum multi-bulk reply depth to 7. + +* Increase the read buffer size from 2k to 16k. + +* Use poll(2) instead of select(2) to support large fds (>= 1024). + +### 0.10.1 + +* Makefile overhaul. Important to check out if you override one or more + variables using environment variables or via arguments to the "make" tool. + +* Issue #45: Fix potential memory leak for a multi bulk reply with 0 elements + being created by the default reply object functions. + +* Issue #43: Don't crash in an asynchronous context when Redis returns an error + reply after the connection has been made (this happens when the maximum + number of connections is reached). + +### 0.10.0 + +* See commit log. + diff --git a/third-party/hiredis/CMakeLists.txt b/third-party/hiredis/CMakeLists.txt new file mode 100644 index 000000000..afda9579f --- /dev/null +++ b/third-party/hiredis/CMakeLists.txt @@ -0,0 +1,42 @@ +# Icinga 2 +# Copyright (C) 2012-2017 Icinga Development Team (https://www.icinga.com/) +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. + +add_library(hiredis SHARED net.c net.h hiredis.c hiredis.h sds.c sds.h async.c async.h read.c read.h) + +if(HAVE_VISIBILITY_HIDDEN) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=default") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=default") +endif() + +set_target_properties ( + hiredis PROPERTIES + FOLDER Lib + VERSION ${SPEC_VERSION} +) + +install( + TARGETS hiredis + RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}/icinga2 +) + +if(APPLE) + install( + TARGETS hiredis + LIBRARY DESTINATION ${CMAKE_INSTALL_BINDIR}/icinga-studio.app/Contents + ) +endif() diff --git a/third-party/hiredis/COPYING b/third-party/hiredis/COPYING new file mode 100644 index 000000000..a5fc97395 --- /dev/null +++ b/third-party/hiredis/COPYING @@ -0,0 +1,29 @@ +Copyright (c) 2009-2011, Salvatore Sanfilippo +Copyright (c) 2010-2011, Pieter Noordhuis + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of Redis nor the names of its contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/third-party/hiredis/Makefile b/third-party/hiredis/Makefile new file mode 100644 index 000000000..cff2a84ce --- /dev/null +++ b/third-party/hiredis/Makefile @@ -0,0 +1,217 @@ +# Hiredis Makefile +# Copyright (C) 2010-2011 Salvatore Sanfilippo +# Copyright (C) 2010-2011 Pieter Noordhuis +# This file is released under the BSD license, see the COPYING file + +OBJ=net.o hiredis.o sds.o async.o read.o +EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib +TESTS=hiredis-test +LIBNAME=libhiredis +PKGCONFNAME=hiredis.pc + +HIREDIS_MAJOR=$(shell grep HIREDIS_MAJOR hiredis.h | awk '{print $$3}') +HIREDIS_MINOR=$(shell grep HIREDIS_MINOR hiredis.h | awk '{print $$3}') +HIREDIS_PATCH=$(shell grep HIREDIS_PATCH hiredis.h | awk '{print $$3}') +HIREDIS_SONAME=$(shell grep HIREDIS_SONAME hiredis.h | awk '{print $$3}') + +# Installation related variables and target +PREFIX?=/usr/local +INCLUDE_PATH?=include/hiredis +LIBRARY_PATH?=lib +PKGCONF_PATH?=pkgconfig +INSTALL_INCLUDE_PATH= $(DESTDIR)$(PREFIX)/$(INCLUDE_PATH) +INSTALL_LIBRARY_PATH= $(DESTDIR)$(PREFIX)/$(LIBRARY_PATH) +INSTALL_PKGCONF_PATH= $(INSTALL_LIBRARY_PATH)/$(PKGCONF_PATH) + +# redis-server configuration used for testing +REDIS_PORT=56379 +REDIS_SERVER=redis-server +define REDIS_TEST_CONFIG + daemonize yes + pidfile /tmp/hiredis-test-redis.pid + port $(REDIS_PORT) + bind 127.0.0.1 + unixsocket /tmp/hiredis-test-redis.sock +endef +export REDIS_TEST_CONFIG + +# Fallback to gcc when $CC is not in $PATH. +CC:=$(shell sh -c 'type $(CC) >/dev/null 2>/dev/null && echo $(CC) || echo gcc') +CXX:=$(shell sh -c 'type $(CXX) >/dev/null 2>/dev/null && echo $(CXX) || echo g++') +OPTIMIZATION?=-O3 +WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings +DEBUG?= -g -ggdb +REAL_CFLAGS=$(OPTIMIZATION) -fPIC $(CFLAGS) $(WARNINGS) $(DEBUG) $(ARCH) +REAL_LDFLAGS=$(LDFLAGS) $(ARCH) + +DYLIBSUFFIX=so +STLIBSUFFIX=a +DYLIB_MINOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME) +DYLIB_MAJOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR) +DYLIBNAME=$(LIBNAME).$(DYLIBSUFFIX) +DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) +STLIBNAME=$(LIBNAME).$(STLIBSUFFIX) +STLIB_MAKE_CMD=ar rcs $(STLIBNAME) + +# Platform-specific overrides +uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') +ifeq ($(uname_S),SunOS) + REAL_LDFLAGS+= -ldl -lnsl -lsocket + DYLIB_MAKE_CMD=$(CC) -G -o $(DYLIBNAME) -h $(DYLIB_MINOR_NAME) $(LDFLAGS) + INSTALL= cp -r +endif +ifeq ($(uname_S),Darwin) + DYLIBSUFFIX=dylib + DYLIB_MINOR_NAME=$(LIBNAME).$(HIREDIS_SONAME).$(DYLIBSUFFIX) + DYLIB_MAKE_CMD=$(CC) -shared -Wl,-install_name,$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) +endif + +all: $(DYLIBNAME) $(STLIBNAME) hiredis-test $(PKGCONFNAME) + +# Deps (use make dep to generate this) +async.o: async.c fmacros.h async.h hiredis.h read.h sds.h net.h dict.c dict.h +dict.o: dict.c fmacros.h dict.h +hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h +net.o: net.c fmacros.h net.h hiredis.h read.h sds.h +read.o: read.c fmacros.h read.h sds.h +sds.o: sds.c sds.h +test.o: test.c fmacros.h hiredis.h read.h sds.h + +$(DYLIBNAME): $(OBJ) + $(DYLIB_MAKE_CMD) $(OBJ) + +$(STLIBNAME): $(OBJ) + $(STLIB_MAKE_CMD) $(OBJ) + +dynamic: $(DYLIBNAME) +static: $(STLIBNAME) + +# Binaries: +hiredis-example-libevent: examples/example-libevent.c adapters/libevent.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -levent $(STLIBNAME) + +hiredis-example-libev: examples/example-libev.c adapters/libev.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -lev $(STLIBNAME) + +hiredis-example-glib: examples/example-glib.c adapters/glib.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) $(shell pkg-config --cflags --libs glib-2.0) -I. $< $(STLIBNAME) + +hiredis-example-ivykis: examples/example-ivykis.c adapters/ivykis.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -livykis $(STLIBNAME) + +hiredis-example-macosx: examples/example-macosx.c adapters/macosx.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) + +ifndef AE_DIR +hiredis-example-ae: + @echo "Please specify AE_DIR (e.g. /src)" + @false +else +hiredis-example-ae: examples/example-ae.c adapters/ae.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(AE_DIR) $< $(AE_DIR)/ae.o $(AE_DIR)/zmalloc.o $(AE_DIR)/../deps/jemalloc/lib/libjemalloc.a -pthread $(STLIBNAME) +endif + +ifndef LIBUV_DIR +hiredis-example-libuv: + @echo "Please specify LIBUV_DIR (e.g. ../libuv/)" + @false +else +hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) +endif + +ifeq ($(and $(QT_MOC),$(QT_INCLUDE_DIR),$(QT_LIBRARY_DIR)),) +hiredis-example-qt: + @echo "Please specify QT_MOC, QT_INCLUDE_DIR AND QT_LIBRARY_DIR" + @false +else +hiredis-example-qt: examples/example-qt.cpp adapters/qt.h $(STLIBNAME) + $(QT_MOC) adapters/qt.h -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore | \ + $(CXX) -x c++ -o qt-adapter-moc.o -c - $(REAL_CFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore + $(QT_MOC) examples/example-qt.h -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore | \ + $(CXX) -x c++ -o qt-example-moc.o -c - $(REAL_CFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore + $(CXX) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore -L$(QT_LIBRARY_DIR) qt-adapter-moc.o qt-example-moc.o $< -pthread $(STLIBNAME) -lQtCore +endif + +hiredis-example: examples/example.c $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< $(STLIBNAME) + +examples: $(EXAMPLES) + +hiredis-test: test.o $(STLIBNAME) + +hiredis-%: %.o $(STLIBNAME) + $(CC) $(REAL_CFLAGS) -o $@ $(REAL_LDFLAGS) $< $(STLIBNAME) + +test: hiredis-test + ./hiredis-test + +check: hiredis-test + @echo "$$REDIS_TEST_CONFIG" | $(REDIS_SERVER) - + $(PRE) ./hiredis-test -h 127.0.0.1 -p $(REDIS_PORT) -s /tmp/hiredis-test-redis.sock || \ + ( kill `cat /tmp/hiredis-test-redis.pid` && false ) + kill `cat /tmp/hiredis-test-redis.pid` + +.c.o: + $(CC) -std=c99 -pedantic -c $(REAL_CFLAGS) $< + +clean: + rm -rf $(DYLIBNAME) $(STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov + +dep: + $(CC) -MM *.c + +ifeq ($(uname_S),SunOS) + INSTALL?= cp -r +endif + +INSTALL?= cp -a + +$(PKGCONFNAME): hiredis.h + @echo "Generating $@ for pkgconfig..." + @echo prefix=$(PREFIX) > $@ + @echo exec_prefix=\$${prefix} >> $@ + @echo libdir=$(PREFIX)/$(LIBRARY_PATH) >> $@ + @echo includedir=$(PREFIX)/$(INCLUDE_PATH) >> $@ + @echo >> $@ + @echo Name: hiredis >> $@ + @echo Description: Minimalistic C client library for Redis. >> $@ + @echo Version: $(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(HIREDIS_PATCH) >> $@ + @echo Libs: -L\$${libdir} -lhiredis >> $@ + @echo Cflags: -I\$${includedir} -D_FILE_OFFSET_BITS=64 >> $@ + +install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) + mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_LIBRARY_PATH) + $(INSTALL) hiredis.h async.h read.h sds.h adapters $(INSTALL_INCLUDE_PATH) + $(INSTALL) $(DYLIBNAME) $(INSTALL_LIBRARY_PATH)/$(DYLIB_MINOR_NAME) + cd $(INSTALL_LIBRARY_PATH) && ln -sf $(DYLIB_MINOR_NAME) $(DYLIBNAME) + $(INSTALL) $(STLIBNAME) $(INSTALL_LIBRARY_PATH) + mkdir -p $(INSTALL_PKGCONF_PATH) + $(INSTALL) $(PKGCONFNAME) $(INSTALL_PKGCONF_PATH) + +32bit: + @echo "" + @echo "WARNING: if this fails under Linux you probably need to install libc6-dev-i386" + @echo "" + $(MAKE) CFLAGS="-m32" LDFLAGS="-m32" + +32bit-vars: + $(eval CFLAGS=-m32) + $(eval LDFLAGS=-m32) + +gprof: + $(MAKE) CFLAGS="-pg" LDFLAGS="-pg" + +gcov: + $(MAKE) CFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS="-fprofile-arcs" + +coverage: gcov + make check + mkdir -p tmp/lcov + lcov -d . -c -o tmp/lcov/hiredis.info + genhtml --legend -o tmp/lcov/report tmp/lcov/hiredis.info + +noopt: + $(MAKE) OPTIMIZATION="" + +.PHONY: all test check clean dep install 32bit 32bit-vars gprof gcov noopt diff --git a/third-party/hiredis/README.md b/third-party/hiredis/README.md new file mode 100644 index 000000000..4f1a58d2a --- /dev/null +++ b/third-party/hiredis/README.md @@ -0,0 +1,392 @@ +[![Build Status](https://travis-ci.org/redis/hiredis.png)](https://travis-ci.org/redis/hiredis) + +# HIREDIS + +Hiredis is a minimalistic C client library for the [Redis](http://redis.io/) database. + +It is minimalistic because it just adds minimal support for the protocol, but +at the same time it uses a high level printf-alike API in order to make it +much higher level than otherwise suggested by its minimal code base and the +lack of explicit bindings for every Redis command. + +Apart from supporting sending commands and receiving replies, it comes with +a reply parser that is decoupled from the I/O layer. It +is a stream parser designed for easy reusability, which can for instance be used +in higher level language bindings for efficient reply parsing. + +Hiredis only supports the binary-safe Redis protocol, so you can use it with any +Redis version >= 1.2.0. + +The library comes with multiple APIs. There is the +*synchronous API*, the *asynchronous API* and the *reply parsing API*. + +## UPGRADING + +Version 0.9.0 is a major overhaul of hiredis in every aspect. However, upgrading existing +code using hiredis should not be a big pain. The key thing to keep in mind when +upgrading is that hiredis >= 0.9.0 uses a `redisContext*` to keep state, in contrast to +the stateless 0.0.1 that only has a file descriptor to work with. + +## Synchronous API + +To consume the synchronous API, there are only a few function calls that need to be introduced: + +```c +redisContext *redisConnect(const char *ip, int port); +void *redisCommand(redisContext *c, const char *format, ...); +void freeReplyObject(void *reply); +``` + +### Connecting + +The function `redisConnect` is used to create a so-called `redisContext`. The +context is where Hiredis holds state for a connection. The `redisContext` +struct has an integer `err` field that is non-zero when the connection is in +an error state. The field `errstr` will contain a string with a description of +the error. More information on errors can be found in the **Errors** section. +After trying to connect to Redis using `redisConnect` you should +check the `err` field to see if establishing the connection was successful: +```c +redisContext *c = redisConnect("127.0.0.1", 6379); +if (c != NULL && c->err) { + printf("Error: %s\n", c->errstr); + // handle error +} +``` + +### Sending commands + +There are several ways to issue commands to Redis. The first that will be introduced is +`redisCommand`. This function takes a format similar to printf. In the simplest form, +it is used like this: +```c +reply = redisCommand(context, "SET foo bar"); +``` + +The specifier `%s` interpolates a string in the command, and uses `strlen` to +determine the length of the string: +```c +reply = redisCommand(context, "SET foo %s", value); +``` +When you need to pass binary safe strings in a command, the `%b` specifier can be +used. Together with a pointer to the string, it requires a `size_t` length argument +of the string: +```c +reply = redisCommand(context, "SET foo %b", value, (size_t) valuelen); +``` +Internally, Hiredis splits the command in different arguments and will +convert it to the protocol used to communicate with Redis. +One or more spaces separates arguments, so you can use the specifiers +anywhere in an argument: +```c +reply = redisCommand(context, "SET key:%s %s", myid, value); +``` + +### Using replies + +The return value of `redisCommand` holds a reply when the command was +successfully executed. When an error occurs, the return value is `NULL` and +the `err` field in the context will be set (see section on **Errors**). +Once an error is returned the context cannot be reused and you should set up +a new connection. + +The standard replies that `redisCommand` are of the type `redisReply`. The +`type` field in the `redisReply` should be used to test what kind of reply +was received: + +* **`REDIS_REPLY_STATUS`**: + * The command replied with a status reply. The status string can be accessed using `reply->str`. + The length of this string can be accessed using `reply->len`. + +* **`REDIS_REPLY_ERROR`**: + * The command replied with an error. The error string can be accessed identical to `REDIS_REPLY_STATUS`. + +* **`REDIS_REPLY_INTEGER`**: + * The command replied with an integer. The integer value can be accessed using the + `reply->integer` field of type `long long`. + +* **`REDIS_REPLY_NIL`**: + * The command replied with a **nil** object. There is no data to access. + +* **`REDIS_REPLY_STRING`**: + * A bulk (string) reply. The value of the reply can be accessed using `reply->str`. + The length of this string can be accessed using `reply->len`. + +* **`REDIS_REPLY_ARRAY`**: + * A multi bulk reply. The number of elements in the multi bulk reply is stored in + `reply->elements`. Every element in the multi bulk reply is a `redisReply` object as well + and can be accessed via `reply->element[..index..]`. + Redis may reply with nested arrays but this is fully supported. + +Replies should be freed using the `freeReplyObject()` function. +Note that this function will take care of freeing sub-reply objects +contained in arrays and nested arrays, so there is no need for the user to +free the sub replies (it is actually harmful and will corrupt the memory). + +**Important:** the current version of hiredis (0.10.0) frees replies when the +asynchronous API is used. This means you should not call `freeReplyObject` when +you use this API. The reply is cleaned up by hiredis _after_ the callback +returns. This behavior will probably change in future releases, so make sure to +keep an eye on the changelog when upgrading (see issue #39). + +### Cleaning up + +To disconnect and free the context the following function can be used: +```c +void redisFree(redisContext *c); +``` +This function immediately closes the socket and then frees the allocations done in +creating the context. + +### Sending commands (cont'd) + +Together with `redisCommand`, the function `redisCommandArgv` can be used to issue commands. +It has the following prototype: +```c +void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen); +``` +It takes the number of arguments `argc`, an array of strings `argv` and the lengths of the +arguments `argvlen`. For convenience, `argvlen` may be set to `NULL` and the function will +use `strlen(3)` on every argument to determine its length. Obviously, when any of the arguments +need to be binary safe, the entire array of lengths `argvlen` should be provided. + +The return value has the same semantic as `redisCommand`. + +### Pipelining + +To explain how Hiredis supports pipelining in a blocking connection, there needs to be +understanding of the internal execution flow. + +When any of the functions in the `redisCommand` family is called, Hiredis first formats the +command according to the Redis protocol. The formatted command is then put in the output buffer +of the context. This output buffer is dynamic, so it can hold any number of commands. +After the command is put in the output buffer, `redisGetReply` is called. This function has the +following two execution paths: + +1. The input buffer is non-empty: + * Try to parse a single reply from the input buffer and return it + * If no reply could be parsed, continue at *2* +2. The input buffer is empty: + * Write the **entire** output buffer to the socket + * Read from the socket until a single reply could be parsed + +The function `redisGetReply` is exported as part of the Hiredis API and can be used when a reply +is expected on the socket. To pipeline commands, the only things that needs to be done is +filling up the output buffer. For this cause, two commands can be used that are identical +to the `redisCommand` family, apart from not returning a reply: +```c +void redisAppendCommand(redisContext *c, const char *format, ...); +void redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen); +``` +After calling either function one or more times, `redisGetReply` can be used to receive the +subsequent replies. The return value for this function is either `REDIS_OK` or `REDIS_ERR`, where +the latter means an error occurred while reading a reply. Just as with the other commands, +the `err` field in the context can be used to find out what the cause of this error is. + +The following examples shows a simple pipeline (resulting in only a single call to `write(2)` and +a single call to `read(2)`): +```c +redisReply *reply; +redisAppendCommand(context,"SET foo bar"); +redisAppendCommand(context,"GET foo"); +redisGetReply(context,&reply); // reply for SET +freeReplyObject(reply); +redisGetReply(context,&reply); // reply for GET +freeReplyObject(reply); +``` +This API can also be used to implement a blocking subscriber: +```c +reply = redisCommand(context,"SUBSCRIBE foo"); +freeReplyObject(reply); +while(redisGetReply(context,&reply) == REDIS_OK) { + // consume message + freeReplyObject(reply); +} +``` +### Errors + +When a function call is not successful, depending on the function either `NULL` or `REDIS_ERR` is +returned. The `err` field inside the context will be non-zero and set to one of the +following constants: + +* **`REDIS_ERR_IO`**: + There was an I/O error while creating the connection, trying to write + to the socket or read from the socket. If you included `errno.h` in your + application, you can use the global `errno` variable to find out what is + wrong. + +* **`REDIS_ERR_EOF`**: + The server closed the connection which resulted in an empty read. + +* **`REDIS_ERR_PROTOCOL`**: + There was an error while parsing the protocol. + +* **`REDIS_ERR_OTHER`**: + Any other error. Currently, it is only used when a specified hostname to connect + to cannot be resolved. + +In every case, the `errstr` field in the context will be set to hold a string representation +of the error. + +## Asynchronous API + +Hiredis comes with an asynchronous API that works easily with any event library. +Examples are bundled that show using Hiredis with [libev](http://software.schmorp.de/pkg/libev.html) +and [libevent](http://monkey.org/~provos/libevent/). + +### Connecting + +The function `redisAsyncConnect` can be used to establish a non-blocking connection to +Redis. It returns a pointer to the newly created `redisAsyncContext` struct. The `err` field +should be checked after creation to see if there were errors creating the connection. +Because the connection that will be created is non-blocking, the kernel is not able to +instantly return if the specified host and port is able to accept a connection. +```c +redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); +if (c->err) { + printf("Error: %s\n", c->errstr); + // handle error +} +``` + +The asynchronous context can hold a disconnect callback function that is called when the +connection is disconnected (either because of an error or per user request). This function should +have the following prototype: +```c +void(const redisAsyncContext *c, int status); +``` +On a disconnect, the `status` argument is set to `REDIS_OK` when disconnection was initiated by the +user, or `REDIS_ERR` when the disconnection was caused by an error. When it is `REDIS_ERR`, the `err` +field in the context can be accessed to find out the cause of the error. + +The context object is always freed after the disconnect callback fired. When a reconnect is needed, +the disconnect callback is a good point to do so. + +Setting the disconnect callback can only be done once per context. For subsequent calls it will +return `REDIS_ERR`. The function to set the disconnect callback has the following prototype: +```c +int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); +``` +### Sending commands and their callbacks + +In an asynchronous context, commands are automatically pipelined due to the nature of an event loop. +Therefore, unlike the synchronous API, there is only a single way to send commands. +Because commands are sent to Redis asynchronously, issuing a command requires a callback function +that is called when the reply is received. Reply callbacks should have the following prototype: +```c +void(redisAsyncContext *c, void *reply, void *privdata); +``` +The `privdata` argument can be used to curry arbitrary data to the callback from the point where +the command is initially queued for execution. + +The functions that can be used to issue commands in an asynchronous context are: +```c +int redisAsyncCommand( + redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, + const char *format, ...); +int redisAsyncCommandArgv( + redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, + int argc, const char **argv, const size_t *argvlen); +``` +Both functions work like their blocking counterparts. The return value is `REDIS_OK` when the command +was successfully added to the output buffer and `REDIS_ERR` otherwise. Example: when the connection +is being disconnected per user-request, no new commands may be added to the output buffer and `REDIS_ERR` is +returned on calls to the `redisAsyncCommand` family. + +If the reply for a command with a `NULL` callback is read, it is immediately freed. When the callback +for a command is non-`NULL`, the memory is freed immediately following the callback: the reply is only +valid for the duration of the callback. + +All pending callbacks are called with a `NULL` reply when the context encountered an error. + +### Disconnecting + +An asynchronous connection can be terminated using: +```c +void redisAsyncDisconnect(redisAsyncContext *ac); +``` +When this function is called, the connection is **not** immediately terminated. Instead, new +commands are no longer accepted and the connection is only terminated when all pending commands +have been written to the socket, their respective replies have been read and their respective +callbacks have been executed. After this, the disconnection callback is executed with the +`REDIS_OK` status and the context object is freed. + +### Hooking it up to event library *X* + +There are a few hooks that need to be set on the context object after it is created. +See the `adapters/` directory for bindings to *libev* and *libevent*. + +## Reply parsing API + +Hiredis comes with a reply parsing API that makes it easy for writing higher +level language bindings. + +The reply parsing API consists of the following functions: +```c +redisReader *redisReaderCreate(void); +void redisReaderFree(redisReader *reader); +int redisReaderFeed(redisReader *reader, const char *buf, size_t len); +int redisReaderGetReply(redisReader *reader, void **reply); +``` +The same set of functions are used internally by hiredis when creating a +normal Redis context, the above API just exposes it to the user for a direct +usage. + +### Usage + +The function `redisReaderCreate` creates a `redisReader` structure that holds a +buffer with unparsed data and state for the protocol parser. + +Incoming data -- most likely from a socket -- can be placed in the internal +buffer of the `redisReader` using `redisReaderFeed`. This function will make a +copy of the buffer pointed to by `buf` for `len` bytes. This data is parsed +when `redisReaderGetReply` is called. This function returns an integer status +and a reply object (as described above) via `void **reply`. The returned status +can be either `REDIS_OK` or `REDIS_ERR`, where the latter means something went +wrong (either a protocol error, or an out of memory error). + +The parser limits the level of nesting for multi bulk payloads to 7. If the +multi bulk nesting level is higher than this, the parser returns an error. + +### Customizing replies + +The function `redisReaderGetReply` creates `redisReply` and makes the function +argument `reply` point to the created `redisReply` variable. For instance, if +the response of type `REDIS_REPLY_STATUS` then the `str` field of `redisReply` +will hold the status as a vanilla C string. However, the functions that are +responsible for creating instances of the `redisReply` can be customized by +setting the `fn` field on the `redisReader` struct. This should be done +immediately after creating the `redisReader`. + +For example, [hiredis-rb](https://github.com/pietern/hiredis-rb/blob/master/ext/hiredis_ext/reader.c) +uses customized reply object functions to create Ruby objects. + +### Reader max buffer + +Both when using the Reader API directly or when using it indirectly via a +normal Redis context, the redisReader structure uses a buffer in order to +accumulate data from the server. +Usually this buffer is destroyed when it is empty and is larger than 16 +KiB in order to avoid wasting memory in unused buffers + +However when working with very big payloads destroying the buffer may slow +down performances considerably, so it is possible to modify the max size of +an idle buffer changing the value of the `maxbuf` field of the reader structure +to the desired value. The special value of 0 means that there is no maximum +value for an idle buffer, so the buffer will never get freed. + +For instance if you have a normal Redis context you can set the maximum idle +buffer to zero (unlimited) just with: +```c +context->reader->maxbuf = 0; +``` +This should be done only in order to maximize performances when working with +large payloads. The context should be set back to `REDIS_READER_MAX_BUF` again +as soon as possible in order to prevent allocation of useless memory. + +## AUTHORS + +Hiredis was written by Salvatore Sanfilippo (antirez at gmail) and +Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. +Hiredis is currently maintained by Matt Stancliff (matt at genges dot com) and +Jan-Erik Rediger (janerik at fnordig dot com) diff --git a/third-party/hiredis/adapters/ae.h b/third-party/hiredis/adapters/ae.h new file mode 100644 index 000000000..5c551c2ed --- /dev/null +++ b/third-party/hiredis/adapters/ae.h @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_AE_H__ +#define __HIREDIS_AE_H__ +#include +#include +#include "../hiredis.h" +#include "../async.h" + +typedef struct redisAeEvents { + redisAsyncContext *context; + aeEventLoop *loop; + int fd; + int reading, writing; +} redisAeEvents; + +static void redisAeReadEvent(aeEventLoop *el, int fd, void *privdata, int mask) { + ((void)el); ((void)fd); ((void)mask); + + redisAeEvents *e = (redisAeEvents*)privdata; + redisAsyncHandleRead(e->context); +} + +static void redisAeWriteEvent(aeEventLoop *el, int fd, void *privdata, int mask) { + ((void)el); ((void)fd); ((void)mask); + + redisAeEvents *e = (redisAeEvents*)privdata; + redisAsyncHandleWrite(e->context); +} + +static void redisAeAddRead(void *privdata) { + redisAeEvents *e = (redisAeEvents*)privdata; + aeEventLoop *loop = e->loop; + if (!e->reading) { + e->reading = 1; + aeCreateFileEvent(loop,e->fd,AE_READABLE,redisAeReadEvent,e); + } +} + +static void redisAeDelRead(void *privdata) { + redisAeEvents *e = (redisAeEvents*)privdata; + aeEventLoop *loop = e->loop; + if (e->reading) { + e->reading = 0; + aeDeleteFileEvent(loop,e->fd,AE_READABLE); + } +} + +static void redisAeAddWrite(void *privdata) { + redisAeEvents *e = (redisAeEvents*)privdata; + aeEventLoop *loop = e->loop; + if (!e->writing) { + e->writing = 1; + aeCreateFileEvent(loop,e->fd,AE_WRITABLE,redisAeWriteEvent,e); + } +} + +static void redisAeDelWrite(void *privdata) { + redisAeEvents *e = (redisAeEvents*)privdata; + aeEventLoop *loop = e->loop; + if (e->writing) { + e->writing = 0; + aeDeleteFileEvent(loop,e->fd,AE_WRITABLE); + } +} + +static void redisAeCleanup(void *privdata) { + redisAeEvents *e = (redisAeEvents*)privdata; + redisAeDelRead(privdata); + redisAeDelWrite(privdata); + free(e); +} + +static int redisAeAttach(aeEventLoop *loop, redisAsyncContext *ac) { + redisContext *c = &(ac->c); + redisAeEvents *e; + + /* Nothing should be attached when something is already attached */ + if (ac->ev.data != NULL) + return REDIS_ERR; + + /* Create container for context and r/w events */ + e = (redisAeEvents*)malloc(sizeof(*e)); + e->context = ac; + e->loop = loop; + e->fd = c->fd; + e->reading = e->writing = 0; + + /* Register functions to start/stop listening for events */ + ac->ev.addRead = redisAeAddRead; + ac->ev.delRead = redisAeDelRead; + ac->ev.addWrite = redisAeAddWrite; + ac->ev.delWrite = redisAeDelWrite; + ac->ev.cleanup = redisAeCleanup; + ac->ev.data = e; + + return REDIS_OK; +} +#endif diff --git a/third-party/hiredis/adapters/glib.h b/third-party/hiredis/adapters/glib.h new file mode 100644 index 000000000..e0a6411d3 --- /dev/null +++ b/third-party/hiredis/adapters/glib.h @@ -0,0 +1,153 @@ +#ifndef __HIREDIS_GLIB_H__ +#define __HIREDIS_GLIB_H__ + +#include + +#include "../hiredis.h" +#include "../async.h" + +typedef struct +{ + GSource source; + redisAsyncContext *ac; + GPollFD poll_fd; +} RedisSource; + +static void +redis_source_add_read (gpointer data) +{ + RedisSource *source = (RedisSource *)data; + g_return_if_fail(source); + source->poll_fd.events |= G_IO_IN; + g_main_context_wakeup(g_source_get_context((GSource *)data)); +} + +static void +redis_source_del_read (gpointer data) +{ + RedisSource *source = (RedisSource *)data; + g_return_if_fail(source); + source->poll_fd.events &= ~G_IO_IN; + g_main_context_wakeup(g_source_get_context((GSource *)data)); +} + +static void +redis_source_add_write (gpointer data) +{ + RedisSource *source = (RedisSource *)data; + g_return_if_fail(source); + source->poll_fd.events |= G_IO_OUT; + g_main_context_wakeup(g_source_get_context((GSource *)data)); +} + +static void +redis_source_del_write (gpointer data) +{ + RedisSource *source = (RedisSource *)data; + g_return_if_fail(source); + source->poll_fd.events &= ~G_IO_OUT; + g_main_context_wakeup(g_source_get_context((GSource *)data)); +} + +static void +redis_source_cleanup (gpointer data) +{ + RedisSource *source = (RedisSource *)data; + + g_return_if_fail(source); + + redis_source_del_read(source); + redis_source_del_write(source); + /* + * It is not our responsibility to remove ourself from the + * current main loop. However, we will remove the GPollFD. + */ + if (source->poll_fd.fd >= 0) { + g_source_remove_poll((GSource *)data, &source->poll_fd); + source->poll_fd.fd = -1; + } +} + +static gboolean +redis_source_prepare (GSource *source, + gint *timeout_) +{ + RedisSource *redis = (RedisSource *)source; + *timeout_ = -1; + return !!(redis->poll_fd.events & redis->poll_fd.revents); +} + +static gboolean +redis_source_check (GSource *source) +{ + RedisSource *redis = (RedisSource *)source; + return !!(redis->poll_fd.events & redis->poll_fd.revents); +} + +static gboolean +redis_source_dispatch (GSource *source, + GSourceFunc callback, + gpointer user_data) +{ + RedisSource *redis = (RedisSource *)source; + + if ((redis->poll_fd.revents & G_IO_OUT)) { + redisAsyncHandleWrite(redis->ac); + redis->poll_fd.revents &= ~G_IO_OUT; + } + + if ((redis->poll_fd.revents & G_IO_IN)) { + redisAsyncHandleRead(redis->ac); + redis->poll_fd.revents &= ~G_IO_IN; + } + + if (callback) { + return callback(user_data); + } + + return TRUE; +} + +static void +redis_source_finalize (GSource *source) +{ + RedisSource *redis = (RedisSource *)source; + + if (redis->poll_fd.fd >= 0) { + g_source_remove_poll(source, &redis->poll_fd); + redis->poll_fd.fd = -1; + } +} + +static GSource * +redis_source_new (redisAsyncContext *ac) +{ + static GSourceFuncs source_funcs = { + .prepare = redis_source_prepare, + .check = redis_source_check, + .dispatch = redis_source_dispatch, + .finalize = redis_source_finalize, + }; + redisContext *c = &ac->c; + RedisSource *source; + + g_return_val_if_fail(ac != NULL, NULL); + + source = (RedisSource *)g_source_new(&source_funcs, sizeof *source); + source->ac = ac; + source->poll_fd.fd = c->fd; + source->poll_fd.events = 0; + source->poll_fd.revents = 0; + g_source_add_poll((GSource *)source, &source->poll_fd); + + ac->ev.addRead = redis_source_add_read; + ac->ev.delRead = redis_source_del_read; + ac->ev.addWrite = redis_source_add_write; + ac->ev.delWrite = redis_source_del_write; + ac->ev.cleanup = redis_source_cleanup; + ac->ev.data = source; + + return (GSource *)source; +} + +#endif /* __HIREDIS_GLIB_H__ */ diff --git a/third-party/hiredis/adapters/ivykis.h b/third-party/hiredis/adapters/ivykis.h new file mode 100644 index 000000000..6a12a868a --- /dev/null +++ b/third-party/hiredis/adapters/ivykis.h @@ -0,0 +1,81 @@ +#ifndef __HIREDIS_IVYKIS_H__ +#define __HIREDIS_IVYKIS_H__ +#include +#include "../hiredis.h" +#include "../async.h" + +typedef struct redisIvykisEvents { + redisAsyncContext *context; + struct iv_fd fd; +} redisIvykisEvents; + +static void redisIvykisReadEvent(void *arg) { + redisAsyncContext *context = (redisAsyncContext *)arg; + redisAsyncHandleRead(context); +} + +static void redisIvykisWriteEvent(void *arg) { + redisAsyncContext *context = (redisAsyncContext *)arg; + redisAsyncHandleWrite(context); +} + +static void redisIvykisAddRead(void *privdata) { + redisIvykisEvents *e = (redisIvykisEvents*)privdata; + iv_fd_set_handler_in(&e->fd, redisIvykisReadEvent); +} + +static void redisIvykisDelRead(void *privdata) { + redisIvykisEvents *e = (redisIvykisEvents*)privdata; + iv_fd_set_handler_in(&e->fd, NULL); +} + +static void redisIvykisAddWrite(void *privdata) { + redisIvykisEvents *e = (redisIvykisEvents*)privdata; + iv_fd_set_handler_out(&e->fd, redisIvykisWriteEvent); +} + +static void redisIvykisDelWrite(void *privdata) { + redisIvykisEvents *e = (redisIvykisEvents*)privdata; + iv_fd_set_handler_out(&e->fd, NULL); +} + +static void redisIvykisCleanup(void *privdata) { + redisIvykisEvents *e = (redisIvykisEvents*)privdata; + + iv_fd_unregister(&e->fd); + free(e); +} + +static int redisIvykisAttach(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + redisIvykisEvents *e; + + /* Nothing should be attached when something is already attached */ + if (ac->ev.data != NULL) + return REDIS_ERR; + + /* Create container for context and r/w events */ + e = (redisIvykisEvents*)malloc(sizeof(*e)); + e->context = ac; + + /* Register functions to start/stop listening for events */ + ac->ev.addRead = redisIvykisAddRead; + ac->ev.delRead = redisIvykisDelRead; + ac->ev.addWrite = redisIvykisAddWrite; + ac->ev.delWrite = redisIvykisDelWrite; + ac->ev.cleanup = redisIvykisCleanup; + ac->ev.data = e; + + /* Initialize and install read/write events */ + IV_FD_INIT(&e->fd); + e->fd.fd = c->fd; + e->fd.handler_in = redisIvykisReadEvent; + e->fd.handler_out = redisIvykisWriteEvent; + e->fd.handler_err = NULL; + e->fd.cookie = e->context; + + iv_fd_register(&e->fd); + + return REDIS_OK; +} +#endif diff --git a/third-party/hiredis/adapters/libev.h b/third-party/hiredis/adapters/libev.h new file mode 100644 index 000000000..2bf8d521f --- /dev/null +++ b/third-party/hiredis/adapters/libev.h @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_LIBEV_H__ +#define __HIREDIS_LIBEV_H__ +#include +#include +#include +#include "../hiredis.h" +#include "../async.h" + +typedef struct redisLibevEvents { + redisAsyncContext *context; + struct ev_loop *loop; + int reading, writing; + ev_io rev, wev; +} redisLibevEvents; + +static void redisLibevReadEvent(EV_P_ ev_io *watcher, int revents) { +#if EV_MULTIPLICITY + ((void)loop); +#endif + ((void)revents); + + redisLibevEvents *e = (redisLibevEvents*)watcher->data; + redisAsyncHandleRead(e->context); +} + +static void redisLibevWriteEvent(EV_P_ ev_io *watcher, int revents) { +#if EV_MULTIPLICITY + ((void)loop); +#endif + ((void)revents); + + redisLibevEvents *e = (redisLibevEvents*)watcher->data; + redisAsyncHandleWrite(e->context); +} + +static void redisLibevAddRead(void *privdata) { + redisLibevEvents *e = (redisLibevEvents*)privdata; + struct ev_loop *loop = e->loop; + ((void)loop); + if (!e->reading) { + e->reading = 1; + ev_io_start(EV_A_ &e->rev); + } +} + +static void redisLibevDelRead(void *privdata) { + redisLibevEvents *e = (redisLibevEvents*)privdata; + struct ev_loop *loop = e->loop; + ((void)loop); + if (e->reading) { + e->reading = 0; + ev_io_stop(EV_A_ &e->rev); + } +} + +static void redisLibevAddWrite(void *privdata) { + redisLibevEvents *e = (redisLibevEvents*)privdata; + struct ev_loop *loop = e->loop; + ((void)loop); + if (!e->writing) { + e->writing = 1; + ev_io_start(EV_A_ &e->wev); + } +} + +static void redisLibevDelWrite(void *privdata) { + redisLibevEvents *e = (redisLibevEvents*)privdata; + struct ev_loop *loop = e->loop; + ((void)loop); + if (e->writing) { + e->writing = 0; + ev_io_stop(EV_A_ &e->wev); + } +} + +static void redisLibevCleanup(void *privdata) { + redisLibevEvents *e = (redisLibevEvents*)privdata; + redisLibevDelRead(privdata); + redisLibevDelWrite(privdata); + free(e); +} + +static int redisLibevAttach(EV_P_ redisAsyncContext *ac) { + redisContext *c = &(ac->c); + redisLibevEvents *e; + + /* Nothing should be attached when something is already attached */ + if (ac->ev.data != NULL) + return REDIS_ERR; + + /* Create container for context and r/w events */ + e = (redisLibevEvents*)malloc(sizeof(*e)); + e->context = ac; +#if EV_MULTIPLICITY + e->loop = loop; +#else + e->loop = NULL; +#endif + e->reading = e->writing = 0; + e->rev.data = e; + e->wev.data = e; + + /* Register functions to start/stop listening for events */ + ac->ev.addRead = redisLibevAddRead; + ac->ev.delRead = redisLibevDelRead; + ac->ev.addWrite = redisLibevAddWrite; + ac->ev.delWrite = redisLibevDelWrite; + ac->ev.cleanup = redisLibevCleanup; + ac->ev.data = e; + + /* Initialize read/write events */ + ev_io_init(&e->rev,redisLibevReadEvent,c->fd,EV_READ); + ev_io_init(&e->wev,redisLibevWriteEvent,c->fd,EV_WRITE); + return REDIS_OK; +} + +#endif diff --git a/third-party/hiredis/adapters/libevent.h b/third-party/hiredis/adapters/libevent.h new file mode 100644 index 000000000..1c2b271bb --- /dev/null +++ b/third-party/hiredis/adapters/libevent.h @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_LIBEVENT_H__ +#define __HIREDIS_LIBEVENT_H__ +#include +#include "../hiredis.h" +#include "../async.h" + +typedef struct redisLibeventEvents { + redisAsyncContext *context; + struct event rev, wev; +} redisLibeventEvents; + +static void redisLibeventReadEvent(int fd, short event, void *arg) { + ((void)fd); ((void)event); + redisLibeventEvents *e = (redisLibeventEvents*)arg; + redisAsyncHandleRead(e->context); +} + +static void redisLibeventWriteEvent(int fd, short event, void *arg) { + ((void)fd); ((void)event); + redisLibeventEvents *e = (redisLibeventEvents*)arg; + redisAsyncHandleWrite(e->context); +} + +static void redisLibeventAddRead(void *privdata) { + redisLibeventEvents *e = (redisLibeventEvents*)privdata; + event_add(&e->rev,NULL); +} + +static void redisLibeventDelRead(void *privdata) { + redisLibeventEvents *e = (redisLibeventEvents*)privdata; + event_del(&e->rev); +} + +static void redisLibeventAddWrite(void *privdata) { + redisLibeventEvents *e = (redisLibeventEvents*)privdata; + event_add(&e->wev,NULL); +} + +static void redisLibeventDelWrite(void *privdata) { + redisLibeventEvents *e = (redisLibeventEvents*)privdata; + event_del(&e->wev); +} + +static void redisLibeventCleanup(void *privdata) { + redisLibeventEvents *e = (redisLibeventEvents*)privdata; + event_del(&e->rev); + event_del(&e->wev); + free(e); +} + +static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { + redisContext *c = &(ac->c); + redisLibeventEvents *e; + + /* Nothing should be attached when something is already attached */ + if (ac->ev.data != NULL) + return REDIS_ERR; + + /* Create container for context and r/w events */ + e = (redisLibeventEvents*)malloc(sizeof(*e)); + e->context = ac; + + /* Register functions to start/stop listening for events */ + ac->ev.addRead = redisLibeventAddRead; + ac->ev.delRead = redisLibeventDelRead; + ac->ev.addWrite = redisLibeventAddWrite; + ac->ev.delWrite = redisLibeventDelWrite; + ac->ev.cleanup = redisLibeventCleanup; + ac->ev.data = e; + + /* Initialize and install read/write events */ + event_set(&e->rev,c->fd,EV_READ,redisLibeventReadEvent,e); + event_set(&e->wev,c->fd,EV_WRITE,redisLibeventWriteEvent,e); + event_base_set(base,&e->rev); + event_base_set(base,&e->wev); + return REDIS_OK; +} +#endif diff --git a/third-party/hiredis/adapters/libuv.h b/third-party/hiredis/adapters/libuv.h new file mode 100644 index 000000000..3cdf3d394 --- /dev/null +++ b/third-party/hiredis/adapters/libuv.h @@ -0,0 +1,121 @@ +#ifndef __HIREDIS_LIBUV_H__ +#define __HIREDIS_LIBUV_H__ +#include +#include +#include "../hiredis.h" +#include "../async.h" +#include + +typedef struct redisLibuvEvents { + redisAsyncContext* context; + uv_poll_t handle; + int events; +} redisLibuvEvents; + + +static void redisLibuvPoll(uv_poll_t* handle, int status, int events) { + redisLibuvEvents* p = (redisLibuvEvents*)handle->data; + + if (status != 0) { + return; + } + + if (events & UV_READABLE) { + redisAsyncHandleRead(p->context); + } + if (events & UV_WRITABLE) { + redisAsyncHandleWrite(p->context); + } +} + + +static void redisLibuvAddRead(void *privdata) { + redisLibuvEvents* p = (redisLibuvEvents*)privdata; + + p->events |= UV_READABLE; + + uv_poll_start(&p->handle, p->events, redisLibuvPoll); +} + + +static void redisLibuvDelRead(void *privdata) { + redisLibuvEvents* p = (redisLibuvEvents*)privdata; + + p->events &= ~UV_READABLE; + + if (p->events) { + uv_poll_start(&p->handle, p->events, redisLibuvPoll); + } else { + uv_poll_stop(&p->handle); + } +} + + +static void redisLibuvAddWrite(void *privdata) { + redisLibuvEvents* p = (redisLibuvEvents*)privdata; + + p->events |= UV_WRITABLE; + + uv_poll_start(&p->handle, p->events, redisLibuvPoll); +} + + +static void redisLibuvDelWrite(void *privdata) { + redisLibuvEvents* p = (redisLibuvEvents*)privdata; + + p->events &= ~UV_WRITABLE; + + if (p->events) { + uv_poll_start(&p->handle, p->events, redisLibuvPoll); + } else { + uv_poll_stop(&p->handle); + } +} + + +static void on_close(uv_handle_t* handle) { + redisLibuvEvents* p = (redisLibuvEvents*)handle->data; + + free(p); +} + + +static void redisLibuvCleanup(void *privdata) { + redisLibuvEvents* p = (redisLibuvEvents*)privdata; + + uv_close((uv_handle_t*)&p->handle, on_close); +} + + +static int redisLibuvAttach(redisAsyncContext* ac, uv_loop_t* loop) { + redisContext *c = &(ac->c); + + if (ac->ev.data != NULL) { + return REDIS_ERR; + } + + ac->ev.addRead = redisLibuvAddRead; + ac->ev.delRead = redisLibuvDelRead; + ac->ev.addWrite = redisLibuvAddWrite; + ac->ev.delWrite = redisLibuvDelWrite; + ac->ev.cleanup = redisLibuvCleanup; + + redisLibuvEvents* p = (redisLibuvEvents*)malloc(sizeof(*p)); + + if (!p) { + return REDIS_ERR; + } + + memset(p, 0, sizeof(*p)); + + if (uv_poll_init(loop, &p->handle, c->fd) != 0) { + return REDIS_ERR; + } + + ac->ev.data = p; + p->handle.data = p; + p->context = ac; + + return REDIS_OK; +} +#endif diff --git a/third-party/hiredis/adapters/macosx.h b/third-party/hiredis/adapters/macosx.h new file mode 100644 index 000000000..72121f606 --- /dev/null +++ b/third-party/hiredis/adapters/macosx.h @@ -0,0 +1,114 @@ +// +// Created by Дмитрий Бахвалов on 13.07.15. +// Copyright (c) 2015 Dmitry Bakhvalov. All rights reserved. +// + +#ifndef __HIREDIS_MACOSX_H__ +#define __HIREDIS_MACOSX_H__ + +#include + +#include "../hiredis.h" +#include "../async.h" + +typedef struct { + redisAsyncContext *context; + CFSocketRef socketRef; + CFRunLoopSourceRef sourceRef; +} RedisRunLoop; + +static int freeRedisRunLoop(RedisRunLoop* redisRunLoop) { + if( redisRunLoop != NULL ) { + if( redisRunLoop->sourceRef != NULL ) { + CFRunLoopSourceInvalidate(redisRunLoop->sourceRef); + CFRelease(redisRunLoop->sourceRef); + } + if( redisRunLoop->socketRef != NULL ) { + CFSocketInvalidate(redisRunLoop->socketRef); + CFRelease(redisRunLoop->socketRef); + } + free(redisRunLoop); + } + return REDIS_ERR; +} + +static void redisMacOSAddRead(void *privdata) { + RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata; + CFSocketEnableCallBacks(redisRunLoop->socketRef, kCFSocketReadCallBack); +} + +static void redisMacOSDelRead(void *privdata) { + RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata; + CFSocketDisableCallBacks(redisRunLoop->socketRef, kCFSocketReadCallBack); +} + +static void redisMacOSAddWrite(void *privdata) { + RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata; + CFSocketEnableCallBacks(redisRunLoop->socketRef, kCFSocketWriteCallBack); +} + +static void redisMacOSDelWrite(void *privdata) { + RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata; + CFSocketDisableCallBacks(redisRunLoop->socketRef, kCFSocketWriteCallBack); +} + +static void redisMacOSCleanup(void *privdata) { + RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata; + freeRedisRunLoop(redisRunLoop); +} + +static void redisMacOSAsyncCallback(CFSocketRef __unused s, CFSocketCallBackType callbackType, CFDataRef __unused address, const void __unused *data, void *info) { + redisAsyncContext* context = (redisAsyncContext*) info; + + switch (callbackType) { + case kCFSocketReadCallBack: + redisAsyncHandleRead(context); + break; + + case kCFSocketWriteCallBack: + redisAsyncHandleWrite(context); + break; + + default: + break; + } +} + +static int redisMacOSAttach(redisAsyncContext *redisAsyncCtx, CFRunLoopRef runLoop) { + redisContext *redisCtx = &(redisAsyncCtx->c); + + /* Nothing should be attached when something is already attached */ + if( redisAsyncCtx->ev.data != NULL ) return REDIS_ERR; + + RedisRunLoop* redisRunLoop = (RedisRunLoop*) calloc(1, sizeof(RedisRunLoop)); + if( !redisRunLoop ) return REDIS_ERR; + + /* Setup redis stuff */ + redisRunLoop->context = redisAsyncCtx; + + redisAsyncCtx->ev.addRead = redisMacOSAddRead; + redisAsyncCtx->ev.delRead = redisMacOSDelRead; + redisAsyncCtx->ev.addWrite = redisMacOSAddWrite; + redisAsyncCtx->ev.delWrite = redisMacOSDelWrite; + redisAsyncCtx->ev.cleanup = redisMacOSCleanup; + redisAsyncCtx->ev.data = redisRunLoop; + + /* Initialize and install read/write events */ + CFSocketContext socketCtx = { 0, redisAsyncCtx, NULL, NULL, NULL }; + + redisRunLoop->socketRef = CFSocketCreateWithNative(NULL, redisCtx->fd, + kCFSocketReadCallBack | kCFSocketWriteCallBack, + redisMacOSAsyncCallback, + &socketCtx); + if( !redisRunLoop->socketRef ) return freeRedisRunLoop(redisRunLoop); + + redisRunLoop->sourceRef = CFSocketCreateRunLoopSource(NULL, redisRunLoop->socketRef, 0); + if( !redisRunLoop->sourceRef ) return freeRedisRunLoop(redisRunLoop); + + CFRunLoopAddSource(runLoop, redisRunLoop->sourceRef, kCFRunLoopDefaultMode); + + return REDIS_OK; +} + +#endif + diff --git a/third-party/hiredis/adapters/qt.h b/third-party/hiredis/adapters/qt.h new file mode 100644 index 000000000..5cc02e6ce --- /dev/null +++ b/third-party/hiredis/adapters/qt.h @@ -0,0 +1,135 @@ +/*- + * Copyright (C) 2014 Pietro Cerutti + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef __HIREDIS_QT_H__ +#define __HIREDIS_QT_H__ +#include +#include "../async.h" + +static void RedisQtAddRead(void *); +static void RedisQtDelRead(void *); +static void RedisQtAddWrite(void *); +static void RedisQtDelWrite(void *); +static void RedisQtCleanup(void *); + +class RedisQtAdapter : public QObject { + + Q_OBJECT + + friend + void RedisQtAddRead(void * adapter) { + RedisQtAdapter * a = static_cast(adapter); + a->addRead(); + } + + friend + void RedisQtDelRead(void * adapter) { + RedisQtAdapter * a = static_cast(adapter); + a->delRead(); + } + + friend + void RedisQtAddWrite(void * adapter) { + RedisQtAdapter * a = static_cast(adapter); + a->addWrite(); + } + + friend + void RedisQtDelWrite(void * adapter) { + RedisQtAdapter * a = static_cast(adapter); + a->delWrite(); + } + + friend + void RedisQtCleanup(void * adapter) { + RedisQtAdapter * a = static_cast(adapter); + a->cleanup(); + } + + public: + RedisQtAdapter(QObject * parent = 0) + : QObject(parent), m_ctx(0), m_read(0), m_write(0) { } + + ~RedisQtAdapter() { + if (m_ctx != 0) { + m_ctx->ev.data = NULL; + } + } + + int setContext(redisAsyncContext * ac) { + if (ac->ev.data != NULL) { + return REDIS_ERR; + } + m_ctx = ac; + m_ctx->ev.data = this; + m_ctx->ev.addRead = RedisQtAddRead; + m_ctx->ev.delRead = RedisQtDelRead; + m_ctx->ev.addWrite = RedisQtAddWrite; + m_ctx->ev.delWrite = RedisQtDelWrite; + m_ctx->ev.cleanup = RedisQtCleanup; + return REDIS_OK; + } + + private: + void addRead() { + if (m_read) return; + m_read = new QSocketNotifier(m_ctx->c.fd, QSocketNotifier::Read, 0); + connect(m_read, SIGNAL(activated(int)), this, SLOT(read())); + } + + void delRead() { + if (!m_read) return; + delete m_read; + m_read = 0; + } + + void addWrite() { + if (m_write) return; + m_write = new QSocketNotifier(m_ctx->c.fd, QSocketNotifier::Write, 0); + connect(m_write, SIGNAL(activated(int)), this, SLOT(write())); + } + + void delWrite() { + if (!m_write) return; + delete m_write; + m_write = 0; + } + + void cleanup() { + delRead(); + delWrite(); + } + + private slots: + void read() { redisAsyncHandleRead(m_ctx); } + void write() { redisAsyncHandleWrite(m_ctx); } + + private: + redisAsyncContext * m_ctx; + QSocketNotifier * m_read; + QSocketNotifier * m_write; +}; + +#endif /* !__HIREDIS_QT_H__ */ diff --git a/third-party/hiredis/async.c b/third-party/hiredis/async.c new file mode 100644 index 000000000..acca29ac4 --- /dev/null +++ b/third-party/hiredis/async.c @@ -0,0 +1,687 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "fmacros.h" +#include +#include +#include +#include +#include +#include +#include "async.h" +#include "net.h" +#include "dict.c" +#include "sds.h" + +#define _EL_ADD_READ(ctx) do { \ + if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \ + } while(0) +#define _EL_DEL_READ(ctx) do { \ + if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \ + } while(0) +#define _EL_ADD_WRITE(ctx) do { \ + if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \ + } while(0) +#define _EL_DEL_WRITE(ctx) do { \ + if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \ + } while(0) +#define _EL_CLEANUP(ctx) do { \ + if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \ + } while(0); + +/* Forward declaration of function in hiredis.c */ +int __redisAppendCommand(redisContext *c, const char *cmd, size_t len); + +/* Functions managing dictionary of callbacks for pub/sub. */ +static unsigned int callbackHash(const void *key) { + return dictGenHashFunction((const unsigned char *)key, + sdslen((const sds)key)); +} + +static void *callbackValDup(void *privdata, const void *src) { + ((void) privdata); + redisCallback *dup = malloc(sizeof(*dup)); + memcpy(dup,src,sizeof(*dup)); + return dup; +} + +static int callbackKeyCompare(void *privdata, const void *key1, const void *key2) { + int l1, l2; + ((void) privdata); + + l1 = sdslen((const sds)key1); + l2 = sdslen((const sds)key2); + if (l1 != l2) return 0; + return memcmp(key1,key2,l1) == 0; +} + +static void callbackKeyDestructor(void *privdata, void *key) { + ((void) privdata); + sdsfree((sds)key); +} + +static void callbackValDestructor(void *privdata, void *val) { + ((void) privdata); + free(val); +} + +static dictType callbackDict = { + callbackHash, + NULL, + callbackValDup, + callbackKeyCompare, + callbackKeyDestructor, + callbackValDestructor +}; + +static redisAsyncContext *redisAsyncInitialize(redisContext *c) { + redisAsyncContext *ac; + + ac = realloc(c,sizeof(redisAsyncContext)); + if (ac == NULL) + return NULL; + + c = &(ac->c); + + /* The regular connect functions will always set the flag REDIS_CONNECTED. + * For the async API, we want to wait until the first write event is + * received up before setting this flag, so reset it here. */ + c->flags &= ~REDIS_CONNECTED; + + ac->err = 0; + ac->errstr = NULL; + ac->data = NULL; + + ac->ev.data = NULL; + ac->ev.addRead = NULL; + ac->ev.delRead = NULL; + ac->ev.addWrite = NULL; + ac->ev.delWrite = NULL; + ac->ev.cleanup = NULL; + + ac->onConnect = NULL; + ac->onDisconnect = NULL; + + ac->replies.head = NULL; + ac->replies.tail = NULL; + ac->sub.invalid.head = NULL; + ac->sub.invalid.tail = NULL; + ac->sub.channels = dictCreate(&callbackDict,NULL); + ac->sub.patterns = dictCreate(&callbackDict,NULL); + return ac; +} + +/* We want the error field to be accessible directly instead of requiring + * an indirection to the redisContext struct. */ +static void __redisAsyncCopyError(redisAsyncContext *ac) { + if (!ac) + return; + + redisContext *c = &(ac->c); + ac->err = c->err; + ac->errstr = c->errstr; +} + +redisAsyncContext *redisAsyncConnect(const char *ip, int port) { + redisContext *c; + redisAsyncContext *ac; + + c = redisConnectNonBlock(ip,port); + if (c == NULL) + return NULL; + + ac = redisAsyncInitialize(c); + if (ac == NULL) { + redisFree(c); + return NULL; + } + + __redisAsyncCopyError(ac); + return ac; +} + +redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, + const char *source_addr) { + redisContext *c = redisConnectBindNonBlock(ip,port,source_addr); + redisAsyncContext *ac = redisAsyncInitialize(c); + __redisAsyncCopyError(ac); + return ac; +} + +redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, + const char *source_addr) { + redisContext *c = redisConnectBindNonBlockWithReuse(ip,port,source_addr); + redisAsyncContext *ac = redisAsyncInitialize(c); + __redisAsyncCopyError(ac); + return ac; +} + +redisAsyncContext *redisAsyncConnectUnix(const char *path) { + redisContext *c; + redisAsyncContext *ac; + + c = redisConnectUnixNonBlock(path); + if (c == NULL) + return NULL; + + ac = redisAsyncInitialize(c); + if (ac == NULL) { + redisFree(c); + return NULL; + } + + __redisAsyncCopyError(ac); + return ac; +} + +int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn) { + if (ac->onConnect == NULL) { + ac->onConnect = fn; + + /* The common way to detect an established connection is to wait for + * the first write event to be fired. This assumes the related event + * library functions are already set. */ + _EL_ADD_WRITE(ac); + return REDIS_OK; + } + return REDIS_ERR; +} + +int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn) { + if (ac->onDisconnect == NULL) { + ac->onDisconnect = fn; + return REDIS_OK; + } + return REDIS_ERR; +} + +/* Helper functions to push/shift callbacks */ +static int __redisPushCallback(redisCallbackList *list, redisCallback *source) { + redisCallback *cb; + + /* Copy callback from stack to heap */ + cb = malloc(sizeof(*cb)); + if (cb == NULL) + return REDIS_ERR_OOM; + + if (source != NULL) { + memcpy(cb,source,sizeof(*cb)); + cb->next = NULL; + } + + /* Store callback in list */ + if (list->head == NULL) + list->head = cb; + if (list->tail != NULL) + list->tail->next = cb; + list->tail = cb; + return REDIS_OK; +} + +static int __redisShiftCallback(redisCallbackList *list, redisCallback *target) { + redisCallback *cb = list->head; + if (cb != NULL) { + list->head = cb->next; + if (cb == list->tail) + list->tail = NULL; + + /* Copy callback from heap to stack */ + if (target != NULL) + memcpy(target,cb,sizeof(*cb)); + free(cb); + return REDIS_OK; + } + return REDIS_ERR; +} + +static void __redisRunCallback(redisAsyncContext *ac, redisCallback *cb, redisReply *reply) { + redisContext *c = &(ac->c); + if (cb->fn != NULL) { + c->flags |= REDIS_IN_CALLBACK; + cb->fn(ac,reply,cb->privdata); + c->flags &= ~REDIS_IN_CALLBACK; + } +} + +/* Helper function to free the context. */ +static void __redisAsyncFree(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + redisCallback cb; + dictIterator *it; + dictEntry *de; + + /* Execute pending callbacks with NULL reply. */ + while (__redisShiftCallback(&ac->replies,&cb) == REDIS_OK) + __redisRunCallback(ac,&cb,NULL); + + /* Execute callbacks for invalid commands */ + while (__redisShiftCallback(&ac->sub.invalid,&cb) == REDIS_OK) + __redisRunCallback(ac,&cb,NULL); + + /* Run subscription callbacks callbacks with NULL reply */ + it = dictGetIterator(ac->sub.channels); + while ((de = dictNext(it)) != NULL) + __redisRunCallback(ac,dictGetEntryVal(de),NULL); + dictReleaseIterator(it); + dictRelease(ac->sub.channels); + + it = dictGetIterator(ac->sub.patterns); + while ((de = dictNext(it)) != NULL) + __redisRunCallback(ac,dictGetEntryVal(de),NULL); + dictReleaseIterator(it); + dictRelease(ac->sub.patterns); + + /* Signal event lib to clean up */ + _EL_CLEANUP(ac); + + /* Execute disconnect callback. When redisAsyncFree() initiated destroying + * this context, the status will always be REDIS_OK. */ + if (ac->onDisconnect && (c->flags & REDIS_CONNECTED)) { + if (c->flags & REDIS_FREEING) { + ac->onDisconnect(ac,REDIS_OK); + } else { + ac->onDisconnect(ac,(ac->err == 0) ? REDIS_OK : REDIS_ERR); + } + } + + /* Cleanup self */ + redisFree(c); +} + +/* Free the async context. When this function is called from a callback, + * control needs to be returned to redisProcessCallbacks() before actual + * free'ing. To do so, a flag is set on the context which is picked up by + * redisProcessCallbacks(). Otherwise, the context is immediately free'd. */ +void redisAsyncFree(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + c->flags |= REDIS_FREEING; + if (!(c->flags & REDIS_IN_CALLBACK)) + __redisAsyncFree(ac); +} + +/* Helper function to make the disconnect happen and clean up. */ +static void __redisAsyncDisconnect(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + + /* Make sure error is accessible if there is any */ + __redisAsyncCopyError(ac); + + if (ac->err == 0) { + /* For clean disconnects, there should be no pending callbacks. */ + assert(__redisShiftCallback(&ac->replies,NULL) == REDIS_ERR); + } else { + /* Disconnection is caused by an error, make sure that pending + * callbacks cannot call new commands. */ + c->flags |= REDIS_DISCONNECTING; + } + + /* For non-clean disconnects, __redisAsyncFree() will execute pending + * callbacks with a NULL-reply. */ + __redisAsyncFree(ac); +} + +/* Tries to do a clean disconnect from Redis, meaning it stops new commands + * from being issued, but tries to flush the output buffer and execute + * callbacks for all remaining replies. When this function is called from a + * callback, there might be more replies and we can safely defer disconnecting + * to redisProcessCallbacks(). Otherwise, we can only disconnect immediately + * when there are no pending callbacks. */ +void redisAsyncDisconnect(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + c->flags |= REDIS_DISCONNECTING; + if (!(c->flags & REDIS_IN_CALLBACK) && ac->replies.head == NULL) + __redisAsyncDisconnect(ac); +} + +static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, redisCallback *dstcb) { + redisContext *c = &(ac->c); + dict *callbacks; + dictEntry *de; + int pvariant; + char *stype; + sds sname; + + /* Custom reply functions are not supported for pub/sub. This will fail + * very hard when they are used... */ + if (reply->type == REDIS_REPLY_ARRAY) { + assert(reply->elements >= 2); + assert(reply->element[0]->type == REDIS_REPLY_STRING); + stype = reply->element[0]->str; + pvariant = (tolower(stype[0]) == 'p') ? 1 : 0; + + if (pvariant) + callbacks = ac->sub.patterns; + else + callbacks = ac->sub.channels; + + /* Locate the right callback */ + assert(reply->element[1]->type == REDIS_REPLY_STRING); + sname = sdsnewlen(reply->element[1]->str,reply->element[1]->len); + de = dictFind(callbacks,sname); + if (de != NULL) { + memcpy(dstcb,dictGetEntryVal(de),sizeof(*dstcb)); + + /* If this is an unsubscribe message, remove it. */ + if (strcasecmp(stype+pvariant,"unsubscribe") == 0) { + dictDelete(callbacks,sname); + + /* If this was the last unsubscribe message, revert to + * non-subscribe mode. */ + assert(reply->element[2]->type == REDIS_REPLY_INTEGER); + if (reply->element[2]->integer == 0) + c->flags &= ~REDIS_SUBSCRIBED; + } + } + sdsfree(sname); + } else { + /* Shift callback for invalid commands. */ + __redisShiftCallback(&ac->sub.invalid,dstcb); + } + return REDIS_OK; +} + +void redisProcessCallbacks(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + redisCallback cb = {NULL, NULL, NULL}; + void *reply = NULL; + int status; + + while((status = redisGetReply(c,&reply)) == REDIS_OK) { + if (reply == NULL) { + /* When the connection is being disconnected and there are + * no more replies, this is the cue to really disconnect. */ + if (c->flags & REDIS_DISCONNECTING && sdslen(c->obuf) == 0 + && ac->replies.head == NULL) { + __redisAsyncDisconnect(ac); + return; + } + + /* If monitor mode, repush callback */ + if(c->flags & REDIS_MONITORING) { + __redisPushCallback(&ac->replies,&cb); + } + + /* When the connection is not being disconnected, simply stop + * trying to get replies and wait for the next loop tick. */ + break; + } + + /* Even if the context is subscribed, pending regular callbacks will + * get a reply before pub/sub messages arrive. */ + if (__redisShiftCallback(&ac->replies,&cb) != REDIS_OK) { + /* + * A spontaneous reply in a not-subscribed context can be the error + * reply that is sent when a new connection exceeds the maximum + * number of allowed connections on the server side. + * + * This is seen as an error instead of a regular reply because the + * server closes the connection after sending it. + * + * To prevent the error from being overwritten by an EOF error the + * connection is closed here. See issue #43. + * + * Another possibility is that the server is loading its dataset. + * In this case we also want to close the connection, and have the + * user wait until the server is ready to take our request. + */ + if (((redisReply*)reply)->type == REDIS_REPLY_ERROR) { + c->err = REDIS_ERR_OTHER; + snprintf(c->errstr,sizeof(c->errstr),"%s",((redisReply*)reply)->str); + c->reader->fn->freeObject(reply); + __redisAsyncDisconnect(ac); + return; + } + /* No more regular callbacks and no errors, the context *must* be subscribed or monitoring. */ + assert((c->flags & REDIS_SUBSCRIBED || c->flags & REDIS_MONITORING)); + if(c->flags & REDIS_SUBSCRIBED) + __redisGetSubscribeCallback(ac,reply,&cb); + } + + if (cb.fn != NULL) { + __redisRunCallback(ac,&cb,reply); + c->reader->fn->freeObject(reply); + + /* Proceed with free'ing when redisAsyncFree() was called. */ + if (c->flags & REDIS_FREEING) { + __redisAsyncFree(ac); + return; + } + } else { + /* No callback for this reply. This can either be a NULL callback, + * or there were no callbacks to begin with. Either way, don't + * abort with an error, but simply ignore it because the client + * doesn't know what the server will spit out over the wire. */ + c->reader->fn->freeObject(reply); + } + } + + /* Disconnect when there was an error reading the reply */ + if (status != REDIS_OK) + __redisAsyncDisconnect(ac); +} + +/* Internal helper function to detect socket status the first time a read or + * write event fires. When connecting was not succesful, the connect callback + * is called with a REDIS_ERR status and the context is free'd. */ +static int __redisAsyncHandleConnect(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + + if (redisCheckSocketError(c) == REDIS_ERR) { + /* Try again later when connect(2) is still in progress. */ + if (errno == EINPROGRESS) + return REDIS_OK; + + if (ac->onConnect) ac->onConnect(ac,REDIS_ERR); + __redisAsyncDisconnect(ac); + return REDIS_ERR; + } + + /* Mark context as connected. */ + c->flags |= REDIS_CONNECTED; + if (ac->onConnect) ac->onConnect(ac,REDIS_OK); + return REDIS_OK; +} + +/* This function should be called when the socket is readable. + * It processes all replies that can be read and executes their callbacks. + */ +void redisAsyncHandleRead(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + + if (!(c->flags & REDIS_CONNECTED)) { + /* Abort connect was not successful. */ + if (__redisAsyncHandleConnect(ac) != REDIS_OK) + return; + /* Try again later when the context is still not connected. */ + if (!(c->flags & REDIS_CONNECTED)) + return; + } + + if (redisBufferRead(c) == REDIS_ERR) { + __redisAsyncDisconnect(ac); + } else { + /* Always re-schedule reads */ + _EL_ADD_READ(ac); + redisProcessCallbacks(ac); + } +} + +void redisAsyncHandleWrite(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + int done = 0; + + if (!(c->flags & REDIS_CONNECTED)) { + /* Abort connect was not successful. */ + if (__redisAsyncHandleConnect(ac) != REDIS_OK) + return; + /* Try again later when the context is still not connected. */ + if (!(c->flags & REDIS_CONNECTED)) + return; + } + + if (redisBufferWrite(c,&done) == REDIS_ERR) { + __redisAsyncDisconnect(ac); + } else { + /* Continue writing when not done, stop writing otherwise */ + if (!done) + _EL_ADD_WRITE(ac); + else + _EL_DEL_WRITE(ac); + + /* Always schedule reads after writes */ + _EL_ADD_READ(ac); + } +} + +/* Sets a pointer to the first argument and its length starting at p. Returns + * the number of bytes to skip to get to the following argument. */ +static const char *nextArgument(const char *start, const char **str, size_t *len) { + const char *p = start; + if (p[0] != '$') { + p = strchr(p,'$'); + if (p == NULL) return NULL; + } + + *len = (int)strtol(p+1,NULL,10); + p = strchr(p,'\r'); + assert(p); + *str = p+2; + return p+2+(*len)+2; +} + +/* Helper function for the redisAsyncCommand* family of functions. Writes a + * formatted command to the output buffer and registers the provided callback + * function with the context. */ +static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len) { + redisContext *c = &(ac->c); + redisCallback cb; + int pvariant, hasnext; + const char *cstr, *astr; + size_t clen, alen; + const char *p; + sds sname; + int ret; + + /* Don't accept new commands when the connection is about to be closed. */ + if (c->flags & (REDIS_DISCONNECTING | REDIS_FREEING)) return REDIS_ERR; + + /* Setup callback */ + cb.fn = fn; + cb.privdata = privdata; + + /* Find out which command will be appended. */ + p = nextArgument(cmd,&cstr,&clen); + assert(p != NULL); + hasnext = (p[0] == '$'); + pvariant = (tolower(cstr[0]) == 'p') ? 1 : 0; + cstr += pvariant; + clen -= pvariant; + + if (hasnext && strncasecmp(cstr,"subscribe\r\n",11) == 0) { + c->flags |= REDIS_SUBSCRIBED; + + /* Add every channel/pattern to the list of subscription callbacks. */ + while ((p = nextArgument(p,&astr,&alen)) != NULL) { + sname = sdsnewlen(astr,alen); + if (pvariant) + ret = dictReplace(ac->sub.patterns,sname,&cb); + else + ret = dictReplace(ac->sub.channels,sname,&cb); + + if (ret == 0) sdsfree(sname); + } + } else if (strncasecmp(cstr,"unsubscribe\r\n",13) == 0) { + /* It is only useful to call (P)UNSUBSCRIBE when the context is + * subscribed to one or more channels or patterns. */ + if (!(c->flags & REDIS_SUBSCRIBED)) return REDIS_ERR; + + /* (P)UNSUBSCRIBE does not have its own response: every channel or + * pattern that is unsubscribed will receive a message. This means we + * should not append a callback function for this command. */ + } else if(strncasecmp(cstr,"monitor\r\n",9) == 0) { + /* Set monitor flag and push callback */ + c->flags |= REDIS_MONITORING; + __redisPushCallback(&ac->replies,&cb); + } else { + if (c->flags & REDIS_SUBSCRIBED) + /* This will likely result in an error reply, but it needs to be + * received and passed to the callback. */ + __redisPushCallback(&ac->sub.invalid,&cb); + else + __redisPushCallback(&ac->replies,&cb); + } + + __redisAppendCommand(c,cmd,len); + + /* Always schedule a write when the write buffer is non-empty */ + _EL_ADD_WRITE(ac); + + return REDIS_OK; +} + +int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap) { + char *cmd; + int len; + int status; + len = redisvFormatCommand(&cmd,format,ap); + + /* We don't want to pass -1 or -2 to future functions as a length. */ + if (len < 0) + return REDIS_ERR; + + status = __redisAsyncCommand(ac,fn,privdata,cmd,len); + free(cmd); + return status; +} + +int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, ...) { + va_list ap; + int status; + va_start(ap,format); + status = redisvAsyncCommand(ac,fn,privdata,format,ap); + va_end(ap); + return status; +} + +int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen) { + sds cmd; + int len; + int status; + len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen); + status = __redisAsyncCommand(ac,fn,privdata,cmd,len); + sdsfree(cmd); + return status; +} + +int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len) { + int status = __redisAsyncCommand(ac,fn,privdata,cmd,len); + return status; +} diff --git a/third-party/hiredis/async.h b/third-party/hiredis/async.h new file mode 100644 index 000000000..59cbf469b --- /dev/null +++ b/third-party/hiredis/async.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_ASYNC_H +#define __HIREDIS_ASYNC_H +#include "hiredis.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct redisAsyncContext; /* need forward declaration of redisAsyncContext */ +struct dict; /* dictionary header is included in async.c */ + +/* Reply callback prototype and container */ +typedef void (redisCallbackFn)(struct redisAsyncContext*, void*, void*); +typedef struct redisCallback { + struct redisCallback *next; /* simple singly linked list */ + redisCallbackFn *fn; + void *privdata; +} redisCallback; + +/* List of callbacks for either regular replies or pub/sub */ +typedef struct redisCallbackList { + redisCallback *head, *tail; +} redisCallbackList; + +/* Connection callback prototypes */ +typedef void (redisDisconnectCallback)(const struct redisAsyncContext*, int status); +typedef void (redisConnectCallback)(const struct redisAsyncContext*, int status); + +/* Context for an async connection to Redis */ +typedef struct redisAsyncContext { + /* Hold the regular context, so it can be realloc'ed. */ + redisContext c; + + /* Setup error flags so they can be used directly. */ + int err; + char *errstr; + + /* Not used by hiredis */ + void *data; + + /* Event library data and hooks */ + struct { + void *data; + + /* Hooks that are called when the library expects to start + * reading/writing. These functions should be idempotent. */ + void (*addRead)(void *privdata); + void (*delRead)(void *privdata); + void (*addWrite)(void *privdata); + void (*delWrite)(void *privdata); + void (*cleanup)(void *privdata); + } ev; + + /* Called when either the connection is terminated due to an error or per + * user request. The status is set accordingly (REDIS_OK, REDIS_ERR). */ + redisDisconnectCallback *onDisconnect; + + /* Called when the first write event was received. */ + redisConnectCallback *onConnect; + + /* Regular command callbacks */ + redisCallbackList replies; + + /* Subscription callbacks */ + struct { + redisCallbackList invalid; + struct dict *channels; + struct dict *patterns; + } sub; +} redisAsyncContext; + +/* Functions that proxy to hiredis */ +redisAsyncContext *redisAsyncConnect(const char *ip, int port); +redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr); +redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, + const char *source_addr); +redisAsyncContext *redisAsyncConnectUnix(const char *path); +int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn); +int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); +void redisAsyncDisconnect(redisAsyncContext *ac); +void redisAsyncFree(redisAsyncContext *ac); + +/* Handle read/write events */ +void redisAsyncHandleRead(redisAsyncContext *ac); +void redisAsyncHandleWrite(redisAsyncContext *ac); + +/* Command functions for an async context. Write the command to the + * output buffer and register the provided callback. */ +int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap); +int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, ...); +int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen); +int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/third-party/hiredis/dict.c b/third-party/hiredis/dict.c new file mode 100644 index 000000000..79b1041ca --- /dev/null +++ b/third-party/hiredis/dict.c @@ -0,0 +1,338 @@ +/* Hash table implementation. + * + * This file implements in memory hash tables with insert/del/replace/find/ + * get-random-element operations. Hash tables will auto resize if needed + * tables of power of two in size are used, collisions are handled by + * chaining. See the source code for more information... :) + * + * Copyright (c) 2006-2010, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "fmacros.h" +#include +#include +#include +#include "dict.h" + +/* -------------------------- private prototypes ---------------------------- */ + +static int _dictExpandIfNeeded(dict *ht); +static unsigned long _dictNextPower(unsigned long size); +static int _dictKeyIndex(dict *ht, const void *key); +static int _dictInit(dict *ht, dictType *type, void *privDataPtr); + +/* -------------------------- hash functions -------------------------------- */ + +/* Generic hash function (a popular one from Bernstein). + * I tested a few and this was the best. */ +static unsigned int dictGenHashFunction(const unsigned char *buf, int len) { + unsigned int hash = 5381; + + while (len--) + hash = ((hash << 5) + hash) + (*buf++); /* hash * 33 + c */ + return hash; +} + +/* ----------------------------- API implementation ------------------------- */ + +/* Reset an hashtable already initialized with ht_init(). + * NOTE: This function should only called by ht_destroy(). */ +static void _dictReset(dict *ht) { + ht->table = NULL; + ht->size = 0; + ht->sizemask = 0; + ht->used = 0; +} + +/* Create a new hash table */ +static dict *dictCreate(dictType *type, void *privDataPtr) { + dict *ht = malloc(sizeof(*ht)); + _dictInit(ht,type,privDataPtr); + return ht; +} + +/* Initialize the hash table */ +static int _dictInit(dict *ht, dictType *type, void *privDataPtr) { + _dictReset(ht); + ht->type = type; + ht->privdata = privDataPtr; + return DICT_OK; +} + +/* Expand or create the hashtable */ +static int dictExpand(dict *ht, unsigned long size) { + dict n; /* the new hashtable */ + unsigned long realsize = _dictNextPower(size), i; + + /* the size is invalid if it is smaller than the number of + * elements already inside the hashtable */ + if (ht->used > size) + return DICT_ERR; + + _dictInit(&n, ht->type, ht->privdata); + n.size = realsize; + n.sizemask = realsize-1; + n.table = calloc(realsize,sizeof(dictEntry*)); + + /* Copy all the elements from the old to the new table: + * note that if the old hash table is empty ht->size is zero, + * so dictExpand just creates an hash table. */ + n.used = ht->used; + for (i = 0; i < ht->size && ht->used > 0; i++) { + dictEntry *he, *nextHe; + + if (ht->table[i] == NULL) continue; + + /* For each hash entry on this slot... */ + he = ht->table[i]; + while(he) { + unsigned int h; + + nextHe = he->next; + /* Get the new element index */ + h = dictHashKey(ht, he->key) & n.sizemask; + he->next = n.table[h]; + n.table[h] = he; + ht->used--; + /* Pass to the next element */ + he = nextHe; + } + } + assert(ht->used == 0); + free(ht->table); + + /* Remap the new hashtable in the old */ + *ht = n; + return DICT_OK; +} + +/* Add an element to the target hash table */ +static int dictAdd(dict *ht, void *key, void *val) { + int index; + dictEntry *entry; + + /* Get the index of the new element, or -1 if + * the element already exists. */ + if ((index = _dictKeyIndex(ht, key)) == -1) + return DICT_ERR; + + /* Allocates the memory and stores key */ + entry = malloc(sizeof(*entry)); + entry->next = ht->table[index]; + ht->table[index] = entry; + + /* Set the hash entry fields. */ + dictSetHashKey(ht, entry, key); + dictSetHashVal(ht, entry, val); + ht->used++; + return DICT_OK; +} + +/* Add an element, discarding the old if the key already exists. + * Return 1 if the key was added from scratch, 0 if there was already an + * element with such key and dictReplace() just performed a value update + * operation. */ +static int dictReplace(dict *ht, void *key, void *val) { + dictEntry *entry, auxentry; + + /* Try to add the element. If the key + * does not exists dictAdd will suceed. */ + if (dictAdd(ht, key, val) == DICT_OK) + return 1; + /* It already exists, get the entry */ + entry = dictFind(ht, key); + /* Free the old value and set the new one */ + /* Set the new value and free the old one. Note that it is important + * to do that in this order, as the value may just be exactly the same + * as the previous one. In this context, think to reference counting, + * you want to increment (set), and then decrement (free), and not the + * reverse. */ + auxentry = *entry; + dictSetHashVal(ht, entry, val); + dictFreeEntryVal(ht, &auxentry); + return 0; +} + +/* Search and remove an element */ +static int dictDelete(dict *ht, const void *key) { + unsigned int h; + dictEntry *de, *prevde; + + if (ht->size == 0) + return DICT_ERR; + h = dictHashKey(ht, key) & ht->sizemask; + de = ht->table[h]; + + prevde = NULL; + while(de) { + if (dictCompareHashKeys(ht,key,de->key)) { + /* Unlink the element from the list */ + if (prevde) + prevde->next = de->next; + else + ht->table[h] = de->next; + + dictFreeEntryKey(ht,de); + dictFreeEntryVal(ht,de); + free(de); + ht->used--; + return DICT_OK; + } + prevde = de; + de = de->next; + } + return DICT_ERR; /* not found */ +} + +/* Destroy an entire hash table */ +static int _dictClear(dict *ht) { + unsigned long i; + + /* Free all the elements */ + for (i = 0; i < ht->size && ht->used > 0; i++) { + dictEntry *he, *nextHe; + + if ((he = ht->table[i]) == NULL) continue; + while(he) { + nextHe = he->next; + dictFreeEntryKey(ht, he); + dictFreeEntryVal(ht, he); + free(he); + ht->used--; + he = nextHe; + } + } + /* Free the table and the allocated cache structure */ + free(ht->table); + /* Re-initialize the table */ + _dictReset(ht); + return DICT_OK; /* never fails */ +} + +/* Clear & Release the hash table */ +static void dictRelease(dict *ht) { + _dictClear(ht); + free(ht); +} + +static dictEntry *dictFind(dict *ht, const void *key) { + dictEntry *he; + unsigned int h; + + if (ht->size == 0) return NULL; + h = dictHashKey(ht, key) & ht->sizemask; + he = ht->table[h]; + while(he) { + if (dictCompareHashKeys(ht, key, he->key)) + return he; + he = he->next; + } + return NULL; +} + +static dictIterator *dictGetIterator(dict *ht) { + dictIterator *iter = malloc(sizeof(*iter)); + + iter->ht = ht; + iter->index = -1; + iter->entry = NULL; + iter->nextEntry = NULL; + return iter; +} + +static dictEntry *dictNext(dictIterator *iter) { + while (1) { + if (iter->entry == NULL) { + iter->index++; + if (iter->index >= + (signed)iter->ht->size) break; + iter->entry = iter->ht->table[iter->index]; + } else { + iter->entry = iter->nextEntry; + } + if (iter->entry) { + /* We need to save the 'next' here, the iterator user + * may delete the entry we are returning. */ + iter->nextEntry = iter->entry->next; + return iter->entry; + } + } + return NULL; +} + +static void dictReleaseIterator(dictIterator *iter) { + free(iter); +} + +/* ------------------------- private functions ------------------------------ */ + +/* Expand the hash table if needed */ +static int _dictExpandIfNeeded(dict *ht) { + /* If the hash table is empty expand it to the intial size, + * if the table is "full" dobule its size. */ + if (ht->size == 0) + return dictExpand(ht, DICT_HT_INITIAL_SIZE); + if (ht->used == ht->size) + return dictExpand(ht, ht->size*2); + return DICT_OK; +} + +/* Our hash table capability is a power of two */ +static unsigned long _dictNextPower(unsigned long size) { + unsigned long i = DICT_HT_INITIAL_SIZE; + + if (size >= LONG_MAX) return LONG_MAX; + while(1) { + if (i >= size) + return i; + i *= 2; + } +} + +/* Returns the index of a free slot that can be populated with + * an hash entry for the given 'key'. + * If the key already exists, -1 is returned. */ +static int _dictKeyIndex(dict *ht, const void *key) { + unsigned int h; + dictEntry *he; + + /* Expand the hashtable if needed */ + if (_dictExpandIfNeeded(ht) == DICT_ERR) + return -1; + /* Compute the key hash value */ + h = dictHashKey(ht, key) & ht->sizemask; + /* Search if this slot does not already contain the given key */ + he = ht->table[h]; + while(he) { + if (dictCompareHashKeys(ht, key, he->key)) + return -1; + he = he->next; + } + return h; +} + diff --git a/third-party/hiredis/dict.h b/third-party/hiredis/dict.h new file mode 100644 index 000000000..95fcd280e --- /dev/null +++ b/third-party/hiredis/dict.h @@ -0,0 +1,126 @@ +/* Hash table implementation. + * + * This file implements in memory hash tables with insert/del/replace/find/ + * get-random-element operations. Hash tables will auto resize if needed + * tables of power of two in size are used, collisions are handled by + * chaining. See the source code for more information... :) + * + * Copyright (c) 2006-2010, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __DICT_H +#define __DICT_H + +#define DICT_OK 0 +#define DICT_ERR 1 + +/* Unused arguments generate annoying warnings... */ +#define DICT_NOTUSED(V) ((void) V) + +typedef struct dictEntry { + void *key; + void *val; + struct dictEntry *next; +} dictEntry; + +typedef struct dictType { + unsigned int (*hashFunction)(const void *key); + void *(*keyDup)(void *privdata, const void *key); + void *(*valDup)(void *privdata, const void *obj); + int (*keyCompare)(void *privdata, const void *key1, const void *key2); + void (*keyDestructor)(void *privdata, void *key); + void (*valDestructor)(void *privdata, void *obj); +} dictType; + +typedef struct dict { + dictEntry **table; + dictType *type; + unsigned long size; + unsigned long sizemask; + unsigned long used; + void *privdata; +} dict; + +typedef struct dictIterator { + dict *ht; + int index; + dictEntry *entry, *nextEntry; +} dictIterator; + +/* This is the initial size of every hash table */ +#define DICT_HT_INITIAL_SIZE 4 + +/* ------------------------------- Macros ------------------------------------*/ +#define dictFreeEntryVal(ht, entry) \ + if ((ht)->type->valDestructor) \ + (ht)->type->valDestructor((ht)->privdata, (entry)->val) + +#define dictSetHashVal(ht, entry, _val_) do { \ + if ((ht)->type->valDup) \ + entry->val = (ht)->type->valDup((ht)->privdata, _val_); \ + else \ + entry->val = (_val_); \ +} while(0) + +#define dictFreeEntryKey(ht, entry) \ + if ((ht)->type->keyDestructor) \ + (ht)->type->keyDestructor((ht)->privdata, (entry)->key) + +#define dictSetHashKey(ht, entry, _key_) do { \ + if ((ht)->type->keyDup) \ + entry->key = (ht)->type->keyDup((ht)->privdata, _key_); \ + else \ + entry->key = (_key_); \ +} while(0) + +#define dictCompareHashKeys(ht, key1, key2) \ + (((ht)->type->keyCompare) ? \ + (ht)->type->keyCompare((ht)->privdata, key1, key2) : \ + (key1) == (key2)) + +#define dictHashKey(ht, key) (ht)->type->hashFunction(key) + +#define dictGetEntryKey(he) ((he)->key) +#define dictGetEntryVal(he) ((he)->val) +#define dictSlots(ht) ((ht)->size) +#define dictSize(ht) ((ht)->used) + +/* API */ +static unsigned int dictGenHashFunction(const unsigned char *buf, int len); +static dict *dictCreate(dictType *type, void *privDataPtr); +static int dictExpand(dict *ht, unsigned long size); +static int dictAdd(dict *ht, void *key, void *val); +static int dictReplace(dict *ht, void *key, void *val); +static int dictDelete(dict *ht, const void *key); +static void dictRelease(dict *ht); +static dictEntry * dictFind(dict *ht, const void *key); +static dictIterator *dictGetIterator(dict *ht); +static dictEntry *dictNext(dictIterator *iter); +static void dictReleaseIterator(dictIterator *iter); + +#endif /* __DICT_H */ diff --git a/third-party/hiredis/examples/example-ae.c b/third-party/hiredis/examples/example-ae.c new file mode 100644 index 000000000..8efa7306a --- /dev/null +++ b/third-party/hiredis/examples/example-ae.c @@ -0,0 +1,62 @@ +#include +#include +#include +#include + +#include +#include +#include + +/* Put event loop in the global scope, so it can be explicitly stopped */ +static aeEventLoop *loop; + +void getCallback(redisAsyncContext *c, void *r, void *privdata) { + redisReply *reply = r; + if (reply == NULL) return; + printf("argv[%s]: %s\n", (char*)privdata, reply->str); + + /* Disconnect after receiving the reply to GET */ + redisAsyncDisconnect(c); +} + +void connectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + aeStop(loop); + return; + } + + printf("Connected...\n"); +} + +void disconnectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + aeStop(loop); + return; + } + + printf("Disconnected...\n"); + aeStop(loop); +} + +int main (int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + + redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); + if (c->err) { + /* Let *c leak for now... */ + printf("Error: %s\n", c->errstr); + return 1; + } + + loop = aeCreateEventLoop(64); + redisAeAttach(loop, c); + redisAsyncSetConnectCallback(c,connectCallback); + redisAsyncSetDisconnectCallback(c,disconnectCallback); + redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1])); + redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); + aeMain(loop); + return 0; +} + diff --git a/third-party/hiredis/examples/example-glib.c b/third-party/hiredis/examples/example-glib.c new file mode 100644 index 000000000..d6e10f8e8 --- /dev/null +++ b/third-party/hiredis/examples/example-glib.c @@ -0,0 +1,73 @@ +#include + +#include +#include +#include + +static GMainLoop *mainloop; + +static void +connect_cb (const redisAsyncContext *ac G_GNUC_UNUSED, + int status) +{ + if (status != REDIS_OK) { + g_printerr("Failed to connect: %s\n", ac->errstr); + g_main_loop_quit(mainloop); + } else { + g_printerr("Connected...\n"); + } +} + +static void +disconnect_cb (const redisAsyncContext *ac G_GNUC_UNUSED, + int status) +{ + if (status != REDIS_OK) { + g_error("Failed to disconnect: %s", ac->errstr); + } else { + g_printerr("Disconnected...\n"); + g_main_loop_quit(mainloop); + } +} + +static void +command_cb(redisAsyncContext *ac, + gpointer r, + gpointer user_data G_GNUC_UNUSED) +{ + redisReply *reply = r; + + if (reply) { + g_print("REPLY: %s\n", reply->str); + } + + redisAsyncDisconnect(ac); +} + +gint +main (gint argc G_GNUC_UNUSED, + gchar *argv[] G_GNUC_UNUSED) +{ + redisAsyncContext *ac; + GMainContext *context = NULL; + GSource *source; + + ac = redisAsyncConnect("127.0.0.1", 6379); + if (ac->err) { + g_printerr("%s\n", ac->errstr); + exit(EXIT_FAILURE); + } + + source = redis_source_new(ac); + mainloop = g_main_loop_new(context, FALSE); + g_source_attach(source, context); + + redisAsyncSetConnectCallback(ac, connect_cb); + redisAsyncSetDisconnectCallback(ac, disconnect_cb); + redisAsyncCommand(ac, command_cb, NULL, "SET key 1234"); + redisAsyncCommand(ac, command_cb, NULL, "GET key"); + + g_main_loop_run(mainloop); + + return EXIT_SUCCESS; +} diff --git a/third-party/hiredis/examples/example-ivykis.c b/third-party/hiredis/examples/example-ivykis.c new file mode 100644 index 000000000..67affcef3 --- /dev/null +++ b/third-party/hiredis/examples/example-ivykis.c @@ -0,0 +1,58 @@ +#include +#include +#include +#include + +#include +#include +#include + +void getCallback(redisAsyncContext *c, void *r, void *privdata) { + redisReply *reply = r; + if (reply == NULL) return; + printf("argv[%s]: %s\n", (char*)privdata, reply->str); + + /* Disconnect after receiving the reply to GET */ + redisAsyncDisconnect(c); +} + +void connectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Connected...\n"); +} + +void disconnectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Disconnected...\n"); +} + +int main (int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + + iv_init(); + + redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); + if (c->err) { + /* Let *c leak for now... */ + printf("Error: %s\n", c->errstr); + return 1; + } + + redisIvykisAttach(c); + redisAsyncSetConnectCallback(c,connectCallback); + redisAsyncSetDisconnectCallback(c,disconnectCallback); + redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1])); + redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); + + iv_main(); + + iv_deinit(); + + return 0; +} diff --git a/third-party/hiredis/examples/example-libev.c b/third-party/hiredis/examples/example-libev.c new file mode 100644 index 000000000..cc8b166ec --- /dev/null +++ b/third-party/hiredis/examples/example-libev.c @@ -0,0 +1,52 @@ +#include +#include +#include +#include + +#include +#include +#include + +void getCallback(redisAsyncContext *c, void *r, void *privdata) { + redisReply *reply = r; + if (reply == NULL) return; + printf("argv[%s]: %s\n", (char*)privdata, reply->str); + + /* Disconnect after receiving the reply to GET */ + redisAsyncDisconnect(c); +} + +void connectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Connected...\n"); +} + +void disconnectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Disconnected...\n"); +} + +int main (int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + + redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); + if (c->err) { + /* Let *c leak for now... */ + printf("Error: %s\n", c->errstr); + return 1; + } + + redisLibevAttach(EV_DEFAULT_ c); + redisAsyncSetConnectCallback(c,connectCallback); + redisAsyncSetDisconnectCallback(c,disconnectCallback); + redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1])); + redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); + ev_loop(EV_DEFAULT_ 0); + return 0; +} diff --git a/third-party/hiredis/examples/example-libevent.c b/third-party/hiredis/examples/example-libevent.c new file mode 100644 index 000000000..d333c22b7 --- /dev/null +++ b/third-party/hiredis/examples/example-libevent.c @@ -0,0 +1,53 @@ +#include +#include +#include +#include + +#include +#include +#include + +void getCallback(redisAsyncContext *c, void *r, void *privdata) { + redisReply *reply = r; + if (reply == NULL) return; + printf("argv[%s]: %s\n", (char*)privdata, reply->str); + + /* Disconnect after receiving the reply to GET */ + redisAsyncDisconnect(c); +} + +void connectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Connected...\n"); +} + +void disconnectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Disconnected...\n"); +} + +int main (int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + struct event_base *base = event_base_new(); + + redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); + if (c->err) { + /* Let *c leak for now... */ + printf("Error: %s\n", c->errstr); + return 1; + } + + redisLibeventAttach(c,base); + redisAsyncSetConnectCallback(c,connectCallback); + redisAsyncSetDisconnectCallback(c,disconnectCallback); + redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1])); + redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); + event_base_dispatch(base); + return 0; +} diff --git a/third-party/hiredis/examples/example-libuv.c b/third-party/hiredis/examples/example-libuv.c new file mode 100644 index 000000000..a5462d410 --- /dev/null +++ b/third-party/hiredis/examples/example-libuv.c @@ -0,0 +1,53 @@ +#include +#include +#include +#include + +#include +#include +#include + +void getCallback(redisAsyncContext *c, void *r, void *privdata) { + redisReply *reply = r; + if (reply == NULL) return; + printf("argv[%s]: %s\n", (char*)privdata, reply->str); + + /* Disconnect after receiving the reply to GET */ + redisAsyncDisconnect(c); +} + +void connectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Connected...\n"); +} + +void disconnectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Disconnected...\n"); +} + +int main (int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + uv_loop_t* loop = uv_default_loop(); + + redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); + if (c->err) { + /* Let *c leak for now... */ + printf("Error: %s\n", c->errstr); + return 1; + } + + redisLibuvAttach(c,loop); + redisAsyncSetConnectCallback(c,connectCallback); + redisAsyncSetDisconnectCallback(c,disconnectCallback); + redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1])); + redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); + uv_run(loop, UV_RUN_DEFAULT); + return 0; +} diff --git a/third-party/hiredis/examples/example-macosx.c b/third-party/hiredis/examples/example-macosx.c new file mode 100644 index 000000000..bc84ed5ba --- /dev/null +++ b/third-party/hiredis/examples/example-macosx.c @@ -0,0 +1,66 @@ +// +// Created by Дмитрий Бахвалов on 13.07.15. +// Copyright (c) 2015 Dmitry Bakhvalov. All rights reserved. +// + +#include + +#include +#include +#include + +void getCallback(redisAsyncContext *c, void *r, void *privdata) { + redisReply *reply = r; + if (reply == NULL) return; + printf("argv[%s]: %s\n", (char*)privdata, reply->str); + + /* Disconnect after receiving the reply to GET */ + redisAsyncDisconnect(c); +} + +void connectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Connected...\n"); +} + +void disconnectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + CFRunLoopStop(CFRunLoopGetCurrent()); + printf("Disconnected...\n"); +} + +int main (int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + + CFRunLoopRef loop = CFRunLoopGetCurrent(); + if( !loop ) { + printf("Error: Cannot get current run loop\n"); + return 1; + } + + redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); + if (c->err) { + /* Let *c leak for now... */ + printf("Error: %s\n", c->errstr); + return 1; + } + + redisMacOSAttach(c, loop); + + redisAsyncSetConnectCallback(c,connectCallback); + redisAsyncSetDisconnectCallback(c,disconnectCallback); + + redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1])); + redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); + + CFRunLoopRun(); + + return 0; +} + diff --git a/third-party/hiredis/examples/example-qt.cpp b/third-party/hiredis/examples/example-qt.cpp new file mode 100644 index 000000000..f524c3f3d --- /dev/null +++ b/third-party/hiredis/examples/example-qt.cpp @@ -0,0 +1,46 @@ +#include +using namespace std; + +#include +#include + +#include "example-qt.h" + +void getCallback(redisAsyncContext *, void * r, void * privdata) { + + redisReply * reply = static_cast(r); + ExampleQt * ex = static_cast(privdata); + if (reply == nullptr || ex == nullptr) return; + + cout << "key: " << reply->str << endl; + + ex->finish(); +} + +void ExampleQt::run() { + + m_ctx = redisAsyncConnect("localhost", 6379); + + if (m_ctx->err) { + cerr << "Error: " << m_ctx->errstr << endl; + redisAsyncFree(m_ctx); + emit finished(); + } + + m_adapter.setContext(m_ctx); + + redisAsyncCommand(m_ctx, NULL, NULL, "SET key %s", m_value); + redisAsyncCommand(m_ctx, getCallback, this, "GET key"); +} + +int main (int argc, char **argv) { + + QCoreApplication app(argc, argv); + + ExampleQt example(argv[argc-1]); + + QObject::connect(&example, SIGNAL(finished()), &app, SLOT(quit())); + QTimer::singleShot(0, &example, SLOT(run())); + + return app.exec(); +} diff --git a/third-party/hiredis/examples/example-qt.h b/third-party/hiredis/examples/example-qt.h new file mode 100644 index 000000000..374f47666 --- /dev/null +++ b/third-party/hiredis/examples/example-qt.h @@ -0,0 +1,32 @@ +#ifndef __HIREDIS_EXAMPLE_QT_H +#define __HIREDIS_EXAMPLE_QT_H + +#include + +class ExampleQt : public QObject { + + Q_OBJECT + + public: + ExampleQt(const char * value, QObject * parent = 0) + : QObject(parent), m_value(value) {} + + signals: + void finished(); + + public slots: + void run(); + + private: + void finish() { emit finished(); } + + private: + const char * m_value; + redisAsyncContext * m_ctx; + RedisQtAdapter m_adapter; + + friend + void getCallback(redisAsyncContext *, void *, void *); +}; + +#endif /* !__HIREDIS_EXAMPLE_QT_H */ diff --git a/third-party/hiredis/examples/example.c b/third-party/hiredis/examples/example.c new file mode 100644 index 000000000..25226a807 --- /dev/null +++ b/third-party/hiredis/examples/example.c @@ -0,0 +1,78 @@ +#include +#include +#include + +#include + +int main(int argc, char **argv) { + unsigned int j; + redisContext *c; + redisReply *reply; + const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + int port = (argc > 2) ? atoi(argv[2]) : 6379; + + struct timeval timeout = { 1, 500000 }; // 1.5 seconds + c = redisConnectWithTimeout(hostname, port, timeout); + if (c == NULL || c->err) { + if (c) { + printf("Connection error: %s\n", c->errstr); + redisFree(c); + } else { + printf("Connection error: can't allocate redis context\n"); + } + exit(1); + } + + /* PING server */ + reply = redisCommand(c,"PING"); + printf("PING: %s\n", reply->str); + freeReplyObject(reply); + + /* Set a key */ + reply = redisCommand(c,"SET %s %s", "foo", "hello world"); + printf("SET: %s\n", reply->str); + freeReplyObject(reply); + + /* Set a key using binary safe API */ + reply = redisCommand(c,"SET %b %b", "bar", (size_t) 3, "hello", (size_t) 5); + printf("SET (binary API): %s\n", reply->str); + freeReplyObject(reply); + + /* Try a GET and two INCR */ + reply = redisCommand(c,"GET foo"); + printf("GET foo: %s\n", reply->str); + freeReplyObject(reply); + + reply = redisCommand(c,"INCR counter"); + printf("INCR counter: %lld\n", reply->integer); + freeReplyObject(reply); + /* again ... */ + reply = redisCommand(c,"INCR counter"); + printf("INCR counter: %lld\n", reply->integer); + freeReplyObject(reply); + + /* Create a list of numbers, from 0 to 9 */ + reply = redisCommand(c,"DEL mylist"); + freeReplyObject(reply); + for (j = 0; j < 10; j++) { + char buf[64]; + + snprintf(buf,64,"%d",j); + reply = redisCommand(c,"LPUSH mylist element-%s", buf); + freeReplyObject(reply); + } + + /* Let's check what we have inside the list */ + reply = redisCommand(c,"LRANGE mylist 0 -1"); + if (reply->type == REDIS_REPLY_ARRAY) { + for (j = 0; j < reply->elements; j++) { + printf("%u) %s\n", j, reply->element[j]->str); + } + } + freeReplyObject(reply); + + /* Disconnects and frees the context */ + redisFree(c); + + return 0; +} diff --git a/third-party/hiredis/fmacros.h b/third-party/hiredis/fmacros.h new file mode 100644 index 000000000..19d7b2193 --- /dev/null +++ b/third-party/hiredis/fmacros.h @@ -0,0 +1,21 @@ +#ifndef __HIREDIS_FMACRO_H +#define __HIREDIS_FMACRO_H + +#if defined(__linux__) +#define _BSD_SOURCE +#define _DEFAULT_SOURCE +#endif + +#if defined(__sun__) +#define _POSIX_C_SOURCE 200112L +#elif defined(__linux__) || defined(__OpenBSD__) || defined(__NetBSD__) +#define _XOPEN_SOURCE 600 +#else +#define _XOPEN_SOURCE +#endif + +#if __APPLE__ && __MACH__ +#define _OSX +#endif + +#endif diff --git a/third-party/hiredis/hiredis.c b/third-party/hiredis/hiredis.c new file mode 100644 index 000000000..73d0251bc --- /dev/null +++ b/third-party/hiredis/hiredis.c @@ -0,0 +1,1021 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2014, Pieter Noordhuis + * Copyright (c) 2015, Matt Stancliff , + * Jan-Erik Rediger + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "fmacros.h" +#include +#include +#include +#include +#include +#include + +#include "hiredis.h" +#include "net.h" +#include "sds.h" + +static redisReply *createReplyObject(int type); +static void *createStringObject(const redisReadTask *task, char *str, size_t len); +static void *createArrayObject(const redisReadTask *task, int elements); +static void *createIntegerObject(const redisReadTask *task, long long value); +static void *createNilObject(const redisReadTask *task); + +/* Default set of functions to build the reply. Keep in mind that such a + * function returning NULL is interpreted as OOM. */ +static redisReplyObjectFunctions defaultFunctions = { + createStringObject, + createArrayObject, + createIntegerObject, + createNilObject, + freeReplyObject +}; + +/* Create a reply object */ +static redisReply *createReplyObject(int type) { + redisReply *r = calloc(1,sizeof(*r)); + + if (r == NULL) + return NULL; + + r->type = type; + return r; +} + +/* Free a reply object */ +void freeReplyObject(void *reply) { + redisReply *r = reply; + size_t j; + + if (r == NULL) + return; + + switch(r->type) { + case REDIS_REPLY_INTEGER: + break; /* Nothing to free */ + case REDIS_REPLY_ARRAY: + if (r->element != NULL) { + for (j = 0; j < r->elements; j++) + if (r->element[j] != NULL) + freeReplyObject(r->element[j]); + free(r->element); + } + break; + case REDIS_REPLY_ERROR: + case REDIS_REPLY_STATUS: + case REDIS_REPLY_STRING: + if (r->str != NULL) + free(r->str); + break; + } + free(r); +} + +static void *createStringObject(const redisReadTask *task, char *str, size_t len) { + redisReply *r, *parent; + char *buf; + + r = createReplyObject(task->type); + if (r == NULL) + return NULL; + + buf = malloc(len+1); + if (buf == NULL) { + freeReplyObject(r); + return NULL; + } + + assert(task->type == REDIS_REPLY_ERROR || + task->type == REDIS_REPLY_STATUS || + task->type == REDIS_REPLY_STRING); + + /* Copy string value */ + memcpy(buf,str,len); + buf[len] = '\0'; + r->str = buf; + r->len = len; + + if (task->parent) { + parent = task->parent->obj; + assert(parent->type == REDIS_REPLY_ARRAY); + parent->element[task->idx] = r; + } + return r; +} + +static void *createArrayObject(const redisReadTask *task, int elements) { + redisReply *r, *parent; + + r = createReplyObject(REDIS_REPLY_ARRAY); + if (r == NULL) + return NULL; + + if (elements > 0) { + r->element = calloc(elements,sizeof(redisReply*)); + if (r->element == NULL) { + freeReplyObject(r); + return NULL; + } + } + + r->elements = elements; + + if (task->parent) { + parent = task->parent->obj; + assert(parent->type == REDIS_REPLY_ARRAY); + parent->element[task->idx] = r; + } + return r; +} + +static void *createIntegerObject(const redisReadTask *task, long long value) { + redisReply *r, *parent; + + r = createReplyObject(REDIS_REPLY_INTEGER); + if (r == NULL) + return NULL; + + r->integer = value; + + if (task->parent) { + parent = task->parent->obj; + assert(parent->type == REDIS_REPLY_ARRAY); + parent->element[task->idx] = r; + } + return r; +} + +static void *createNilObject(const redisReadTask *task) { + redisReply *r, *parent; + + r = createReplyObject(REDIS_REPLY_NIL); + if (r == NULL) + return NULL; + + if (task->parent) { + parent = task->parent->obj; + assert(parent->type == REDIS_REPLY_ARRAY); + parent->element[task->idx] = r; + } + return r; +} + +/* Return the number of digits of 'v' when converted to string in radix 10. + * Implementation borrowed from link in redis/src/util.c:string2ll(). */ +static uint32_t countDigits(uint64_t v) { + uint32_t result = 1; + for (;;) { + if (v < 10) return result; + if (v < 100) return result + 1; + if (v < 1000) return result + 2; + if (v < 10000) return result + 3; + v /= 10000U; + result += 4; + } +} + +/* Helper that calculates the bulk length given a certain string length. */ +static size_t bulklen(size_t len) { + return 1+countDigits(len)+2+len+2; +} + +int redisvFormatCommand(char **target, const char *format, va_list ap) { + const char *c = format; + char *cmd = NULL; /* final command */ + int pos; /* position in final command */ + sds curarg, newarg; /* current argument */ + int touched = 0; /* was the current argument touched? */ + char **curargv = NULL, **newargv = NULL; + int argc = 0; + int totlen = 0; + int error_type = 0; /* 0 = no error; -1 = memory error; -2 = format error */ + int j; + + /* Abort if there is not target to set */ + if (target == NULL) + return -1; + + /* Build the command string accordingly to protocol */ + curarg = sdsempty(); + if (curarg == NULL) + return -1; + + while(*c != '\0') { + if (*c != '%' || c[1] == '\0') { + if (*c == ' ') { + if (touched) { + newargv = realloc(curargv,sizeof(char*)*(argc+1)); + if (newargv == NULL) goto memory_err; + curargv = newargv; + curargv[argc++] = curarg; + totlen += bulklen(sdslen(curarg)); + + /* curarg is put in argv so it can be overwritten. */ + curarg = sdsempty(); + if (curarg == NULL) goto memory_err; + touched = 0; + } + } else { + newarg = sdscatlen(curarg,c,1); + if (newarg == NULL) goto memory_err; + curarg = newarg; + touched = 1; + } + } else { + char *arg; + size_t size; + + /* Set newarg so it can be checked even if it is not touched. */ + newarg = curarg; + + switch(c[1]) { + case 's': + arg = va_arg(ap,char*); + size = strlen(arg); + if (size > 0) + newarg = sdscatlen(curarg,arg,size); + break; + case 'b': + arg = va_arg(ap,char*); + size = va_arg(ap,size_t); + if (size > 0) + newarg = sdscatlen(curarg,arg,size); + break; + case '%': + newarg = sdscat(curarg,"%"); + break; + default: + /* Try to detect printf format */ + { + static const char intfmts[] = "diouxX"; + static const char flags[] = "#0-+ "; + char _format[16]; + const char *_p = c+1; + size_t _l = 0; + va_list _cpy; + + /* Flags */ + while (*_p != '\0' && strchr(flags,*_p) != NULL) _p++; + + /* Field width */ + while (*_p != '\0' && isdigit(*_p)) _p++; + + /* Precision */ + if (*_p == '.') { + _p++; + while (*_p != '\0' && isdigit(*_p)) _p++; + } + + /* Copy va_list before consuming with va_arg */ + va_copy(_cpy,ap); + + /* Integer conversion (without modifiers) */ + if (strchr(intfmts,*_p) != NULL) { + va_arg(ap,int); + goto fmt_valid; + } + + /* Double conversion (without modifiers) */ + if (strchr("eEfFgGaA",*_p) != NULL) { + va_arg(ap,double); + goto fmt_valid; + } + + /* Size: char */ + if (_p[0] == 'h' && _p[1] == 'h') { + _p += 2; + if (*_p != '\0' && strchr(intfmts,*_p) != NULL) { + va_arg(ap,int); /* char gets promoted to int */ + goto fmt_valid; + } + goto fmt_invalid; + } + + /* Size: short */ + if (_p[0] == 'h') { + _p += 1; + if (*_p != '\0' && strchr(intfmts,*_p) != NULL) { + va_arg(ap,int); /* short gets promoted to int */ + goto fmt_valid; + } + goto fmt_invalid; + } + + /* Size: long long */ + if (_p[0] == 'l' && _p[1] == 'l') { + _p += 2; + if (*_p != '\0' && strchr(intfmts,*_p) != NULL) { + va_arg(ap,long long); + goto fmt_valid; + } + goto fmt_invalid; + } + + /* Size: long */ + if (_p[0] == 'l') { + _p += 1; + if (*_p != '\0' && strchr(intfmts,*_p) != NULL) { + va_arg(ap,long); + goto fmt_valid; + } + goto fmt_invalid; + } + + fmt_invalid: + va_end(_cpy); + goto format_err; + + fmt_valid: + _l = (_p+1)-c; + if (_l < sizeof(_format)-2) { + memcpy(_format,c,_l); + _format[_l] = '\0'; + newarg = sdscatvprintf(curarg,_format,_cpy); + + /* Update current position (note: outer blocks + * increment c twice so compensate here) */ + c = _p-1; + } + + va_end(_cpy); + break; + } + } + + if (newarg == NULL) goto memory_err; + curarg = newarg; + + touched = 1; + c++; + } + c++; + } + + /* Add the last argument if needed */ + if (touched) { + newargv = realloc(curargv,sizeof(char*)*(argc+1)); + if (newargv == NULL) goto memory_err; + curargv = newargv; + curargv[argc++] = curarg; + totlen += bulklen(sdslen(curarg)); + } else { + sdsfree(curarg); + } + + /* Clear curarg because it was put in curargv or was free'd. */ + curarg = NULL; + + /* Add bytes needed to hold multi bulk count */ + totlen += 1+countDigits(argc)+2; + + /* Build the command at protocol level */ + cmd = malloc(totlen+1); + if (cmd == NULL) goto memory_err; + + pos = sprintf(cmd,"*%d\r\n",argc); + for (j = 0; j < argc; j++) { + pos += sprintf(cmd+pos,"$%zu\r\n",sdslen(curargv[j])); + memcpy(cmd+pos,curargv[j],sdslen(curargv[j])); + pos += sdslen(curargv[j]); + sdsfree(curargv[j]); + cmd[pos++] = '\r'; + cmd[pos++] = '\n'; + } + assert(pos == totlen); + cmd[pos] = '\0'; + + free(curargv); + *target = cmd; + return totlen; + +format_err: + error_type = -2; + goto cleanup; + +memory_err: + error_type = -1; + goto cleanup; + +cleanup: + if (curargv) { + while(argc--) + sdsfree(curargv[argc]); + free(curargv); + } + + sdsfree(curarg); + + /* No need to check cmd since it is the last statement that can fail, + * but do it anyway to be as defensive as possible. */ + if (cmd != NULL) + free(cmd); + + return error_type; +} + +/* Format a command according to the Redis protocol. This function + * takes a format similar to printf: + * + * %s represents a C null terminated string you want to interpolate + * %b represents a binary safe string + * + * When using %b you need to provide both the pointer to the string + * and the length in bytes as a size_t. Examples: + * + * len = redisFormatCommand(target, "GET %s", mykey); + * len = redisFormatCommand(target, "SET %s %b", mykey, myval, myvallen); + */ +int redisFormatCommand(char **target, const char *format, ...) { + va_list ap; + int len; + va_start(ap,format); + len = redisvFormatCommand(target,format,ap); + va_end(ap); + + /* The API says "-1" means bad result, but we now also return "-2" in some + * cases. Force the return value to always be -1. */ + if (len < 0) + len = -1; + + return len; +} + +/* Format a command according to the Redis protocol using an sds string and + * sdscatfmt for the processing of arguments. This function takes the + * number of arguments, an array with arguments and an array with their + * lengths. If the latter is set to NULL, strlen will be used to compute the + * argument lengths. + */ +int redisFormatSdsCommandArgv(sds *target, int argc, const char **argv, + const size_t *argvlen) +{ + sds cmd; + unsigned long long totlen; + int j; + size_t len; + + /* Abort on a NULL target */ + if (target == NULL) + return -1; + + /* Calculate our total size */ + totlen = 1+countDigits(argc)+2; + for (j = 0; j < argc; j++) { + len = argvlen ? argvlen[j] : strlen(argv[j]); + totlen += bulklen(len); + } + + /* Use an SDS string for command construction */ + cmd = sdsempty(); + if (cmd == NULL) + return -1; + + /* We already know how much storage we need */ + cmd = sdsMakeRoomFor(cmd, totlen); + if (cmd == NULL) + return -1; + + /* Construct command */ + cmd = sdscatfmt(cmd, "*%i\r\n", argc); + for (j=0; j < argc; j++) { + len = argvlen ? argvlen[j] : strlen(argv[j]); + cmd = sdscatfmt(cmd, "$%T\r\n", len); + cmd = sdscatlen(cmd, argv[j], len); + cmd = sdscatlen(cmd, "\r\n", sizeof("\r\n")-1); + } + + assert(sdslen(cmd)==totlen); + + *target = cmd; + return totlen; +} + +void redisFreeSdsCommand(sds cmd) { + sdsfree(cmd); +} + +/* Format a command according to the Redis protocol. This function takes the + * number of arguments, an array with arguments and an array with their + * lengths. If the latter is set to NULL, strlen will be used to compute the + * argument lengths. + */ +int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen) { + char *cmd = NULL; /* final command */ + int pos; /* position in final command */ + size_t len; + int totlen, j; + + /* Abort on a NULL target */ + if (target == NULL) + return -1; + + /* Calculate number of bytes needed for the command */ + totlen = 1+countDigits(argc)+2; + for (j = 0; j < argc; j++) { + len = argvlen ? argvlen[j] : strlen(argv[j]); + totlen += bulklen(len); + } + + /* Build the command at protocol level */ + cmd = malloc(totlen+1); + if (cmd == NULL) + return -1; + + pos = sprintf(cmd,"*%d\r\n",argc); + for (j = 0; j < argc; j++) { + len = argvlen ? argvlen[j] : strlen(argv[j]); + pos += sprintf(cmd+pos,"$%zu\r\n",len); + memcpy(cmd+pos,argv[j],len); + pos += len; + cmd[pos++] = '\r'; + cmd[pos++] = '\n'; + } + assert(pos == totlen); + cmd[pos] = '\0'; + + *target = cmd; + return totlen; +} + +void redisFreeCommand(char *cmd) { + free(cmd); +} + +void __redisSetError(redisContext *c, int type, const char *str) { + size_t len; + + c->err = type; + if (str != NULL) { + len = strlen(str); + len = len < (sizeof(c->errstr)-1) ? len : (sizeof(c->errstr)-1); + memcpy(c->errstr,str,len); + c->errstr[len] = '\0'; + } else { + /* Only REDIS_ERR_IO may lack a description! */ + assert(type == REDIS_ERR_IO); + __redis_strerror_r(errno, c->errstr, sizeof(c->errstr)); + } +} + +redisReader *redisReaderCreate(void) { + return redisReaderCreateWithFunctions(&defaultFunctions); +} + +static redisContext *redisContextInit(void) { + redisContext *c; + + c = calloc(1,sizeof(redisContext)); + if (c == NULL) + return NULL; + + c->err = 0; + c->errstr[0] = '\0'; + c->obuf = sdsempty(); + c->reader = redisReaderCreate(); + c->tcp.host = NULL; + c->tcp.source_addr = NULL; + c->unix_sock.path = NULL; + c->timeout = NULL; + + if (c->obuf == NULL || c->reader == NULL) { + redisFree(c); + return NULL; + } + + return c; +} + +void redisFree(redisContext *c) { + if (c == NULL) + return; + if (c->fd > 0) + close(c->fd); + if (c->obuf != NULL) + sdsfree(c->obuf); + if (c->reader != NULL) + redisReaderFree(c->reader); + if (c->tcp.host) + free(c->tcp.host); + if (c->tcp.source_addr) + free(c->tcp.source_addr); + if (c->unix_sock.path) + free(c->unix_sock.path); + if (c->timeout) + free(c->timeout); + free(c); +} + +int redisFreeKeepFd(redisContext *c) { + int fd = c->fd; + c->fd = -1; + redisFree(c); + return fd; +} + +int redisReconnect(redisContext *c) { + c->err = 0; + memset(c->errstr, '\0', strlen(c->errstr)); + + if (c->fd > 0) { + close(c->fd); + } + + sdsfree(c->obuf); + redisReaderFree(c->reader); + + c->obuf = sdsempty(); + c->reader = redisReaderCreate(); + + if (c->connection_type == REDIS_CONN_TCP) { + return redisContextConnectBindTcp(c, c->tcp.host, c->tcp.port, + c->timeout, c->tcp.source_addr); + } else if (c->connection_type == REDIS_CONN_UNIX) { + return redisContextConnectUnix(c, c->unix_sock.path, c->timeout); + } else { + /* Something bad happened here and shouldn't have. There isn't + enough information in the context to reconnect. */ + __redisSetError(c,REDIS_ERR_OTHER,"Not enough information to reconnect"); + } + + return REDIS_ERR; +} + +/* Connect to a Redis instance. On error the field error in the returned + * context will be set to the return value of the error function. + * When no set of reply functions is given, the default set will be used. */ +redisContext *redisConnect(const char *ip, int port) { + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + + c->flags |= REDIS_BLOCK; + redisContextConnectTcp(c,ip,port,NULL); + return c; +} + +redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv) { + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + + c->flags |= REDIS_BLOCK; + redisContextConnectTcp(c,ip,port,&tv); + return c; +} + +redisContext *redisConnectNonBlock(const char *ip, int port) { + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + + c->flags &= ~REDIS_BLOCK; + redisContextConnectTcp(c,ip,port,NULL); + return c; +} + +redisContext *redisConnectBindNonBlock(const char *ip, int port, + const char *source_addr) { + redisContext *c = redisContextInit(); + c->flags &= ~REDIS_BLOCK; + redisContextConnectBindTcp(c,ip,port,NULL,source_addr); + return c; +} + +redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, + const char *source_addr) { + redisContext *c = redisContextInit(); + c->flags &= ~REDIS_BLOCK; + c->flags |= REDIS_REUSEADDR; + redisContextConnectBindTcp(c,ip,port,NULL,source_addr); + return c; +} + +redisContext *redisConnectUnix(const char *path) { + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + + c->flags |= REDIS_BLOCK; + redisContextConnectUnix(c,path,NULL); + return c; +} + +redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv) { + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + + c->flags |= REDIS_BLOCK; + redisContextConnectUnix(c,path,&tv); + return c; +} + +redisContext *redisConnectUnixNonBlock(const char *path) { + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + + c->flags &= ~REDIS_BLOCK; + redisContextConnectUnix(c,path,NULL); + return c; +} + +redisContext *redisConnectFd(int fd) { + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + + c->fd = fd; + c->flags |= REDIS_BLOCK | REDIS_CONNECTED; + return c; +} + +/* Set read/write timeout on a blocking socket. */ +int redisSetTimeout(redisContext *c, const struct timeval tv) { + if (c->flags & REDIS_BLOCK) + return redisContextSetTimeout(c,tv); + return REDIS_ERR; +} + +/* Enable connection KeepAlive. */ +int redisEnableKeepAlive(redisContext *c) { + if (redisKeepAlive(c, REDIS_KEEPALIVE_INTERVAL) != REDIS_OK) + return REDIS_ERR; + return REDIS_OK; +} + +/* Use this function to handle a read event on the descriptor. It will try + * and read some bytes from the socket and feed them to the reply parser. + * + * After this function is called, you may use redisContextReadReply to + * see if there is a reply available. */ +int redisBufferRead(redisContext *c) { + char buf[1024*16]; + int nread; + + /* Return early when the context has seen an error. */ + if (c->err) + return REDIS_ERR; + + nread = read(c->fd,buf,sizeof(buf)); + if (nread == -1) { + if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { + /* Try again later */ + } else { + __redisSetError(c,REDIS_ERR_IO,NULL); + return REDIS_ERR; + } + } else if (nread == 0) { + __redisSetError(c,REDIS_ERR_EOF,"Server closed the connection"); + return REDIS_ERR; + } else { + if (redisReaderFeed(c->reader,buf,nread) != REDIS_OK) { + __redisSetError(c,c->reader->err,c->reader->errstr); + return REDIS_ERR; + } + } + return REDIS_OK; +} + +/* Write the output buffer to the socket. + * + * Returns REDIS_OK when the buffer is empty, or (a part of) the buffer was + * succesfully written to the socket. When the buffer is empty after the + * write operation, "done" is set to 1 (if given). + * + * Returns REDIS_ERR if an error occured trying to write and sets + * c->errstr to hold the appropriate error string. + */ +int redisBufferWrite(redisContext *c, int *done) { + int nwritten; + + /* Return early when the context has seen an error. */ + if (c->err) + return REDIS_ERR; + + if (sdslen(c->obuf) > 0) { + nwritten = write(c->fd,c->obuf,sdslen(c->obuf)); + if (nwritten == -1) { + if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { + /* Try again later */ + } else { + __redisSetError(c,REDIS_ERR_IO,NULL); + return REDIS_ERR; + } + } else if (nwritten > 0) { + if (nwritten == (signed)sdslen(c->obuf)) { + sdsfree(c->obuf); + c->obuf = sdsempty(); + } else { + sdsrange(c->obuf,nwritten,-1); + } + } + } + if (done != NULL) *done = (sdslen(c->obuf) == 0); + return REDIS_OK; +} + +/* Internal helper function to try and get a reply from the reader, + * or set an error in the context otherwise. */ +int redisGetReplyFromReader(redisContext *c, void **reply) { + if (redisReaderGetReply(c->reader,reply) == REDIS_ERR) { + __redisSetError(c,c->reader->err,c->reader->errstr); + return REDIS_ERR; + } + return REDIS_OK; +} + +int redisGetReply(redisContext *c, void **reply) { + int wdone = 0; + void *aux = NULL; + + /* Try to read pending replies */ + if (redisGetReplyFromReader(c,&aux) == REDIS_ERR) + return REDIS_ERR; + + /* For the blocking context, flush output buffer and read reply */ + if (aux == NULL && c->flags & REDIS_BLOCK) { + /* Write until done */ + do { + if (redisBufferWrite(c,&wdone) == REDIS_ERR) + return REDIS_ERR; + } while (!wdone); + + /* Read until there is a reply */ + do { + if (redisBufferRead(c) == REDIS_ERR) + return REDIS_ERR; + if (redisGetReplyFromReader(c,&aux) == REDIS_ERR) + return REDIS_ERR; + } while (aux == NULL); + } + + /* Set reply object */ + if (reply != NULL) *reply = aux; + return REDIS_OK; +} + + +/* Helper function for the redisAppendCommand* family of functions. + * + * Write a formatted command to the output buffer. When this family + * is used, you need to call redisGetReply yourself to retrieve + * the reply (or replies in pub/sub). + */ +int __redisAppendCommand(redisContext *c, const char *cmd, size_t len) { + sds newbuf; + + newbuf = sdscatlen(c->obuf,cmd,len); + if (newbuf == NULL) { + __redisSetError(c,REDIS_ERR_OOM,"Out of memory"); + return REDIS_ERR; + } + + c->obuf = newbuf; + return REDIS_OK; +} + +int redisAppendFormattedCommand(redisContext *c, const char *cmd, size_t len) { + + if (__redisAppendCommand(c, cmd, len) != REDIS_OK) { + return REDIS_ERR; + } + + return REDIS_OK; +} + +int redisvAppendCommand(redisContext *c, const char *format, va_list ap) { + char *cmd; + int len; + + len = redisvFormatCommand(&cmd,format,ap); + if (len == -1) { + __redisSetError(c,REDIS_ERR_OOM,"Out of memory"); + return REDIS_ERR; + } else if (len == -2) { + __redisSetError(c,REDIS_ERR_OTHER,"Invalid format string"); + return REDIS_ERR; + } + + if (__redisAppendCommand(c,cmd,len) != REDIS_OK) { + free(cmd); + return REDIS_ERR; + } + + free(cmd); + return REDIS_OK; +} + +int redisAppendCommand(redisContext *c, const char *format, ...) { + va_list ap; + int ret; + + va_start(ap,format); + ret = redisvAppendCommand(c,format,ap); + va_end(ap); + return ret; +} + +int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) { + sds cmd; + int len; + + len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen); + if (len == -1) { + __redisSetError(c,REDIS_ERR_OOM,"Out of memory"); + return REDIS_ERR; + } + + if (__redisAppendCommand(c,cmd,len) != REDIS_OK) { + sdsfree(cmd); + return REDIS_ERR; + } + + sdsfree(cmd); + return REDIS_OK; +} + +/* Helper function for the redisCommand* family of functions. + * + * Write a formatted command to the output buffer. If the given context is + * blocking, immediately read the reply into the "reply" pointer. When the + * context is non-blocking, the "reply" pointer will not be used and the + * command is simply appended to the write buffer. + * + * Returns the reply when a reply was succesfully retrieved. Returns NULL + * otherwise. When NULL is returned in a blocking context, the error field + * in the context will be set. + */ +static void *__redisBlockForReply(redisContext *c) { + void *reply; + + if (c->flags & REDIS_BLOCK) { + if (redisGetReply(c,&reply) != REDIS_OK) + return NULL; + return reply; + } + return NULL; +} + +void *redisvCommand(redisContext *c, const char *format, va_list ap) { + if (redisvAppendCommand(c,format,ap) != REDIS_OK) + return NULL; + return __redisBlockForReply(c); +} + +void *redisCommand(redisContext *c, const char *format, ...) { + va_list ap; + void *reply = NULL; + va_start(ap,format); + reply = redisvCommand(c,format,ap); + va_end(ap); + return reply; +} + +void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) { + if (redisAppendCommandArgv(c,argc,argv,argvlen) != REDIS_OK) + return NULL; + return __redisBlockForReply(c); +} diff --git a/third-party/hiredis/hiredis.h b/third-party/hiredis/hiredis.h new file mode 100644 index 000000000..fe267b9b3 --- /dev/null +++ b/third-party/hiredis/hiredis.h @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2014, Pieter Noordhuis + * Copyright (c) 2015, Matt Stancliff , + * Jan-Erik Rediger + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_H +#define __HIREDIS_H +#include "read.h" +#include /* for va_list */ +#include /* for struct timeval */ +#include /* uintXX_t, etc */ +#include "sds.h" /* for sds */ + +#define HIREDIS_MAJOR 0 +#define HIREDIS_MINOR 13 +#define HIREDIS_PATCH 3 +#define HIREDIS_SONAME 0.13 + +/* Connection type can be blocking or non-blocking and is set in the + * least significant bit of the flags field in redisContext. */ +#define REDIS_BLOCK 0x1 + +/* Connection may be disconnected before being free'd. The second bit + * in the flags field is set when the context is connected. */ +#define REDIS_CONNECTED 0x2 + +/* The async API might try to disconnect cleanly and flush the output + * buffer and read all subsequent replies before disconnecting. + * This flag means no new commands can come in and the connection + * should be terminated once all replies have been read. */ +#define REDIS_DISCONNECTING 0x4 + +/* Flag specific to the async API which means that the context should be clean + * up as soon as possible. */ +#define REDIS_FREEING 0x8 + +/* Flag that is set when an async callback is executed. */ +#define REDIS_IN_CALLBACK 0x10 + +/* Flag that is set when the async context has one or more subscriptions. */ +#define REDIS_SUBSCRIBED 0x20 + +/* Flag that is set when monitor mode is active */ +#define REDIS_MONITORING 0x40 + +/* Flag that is set when we should set SO_REUSEADDR before calling bind() */ +#define REDIS_REUSEADDR 0x80 + +#define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */ + +/* number of times we retry to connect in the case of EADDRNOTAVAIL and + * SO_REUSEADDR is being used. */ +#define REDIS_CONNECT_RETRIES 10 + +/* strerror_r has two completely different prototypes and behaviors + * depending on system issues, so we need to operate on the error buffer + * differently depending on which strerror_r we're using. */ +#ifndef _GNU_SOURCE +/* "regular" POSIX strerror_r that does the right thing. */ +#define __redis_strerror_r(errno, buf, len) \ + do { \ + strerror_r((errno), (buf), (len)); \ + } while (0) +#else +/* "bad" GNU strerror_r we need to clean up after. */ +#define __redis_strerror_r(errno, buf, len) \ + do { \ + char *err_str = strerror_r((errno), (buf), (len)); \ + /* If return value _isn't_ the start of the buffer we passed in, \ + * then GNU strerror_r returned an internal static buffer and we \ + * need to copy the result into our private buffer. */ \ + if (err_str != (buf)) { \ + buf[(len)] = '\0'; \ + strncat((buf), err_str, ((len) - 1)); \ + } \ + } while (0) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* This is the reply object returned by redisCommand() */ +typedef struct redisReply { + int type; /* REDIS_REPLY_* */ + long long integer; /* The integer when type is REDIS_REPLY_INTEGER */ + int len; /* Length of string */ + char *str; /* Used for both REDIS_REPLY_ERROR and REDIS_REPLY_STRING */ + size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */ + struct redisReply **element; /* elements vector for REDIS_REPLY_ARRAY */ +} redisReply; + +redisReader *redisReaderCreate(void); + +/* Function to free the reply objects hiredis returns by default. */ +void freeReplyObject(void *reply); + +/* Functions to format a command according to the protocol. */ +int redisvFormatCommand(char **target, const char *format, va_list ap); +int redisFormatCommand(char **target, const char *format, ...); +int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen); +int redisFormatSdsCommandArgv(sds *target, int argc, const char ** argv, const size_t *argvlen); +void redisFreeCommand(char *cmd); +void redisFreeSdsCommand(sds cmd); + +enum redisConnectionType { + REDIS_CONN_TCP, + REDIS_CONN_UNIX, +}; + +/* Context for a connection to Redis */ +typedef struct redisContext { + int err; /* Error flags, 0 when there is no error */ + char errstr[128]; /* String representation of error when applicable */ + int fd; + int flags; + char *obuf; /* Write buffer */ + redisReader *reader; /* Protocol reader */ + + enum redisConnectionType connection_type; + struct timeval *timeout; + + struct { + char *host; + char *source_addr; + int port; + } tcp; + + struct { + char *path; + } unix_sock; + +} redisContext; + +redisContext *redisConnect(const char *ip, int port); +redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv); +redisContext *redisConnectNonBlock(const char *ip, int port); +redisContext *redisConnectBindNonBlock(const char *ip, int port, + const char *source_addr); +redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, + const char *source_addr); +redisContext *redisConnectUnix(const char *path); +redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv); +redisContext *redisConnectUnixNonBlock(const char *path); +redisContext *redisConnectFd(int fd); + +/** + * Reconnect the given context using the saved information. + * + * This re-uses the exact same connect options as in the initial connection. + * host, ip (or path), timeout and bind address are reused, + * flags are used unmodified from the existing context. + * + * Returns REDIS_OK on successfull connect or REDIS_ERR otherwise. + */ +int redisReconnect(redisContext *c); + +int redisSetTimeout(redisContext *c, const struct timeval tv); +int redisEnableKeepAlive(redisContext *c); +void redisFree(redisContext *c); +int redisFreeKeepFd(redisContext *c); +int redisBufferRead(redisContext *c); +int redisBufferWrite(redisContext *c, int *done); + +/* In a blocking context, this function first checks if there are unconsumed + * replies to return and returns one if so. Otherwise, it flushes the output + * buffer to the socket and reads until it has a reply. In a non-blocking + * context, it will return unconsumed replies until there are no more. */ +int redisGetReply(redisContext *c, void **reply); +int redisGetReplyFromReader(redisContext *c, void **reply); + +/* Write a formatted command to the output buffer. Use these functions in blocking mode + * to get a pipeline of commands. */ +int redisAppendFormattedCommand(redisContext *c, const char *cmd, size_t len); + +/* Write a command to the output buffer. Use these functions in blocking mode + * to get a pipeline of commands. */ +int redisvAppendCommand(redisContext *c, const char *format, va_list ap); +int redisAppendCommand(redisContext *c, const char *format, ...); +int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen); + +/* Issue a command to Redis. In a blocking context, it is identical to calling + * redisAppendCommand, followed by redisGetReply. The function will return + * NULL if there was an error in performing the request, otherwise it will + * return the reply. In a non-blocking context, it is identical to calling + * only redisAppendCommand and will always return NULL. */ +void *redisvCommand(redisContext *c, const char *format, va_list ap); +void *redisCommand(redisContext *c, const char *format, ...); +void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/third-party/hiredis/net.c b/third-party/hiredis/net.c new file mode 100644 index 000000000..60a2dc754 --- /dev/null +++ b/third-party/hiredis/net.c @@ -0,0 +1,458 @@ +/* Extracted from anet.c to work properly with Hiredis error reporting. + * + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2014, Pieter Noordhuis + * Copyright (c) 2015, Matt Stancliff , + * Jan-Erik Rediger + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "fmacros.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "net.h" +#include "sds.h" + +/* Defined in hiredis.c */ +void __redisSetError(redisContext *c, int type, const char *str); + +static void redisContextCloseFd(redisContext *c) { + if (c && c->fd >= 0) { + close(c->fd); + c->fd = -1; + } +} + +static void __redisSetErrorFromErrno(redisContext *c, int type, const char *prefix) { + char buf[128] = { 0 }; + size_t len = 0; + + if (prefix != NULL) + len = snprintf(buf,sizeof(buf),"%s: ",prefix); + __redis_strerror_r(errno, (char *)(buf + len), sizeof(buf) - len); + __redisSetError(c,type,buf); +} + +static int redisSetReuseAddr(redisContext *c) { + int on = 1; + if (setsockopt(c->fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) { + __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); + redisContextCloseFd(c); + return REDIS_ERR; + } + return REDIS_OK; +} + +static int redisCreateSocket(redisContext *c, int type) { + int s; + if ((s = socket(type, SOCK_STREAM, 0)) == -1) { + __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); + return REDIS_ERR; + } + c->fd = s; + if (type == AF_INET) { + if (redisSetReuseAddr(c) == REDIS_ERR) { + return REDIS_ERR; + } + } + return REDIS_OK; +} + +static int redisSetBlocking(redisContext *c, int blocking) { + int flags; + + /* Set the socket nonblocking. + * Note that fcntl(2) for F_GETFL and F_SETFL can't be + * interrupted by a signal. */ + if ((flags = fcntl(c->fd, F_GETFL)) == -1) { + __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_GETFL)"); + redisContextCloseFd(c); + return REDIS_ERR; + } + + if (blocking) + flags &= ~O_NONBLOCK; + else + flags |= O_NONBLOCK; + + if (fcntl(c->fd, F_SETFL, flags) == -1) { + __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_SETFL)"); + redisContextCloseFd(c); + return REDIS_ERR; + } + return REDIS_OK; +} + +int redisKeepAlive(redisContext *c, int interval) { + int val = 1; + int fd = c->fd; + + if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1){ + __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); + return REDIS_ERR; + } + + val = interval; + +#ifdef _OSX + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE, &val, sizeof(val)) < 0) { + __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); + return REDIS_ERR; + } +#else +#if defined(__GLIBC__) && !defined(__FreeBSD_kernel__) + val = interval; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &val, sizeof(val)) < 0) { + __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); + return REDIS_ERR; + } + + val = interval/3; + if (val == 0) val = 1; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &val, sizeof(val)) < 0) { + __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); + return REDIS_ERR; + } + + val = 3; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &val, sizeof(val)) < 0) { + __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); + return REDIS_ERR; + } +#endif +#endif + + return REDIS_OK; +} + +static int redisSetTcpNoDelay(redisContext *c) { + int yes = 1; + if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) { + __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(TCP_NODELAY)"); + redisContextCloseFd(c); + return REDIS_ERR; + } + return REDIS_OK; +} + +#define __MAX_MSEC (((LONG_MAX) - 999) / 1000) + +static int redisContextWaitReady(redisContext *c, const struct timeval *timeout) { + struct pollfd wfd[1]; + long msec; + + msec = -1; + wfd[0].fd = c->fd; + wfd[0].events = POLLOUT; + + /* Only use timeout when not NULL. */ + if (timeout != NULL) { + if (timeout->tv_usec > 1000000 || timeout->tv_sec > __MAX_MSEC) { + __redisSetErrorFromErrno(c, REDIS_ERR_IO, NULL); + redisContextCloseFd(c); + return REDIS_ERR; + } + + msec = (timeout->tv_sec * 1000) + ((timeout->tv_usec + 999) / 1000); + + if (msec < 0 || msec > INT_MAX) { + msec = INT_MAX; + } + } + + if (errno == EINPROGRESS) { + int res; + + if ((res = poll(wfd, 1, msec)) == -1) { + __redisSetErrorFromErrno(c, REDIS_ERR_IO, "poll(2)"); + redisContextCloseFd(c); + return REDIS_ERR; + } else if (res == 0) { + errno = ETIMEDOUT; + __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); + redisContextCloseFd(c); + return REDIS_ERR; + } + + if (redisCheckSocketError(c) != REDIS_OK) + return REDIS_ERR; + + return REDIS_OK; + } + + __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); + redisContextCloseFd(c); + return REDIS_ERR; +} + +int redisCheckSocketError(redisContext *c) { + int err = 0; + socklen_t errlen = sizeof(err); + + if (getsockopt(c->fd, SOL_SOCKET, SO_ERROR, &err, &errlen) == -1) { + __redisSetErrorFromErrno(c,REDIS_ERR_IO,"getsockopt(SO_ERROR)"); + return REDIS_ERR; + } + + if (err) { + errno = err; + __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); + return REDIS_ERR; + } + + return REDIS_OK; +} + +int redisContextSetTimeout(redisContext *c, const struct timeval tv) { + if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,&tv,sizeof(tv)) == -1) { + __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_RCVTIMEO)"); + return REDIS_ERR; + } + if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,&tv,sizeof(tv)) == -1) { + __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_SNDTIMEO)"); + return REDIS_ERR; + } + return REDIS_OK; +} + +static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, + const struct timeval *timeout, + const char *source_addr) { + int s, rv, n; + char _port[6]; /* strlen("65535"); */ + struct addrinfo hints, *servinfo, *bservinfo, *p, *b; + int blocking = (c->flags & REDIS_BLOCK); + int reuseaddr = (c->flags & REDIS_REUSEADDR); + int reuses = 0; + + c->connection_type = REDIS_CONN_TCP; + c->tcp.port = port; + + /* We need to take possession of the passed parameters + * to make them reusable for a reconnect. + * We also carefully check we don't free data we already own, + * as in the case of the reconnect method. + * + * This is a bit ugly, but atleast it works and doesn't leak memory. + **/ + if (c->tcp.host != addr) { + if (c->tcp.host) + free(c->tcp.host); + + c->tcp.host = strdup(addr); + } + + if (timeout) { + if (c->timeout != timeout) { + if (c->timeout == NULL) + c->timeout = malloc(sizeof(struct timeval)); + + memcpy(c->timeout, timeout, sizeof(struct timeval)); + } + } else { + if (c->timeout) + free(c->timeout); + c->timeout = NULL; + } + + if (source_addr == NULL) { + free(c->tcp.source_addr); + c->tcp.source_addr = NULL; + } else if (c->tcp.source_addr != source_addr) { + free(c->tcp.source_addr); + c->tcp.source_addr = strdup(source_addr); + } + + snprintf(_port, 6, "%d", port); + memset(&hints,0,sizeof(hints)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + + /* Try with IPv6 if no IPv4 address was found. We do it in this order since + * in a Redis client you can't afford to test if you have IPv6 connectivity + * as this would add latency to every connect. Otherwise a more sensible + * route could be: Use IPv6 if both addresses are available and there is IPv6 + * connectivity. */ + if ((rv = getaddrinfo(c->tcp.host,_port,&hints,&servinfo)) != 0) { + hints.ai_family = AF_INET6; + if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) { + __redisSetError(c,REDIS_ERR_OTHER,gai_strerror(rv)); + return REDIS_ERR; + } + } + for (p = servinfo; p != NULL; p = p->ai_next) { +addrretry: + if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) + continue; + + c->fd = s; + if (redisSetBlocking(c,0) != REDIS_OK) + goto error; + if (c->tcp.source_addr) { + int bound = 0; + /* Using getaddrinfo saves us from self-determining IPv4 vs IPv6 */ + if ((rv = getaddrinfo(c->tcp.source_addr, NULL, &hints, &bservinfo)) != 0) { + char buf[128]; + snprintf(buf,sizeof(buf),"Can't get addr: %s",gai_strerror(rv)); + __redisSetError(c,REDIS_ERR_OTHER,buf); + goto error; + } + + if (reuseaddr) { + n = 1; + if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char*) &n, + sizeof(n)) < 0) { + goto error; + } + } + + for (b = bservinfo; b != NULL; b = b->ai_next) { + if (bind(s,b->ai_addr,b->ai_addrlen) != -1) { + bound = 1; + break; + } + } + freeaddrinfo(bservinfo); + if (!bound) { + char buf[128]; + snprintf(buf,sizeof(buf),"Can't bind socket: %s",strerror(errno)); + __redisSetError(c,REDIS_ERR_OTHER,buf); + goto error; + } + } + if (connect(s,p->ai_addr,p->ai_addrlen) == -1) { + if (errno == EHOSTUNREACH) { + redisContextCloseFd(c); + continue; + } else if (errno == EINPROGRESS && !blocking) { + /* This is ok. */ + } else if (errno == EADDRNOTAVAIL && reuseaddr) { + if (++reuses >= REDIS_CONNECT_RETRIES) { + goto error; + } else { + goto addrretry; + } + } else { + if (redisContextWaitReady(c,c->timeout) != REDIS_OK) + goto error; + } + } + if (blocking && redisSetBlocking(c,1) != REDIS_OK) + goto error; + if (redisSetTcpNoDelay(c) != REDIS_OK) + goto error; + + c->flags |= REDIS_CONNECTED; + rv = REDIS_OK; + goto end; + } + if (p == NULL) { + char buf[128]; + snprintf(buf,sizeof(buf),"Can't create socket: %s",strerror(errno)); + __redisSetError(c,REDIS_ERR_OTHER,buf); + goto error; + } + +error: + rv = REDIS_ERR; +end: + freeaddrinfo(servinfo); + return rv; // Need to return REDIS_OK if alright +} + +int redisContextConnectTcp(redisContext *c, const char *addr, int port, + const struct timeval *timeout) { + return _redisContextConnectTcp(c, addr, port, timeout, NULL); +} + +int redisContextConnectBindTcp(redisContext *c, const char *addr, int port, + const struct timeval *timeout, + const char *source_addr) { + return _redisContextConnectTcp(c, addr, port, timeout, source_addr); +} + +int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout) { + int blocking = (c->flags & REDIS_BLOCK); + struct sockaddr_un sa; + + if (redisCreateSocket(c,AF_LOCAL) < 0) + return REDIS_ERR; + if (redisSetBlocking(c,0) != REDIS_OK) + return REDIS_ERR; + + c->connection_type = REDIS_CONN_UNIX; + if (c->unix_sock.path != path) + c->unix_sock.path = strdup(path); + + if (timeout) { + if (c->timeout != timeout) { + if (c->timeout == NULL) + c->timeout = malloc(sizeof(struct timeval)); + + memcpy(c->timeout, timeout, sizeof(struct timeval)); + } + } else { + if (c->timeout) + free(c->timeout); + c->timeout = NULL; + } + + sa.sun_family = AF_LOCAL; + strncpy(sa.sun_path,path,sizeof(sa.sun_path)-1); + if (connect(c->fd, (struct sockaddr*)&sa, sizeof(sa)) == -1) { + if (errno == EINPROGRESS && !blocking) { + /* This is ok. */ + } else { + if (redisContextWaitReady(c,c->timeout) != REDIS_OK) + return REDIS_ERR; + } + } + + /* Reset socket to be blocking after connect(2). */ + if (blocking && redisSetBlocking(c,1) != REDIS_OK) + return REDIS_ERR; + + c->flags |= REDIS_CONNECTED; + return REDIS_OK; +} diff --git a/third-party/hiredis/net.h b/third-party/hiredis/net.h new file mode 100644 index 000000000..2f1a0bf85 --- /dev/null +++ b/third-party/hiredis/net.h @@ -0,0 +1,53 @@ +/* Extracted from anet.c to work properly with Hiredis error reporting. + * + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2014, Pieter Noordhuis + * Copyright (c) 2015, Matt Stancliff , + * Jan-Erik Rediger + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __NET_H +#define __NET_H + +#include "hiredis.h" + +#if defined(__sun) +#define AF_LOCAL AF_UNIX +#endif + +int redisCheckSocketError(redisContext *c); +int redisContextSetTimeout(redisContext *c, const struct timeval tv); +int redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout); +int redisContextConnectBindTcp(redisContext *c, const char *addr, int port, + const struct timeval *timeout, + const char *source_addr); +int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout); +int redisKeepAlive(redisContext *c, int interval); + +#endif diff --git a/third-party/hiredis/read.c b/third-party/hiredis/read.c new file mode 100644 index 000000000..df1a467a9 --- /dev/null +++ b/third-party/hiredis/read.c @@ -0,0 +1,525 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#include "fmacros.h" +#include +#include +#ifndef _MSC_VER +#include +#endif +#include +#include +#include + +#include "read.h" +#include "sds.h" + +static void __redisReaderSetError(redisReader *r, int type, const char *str) { + size_t len; + + if (r->reply != NULL && r->fn && r->fn->freeObject) { + r->fn->freeObject(r->reply); + r->reply = NULL; + } + + /* Clear input buffer on errors. */ + if (r->buf != NULL) { + sdsfree(r->buf); + r->buf = NULL; + r->pos = r->len = 0; + } + + /* Reset task stack. */ + r->ridx = -1; + + /* Set error. */ + r->err = type; + len = strlen(str); + len = len < (sizeof(r->errstr)-1) ? len : (sizeof(r->errstr)-1); + memcpy(r->errstr,str,len); + r->errstr[len] = '\0'; +} + +static size_t chrtos(char *buf, size_t size, char byte) { + size_t len = 0; + + switch(byte) { + case '\\': + case '"': + len = snprintf(buf,size,"\"\\%c\"",byte); + break; + case '\n': len = snprintf(buf,size,"\"\\n\""); break; + case '\r': len = snprintf(buf,size,"\"\\r\""); break; + case '\t': len = snprintf(buf,size,"\"\\t\""); break; + case '\a': len = snprintf(buf,size,"\"\\a\""); break; + case '\b': len = snprintf(buf,size,"\"\\b\""); break; + default: + if (isprint(byte)) + len = snprintf(buf,size,"\"%c\"",byte); + else + len = snprintf(buf,size,"\"\\x%02x\"",(unsigned char)byte); + break; + } + + return len; +} + +static void __redisReaderSetErrorProtocolByte(redisReader *r, char byte) { + char cbuf[8], sbuf[128]; + + chrtos(cbuf,sizeof(cbuf),byte); + snprintf(sbuf,sizeof(sbuf), + "Protocol error, got %s as reply type byte", cbuf); + __redisReaderSetError(r,REDIS_ERR_PROTOCOL,sbuf); +} + +static void __redisReaderSetErrorOOM(redisReader *r) { + __redisReaderSetError(r,REDIS_ERR_OOM,"Out of memory"); +} + +static char *readBytes(redisReader *r, unsigned int bytes) { + char *p; + if (r->len-r->pos >= bytes) { + p = r->buf+r->pos; + r->pos += bytes; + return p; + } + return NULL; +} + +/* Find pointer to \r\n. */ +static char *seekNewline(char *s, size_t len) { + int pos = 0; + int _len = len-1; + + /* Position should be < len-1 because the character at "pos" should be + * followed by a \n. Note that strchr cannot be used because it doesn't + * allow to search a limited length and the buffer that is being searched + * might not have a trailing NULL character. */ + while (pos < _len) { + while(pos < _len && s[pos] != '\r') pos++; + if (s[pos] != '\r') { + /* Not found. */ + return NULL; + } else { + if (s[pos+1] == '\n') { + /* Found. */ + return s+pos; + } else { + /* Continue searching. */ + pos++; + } + } + } + return NULL; +} + +/* Read a long long value starting at *s, under the assumption that it will be + * terminated by \r\n. Ambiguously returns -1 for unexpected input. */ +static long long readLongLong(char *s) { + long long v = 0; + int dec, mult = 1; + char c; + + if (*s == '-') { + mult = -1; + s++; + } else if (*s == '+') { + mult = 1; + s++; + } + + while ((c = *(s++)) != '\r') { + dec = c - '0'; + if (dec >= 0 && dec < 10) { + v *= 10; + v += dec; + } else { + /* Should not happen... */ + return -1; + } + } + + return mult*v; +} + +static char *readLine(redisReader *r, int *_len) { + char *p, *s; + int len; + + p = r->buf+r->pos; + s = seekNewline(p,(r->len-r->pos)); + if (s != NULL) { + len = s-(r->buf+r->pos); + r->pos += len+2; /* skip \r\n */ + if (_len) *_len = len; + return p; + } + return NULL; +} + +static void moveToNextTask(redisReader *r) { + redisReadTask *cur, *prv; + while (r->ridx >= 0) { + /* Return a.s.a.p. when the stack is now empty. */ + if (r->ridx == 0) { + r->ridx--; + return; + } + + cur = &(r->rstack[r->ridx]); + prv = &(r->rstack[r->ridx-1]); + assert(prv->type == REDIS_REPLY_ARRAY); + if (cur->idx == prv->elements-1) { + r->ridx--; + } else { + /* Reset the type because the next item can be anything */ + assert(cur->idx < prv->elements); + cur->type = -1; + cur->elements = -1; + cur->idx++; + return; + } + } +} + +static int processLineItem(redisReader *r) { + redisReadTask *cur = &(r->rstack[r->ridx]); + void *obj; + char *p; + int len; + + if ((p = readLine(r,&len)) != NULL) { + if (cur->type == REDIS_REPLY_INTEGER) { + if (r->fn && r->fn->createInteger) + obj = r->fn->createInteger(cur,readLongLong(p)); + else + obj = (void*)REDIS_REPLY_INTEGER; + } else { + /* Type will be error or status. */ + if (r->fn && r->fn->createString) + obj = r->fn->createString(cur,p,len); + else + obj = (void*)(size_t)(cur->type); + } + + if (obj == NULL) { + __redisReaderSetErrorOOM(r); + return REDIS_ERR; + } + + /* Set reply if this is the root object. */ + if (r->ridx == 0) r->reply = obj; + moveToNextTask(r); + return REDIS_OK; + } + + return REDIS_ERR; +} + +static int processBulkItem(redisReader *r) { + redisReadTask *cur = &(r->rstack[r->ridx]); + void *obj = NULL; + char *p, *s; + long len; + unsigned long bytelen; + int success = 0; + + p = r->buf+r->pos; + s = seekNewline(p,r->len-r->pos); + if (s != NULL) { + p = r->buf+r->pos; + bytelen = s-(r->buf+r->pos)+2; /* include \r\n */ + len = readLongLong(p); + + if (len < 0) { + /* The nil object can always be created. */ + if (r->fn && r->fn->createNil) + obj = r->fn->createNil(cur); + else + obj = (void*)REDIS_REPLY_NIL; + success = 1; + } else { + /* Only continue when the buffer contains the entire bulk item. */ + bytelen += len+2; /* include \r\n */ + if (r->pos+bytelen <= r->len) { + if (r->fn && r->fn->createString) + obj = r->fn->createString(cur,s+2,len); + else + obj = (void*)REDIS_REPLY_STRING; + success = 1; + } + } + + /* Proceed when obj was created. */ + if (success) { + if (obj == NULL) { + __redisReaderSetErrorOOM(r); + return REDIS_ERR; + } + + r->pos += bytelen; + + /* Set reply if this is the root object. */ + if (r->ridx == 0) r->reply = obj; + moveToNextTask(r); + return REDIS_OK; + } + } + + return REDIS_ERR; +} + +static int processMultiBulkItem(redisReader *r) { + redisReadTask *cur = &(r->rstack[r->ridx]); + void *obj; + char *p; + long elements; + int root = 0; + + /* Set error for nested multi bulks with depth > 7 */ + if (r->ridx == 8) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "No support for nested multi bulk replies with depth > 7"); + return REDIS_ERR; + } + + if ((p = readLine(r,NULL)) != NULL) { + elements = readLongLong(p); + root = (r->ridx == 0); + + if (elements == -1) { + if (r->fn && r->fn->createNil) + obj = r->fn->createNil(cur); + else + obj = (void*)REDIS_REPLY_NIL; + + if (obj == NULL) { + __redisReaderSetErrorOOM(r); + return REDIS_ERR; + } + + moveToNextTask(r); + } else { + if (r->fn && r->fn->createArray) + obj = r->fn->createArray(cur,elements); + else + obj = (void*)REDIS_REPLY_ARRAY; + + if (obj == NULL) { + __redisReaderSetErrorOOM(r); + return REDIS_ERR; + } + + /* Modify task stack when there are more than 0 elements. */ + if (elements > 0) { + cur->elements = elements; + cur->obj = obj; + r->ridx++; + r->rstack[r->ridx].type = -1; + r->rstack[r->ridx].elements = -1; + r->rstack[r->ridx].idx = 0; + r->rstack[r->ridx].obj = NULL; + r->rstack[r->ridx].parent = cur; + r->rstack[r->ridx].privdata = r->privdata; + } else { + moveToNextTask(r); + } + } + + /* Set reply if this is the root object. */ + if (root) r->reply = obj; + return REDIS_OK; + } + + return REDIS_ERR; +} + +static int processItem(redisReader *r) { + redisReadTask *cur = &(r->rstack[r->ridx]); + char *p; + + /* check if we need to read type */ + if (cur->type < 0) { + if ((p = readBytes(r,1)) != NULL) { + switch (p[0]) { + case '-': + cur->type = REDIS_REPLY_ERROR; + break; + case '+': + cur->type = REDIS_REPLY_STATUS; + break; + case ':': + cur->type = REDIS_REPLY_INTEGER; + break; + case '$': + cur->type = REDIS_REPLY_STRING; + break; + case '*': + cur->type = REDIS_REPLY_ARRAY; + break; + default: + __redisReaderSetErrorProtocolByte(r,*p); + return REDIS_ERR; + } + } else { + /* could not consume 1 byte */ + return REDIS_ERR; + } + } + + /* process typed item */ + switch(cur->type) { + case REDIS_REPLY_ERROR: + case REDIS_REPLY_STATUS: + case REDIS_REPLY_INTEGER: + return processLineItem(r); + case REDIS_REPLY_STRING: + return processBulkItem(r); + case REDIS_REPLY_ARRAY: + return processMultiBulkItem(r); + default: + assert(NULL); + return REDIS_ERR; /* Avoid warning. */ + } +} + +redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn) { + redisReader *r; + + r = calloc(sizeof(redisReader),1); + if (r == NULL) + return NULL; + + r->err = 0; + r->errstr[0] = '\0'; + r->fn = fn; + r->buf = sdsempty(); + r->maxbuf = REDIS_READER_MAX_BUF; + if (r->buf == NULL) { + free(r); + return NULL; + } + + r->ridx = -1; + return r; +} + +void redisReaderFree(redisReader *r) { + if (r->reply != NULL && r->fn && r->fn->freeObject) + r->fn->freeObject(r->reply); + if (r->buf != NULL) + sdsfree(r->buf); + free(r); +} + +int redisReaderFeed(redisReader *r, const char *buf, size_t len) { + sds newbuf; + + /* Return early when this reader is in an erroneous state. */ + if (r->err) + return REDIS_ERR; + + /* Copy the provided buffer. */ + if (buf != NULL && len >= 1) { + /* Destroy internal buffer when it is empty and is quite large. */ + if (r->len == 0 && r->maxbuf != 0 && sdsavail(r->buf) > r->maxbuf) { + sdsfree(r->buf); + r->buf = sdsempty(); + r->pos = 0; + + /* r->buf should not be NULL since we just free'd a larger one. */ + assert(r->buf != NULL); + } + + newbuf = sdscatlen(r->buf,buf,len); + if (newbuf == NULL) { + __redisReaderSetErrorOOM(r); + return REDIS_ERR; + } + + r->buf = newbuf; + r->len = sdslen(r->buf); + } + + return REDIS_OK; +} + +int redisReaderGetReply(redisReader *r, void **reply) { + /* Default target pointer to NULL. */ + if (reply != NULL) + *reply = NULL; + + /* Return early when this reader is in an erroneous state. */ + if (r->err) + return REDIS_ERR; + + /* When the buffer is empty, there will never be a reply. */ + if (r->len == 0) + return REDIS_OK; + + /* Set first item to process when the stack is empty. */ + if (r->ridx == -1) { + r->rstack[0].type = -1; + r->rstack[0].elements = -1; + r->rstack[0].idx = -1; + r->rstack[0].obj = NULL; + r->rstack[0].parent = NULL; + r->rstack[0].privdata = r->privdata; + r->ridx = 0; + } + + /* Process items in reply. */ + while (r->ridx >= 0) + if (processItem(r) != REDIS_OK) + break; + + /* Return ASAP when an error occurred. */ + if (r->err) + return REDIS_ERR; + + /* Discard part of the buffer when we've consumed at least 1k, to avoid + * doing unnecessary calls to memmove() in sds.c. */ + if (r->pos >= 1024) { + sdsrange(r->buf,r->pos,-1); + r->pos = 0; + r->len = sdslen(r->buf); + } + + /* Emit a reply when there is one. */ + if (r->ridx == -1) { + if (reply != NULL) + *reply = r->reply; + r->reply = NULL; + } + return REDIS_OK; +} diff --git a/third-party/hiredis/read.h b/third-party/hiredis/read.h new file mode 100644 index 000000000..635a872cd --- /dev/null +++ b/third-party/hiredis/read.h @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#ifndef __HIREDIS_READ_H +#define __HIREDIS_READ_H +#include /* for size_t */ + +#define REDIS_ERR -1 +#define REDIS_OK 0 + +/* When an error occurs, the err flag in a context is set to hold the type of + * error that occured. REDIS_ERR_IO means there was an I/O error and you + * should use the "errno" variable to find out what is wrong. + * For other values, the "errstr" field will hold a description. */ +#define REDIS_ERR_IO 1 /* Error in read or write */ +#define REDIS_ERR_EOF 3 /* End of file */ +#define REDIS_ERR_PROTOCOL 4 /* Protocol error */ +#define REDIS_ERR_OOM 5 /* Out of memory */ +#define REDIS_ERR_OTHER 2 /* Everything else... */ + +#define REDIS_REPLY_STRING 1 +#define REDIS_REPLY_ARRAY 2 +#define REDIS_REPLY_INTEGER 3 +#define REDIS_REPLY_NIL 4 +#define REDIS_REPLY_STATUS 5 +#define REDIS_REPLY_ERROR 6 + +#define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct redisReadTask { + int type; + int elements; /* number of elements in multibulk container */ + int idx; /* index in parent (array) object */ + void *obj; /* holds user-generated value for a read task */ + struct redisReadTask *parent; /* parent task */ + void *privdata; /* user-settable arbitrary field */ +} redisReadTask; + +typedef struct redisReplyObjectFunctions { + void *(*createString)(const redisReadTask*, char*, size_t); + void *(*createArray)(const redisReadTask*, int); + void *(*createInteger)(const redisReadTask*, long long); + void *(*createNil)(const redisReadTask*); + void (*freeObject)(void*); +} redisReplyObjectFunctions; + +typedef struct redisReader { + int err; /* Error flags, 0 when there is no error */ + char errstr[128]; /* String representation of error when applicable */ + + char *buf; /* Read buffer */ + size_t pos; /* Buffer cursor */ + size_t len; /* Buffer length */ + size_t maxbuf; /* Max length of unused buffer */ + + redisReadTask rstack[9]; + int ridx; /* Index of current read task */ + void *reply; /* Temporary reply pointer */ + + redisReplyObjectFunctions *fn; + void *privdata; +} redisReader; + +/* Public API for the protocol parser. */ +redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn); +void redisReaderFree(redisReader *r); +int redisReaderFeed(redisReader *r, const char *buf, size_t len); +int redisReaderGetReply(redisReader *r, void **reply); + +/* Backwards compatibility, can be removed on big version bump. */ +#define redisReplyReaderCreate redisReaderCreate +#define redisReplyReaderFree redisReaderFree +#define redisReplyReaderFeed redisReaderFeed +#define redisReplyReaderGetReply redisReaderGetReply +#define redisReplyReaderSetPrivdata(_r, _p) (int)(((redisReader*)(_r))->privdata = (_p)) +#define redisReplyReaderGetObject(_r) (((redisReader*)(_r))->reply) +#define redisReplyReaderGetError(_r) (((redisReader*)(_r))->errstr) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/third-party/hiredis/sds.c b/third-party/hiredis/sds.c new file mode 100644 index 000000000..5d55b7792 --- /dev/null +++ b/third-party/hiredis/sds.c @@ -0,0 +1,1095 @@ +/* SDS (Simple Dynamic Strings), A C dynamic strings library. + * + * Copyright (c) 2006-2014, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +#include "sds.h" + +/* Create a new sds string with the content specified by the 'init' pointer + * and 'initlen'. + * If NULL is used for 'init' the string is initialized with zero bytes. + * + * The string is always null-termined (all the sds strings are, always) so + * even if you create an sds string with: + * + * mystring = sdsnewlen("abc",3"); + * + * You can print the string with printf() as there is an implicit \0 at the + * end of the string. However the string is binary safe and can contain + * \0 characters in the middle, as the length is stored in the sds header. */ +sds sdsnewlen(const void *init, size_t initlen) { + struct sdshdr *sh; + + if (init) { + sh = malloc(sizeof *sh+initlen+1); + } else { + sh = calloc(sizeof *sh+initlen+1,1); + } + if (sh == NULL) return NULL; + sh->len = initlen; + sh->free = 0; + if (initlen && init) + memcpy(sh->buf, init, initlen); + sh->buf[initlen] = '\0'; + return (char*)sh->buf; +} + +/* Create an empty (zero length) sds string. Even in this case the string + * always has an implicit null term. */ +sds sdsempty(void) { + return sdsnewlen("",0); +} + +/* Create a new sds string starting from a null termined C string. */ +sds sdsnew(const char *init) { + size_t initlen = (init == NULL) ? 0 : strlen(init); + return sdsnewlen(init, initlen); +} + +/* Duplicate an sds string. */ +sds sdsdup(const sds s) { + return sdsnewlen(s, sdslen(s)); +} + +/* Free an sds string. No operation is performed if 's' is NULL. */ +void sdsfree(sds s) { + if (s == NULL) return; + free(s-sizeof(struct sdshdr)); +} + +/* Set the sds string length to the length as obtained with strlen(), so + * considering as content only up to the first null term character. + * + * This function is useful when the sds string is hacked manually in some + * way, like in the following example: + * + * s = sdsnew("foobar"); + * s[2] = '\0'; + * sdsupdatelen(s); + * printf("%d\n", sdslen(s)); + * + * The output will be "2", but if we comment out the call to sdsupdatelen() + * the output will be "6" as the string was modified but the logical length + * remains 6 bytes. */ +void sdsupdatelen(sds s) { + struct sdshdr *sh = (void*) (s-sizeof *sh); + int reallen = strlen(s); + sh->free += (sh->len-reallen); + sh->len = reallen; +} + +/* Modify an sds string on-place to make it empty (zero length). + * However all the existing buffer is not discarded but set as free space + * so that next append operations will not require allocations up to the + * number of bytes previously available. */ +void sdsclear(sds s) { + struct sdshdr *sh = (void*) (s-sizeof *sh); + sh->free += sh->len; + sh->len = 0; + sh->buf[0] = '\0'; +} + +/* Enlarge the free space at the end of the sds string so that the caller + * is sure that after calling this function can overwrite up to addlen + * bytes after the end of the string, plus one more byte for nul term. + * + * Note: this does not change the *length* of the sds string as returned + * by sdslen(), but only the free buffer space we have. */ +sds sdsMakeRoomFor(sds s, size_t addlen) { + struct sdshdr *sh, *newsh; + size_t free = sdsavail(s); + size_t len, newlen; + + if (free >= addlen) return s; + len = sdslen(s); + sh = (void*) (s-sizeof *sh); + newlen = (len+addlen); + if (newlen < SDS_MAX_PREALLOC) + newlen *= 2; + else + newlen += SDS_MAX_PREALLOC; + newsh = realloc(sh, sizeof *newsh+newlen+1); + if (newsh == NULL) return NULL; + + newsh->free = newlen - len; + return newsh->buf; +} + +/* Reallocate the sds string so that it has no free space at the end. The + * contained string remains not altered, but next concatenation operations + * will require a reallocation. + * + * After the call, the passed sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdsRemoveFreeSpace(sds s) { + struct sdshdr *sh; + + sh = (void*) (s-sizeof *sh); + sh = realloc(sh, sizeof *sh+sh->len+1); + sh->free = 0; + return sh->buf; +} + +/* Return the total size of the allocation of the specifed sds string, + * including: + * 1) The sds header before the pointer. + * 2) The string. + * 3) The free buffer at the end if any. + * 4) The implicit null term. + */ +size_t sdsAllocSize(sds s) { + struct sdshdr *sh = (void*) (s-sizeof *sh); + + return sizeof(*sh)+sh->len+sh->free+1; +} + +/* Increment the sds length and decrements the left free space at the + * end of the string according to 'incr'. Also set the null term + * in the new end of the string. + * + * This function is used in order to fix the string length after the + * user calls sdsMakeRoomFor(), writes something after the end of + * the current string, and finally needs to set the new length. + * + * Note: it is possible to use a negative increment in order to + * right-trim the string. + * + * Usage example: + * + * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the + * following schema, to cat bytes coming from the kernel to the end of an + * sds string without copying into an intermediate buffer: + * + * oldlen = sdslen(s); + * s = sdsMakeRoomFor(s, BUFFER_SIZE); + * nread = read(fd, s+oldlen, BUFFER_SIZE); + * ... check for nread <= 0 and handle it ... + * sdsIncrLen(s, nread); + */ +void sdsIncrLen(sds s, int incr) { + struct sdshdr *sh = (void*) (s-sizeof *sh); + + assert(sh->free >= incr); + sh->len += incr; + sh->free -= incr; + assert(sh->free >= 0); + s[sh->len] = '\0'; +} + +/* Grow the sds to have the specified length. Bytes that were not part of + * the original length of the sds will be set to zero. + * + * if the specified length is smaller than the current length, no operation + * is performed. */ +sds sdsgrowzero(sds s, size_t len) { + struct sdshdr *sh = (void*) (s-sizeof *sh); + size_t totlen, curlen = sh->len; + + if (len <= curlen) return s; + s = sdsMakeRoomFor(s,len-curlen); + if (s == NULL) return NULL; + + /* Make sure added region doesn't contain garbage */ + sh = (void*)(s-sizeof *sh); + memset(s+curlen,0,(len-curlen+1)); /* also set trailing \0 byte */ + totlen = sh->len+sh->free; + sh->len = len; + sh->free = totlen-sh->len; + return s; +} + +/* Append the specified binary-safe string pointed by 't' of 'len' bytes to the + * end of the specified sds string 's'. + * + * After the call, the passed sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdscatlen(sds s, const void *t, size_t len) { + struct sdshdr *sh; + size_t curlen = sdslen(s); + + s = sdsMakeRoomFor(s,len); + if (s == NULL) return NULL; + sh = (void*) (s-sizeof *sh); + memcpy(s+curlen, t, len); + sh->len = curlen+len; + sh->free = sh->free-len; + s[curlen+len] = '\0'; + return s; +} + +/* Append the specified null termianted C string to the sds string 's'. + * + * After the call, the passed sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdscat(sds s, const char *t) { + return sdscatlen(s, t, strlen(t)); +} + +/* Append the specified sds 't' to the existing sds 's'. + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdscatsds(sds s, const sds t) { + return sdscatlen(s, t, sdslen(t)); +} + +/* Destructively modify the sds string 's' to hold the specified binary + * safe string pointed by 't' of length 'len' bytes. */ +sds sdscpylen(sds s, const char *t, size_t len) { + struct sdshdr *sh = (void*) (s-sizeof *sh); + size_t totlen = sh->free+sh->len; + + if (totlen < len) { + s = sdsMakeRoomFor(s,len-sh->len); + if (s == NULL) return NULL; + sh = (void*) (s-sizeof *sh); + totlen = sh->free+sh->len; + } + memcpy(s, t, len); + s[len] = '\0'; + sh->len = len; + sh->free = totlen-len; + return s; +} + +/* Like sdscpylen() but 't' must be a null-termined string so that the length + * of the string is obtained with strlen(). */ +sds sdscpy(sds s, const char *t) { + return sdscpylen(s, t, strlen(t)); +} + +/* Helper for sdscatlonglong() doing the actual number -> string + * conversion. 's' must point to a string with room for at least + * SDS_LLSTR_SIZE bytes. + * + * The function returns the lenght of the null-terminated string + * representation stored at 's'. */ +#define SDS_LLSTR_SIZE 21 +int sdsll2str(char *s, long long value) { + char *p, aux; + unsigned long long v; + size_t l; + + /* Generate the string representation, this method produces + * an reversed string. */ + v = (value < 0) ? -value : value; + p = s; + do { + *p++ = '0'+(v%10); + v /= 10; + } while(v); + if (value < 0) *p++ = '-'; + + /* Compute length and add null term. */ + l = p-s; + *p = '\0'; + + /* Reverse the string. */ + p--; + while(s < p) { + aux = *s; + *s = *p; + *p = aux; + s++; + p--; + } + return l; +} + +/* Identical sdsll2str(), but for unsigned long long type. */ +int sdsull2str(char *s, unsigned long long v) { + char *p, aux; + size_t l; + + /* Generate the string representation, this method produces + * an reversed string. */ + p = s; + do { + *p++ = '0'+(v%10); + v /= 10; + } while(v); + + /* Compute length and add null term. */ + l = p-s; + *p = '\0'; + + /* Reverse the string. */ + p--; + while(s < p) { + aux = *s; + *s = *p; + *p = aux; + s++; + p--; + } + return l; +} + +/* Like sdscatpritf() but gets va_list instead of being variadic. */ +sds sdscatvprintf(sds s, const char *fmt, va_list ap) { + va_list cpy; + char *buf, *t; + size_t buflen = 16; + + while(1) { + buf = malloc(buflen); + if (buf == NULL) return NULL; + buf[buflen-2] = '\0'; + va_copy(cpy,ap); + vsnprintf(buf, buflen, fmt, cpy); + if (buf[buflen-2] != '\0') { + free(buf); + buflen *= 2; + continue; + } + break; + } + t = sdscat(s, buf); + free(buf); + return t; +} + +/* Append to the sds string 's' a string obtained using printf-alike format + * specifier. + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. + * + * Example: + * + * s = sdsnew("Sum is: "); + * s = sdscatprintf(s,"%d+%d = %d",a,b,a+b); + * + * Often you need to create a string from scratch with the printf-alike + * format. When this is the need, just use sdsempty() as the target string: + * + * s = sdscatprintf(sdsempty(), "... your format ...", args); + */ +sds sdscatprintf(sds s, const char *fmt, ...) { + va_list ap; + char *t; + va_start(ap, fmt); + t = sdscatvprintf(s,fmt,ap); + va_end(ap); + return t; +} + +/* This function is similar to sdscatprintf, but much faster as it does + * not rely on sprintf() family functions implemented by the libc that + * are often very slow. Moreover directly handling the sds string as + * new data is concatenated provides a performance improvement. + * + * However this function only handles an incompatible subset of printf-alike + * format specifiers: + * + * %s - C String + * %S - SDS string + * %i - signed int + * %I - 64 bit signed integer (long long, int64_t) + * %u - unsigned int + * %U - 64 bit unsigned integer (unsigned long long, uint64_t) + * %T - A size_t variable. + * %% - Verbatim "%" character. + */ +sds sdscatfmt(sds s, char const *fmt, ...) { + struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); + size_t initlen = sdslen(s); + const char *f = fmt; + int i; + va_list ap; + + va_start(ap,fmt); + f = fmt; /* Next format specifier byte to process. */ + i = initlen; /* Position of the next byte to write to dest str. */ + while(*f) { + char next, *str; + int l; + long long num; + unsigned long long unum; + + /* Make sure there is always space for at least 1 char. */ + if (sh->free == 0) { + s = sdsMakeRoomFor(s,1); + sh = (void*) (s-(sizeof(struct sdshdr))); + } + + switch(*f) { + case '%': + next = *(f+1); + f++; + switch(next) { + case 's': + case 'S': + str = va_arg(ap,char*); + l = (next == 's') ? strlen(str) : sdslen(str); + if (sh->free < l) { + s = sdsMakeRoomFor(s,l); + sh = (void*) (s-(sizeof(struct sdshdr))); + } + memcpy(s+i,str,l); + sh->len += l; + sh->free -= l; + i += l; + break; + case 'i': + case 'I': + if (next == 'i') + num = va_arg(ap,int); + else + num = va_arg(ap,long long); + { + char buf[SDS_LLSTR_SIZE]; + l = sdsll2str(buf,num); + if (sh->free < l) { + s = sdsMakeRoomFor(s,l); + sh = (void*) (s-(sizeof(struct sdshdr))); + } + memcpy(s+i,buf,l); + sh->len += l; + sh->free -= l; + i += l; + } + break; + case 'u': + case 'U': + case 'T': + if (next == 'u') + unum = va_arg(ap,unsigned int); + else if(next == 'U') + unum = va_arg(ap,unsigned long long); + else + unum = (unsigned long long)va_arg(ap,size_t); + { + char buf[SDS_LLSTR_SIZE]; + l = sdsull2str(buf,unum); + if (sh->free < l) { + s = sdsMakeRoomFor(s,l); + sh = (void*) (s-(sizeof(struct sdshdr))); + } + memcpy(s+i,buf,l); + sh->len += l; + sh->free -= l; + i += l; + } + break; + default: /* Handle %% and generally %. */ + s[i++] = next; + sh->len += 1; + sh->free -= 1; + break; + } + break; + default: + s[i++] = *f; + sh->len += 1; + sh->free -= 1; + break; + } + f++; + } + va_end(ap); + + /* Add null-term */ + s[i] = '\0'; + return s; +} + + +/* Remove the part of the string from left and from right composed just of + * contiguous characters found in 'cset', that is a null terminted C string. + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. + * + * Example: + * + * s = sdsnew("AA...AA.a.aa.aHelloWorld :::"); + * s = sdstrim(s,"A. :"); + * printf("%s\n", s); + * + * Output will be just "Hello World". + */ +void sdstrim(sds s, const char *cset) { + struct sdshdr *sh = (void*) (s-sizeof *sh); + char *start, *end, *sp, *ep; + size_t len; + + sp = start = s; + ep = end = s+sdslen(s)-1; + while(sp <= end && strchr(cset, *sp)) sp++; + while(ep > start && strchr(cset, *ep)) ep--; + len = (sp > ep) ? 0 : ((ep-sp)+1); + if (sh->buf != sp) memmove(sh->buf, sp, len); + sh->buf[len] = '\0'; + sh->free = sh->free+(sh->len-len); + sh->len = len; +} + +/* Turn the string into a smaller (or equal) string containing only the + * substring specified by the 'start' and 'end' indexes. + * + * start and end can be negative, where -1 means the last character of the + * string, -2 the penultimate character, and so forth. + * + * The interval is inclusive, so the start and end characters will be part + * of the resulting string. + * + * The string is modified in-place. + * + * Example: + * + * s = sdsnew("Hello World"); + * sdsrange(s,1,-1); => "ello World" + */ +void sdsrange(sds s, int start, int end) { + struct sdshdr *sh = (void*) (s-sizeof *sh); + size_t newlen, len = sdslen(s); + + if (len == 0) return; + if (start < 0) { + start = len+start; + if (start < 0) start = 0; + } + if (end < 0) { + end = len+end; + if (end < 0) end = 0; + } + newlen = (start > end) ? 0 : (end-start)+1; + if (newlen != 0) { + if (start >= (signed)len) { + newlen = 0; + } else if (end >= (signed)len) { + end = len-1; + newlen = (start > end) ? 0 : (end-start)+1; + } + } else { + start = 0; + } + if (start && newlen) memmove(sh->buf, sh->buf+start, newlen); + sh->buf[newlen] = 0; + sh->free = sh->free+(sh->len-newlen); + sh->len = newlen; +} + +/* Apply tolower() to every character of the sds string 's'. */ +void sdstolower(sds s) { + int len = sdslen(s), j; + + for (j = 0; j < len; j++) s[j] = tolower(s[j]); +} + +/* Apply toupper() to every character of the sds string 's'. */ +void sdstoupper(sds s) { + int len = sdslen(s), j; + + for (j = 0; j < len; j++) s[j] = toupper(s[j]); +} + +/* Compare two sds strings s1 and s2 with memcmp(). + * + * Return value: + * + * 1 if s1 > s2. + * -1 if s1 < s2. + * 0 if s1 and s2 are exactly the same binary string. + * + * If two strings share exactly the same prefix, but one of the two has + * additional characters, the longer string is considered to be greater than + * the smaller one. */ +int sdscmp(const sds s1, const sds s2) { + size_t l1, l2, minlen; + int cmp; + + l1 = sdslen(s1); + l2 = sdslen(s2); + minlen = (l1 < l2) ? l1 : l2; + cmp = memcmp(s1,s2,minlen); + if (cmp == 0) return l1-l2; + return cmp; +} + +/* Split 's' with separator in 'sep'. An array + * of sds strings is returned. *count will be set + * by reference to the number of tokens returned. + * + * On out of memory, zero length string, zero length + * separator, NULL is returned. + * + * Note that 'sep' is able to split a string using + * a multi-character separator. For example + * sdssplit("foo_-_bar","_-_"); will return two + * elements "foo" and "bar". + * + * This version of the function is binary-safe but + * requires length arguments. sdssplit() is just the + * same function but for zero-terminated strings. + */ +sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) { + int elements = 0, slots = 5, start = 0, j; + sds *tokens; + + if (seplen < 1 || len < 0) return NULL; + + tokens = malloc(sizeof(sds)*slots); + if (tokens == NULL) return NULL; + + if (len == 0) { + *count = 0; + return tokens; + } + for (j = 0; j < (len-(seplen-1)); j++) { + /* make sure there is room for the next element and the final one */ + if (slots < elements+2) { + sds *newtokens; + + slots *= 2; + newtokens = realloc(tokens,sizeof(sds)*slots); + if (newtokens == NULL) goto cleanup; + tokens = newtokens; + } + /* search the separator */ + if ((seplen == 1 && *(s+j) == sep[0]) || (memcmp(s+j,sep,seplen) == 0)) { + tokens[elements] = sdsnewlen(s+start,j-start); + if (tokens[elements] == NULL) goto cleanup; + elements++; + start = j+seplen; + j = j+seplen-1; /* skip the separator */ + } + } + /* Add the final element. We are sure there is room in the tokens array. */ + tokens[elements] = sdsnewlen(s+start,len-start); + if (tokens[elements] == NULL) goto cleanup; + elements++; + *count = elements; + return tokens; + +cleanup: + { + int i; + for (i = 0; i < elements; i++) sdsfree(tokens[i]); + free(tokens); + *count = 0; + return NULL; + } +} + +/* Free the result returned by sdssplitlen(), or do nothing if 'tokens' is NULL. */ +void sdsfreesplitres(sds *tokens, int count) { + if (!tokens) return; + while(count--) + sdsfree(tokens[count]); + free(tokens); +} + +/* Create an sds string from a long long value. It is much faster than: + * + * sdscatprintf(sdsempty(),"%lld\n", value); + */ +sds sdsfromlonglong(long long value) { + char buf[32], *p; + unsigned long long v; + + v = (value < 0) ? -value : value; + p = buf+31; /* point to the last character */ + do { + *p-- = '0'+(v%10); + v /= 10; + } while(v); + if (value < 0) *p-- = '-'; + p++; + return sdsnewlen(p,32-(p-buf)); +} + +/* Append to the sds string "s" an escaped string representation where + * all the non-printable characters (tested with isprint()) are turned into + * escapes in the form "\n\r\a...." or "\x". + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdscatrepr(sds s, const char *p, size_t len) { + s = sdscatlen(s,"\"",1); + while(len--) { + switch(*p) { + case '\\': + case '"': + s = sdscatprintf(s,"\\%c",*p); + break; + case '\n': s = sdscatlen(s,"\\n",2); break; + case '\r': s = sdscatlen(s,"\\r",2); break; + case '\t': s = sdscatlen(s,"\\t",2); break; + case '\a': s = sdscatlen(s,"\\a",2); break; + case '\b': s = sdscatlen(s,"\\b",2); break; + default: + if (isprint(*p)) + s = sdscatprintf(s,"%c",*p); + else + s = sdscatprintf(s,"\\x%02x",(unsigned char)*p); + break; + } + p++; + } + return sdscatlen(s,"\"",1); +} + +/* Helper function for sdssplitargs() that returns non zero if 'c' + * is a valid hex digit. */ +int is_hex_digit(char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F'); +} + +/* Helper function for sdssplitargs() that converts a hex digit into an + * integer from 0 to 15 */ +int hex_digit_to_int(char c) { + switch(c) { + case '0': return 0; + case '1': return 1; + case '2': return 2; + case '3': return 3; + case '4': return 4; + case '5': return 5; + case '6': return 6; + case '7': return 7; + case '8': return 8; + case '9': return 9; + case 'a': case 'A': return 10; + case 'b': case 'B': return 11; + case 'c': case 'C': return 12; + case 'd': case 'D': return 13; + case 'e': case 'E': return 14; + case 'f': case 'F': return 15; + default: return 0; + } +} + +/* Split a line into arguments, where every argument can be in the + * following programming-language REPL-alike form: + * + * foo bar "newline are supported\n" and "\xff\x00otherstuff" + * + * The number of arguments is stored into *argc, and an array + * of sds is returned. + * + * The caller should free the resulting array of sds strings with + * sdsfreesplitres(). + * + * Note that sdscatrepr() is able to convert back a string into + * a quoted string in the same format sdssplitargs() is able to parse. + * + * The function returns the allocated tokens on success, even when the + * input string is empty, or NULL if the input contains unbalanced + * quotes or closed quotes followed by non space characters + * as in: "foo"bar or "foo' + */ +sds *sdssplitargs(const char *line, int *argc) { + const char *p = line; + char *current = NULL; + char **vector = NULL; + + *argc = 0; + while(1) { + /* skip blanks */ + while(*p && isspace(*p)) p++; + if (*p) { + /* get a token */ + int inq=0; /* set to 1 if we are in "quotes" */ + int insq=0; /* set to 1 if we are in 'single quotes' */ + int done=0; + + if (current == NULL) current = sdsempty(); + while(!done) { + if (inq) { + if (*p == '\\' && *(p+1) == 'x' && + is_hex_digit(*(p+2)) && + is_hex_digit(*(p+3))) + { + unsigned char byte; + + byte = (hex_digit_to_int(*(p+2))*16)+ + hex_digit_to_int(*(p+3)); + current = sdscatlen(current,(char*)&byte,1); + p += 3; + } else if (*p == '\\' && *(p+1)) { + char c; + + p++; + switch(*p) { + case 'n': c = '\n'; break; + case 'r': c = '\r'; break; + case 't': c = '\t'; break; + case 'b': c = '\b'; break; + case 'a': c = '\a'; break; + default: c = *p; break; + } + current = sdscatlen(current,&c,1); + } else if (*p == '"') { + /* closing quote must be followed by a space or + * nothing at all. */ + if (*(p+1) && !isspace(*(p+1))) goto err; + done=1; + } else if (!*p) { + /* unterminated quotes */ + goto err; + } else { + current = sdscatlen(current,p,1); + } + } else if (insq) { + if (*p == '\\' && *(p+1) == '\'') { + p++; + current = sdscatlen(current,"'",1); + } else if (*p == '\'') { + /* closing quote must be followed by a space or + * nothing at all. */ + if (*(p+1) && !isspace(*(p+1))) goto err; + done=1; + } else if (!*p) { + /* unterminated quotes */ + goto err; + } else { + current = sdscatlen(current,p,1); + } + } else { + switch(*p) { + case ' ': + case '\n': + case '\r': + case '\t': + case '\0': + done=1; + break; + case '"': + inq=1; + break; + case '\'': + insq=1; + break; + default: + current = sdscatlen(current,p,1); + break; + } + } + if (*p) p++; + } + /* add the token to the vector */ + vector = realloc(vector,((*argc)+1)*sizeof(char*)); + vector[*argc] = current; + (*argc)++; + current = NULL; + } else { + /* Even on empty input string return something not NULL. */ + if (vector == NULL) vector = malloc(sizeof(void*)); + return vector; + } + } + +err: + while((*argc)--) + sdsfree(vector[*argc]); + free(vector); + if (current) sdsfree(current); + *argc = 0; + return NULL; +} + +/* Modify the string substituting all the occurrences of the set of + * characters specified in the 'from' string to the corresponding character + * in the 'to' array. + * + * For instance: sdsmapchars(mystring, "ho", "01", 2) + * will have the effect of turning the string "hello" into "0ell1". + * + * The function returns the sds string pointer, that is always the same + * as the input pointer since no resize is needed. */ +sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) { + size_t j, i, l = sdslen(s); + + for (j = 0; j < l; j++) { + for (i = 0; i < setlen; i++) { + if (s[j] == from[i]) { + s[j] = to[i]; + break; + } + } + } + return s; +} + +/* Join an array of C strings using the specified separator (also a C string). + * Returns the result as an sds string. */ +sds sdsjoin(char **argv, int argc, char *sep, size_t seplen) { + sds join = sdsempty(); + int j; + + for (j = 0; j < argc; j++) { + join = sdscat(join, argv[j]); + if (j != argc-1) join = sdscatlen(join,sep,seplen); + } + return join; +} + +/* Like sdsjoin, but joins an array of SDS strings. */ +sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) { + sds join = sdsempty(); + int j; + + for (j = 0; j < argc; j++) { + join = sdscatsds(join, argv[j]); + if (j != argc-1) join = sdscatlen(join,sep,seplen); + } + return join; +} + +#ifdef SDS_TEST_MAIN +#include +#include "testhelp.h" + +int main(void) { + { + struct sdshdr *sh; + sds x = sdsnew("foo"), y; + + test_cond("Create a string and obtain the length", + sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0) + + sdsfree(x); + x = sdsnewlen("foo",2); + test_cond("Create a string with specified length", + sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0) + + x = sdscat(x,"bar"); + test_cond("Strings concatenation", + sdslen(x) == 5 && memcmp(x,"fobar\0",6) == 0); + + x = sdscpy(x,"a"); + test_cond("sdscpy() against an originally longer string", + sdslen(x) == 1 && memcmp(x,"a\0",2) == 0) + + x = sdscpy(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk"); + test_cond("sdscpy() against an originally shorter string", + sdslen(x) == 33 && + memcmp(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk\0",33) == 0) + + sdsfree(x); + x = sdscatprintf(sdsempty(),"%d",123); + test_cond("sdscatprintf() seems working in the base case", + sdslen(x) == 3 && memcmp(x,"123\0",4) ==0) + + sdsfree(x); + x = sdsnew("xxciaoyyy"); + sdstrim(x,"xy"); + test_cond("sdstrim() correctly trims characters", + sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0) + + y = sdsdup(x); + sdsrange(y,1,1); + test_cond("sdsrange(...,1,1)", + sdslen(y) == 1 && memcmp(y,"i\0",2) == 0) + + sdsfree(y); + y = sdsdup(x); + sdsrange(y,1,-1); + test_cond("sdsrange(...,1,-1)", + sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) + + sdsfree(y); + y = sdsdup(x); + sdsrange(y,-2,-1); + test_cond("sdsrange(...,-2,-1)", + sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0) + + sdsfree(y); + y = sdsdup(x); + sdsrange(y,2,1); + test_cond("sdsrange(...,2,1)", + sdslen(y) == 0 && memcmp(y,"\0",1) == 0) + + sdsfree(y); + y = sdsdup(x); + sdsrange(y,1,100); + test_cond("sdsrange(...,1,100)", + sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) + + sdsfree(y); + y = sdsdup(x); + sdsrange(y,100,100); + test_cond("sdsrange(...,100,100)", + sdslen(y) == 0 && memcmp(y,"\0",1) == 0) + + sdsfree(y); + sdsfree(x); + x = sdsnew("foo"); + y = sdsnew("foa"); + test_cond("sdscmp(foo,foa)", sdscmp(x,y) > 0) + + sdsfree(y); + sdsfree(x); + x = sdsnew("bar"); + y = sdsnew("bar"); + test_cond("sdscmp(bar,bar)", sdscmp(x,y) == 0) + + sdsfree(y); + sdsfree(x); + x = sdsnew("aar"); + y = sdsnew("bar"); + test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0) + + sdsfree(y); + sdsfree(x); + x = sdsnewlen("\a\n\0foo\r",7); + y = sdscatrepr(sdsempty(),x,sdslen(x)); + test_cond("sdscatrepr(...data...)", + memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0) + + { + int oldfree; + + sdsfree(x); + x = sdsnew("0"); + sh = (void*) (x-(sizeof(struct sdshdr))); + test_cond("sdsnew() free/len buffers", sh->len == 1 && sh->free == 0); + x = sdsMakeRoomFor(x,1); + sh = (void*) (x-(sizeof(struct sdshdr))); + test_cond("sdsMakeRoomFor()", sh->len == 1 && sh->free > 0); + oldfree = sh->free; + x[1] = '1'; + sdsIncrLen(x,1); + test_cond("sdsIncrLen() -- content", x[0] == '0' && x[1] == '1'); + test_cond("sdsIncrLen() -- len", sh->len == 2); + test_cond("sdsIncrLen() -- free", sh->free == oldfree-1); + } + } + test_report() + return 0; +} +#endif diff --git a/third-party/hiredis/sds.h b/third-party/hiredis/sds.h new file mode 100644 index 000000000..19a2abd31 --- /dev/null +++ b/third-party/hiredis/sds.h @@ -0,0 +1,105 @@ +/* SDS (Simple Dynamic Strings), A C dynamic strings library. + * + * Copyright (c) 2006-2014, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SDS_H +#define __SDS_H + +#define SDS_MAX_PREALLOC (1024*1024) + +#include +#include +#ifdef _MSC_VER +#include "win32.h" +#endif + +typedef char *sds; + +struct sdshdr { + int len; + int free; + char buf[]; +}; + +static inline size_t sdslen(const sds s) { + struct sdshdr *sh = (struct sdshdr *)(s-sizeof *sh); + return sh->len; +} + +static inline size_t sdsavail(const sds s) { + struct sdshdr *sh = (struct sdshdr *)(s-sizeof *sh); + return sh->free; +} + +sds sdsnewlen(const void *init, size_t initlen); +sds sdsnew(const char *init); +sds sdsempty(void); +size_t sdslen(const sds s); +sds sdsdup(const sds s); +void sdsfree(sds s); +size_t sdsavail(const sds s); +sds sdsgrowzero(sds s, size_t len); +sds sdscatlen(sds s, const void *t, size_t len); +sds sdscat(sds s, const char *t); +sds sdscatsds(sds s, const sds t); +sds sdscpylen(sds s, const char *t, size_t len); +sds sdscpy(sds s, const char *t); + +sds sdscatvprintf(sds s, const char *fmt, va_list ap); +#ifdef __GNUC__ +sds sdscatprintf(sds s, const char *fmt, ...) + __attribute__((format(printf, 2, 3))); +#else +sds sdscatprintf(sds s, const char *fmt, ...); +#endif + +sds sdscatfmt(sds s, char const *fmt, ...); +void sdstrim(sds s, const char *cset); +void sdsrange(sds s, int start, int end); +void sdsupdatelen(sds s); +void sdsclear(sds s); +int sdscmp(const sds s1, const sds s2); +sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count); +void sdsfreesplitres(sds *tokens, int count); +void sdstolower(sds s); +void sdstoupper(sds s); +sds sdsfromlonglong(long long value); +sds sdscatrepr(sds s, const char *p, size_t len); +sds *sdssplitargs(const char *line, int *argc); +sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen); +sds sdsjoin(char **argv, int argc, char *sep, size_t seplen); +sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen); + +/* Low level functions exposed to the user API */ +sds sdsMakeRoomFor(sds s, size_t addlen); +void sdsIncrLen(sds s, int incr); +sds sdsRemoveFreeSpace(sds s); +size_t sdsAllocSize(sds s); + +#endif diff --git a/third-party/hiredis/test.c b/third-party/hiredis/test.c new file mode 100644 index 000000000..d5e82170d --- /dev/null +++ b/third-party/hiredis/test.c @@ -0,0 +1,807 @@ +#include "fmacros.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hiredis.h" +#include "net.h" + +enum connection_type { + CONN_TCP, + CONN_UNIX, + CONN_FD +}; + +struct config { + enum connection_type type; + + struct { + const char *host; + int port; + struct timeval timeout; + } tcp; + + struct { + const char *path; + } unix; +}; + +/* The following lines make up our testing "framework" :) */ +static int tests = 0, fails = 0; +#define test(_s) { printf("#%02d ", ++tests); printf(_s); } +#define test_cond(_c) if(_c) printf("\033[0;32mPASSED\033[0;0m\n"); else {printf("\033[0;31mFAILED\033[0;0m\n"); fails++;} + +static long long usec(void) { + struct timeval tv; + gettimeofday(&tv,NULL); + return (((long long)tv.tv_sec)*1000000)+tv.tv_usec; +} + +/* The assert() calls below have side effects, so we need assert() + * even if we are compiling without asserts (-DNDEBUG). */ +#ifdef NDEBUG +#undef assert +#define assert(e) (void)(e) +#endif + +static redisContext *select_database(redisContext *c) { + redisReply *reply; + + /* Switch to DB 9 for testing, now that we know we can chat. */ + reply = redisCommand(c,"SELECT 9"); + assert(reply != NULL); + freeReplyObject(reply); + + /* Make sure the DB is emtpy */ + reply = redisCommand(c,"DBSIZE"); + assert(reply != NULL); + if (reply->type == REDIS_REPLY_INTEGER && reply->integer == 0) { + /* Awesome, DB 9 is empty and we can continue. */ + freeReplyObject(reply); + } else { + printf("Database #9 is not empty, test can not continue\n"); + exit(1); + } + + return c; +} + +static int disconnect(redisContext *c, int keep_fd) { + redisReply *reply; + + /* Make sure we're on DB 9. */ + reply = redisCommand(c,"SELECT 9"); + assert(reply != NULL); + freeReplyObject(reply); + reply = redisCommand(c,"FLUSHDB"); + assert(reply != NULL); + freeReplyObject(reply); + + /* Free the context as well, but keep the fd if requested. */ + if (keep_fd) + return redisFreeKeepFd(c); + redisFree(c); + return -1; +} + +static redisContext *connect(struct config config) { + redisContext *c = NULL; + + if (config.type == CONN_TCP) { + c = redisConnect(config.tcp.host, config.tcp.port); + } else if (config.type == CONN_UNIX) { + c = redisConnectUnix(config.unix.path); + } else if (config.type == CONN_FD) { + /* Create a dummy connection just to get an fd to inherit */ + redisContext *dummy_ctx = redisConnectUnix(config.unix.path); + if (dummy_ctx) { + int fd = disconnect(dummy_ctx, 1); + printf("Connecting to inherited fd %d\n", fd); + c = redisConnectFd(fd); + } + } else { + assert(NULL); + } + + if (c == NULL) { + printf("Connection error: can't allocate redis context\n"); + exit(1); + } else if (c->err) { + printf("Connection error: %s\n", c->errstr); + redisFree(c); + exit(1); + } + + return select_database(c); +} + +static void test_format_commands(void) { + char *cmd; + int len; + + test("Format command without interpolation: "); + len = redisFormatCommand(&cmd,"SET foo bar"); + test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && + len == 4+4+(3+2)+4+(3+2)+4+(3+2)); + free(cmd); + + test("Format command with %%s string interpolation: "); + len = redisFormatCommand(&cmd,"SET %s %s","foo","bar"); + test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && + len == 4+4+(3+2)+4+(3+2)+4+(3+2)); + free(cmd); + + test("Format command with %%s and an empty string: "); + len = redisFormatCommand(&cmd,"SET %s %s","foo",""); + test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$0\r\n\r\n",len) == 0 && + len == 4+4+(3+2)+4+(3+2)+4+(0+2)); + free(cmd); + + test("Format command with an empty string in between proper interpolations: "); + len = redisFormatCommand(&cmd,"SET %s %s","","foo"); + test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$0\r\n\r\n$3\r\nfoo\r\n",len) == 0 && + len == 4+4+(3+2)+4+(0+2)+4+(3+2)); + free(cmd); + + test("Format command with %%b string interpolation: "); + len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"b\0r",(size_t)3); + test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nb\0r\r\n",len) == 0 && + len == 4+4+(3+2)+4+(3+2)+4+(3+2)); + free(cmd); + + test("Format command with %%b and an empty string: "); + len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"",(size_t)0); + test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$0\r\n\r\n",len) == 0 && + len == 4+4+(3+2)+4+(3+2)+4+(0+2)); + free(cmd); + + test("Format command with literal %%: "); + len = redisFormatCommand(&cmd,"SET %% %%"); + test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$1\r\n%\r\n$1\r\n%\r\n",len) == 0 && + len == 4+4+(3+2)+4+(1+2)+4+(1+2)); + free(cmd); + + /* Vararg width depends on the type. These tests make sure that the + * width is correctly determined using the format and subsequent varargs + * can correctly be interpolated. */ +#define INTEGER_WIDTH_TEST(fmt, type) do { \ + type value = 123; \ + test("Format command with printf-delegation (" #type "): "); \ + len = redisFormatCommand(&cmd,"key:%08" fmt " str:%s", value, "hello"); \ + test_cond(strncmp(cmd,"*2\r\n$12\r\nkey:00000123\r\n$9\r\nstr:hello\r\n",len) == 0 && \ + len == 4+5+(12+2)+4+(9+2)); \ + free(cmd); \ +} while(0) + +#define FLOAT_WIDTH_TEST(type) do { \ + type value = 123.0; \ + test("Format command with printf-delegation (" #type "): "); \ + len = redisFormatCommand(&cmd,"key:%08.3f str:%s", value, "hello"); \ + test_cond(strncmp(cmd,"*2\r\n$12\r\nkey:0123.000\r\n$9\r\nstr:hello\r\n",len) == 0 && \ + len == 4+5+(12+2)+4+(9+2)); \ + free(cmd); \ +} while(0) + + INTEGER_WIDTH_TEST("d", int); + INTEGER_WIDTH_TEST("hhd", char); + INTEGER_WIDTH_TEST("hd", short); + INTEGER_WIDTH_TEST("ld", long); + INTEGER_WIDTH_TEST("lld", long long); + INTEGER_WIDTH_TEST("u", unsigned int); + INTEGER_WIDTH_TEST("hhu", unsigned char); + INTEGER_WIDTH_TEST("hu", unsigned short); + INTEGER_WIDTH_TEST("lu", unsigned long); + INTEGER_WIDTH_TEST("llu", unsigned long long); + FLOAT_WIDTH_TEST(float); + FLOAT_WIDTH_TEST(double); + + test("Format command with invalid printf format: "); + len = redisFormatCommand(&cmd,"key:%08p %b",(void*)1234,"foo",(size_t)3); + test_cond(len == -1); + + const char *argv[3]; + argv[0] = "SET"; + argv[1] = "foo\0xxx"; + argv[2] = "bar"; + size_t lens[3] = { 3, 7, 3 }; + int argc = 3; + + test("Format command by passing argc/argv without lengths: "); + len = redisFormatCommandArgv(&cmd,argc,argv,NULL); + test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && + len == 4+4+(3+2)+4+(3+2)+4+(3+2)); + free(cmd); + + test("Format command by passing argc/argv with lengths: "); + len = redisFormatCommandArgv(&cmd,argc,argv,lens); + test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 && + len == 4+4+(3+2)+4+(7+2)+4+(3+2)); + free(cmd); +} + +static void test_append_formatted_commands(struct config config) { + redisContext *c; + redisReply *reply; + char *cmd; + int len; + + c = connect(config); + + test("Append format command: "); + + len = redisFormatCommand(&cmd, "SET foo bar"); + + test_cond(redisAppendFormattedCommand(c, cmd, len) == REDIS_OK); + + assert(redisGetReply(c, (void*)&reply) == REDIS_OK); + + free(cmd); + freeReplyObject(reply); + + disconnect(c, 0); +} + +static void test_reply_reader(void) { + redisReader *reader; + void *reply; + int ret; + int i; + + test("Error handling in reply parser: "); + reader = redisReaderCreate(); + redisReaderFeed(reader,(char*)"@foo\r\n",6); + ret = redisReaderGetReply(reader,NULL); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr,"Protocol error, got \"@\" as reply type byte") == 0); + redisReaderFree(reader); + + /* when the reply already contains multiple items, they must be free'd + * on an error. valgrind will bark when this doesn't happen. */ + test("Memory cleanup in reply parser: "); + reader = redisReaderCreate(); + redisReaderFeed(reader,(char*)"*2\r\n",4); + redisReaderFeed(reader,(char*)"$5\r\nhello\r\n",11); + redisReaderFeed(reader,(char*)"@foo\r\n",6); + ret = redisReaderGetReply(reader,NULL); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr,"Protocol error, got \"@\" as reply type byte") == 0); + redisReaderFree(reader); + + test("Set error on nested multi bulks with depth > 7: "); + reader = redisReaderCreate(); + + for (i = 0; i < 9; i++) { + redisReaderFeed(reader,(char*)"*1\r\n",4); + } + + ret = redisReaderGetReply(reader,NULL); + test_cond(ret == REDIS_ERR && + strncasecmp(reader->errstr,"No support for",14) == 0); + redisReaderFree(reader); + + test("Works with NULL functions for reply: "); + reader = redisReaderCreate(); + reader->fn = NULL; + redisReaderFeed(reader,(char*)"+OK\r\n",5); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && reply == (void*)REDIS_REPLY_STATUS); + redisReaderFree(reader); + + test("Works when a single newline (\\r\\n) covers two calls to feed: "); + reader = redisReaderCreate(); + reader->fn = NULL; + redisReaderFeed(reader,(char*)"+OK\r",4); + ret = redisReaderGetReply(reader,&reply); + assert(ret == REDIS_OK && reply == NULL); + redisReaderFeed(reader,(char*)"\n",1); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && reply == (void*)REDIS_REPLY_STATUS); + redisReaderFree(reader); + + test("Don't reset state after protocol error: "); + reader = redisReaderCreate(); + reader->fn = NULL; + redisReaderFeed(reader,(char*)"x",1); + ret = redisReaderGetReply(reader,&reply); + assert(ret == REDIS_ERR); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && reply == NULL); + redisReaderFree(reader); + + /* Regression test for issue #45 on GitHub. */ + test("Don't do empty allocation for empty multi bulk: "); + reader = redisReaderCreate(); + redisReaderFeed(reader,(char*)"*0\r\n",4); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_ARRAY && + ((redisReply*)reply)->elements == 0); + freeReplyObject(reply); + redisReaderFree(reader); +} + +static void test_free_null(void) { + void *redisContext = NULL; + void *reply = NULL; + + test("Don't fail when redisFree is passed a NULL value: "); + redisFree(redisContext); + test_cond(redisContext == NULL); + + test("Don't fail when freeReplyObject is passed a NULL value: "); + freeReplyObject(reply); + test_cond(reply == NULL); +} + +static void test_blocking_connection_errors(void) { + redisContext *c; + + test("Returns error when host cannot be resolved: "); + c = redisConnect((char*)"idontexist.test", 6379); + test_cond(c->err == REDIS_ERR_OTHER && + (strcmp(c->errstr,"Name or service not known") == 0 || + strcmp(c->errstr,"Can't resolve: idontexist.test") == 0 || + strcmp(c->errstr,"nodename nor servname provided, or not known") == 0 || + strcmp(c->errstr,"No address associated with hostname") == 0 || + strcmp(c->errstr,"Temporary failure in name resolution") == 0 || + strcmp(c->errstr,"hostname nor servname provided, or not known") == 0 || + strcmp(c->errstr,"no address associated with name") == 0)); + redisFree(c); + + test("Returns error when the port is not open: "); + c = redisConnect((char*)"localhost", 1); + test_cond(c->err == REDIS_ERR_IO && + strcmp(c->errstr,"Connection refused") == 0); + redisFree(c); + + test("Returns error when the unix socket path doesn't accept connections: "); + c = redisConnectUnix((char*)"/tmp/idontexist.sock"); + test_cond(c->err == REDIS_ERR_IO); /* Don't care about the message... */ + redisFree(c); +} + +static void test_blocking_connection(struct config config) { + redisContext *c; + redisReply *reply; + + c = connect(config); + + test("Is able to deliver commands: "); + reply = redisCommand(c,"PING"); + test_cond(reply->type == REDIS_REPLY_STATUS && + strcasecmp(reply->str,"pong") == 0) + freeReplyObject(reply); + + test("Is a able to send commands verbatim: "); + reply = redisCommand(c,"SET foo bar"); + test_cond (reply->type == REDIS_REPLY_STATUS && + strcasecmp(reply->str,"ok") == 0) + freeReplyObject(reply); + + test("%%s String interpolation works: "); + reply = redisCommand(c,"SET %s %s","foo","hello world"); + freeReplyObject(reply); + reply = redisCommand(c,"GET foo"); + test_cond(reply->type == REDIS_REPLY_STRING && + strcmp(reply->str,"hello world") == 0); + freeReplyObject(reply); + + test("%%b String interpolation works: "); + reply = redisCommand(c,"SET %b %b","foo",(size_t)3,"hello\x00world",(size_t)11); + freeReplyObject(reply); + reply = redisCommand(c,"GET foo"); + test_cond(reply->type == REDIS_REPLY_STRING && + memcmp(reply->str,"hello\x00world",11) == 0) + + test("Binary reply length is correct: "); + test_cond(reply->len == 11) + freeReplyObject(reply); + + test("Can parse nil replies: "); + reply = redisCommand(c,"GET nokey"); + test_cond(reply->type == REDIS_REPLY_NIL) + freeReplyObject(reply); + + /* test 7 */ + test("Can parse integer replies: "); + reply = redisCommand(c,"INCR mycounter"); + test_cond(reply->type == REDIS_REPLY_INTEGER && reply->integer == 1) + freeReplyObject(reply); + + test("Can parse multi bulk replies: "); + freeReplyObject(redisCommand(c,"LPUSH mylist foo")); + freeReplyObject(redisCommand(c,"LPUSH mylist bar")); + reply = redisCommand(c,"LRANGE mylist 0 -1"); + test_cond(reply->type == REDIS_REPLY_ARRAY && + reply->elements == 2 && + !memcmp(reply->element[0]->str,"bar",3) && + !memcmp(reply->element[1]->str,"foo",3)) + freeReplyObject(reply); + + /* m/e with multi bulk reply *before* other reply. + * specifically test ordering of reply items to parse. */ + test("Can handle nested multi bulk replies: "); + freeReplyObject(redisCommand(c,"MULTI")); + freeReplyObject(redisCommand(c,"LRANGE mylist 0 -1")); + freeReplyObject(redisCommand(c,"PING")); + reply = (redisCommand(c,"EXEC")); + test_cond(reply->type == REDIS_REPLY_ARRAY && + reply->elements == 2 && + reply->element[0]->type == REDIS_REPLY_ARRAY && + reply->element[0]->elements == 2 && + !memcmp(reply->element[0]->element[0]->str,"bar",3) && + !memcmp(reply->element[0]->element[1]->str,"foo",3) && + reply->element[1]->type == REDIS_REPLY_STATUS && + strcasecmp(reply->element[1]->str,"pong") == 0); + freeReplyObject(reply); + + disconnect(c, 0); +} + +static void test_blocking_connection_timeouts(struct config config) { + redisContext *c; + redisReply *reply; + ssize_t s; + const char *cmd = "DEBUG SLEEP 3\r\n"; + struct timeval tv; + + c = connect(config); + test("Successfully completes a command when the timeout is not exceeded: "); + reply = redisCommand(c,"SET foo fast"); + freeReplyObject(reply); + tv.tv_sec = 0; + tv.tv_usec = 10000; + redisSetTimeout(c, tv); + reply = redisCommand(c, "GET foo"); + test_cond(reply != NULL && reply->type == REDIS_REPLY_STRING && memcmp(reply->str, "fast", 4) == 0); + freeReplyObject(reply); + disconnect(c, 0); + + c = connect(config); + test("Does not return a reply when the command times out: "); + s = write(c->fd, cmd, strlen(cmd)); + tv.tv_sec = 0; + tv.tv_usec = 10000; + redisSetTimeout(c, tv); + reply = redisCommand(c, "GET foo"); + test_cond(s > 0 && reply == NULL && c->err == REDIS_ERR_IO && strcmp(c->errstr, "Resource temporarily unavailable") == 0); + freeReplyObject(reply); + + test("Reconnect properly reconnects after a timeout: "); + redisReconnect(c); + reply = redisCommand(c, "PING"); + test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); + freeReplyObject(reply); + + test("Reconnect properly uses owned parameters: "); + config.tcp.host = "foo"; + config.unix.path = "foo"; + redisReconnect(c); + reply = redisCommand(c, "PING"); + test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); + freeReplyObject(reply); + + disconnect(c, 0); +} + +static void test_blocking_io_errors(struct config config) { + redisContext *c; + redisReply *reply; + void *_reply; + int major, minor; + + /* Connect to target given by config. */ + c = connect(config); + { + /* Find out Redis version to determine the path for the next test */ + const char *field = "redis_version:"; + char *p, *eptr; + + reply = redisCommand(c,"INFO"); + p = strstr(reply->str,field); + major = strtol(p+strlen(field),&eptr,10); + p = eptr+1; /* char next to the first "." */ + minor = strtol(p,&eptr,10); + freeReplyObject(reply); + } + + test("Returns I/O error when the connection is lost: "); + reply = redisCommand(c,"QUIT"); + if (major > 2 || (major == 2 && minor > 0)) { + /* > 2.0 returns OK on QUIT and read() should be issued once more + * to know the descriptor is at EOF. */ + test_cond(strcasecmp(reply->str,"OK") == 0 && + redisGetReply(c,&_reply) == REDIS_ERR); + freeReplyObject(reply); + } else { + test_cond(reply == NULL); + } + + /* On 2.0, QUIT will cause the connection to be closed immediately and + * the read(2) for the reply on QUIT will set the error to EOF. + * On >2.0, QUIT will return with OK and another read(2) needed to be + * issued to find out the socket was closed by the server. In both + * conditions, the error will be set to EOF. */ + assert(c->err == REDIS_ERR_EOF && + strcmp(c->errstr,"Server closed the connection") == 0); + redisFree(c); + + c = connect(config); + test("Returns I/O error on socket timeout: "); + struct timeval tv = { 0, 1000 }; + assert(redisSetTimeout(c,tv) == REDIS_OK); + test_cond(redisGetReply(c,&_reply) == REDIS_ERR && + c->err == REDIS_ERR_IO && errno == EAGAIN); + redisFree(c); +} + +static void test_invalid_timeout_errors(struct config config) { + redisContext *c; + + test("Set error when an invalid timeout usec value is given to redisConnectWithTimeout: "); + + config.tcp.timeout.tv_sec = 0; + config.tcp.timeout.tv_usec = 10000001; + + c = redisConnectWithTimeout(config.tcp.host, config.tcp.port, config.tcp.timeout); + + test_cond(c->err == REDIS_ERR_IO); + redisFree(c); + + test("Set error when an invalid timeout sec value is given to redisConnectWithTimeout: "); + + config.tcp.timeout.tv_sec = (((LONG_MAX) - 999) / 1000) + 1; + config.tcp.timeout.tv_usec = 0; + + c = redisConnectWithTimeout(config.tcp.host, config.tcp.port, config.tcp.timeout); + + test_cond(c->err == REDIS_ERR_IO); + redisFree(c); +} + +static void test_throughput(struct config config) { + redisContext *c = connect(config); + redisReply **replies; + int i, num; + long long t1, t2; + + test("Throughput:\n"); + for (i = 0; i < 500; i++) + freeReplyObject(redisCommand(c,"LPUSH mylist foo")); + + num = 1000; + replies = malloc(sizeof(redisReply*)*num); + t1 = usec(); + for (i = 0; i < num; i++) { + replies[i] = redisCommand(c,"PING"); + assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_STATUS); + } + t2 = usec(); + for (i = 0; i < num; i++) freeReplyObject(replies[i]); + free(replies); + printf("\t(%dx PING: %.3fs)\n", num, (t2-t1)/1000000.0); + + replies = malloc(sizeof(redisReply*)*num); + t1 = usec(); + for (i = 0; i < num; i++) { + replies[i] = redisCommand(c,"LRANGE mylist 0 499"); + assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_ARRAY); + assert(replies[i] != NULL && replies[i]->elements == 500); + } + t2 = usec(); + for (i = 0; i < num; i++) freeReplyObject(replies[i]); + free(replies); + printf("\t(%dx LRANGE with 500 elements: %.3fs)\n", num, (t2-t1)/1000000.0); + + num = 10000; + replies = malloc(sizeof(redisReply*)*num); + for (i = 0; i < num; i++) + redisAppendCommand(c,"PING"); + t1 = usec(); + for (i = 0; i < num; i++) { + assert(redisGetReply(c, (void*)&replies[i]) == REDIS_OK); + assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_STATUS); + } + t2 = usec(); + for (i = 0; i < num; i++) freeReplyObject(replies[i]); + free(replies); + printf("\t(%dx PING (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0); + + replies = malloc(sizeof(redisReply*)*num); + for (i = 0; i < num; i++) + redisAppendCommand(c,"LRANGE mylist 0 499"); + t1 = usec(); + for (i = 0; i < num; i++) { + assert(redisGetReply(c, (void*)&replies[i]) == REDIS_OK); + assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_ARRAY); + assert(replies[i] != NULL && replies[i]->elements == 500); + } + t2 = usec(); + for (i = 0; i < num; i++) freeReplyObject(replies[i]); + free(replies); + printf("\t(%dx LRANGE with 500 elements (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0); + + disconnect(c, 0); +} + +// static long __test_callback_flags = 0; +// static void __test_callback(redisContext *c, void *privdata) { +// ((void)c); +// /* Shift to detect execution order */ +// __test_callback_flags <<= 8; +// __test_callback_flags |= (long)privdata; +// } +// +// static void __test_reply_callback(redisContext *c, redisReply *reply, void *privdata) { +// ((void)c); +// /* Shift to detect execution order */ +// __test_callback_flags <<= 8; +// __test_callback_flags |= (long)privdata; +// if (reply) freeReplyObject(reply); +// } +// +// static redisContext *__connect_nonblock() { +// /* Reset callback flags */ +// __test_callback_flags = 0; +// return redisConnectNonBlock("127.0.0.1", port, NULL); +// } +// +// static void test_nonblocking_connection() { +// redisContext *c; +// int wdone = 0; +// +// test("Calls command callback when command is issued: "); +// c = __connect_nonblock(); +// redisSetCommandCallback(c,__test_callback,(void*)1); +// redisCommand(c,"PING"); +// test_cond(__test_callback_flags == 1); +// redisFree(c); +// +// test("Calls disconnect callback on redisDisconnect: "); +// c = __connect_nonblock(); +// redisSetDisconnectCallback(c,__test_callback,(void*)2); +// redisDisconnect(c); +// test_cond(__test_callback_flags == 2); +// redisFree(c); +// +// test("Calls disconnect callback and free callback on redisFree: "); +// c = __connect_nonblock(); +// redisSetDisconnectCallback(c,__test_callback,(void*)2); +// redisSetFreeCallback(c,__test_callback,(void*)4); +// redisFree(c); +// test_cond(__test_callback_flags == ((2 << 8) | 4)); +// +// test("redisBufferWrite against empty write buffer: "); +// c = __connect_nonblock(); +// test_cond(redisBufferWrite(c,&wdone) == REDIS_OK && wdone == 1); +// redisFree(c); +// +// test("redisBufferWrite against not yet connected fd: "); +// c = __connect_nonblock(); +// redisCommand(c,"PING"); +// test_cond(redisBufferWrite(c,NULL) == REDIS_ERR && +// strncmp(c->error,"write:",6) == 0); +// redisFree(c); +// +// test("redisBufferWrite against closed fd: "); +// c = __connect_nonblock(); +// redisCommand(c,"PING"); +// redisDisconnect(c); +// test_cond(redisBufferWrite(c,NULL) == REDIS_ERR && +// strncmp(c->error,"write:",6) == 0); +// redisFree(c); +// +// test("Process callbacks in the right sequence: "); +// c = __connect_nonblock(); +// redisCommandWithCallback(c,__test_reply_callback,(void*)1,"PING"); +// redisCommandWithCallback(c,__test_reply_callback,(void*)2,"PING"); +// redisCommandWithCallback(c,__test_reply_callback,(void*)3,"PING"); +// +// /* Write output buffer */ +// wdone = 0; +// while(!wdone) { +// usleep(500); +// redisBufferWrite(c,&wdone); +// } +// +// /* Read until at least one callback is executed (the 3 replies will +// * arrive in a single packet, causing all callbacks to be executed in +// * a single pass). */ +// while(__test_callback_flags == 0) { +// assert(redisBufferRead(c) == REDIS_OK); +// redisProcessCallbacks(c); +// } +// test_cond(__test_callback_flags == 0x010203); +// redisFree(c); +// +// test("redisDisconnect executes pending callbacks with NULL reply: "); +// c = __connect_nonblock(); +// redisSetDisconnectCallback(c,__test_callback,(void*)1); +// redisCommandWithCallback(c,__test_reply_callback,(void*)2,"PING"); +// redisDisconnect(c); +// test_cond(__test_callback_flags == 0x0201); +// redisFree(c); +// } + +int main(int argc, char **argv) { + struct config cfg = { + .tcp = { + .host = "127.0.0.1", + .port = 6379 + }, + .unix = { + .path = "/tmp/redis.sock" + } + }; + int throughput = 1; + int test_inherit_fd = 1; + + /* Ignore broken pipe signal (for I/O error tests). */ + signal(SIGPIPE, SIG_IGN); + + /* Parse command line options. */ + argv++; argc--; + while (argc) { + if (argc >= 2 && !strcmp(argv[0],"-h")) { + argv++; argc--; + cfg.tcp.host = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"-p")) { + argv++; argc--; + cfg.tcp.port = atoi(argv[0]); + } else if (argc >= 2 && !strcmp(argv[0],"-s")) { + argv++; argc--; + cfg.unix.path = argv[0]; + } else if (argc >= 1 && !strcmp(argv[0],"--skip-throughput")) { + throughput = 0; + } else if (argc >= 1 && !strcmp(argv[0],"--skip-inherit-fd")) { + test_inherit_fd = 0; + } else { + fprintf(stderr, "Invalid argument: %s\n", argv[0]); + exit(1); + } + argv++; argc--; + } + + test_format_commands(); + test_reply_reader(); + test_blocking_connection_errors(); + test_free_null(); + + printf("\nTesting against TCP connection (%s:%d):\n", cfg.tcp.host, cfg.tcp.port); + cfg.type = CONN_TCP; + test_blocking_connection(cfg); + test_blocking_connection_timeouts(cfg); + test_blocking_io_errors(cfg); + test_invalid_timeout_errors(cfg); + test_append_formatted_commands(cfg); + if (throughput) test_throughput(cfg); + + printf("\nTesting against Unix socket connection (%s):\n", cfg.unix.path); + cfg.type = CONN_UNIX; + test_blocking_connection(cfg); + test_blocking_connection_timeouts(cfg); + test_blocking_io_errors(cfg); + if (throughput) test_throughput(cfg); + + if (test_inherit_fd) { + printf("\nTesting against inherited fd (%s):\n", cfg.unix.path); + cfg.type = CONN_FD; + test_blocking_connection(cfg); + } + + + if (fails) { + printf("*** %d TESTS FAILED ***\n", fails); + return 1; + } + + printf("ALL TESTS PASSED\n"); + return 0; +} diff --git a/third-party/hiredis/win32.h b/third-party/hiredis/win32.h new file mode 100644 index 000000000..1a27c18f2 --- /dev/null +++ b/third-party/hiredis/win32.h @@ -0,0 +1,42 @@ +#ifndef _WIN32_HELPER_INCLUDE +#define _WIN32_HELPER_INCLUDE +#ifdef _MSC_VER + +#ifndef inline +#define inline __inline +#endif + +#ifndef va_copy +#define va_copy(d,s) ((d) = (s)) +#endif + +#ifndef snprintf +#define snprintf c99_snprintf + +__inline int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap) +{ + int count = -1; + + if (size != 0) + count = _vsnprintf_s(str, size, _TRUNCATE, format, ap); + if (count == -1) + count = _vscprintf(format, ap); + + return count; +} + +__inline int c99_snprintf(char* str, size_t size, const char* format, ...) +{ + int count; + va_list ap; + + va_start(ap, format); + count = c99_vsnprintf(str, size, format, ap); + va_end(ap); + + return count; +} +#endif + +#endif +#endif \ No newline at end of file From fff1049ba0ba1d867c8ec00e6081ff830471dd36 Mon Sep 17 00:00:00 2001 From: Gunnar Beutner Date: Mon, 2 Oct 2017 09:59:11 +0200 Subject: [PATCH 002/219] Ensure that the WorkQueue name is set --- lib/redis/rediswriter.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index 70403068b..af3f3e949 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -61,6 +61,8 @@ void RedisWriter::Start(bool runtimeCreated) m_StatsTimer->OnTimerExpired.connect(boost::bind(&RedisWriter::PublishStatsTimerHandler, this)); m_StatsTimer->Start(); + m_WorkQueue.SetName("RedisWriter"); + boost::thread thread(boost::bind(&RedisWriter::HandleEvents, this)); thread.detach(); } From b6c86f98ec6ba99ab343002464145663dc066dd5 Mon Sep 17 00:00:00 2001 From: Gunnar Beutner Date: Mon, 2 Oct 2017 10:05:56 +0200 Subject: [PATCH 003/219] Clean up the code a bit --- lib/redis/rediswriter-status.cpp | 10 ++++++--- lib/redis/rediswriter-utility.cpp | 37 ++++++++++++++----------------- lib/redis/rediswriter.hpp | 11 ++++----- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/lib/redis/rediswriter-status.cpp b/lib/redis/rediswriter-status.cpp index 8c830fbc8..09700638d 100644 --- a/lib/redis/rediswriter-status.cpp +++ b/lib/redis/rediswriter-status.cpp @@ -135,7 +135,11 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, const String } checkSum->Set("properties_checksum", CalculateCheckSumProperties(object)); - checkSum->Set("vars_checksum", CalculateCheckSumVars(object)); + + CustomVarObject::Ptr customVarObject = dynamic_pointer_cast(object); + + if (customVarObject) + checkSum->Set("vars_checksum", CalculateCheckSumVars(customVarObject)); String checkSumBody = JsonEncode(checkSum); @@ -188,7 +192,7 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, const String if (checkable) { Dictionary::Ptr attrs = new Dictionary(); String tableName; - String objectCheckSum = CalculateCheckSumString(objectName, true); //store binary checksum here + String objectCheckSum = CalculateCheckSumString(objectName); Host::Ptr host; Service::Ptr service; @@ -198,7 +202,7 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, const String if (service) { tableName = "servicestate"; attrs->Set("service_checksum", objectCheckSum); - attrs->Set("host_checksum", CalculateCheckSumString(host->GetName(), true)); + attrs->Set("host_checksum", CalculateCheckSumString(host->GetName())); } else { tableName = "hoststate"; attrs->Set("host_checksum", objectCheckSum); diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index e91edc800..33cd49022 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -36,46 +36,43 @@ String RedisWriter::FormatCheckSumBinary(const String& str) return output; } -String RedisWriter::CalculateCheckSumString(const String& str, bool binary) +String RedisWriter::CalculateCheckSumString(const String& str) { - return SHA1(str, binary); + return SHA1(str); } -String RedisWriter::CalculateCheckSumGroups(const Array::Ptr& groups, bool binary) +String RedisWriter::CalculateCheckSumGroups(const Array::Ptr& groups) { String output; - ObjectLock olock(groups); + { + ObjectLock olock(groups); - for (const String& group : groups) { - output += SHA1(group, true); //binary checksum required here + for (const String& group : groups) { + output += SHA1(group); + } } - return SHA1(output, binary); + return SHA1(output); } -String RedisWriter::CalculateCheckSumProperties(const ConfigObject::Ptr& object, bool binary) +String RedisWriter::CalculateCheckSumProperties(const ConfigObject::Ptr& object) { //TODO: consider precision of 6 for double values; use specific config fields for hashing? - return HashValue(object, binary); + return HashValue(object); } -String RedisWriter::CalculateCheckSumVars(const ConfigObject::Ptr& object, bool binary) +String RedisWriter::CalculateCheckSumVars(const CustomVarObject::Ptr& object) { - CustomVarObject::Ptr customVarObject = dynamic_pointer_cast(object); - - if (!customVarObject) - return HashValue(Empty, binary); - - Dictionary::Ptr vars = customVarObject->GetVars(); + Dictionary::Ptr vars = object->GetVars(); if (!vars) - return HashValue(Empty, binary); + return HashValue(Empty); - return HashValue(vars, binary); + return HashValue(vars); } -String RedisWriter::HashValue(const Value& value, bool binary) +String RedisWriter::HashValue(const Value& value) { Value temp; @@ -86,7 +83,7 @@ String RedisWriter::HashValue(const Value& value, bool binary) else temp = value; - return SHA1(JsonEncode(temp), binary); + return SHA1(JsonEncode(temp)); } Dictionary::Ptr RedisWriter::SerializeObjectAttrs(const Object::Ptr& object, int fieldType) diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 07f56a15f..48ea8d238 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -21,6 +21,7 @@ #define REDISWRITER_H #include "redis/rediswriter.thpp" +#include "icinga/customvarobject.hpp" #include "remote/messageorigin.hpp" #include "base/timer.hpp" #include "base/workqueue.hpp" @@ -70,12 +71,12 @@ private: /* utilities */ static String FormatCheckSumBinary(const String& str); - static String CalculateCheckSumString(const String& str, bool binary = false); - static String CalculateCheckSumGroups(const Array::Ptr& groups, bool binary = false); - static String CalculateCheckSumProperties(const ConfigObject::Ptr& object, bool binary = false); - static String CalculateCheckSumVars(const ConfigObject::Ptr& object, bool binary = false); + static String CalculateCheckSumString(const String& str); + static String CalculateCheckSumGroups(const Array::Ptr& groups); + static String CalculateCheckSumProperties(const ConfigObject::Ptr& object); + static String CalculateCheckSumVars(const CustomVarObject::Ptr& object); - static String HashValue(const Value& value, bool binary = false); + static String HashValue(const Value& value); static Dictionary::Ptr SerializeObjectAttrs(const Object::Ptr& object, int fieldType); static void StateChangedHandler(const ConfigObject::Ptr& object); From 240e0476935b745ce9920ad473cf01449a5023e7 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Tue, 10 Oct 2017 12:03:03 +0200 Subject: [PATCH 004/219] Add development notes --- doc/99-redis.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/99-redis.md diff --git a/doc/99-redis.md b/doc/99-redis.md new file mode 100644 index 000000000..5da9cfdb7 --- /dev/null +++ b/doc/99-redis.md @@ -0,0 +1,13 @@ +### Keys + +All Keys are prefixed with "icinga:" + +Key | Type | Description | Dev notes +-----------------------|--------|-----------------|----------- +status:{feature} | Hash | Feature status | Currently the hash only contains one key (name) which returns a json string +{type}state.{SHA1} | String | Host state | json string of the current state of the object SHA1 is of the object name +config:{type} | Hash | Config | Has all the config with name as key and json string of config as value +config:{type}:checksum | Hash | Checksums | Key is name, returns a json encoded string of checksums +subscription:{name} | String | sub description | json string describing the subsciption +event:{name} | List | sub output | Publish endpoint for subscription of the same name +:trigger::configchange | List | ? | I have no idea what this is or where it comes from From 1ada53dd57602aed14e0a273598936fb74b0f5c7 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Wed, 11 Oct 2017 14:19:42 +0200 Subject: [PATCH 005/219] Trim event queue TODO: make configurable refs #5119 --- lib/redis/rediswriter.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index af3f3e949..c06403996 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -25,6 +25,9 @@ using namespace icinga; +//TODO Make configurable and figure out a sane default +#define MAX_EVENTS 5000 + REGISTER_TYPE(RedisWriter); RedisWriter::RedisWriter(void) @@ -284,7 +287,10 @@ void RedisWriter::HandleEvent(const Dictionary::Ptr& event) String body = JsonEncode(event); + ExecuteQuery({ "MULTI" }); ExecuteQuery({ "LPUSH", "icinga:event:" + name, body }); + ExecuteQuery({ "LTRIM", "icinga:event:" + name, 0, MAX_EVENTS - 1 }); + ExecuteQuery({ "EXEC" }); } } From 7c8412928e9a42f10ae99cfe167f8ec9a08c5be8 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Wed, 11 Oct 2017 16:03:43 +0200 Subject: [PATCH 006/219] Subscriptions fixes #5656 --- lib/redis/rediswriter.cpp | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index c06403996..3eca8ec40 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -156,7 +156,7 @@ void RedisWriter::UpdateSubscriptions(void) Log(LogInformation, "RedisWriter", "Updating Redis subscriptions"); - std::map subscriptions; + Array::Ptr keys = new Array(); long long cursor = 0; do { @@ -173,42 +173,36 @@ void RedisWriter::UpdateSubscriptions(void) for (size_t i = 0; i < keysReply->elements; i++) { redisReply *keyReply = keysReply->element[i]; VERIFY(keyReply->type == REDIS_REPLY_STRING); - - boost::shared_ptr vreply = ExecuteQuery({ "GET", keyReply->str }); - - subscriptions[keyReply->str] = vreply->str; + keys->Add(keyReply->str); } } while (cursor != 0); m_Subscriptions.clear(); - for (const std::pair& kv : subscriptions) { - const String& key = kv.first.SubStr(20); /* removes the "icinga:subscription: prefix */ - const String& value = kv.second; - + ObjectLock olock(keys); + for (String key : keys) { try { - Dictionary::Ptr subscriptionInfo = JsonDecode(value); - - Log(LogInformation, "RedisWriter") - << "Subscriber Info - Key: " << key << " Value: " << Value(subscriptionInfo); + boost::shared_ptr redisReply = ExecuteQuery({ "LRANGE", key, "0", "-1" }); + VERIFY(redisReply->type == REDIS_REPLY_ARRAY); RedisSubscriptionInfo rsi; + Array::Ptr printer = new Array(); + for (size_t j = 0; j < redisReply->elements; j++) { + rsi.EventTypes.insert(redisReply->element[j]->str); + printer->Add(redisReply->element[j]->str); + } + Log(LogInformation, "RedisWriter") + << "Subscriber Info - Key: " << key << " Value: " << printer->ToString(); - Array::Ptr types = subscriptionInfo->Get("types"); + m_Subscriptions[key.SubStr(20)] = rsi; - if (types) - rsi.EventTypes = types->ToSet(); - - m_Subscriptions[key] = rsi; } catch (const std::exception& ex) { Log(LogWarning, "RedisWriter") - << "Invalid Redis subscriber info for subscriber '" << key << "': " << DiagnosticInformation(ex); + << "Invalid Redis subscriber info for subscriber '" << key << "': " << DiagnosticInformation(ex); continue; } - //TODO } - Log(LogInformation, "RedisWriter") << "Current Redis event subscriptions: " << m_Subscriptions.size(); } @@ -289,7 +283,7 @@ void RedisWriter::HandleEvent(const Dictionary::Ptr& event) ExecuteQuery({ "MULTI" }); ExecuteQuery({ "LPUSH", "icinga:event:" + name, body }); - ExecuteQuery({ "LTRIM", "icinga:event:" + name, 0, MAX_EVENTS - 1 }); + ExecuteQuery({ "LTRIM", "icinga:event:" + name, "0", String(MAX_EVENTS - 1)}); ExecuteQuery({ "EXEC" }); } } From 1725038ca8a29f37ddff4650069d34fd3cd2dc5e Mon Sep 17 00:00:00 2001 From: Gunnar Beutner Date: Thu, 12 Oct 2017 08:34:17 +0200 Subject: [PATCH 007/219] Implement pipelining support for Redis queries fixes #5670 --- lib/redis/rediswriter.cpp | 59 ++++++++++++++++++++++++++++++++++++++- lib/redis/rediswriter.hpp | 1 + 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index 3eca8ec40..b89e59bf1 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -334,5 +334,62 @@ boost::shared_ptr RedisWriter::ExecuteQuery(const std::vector(reply); + return boost::shared_ptr(reply, freeReplyObject); +} + +std::vector > RedisWriter::ExecuteQueries(const std::vector >& queries) +{ + const char **argv; + size_t *argvlen; + + for (const auto& query : queries) { + argv = new const char *[query.size()]; + argvlen = new size_t[query.size()]; + + for (std::vector::size_type i = 0; i < query.size(); i++) { + argv[i] = query[i].CStr(); + argvlen[i] = query[i].GetLength(); + } + + redisAppendCommandArgv(m_Context, query.size(), argv, argvlen); + + delete [] argv; + delete [] argvlen; + } + + std::vector > replies; + + for (size_t i = 0; i < queries.size(); i++) { + redisReply *rawReply; + + if (redisGetReply(m_Context, reinterpret_cast(&rawReply)) == REDIS_ERR) { + BOOST_THROW_EXCEPTION( + redis_error() + << errinfo_message("redisGetReply() failed") + ); + } + + boost::shared_ptr reply(rawReply, freeReplyObject); + replies.push_back(reply); + } + + for (size_t i = 0; i < queries.size(); i++) { + const auto& query = queries[i]; + const auto& reply = replies[i]; + + if (reply->type == REDIS_REPLY_ERROR) { + Log(LogCritical, "RedisWriter") + << "Redis query failed: " << reply->str; + + String msg = reply->str; + + BOOST_THROW_EXCEPTION( + redis_error() + << errinfo_message(msg) + << errinfo_redis_query(Utility::Join(Array::FromVector(query), ' ', false)) + ); + } + } + + return replies; } diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 48ea8d238..ca43a476a 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -88,6 +88,7 @@ private: void ExceptionHandler(boost::exception_ptr exp); boost::shared_ptr ExecuteQuery(const std::vector& query); + std::vector > ExecuteQueries(const std::vector >& queries); Timer::Ptr m_StatsTimer; Timer::Ptr m_ReconnectTimer; From 06211c3ac7c7d5e614f4d86cd5fed3ee705db5b8 Mon Sep 17 00:00:00 2001 From: Gunnar Beutner Date: Thu, 12 Oct 2017 08:51:13 +0200 Subject: [PATCH 008/219] Implement support for the new config/state schema fixes #5671 --- lib/redis/rediswriter-status.cpp | 335 +++++++++++++++++------------- lib/redis/rediswriter-utility.cpp | 19 +- lib/redis/rediswriter.hpp | 9 +- 3 files changed, 214 insertions(+), 149 deletions(-) diff --git a/lib/redis/rediswriter-status.cpp b/lib/redis/rediswriter-status.cpp index 09700638d..6b5ebbb0b 100644 --- a/lib/redis/rediswriter-status.cpp +++ b/lib/redis/rediswriter-status.cpp @@ -26,6 +26,7 @@ #include "base/serializer.hpp" #include "base/tlsutility.hpp" #include "base/initialize.hpp" +#include "base/convert.hpp" using namespace icinga; @@ -49,7 +50,6 @@ void RedisWriter::ConfigStaticInitialize(void) { /* triggered in ProcessCheckResult(), requires UpdateNextCheck() to be called before */ ConfigObject::OnStateChanged.connect(boost::bind(&RedisWriter::StateChangedHandler, _1)); - CustomVarObject::OnVarsChanged.connect(boost::bind(&RedisWriter::VarsChangedHandler, _1)); /* triggered on create, update and delete objects */ ConfigObject::OnActiveChanged.connect(boost::bind(&RedisWriter::VersionChangedHandler, _1)); @@ -62,8 +62,57 @@ void RedisWriter::UpdateAllConfigObjects(void) double startTime = Utility::GetTime(); - //TODO: "Publish" the config dump by adding another event, globally or by object - ExecuteQuery({ "MULTI" }); + std::vector deleteQuery({ "DEL" }); + long long cursor = 0; + + const String keyPrefix = "icinga:config:"; + + do { + boost::shared_ptr reply = ExecuteQuery({ "SCAN", Convert::ToString(cursor), "MATCH", keyPrefix + "*", "COUNT", "1000" }); + + VERIFY(reply->type == REDIS_REPLY_ARRAY); + VERIFY(reply->elements % 2 == 0); + + redisReply *cursorReply = reply->element[0]; + cursor = Convert::ToLong(cursorReply->str); + + redisReply *keysReply = reply->element[1]; + + for (size_t i = 0; i < keysReply->elements; i++) { + redisReply *keyReply = keysReply->element[i]; + VERIFY(keyReply->type == REDIS_REPLY_STRING); + + String key = keyReply->str; + String namePair = key.SubStr(keyPrefix.GetLength()); + + String::SizeType pos = namePair.FindFirstOf(":"); + + if (pos == String::NPos) + continue; + + String type = namePair.SubStr(0, pos); + String name = namePair.SubStr(pos + 1); + + Type::Ptr ptype = Type::GetByName(type); + + if (!ptype) + continue; + + ConfigType *ctype = dynamic_cast(ptype.get()); + + if (!ctype) + continue; + + if (ctype->GetObject(name)) + continue; + + deleteQuery.push_back("icinga:config:" + type + ":" + name); + deleteQuery.push_back("icinga:status:" + type + ":" + name); + } + } while (cursor != 0); + + if (deleteQuery.size() > 1) + ExecuteQuery(deleteQuery); for (const Type::Ptr& type : Type::GetAllTypes()) { ConfigType *ctype = dynamic_cast(type.get()); @@ -77,21 +126,19 @@ void RedisWriter::UpdateAllConfigObjects(void) /* fetch all objects and dump them */ for (const ConfigObject::Ptr& object : ctype->GetObjects()) { - SendConfigUpdate(object, typeName); - SendStatusUpdate(object, typeName); + SendConfigUpdate(object, false); + SendStatusUpdate(object, false); } /* publish config type dump finished */ ExecuteQuery({ "PUBLISH", "icinga:config:dump", typeName }); } - ExecuteQuery({ "EXEC" }); - Log(LogInformation, "RedisWriter") << "Initial config/status dump finished in " << Utility::GetTime() - startTime << " seconds."; } -void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, const String& typeName, bool runtimeUpdate) +void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTransaction, bool runtimeUpdate) { AssertOnWorkQueue(); @@ -104,55 +151,62 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, const String return; */ - /* Serialize config object attributes */ - Dictionary::Ptr objectAttrs = SerializeObjectAttrs(object, FAConfig); + if (useTransaction) + ExecuteQuery({ "MULTI" }); - String jsonBody = JsonEncode(objectAttrs); + UpdateObjectAttrs("icinga:config:", object, FAConfig); - String objectName = object->GetName(); +// /* Serialize config object attributes */ +// Dictionary::Ptr objectAttrs = SerializeObjectAttrs(object, FAConfig); +// +// String jsonBody = JsonEncode(objectAttrs); +// +// String objectName = object->GetName(); +// +// ExecuteQuery({ "HSET", "icinga:config:" + typeName, objectName, jsonBody }); +// +// /* check sums */ +// /* hset icinga:config:Host:checksums localhost { "name_checksum": "...", "properties_checksum": "...", "groups_checksum": "...", "vars_checksum": null } */ +// Dictionary::Ptr checkSum = new Dictionary(); +// +// checkSum->Set("name_checksum", CalculateCheckSumString(object->GetName())); +// +// // TODO: move this elsewhere +// Checkable::Ptr checkable = dynamic_pointer_cast(object); +// +// if (checkable) { +// Host::Ptr host; +// Service::Ptr service; +// +// tie(host, service) = GetHostService(checkable); +// +// if (service) +// checkSum->Set("groups_checksum", CalculateCheckSumGroups(service->GetGroups())); +// else +// checkSum->Set("groups_checksum", CalculateCheckSumGroups(host->GetGroups())); +// } +// +// checkSum->Set("properties_checksum", CalculateCheckSumProperties(object)); +// +// CustomVarObject::Ptr customVarObject = dynamic_pointer_cast(object); +// +// if (customVarObject) +// checkSum->Set("vars_checksum", CalculateCheckSumVars(customVarObject)); +// +// String checkSumBody = JsonEncode(checkSum); +// +// ExecuteQuery({ "HSET", "icinga:config:" + typeName + ":checksum", objectName, checkSumBody }); - ExecuteQuery({ "HSET", "icinga:config:" + typeName, objectName, jsonBody }); - - /* check sums */ - /* hset icinga:config:Host:checksums localhost { "name_checksum": "...", "properties_checksum": "...", "groups_checksum": "...", "vars_checksum": null } */ - Dictionary::Ptr checkSum = new Dictionary(); - - checkSum->Set("name_checksum", CalculateCheckSumString(object->GetName())); - - // TODO: move this elsewhere - Checkable::Ptr checkable = dynamic_pointer_cast(object); - - if (checkable) { - Host::Ptr host; - Service::Ptr service; - - tie(host, service) = GetHostService(checkable); - - if (service) - checkSum->Set("groups_checksum", CalculateCheckSumGroups(service->GetGroups())); - else - checkSum->Set("groups_checksum", CalculateCheckSumGroups(host->GetGroups())); + if (runtimeUpdate) { + Type::Ptr type = object->GetReflectionType(); + ExecuteQuery({ "PUBLISH", "icinga:config:update", type->GetName() + ":" + object->GetName() }); } - checkSum->Set("properties_checksum", CalculateCheckSumProperties(object)); - - CustomVarObject::Ptr customVarObject = dynamic_pointer_cast(object); - - if (customVarObject) - checkSum->Set("vars_checksum", CalculateCheckSumVars(customVarObject)); - - String checkSumBody = JsonEncode(checkSum); - - ExecuteQuery({ "HSET", "icinga:config:" + typeName + ":checksum", objectName, checkSumBody }); - - /* publish runtime updated objects immediately */ - if (!runtimeUpdate) - return; - - ExecuteQuery({ "PUBLISH", "icinga:config:update", typeName + ":" + objectName + "!" + checkSumBody }); + if (useTransaction) + ExecuteQuery({ "EXEC" }); } -void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object, const String& typeName) +void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) { AssertOnWorkQueue(); @@ -160,16 +214,18 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object, const String if (!m_Context) return; + String typeName = object->GetReflectionType()->GetName(); String objectName = object->GetName(); - ExecuteQuery({ "HDEL", "icinga:config:" + typeName, objectName }); - ExecuteQuery({ "HDEL", "icinga:config:" + typeName + ":checksum", objectName }); - ExecuteQuery({ "HDEL", "icinga:status:" + typeName, objectName }); + ExecuteQueries({ + { "DEL", "icinga:config:" + typeName, objectName }, + { "DEL", "icinga:status:" + typeName, objectName }, + { "PUBLISH", "icinga:config:delete", typeName + ":" + objectName } + }); - ExecuteQuery({ "PUBLISH", "icinga:config:delete", typeName + ":" + objectName }); } -void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, const String& typeName) +void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, bool useTransaction) { AssertOnWorkQueue(); @@ -177,82 +233,90 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, const String if (!m_Context) return; - /* Serialize config object attributes */ - Dictionary::Ptr objectAttrs = SerializeObjectAttrs(object, FAState); + if (useTransaction) + ExecuteQuery({ "MULTI" }); - String jsonBody = JsonEncode(objectAttrs); + UpdateObjectAttrs("icinga:status:", object, FAState); - String objectName = object->GetName(); + if (useTransaction) + ExecuteQuery({ "EXEC" }); - ExecuteQuery({ "HSET", "icinga:status:" + typeName, objectName, jsonBody }); - - /* Icinga DB part for Icinga Web 2 */ - Checkable::Ptr checkable = dynamic_pointer_cast(object); - - if (checkable) { - Dictionary::Ptr attrs = new Dictionary(); - String tableName; - String objectCheckSum = CalculateCheckSumString(objectName); - - Host::Ptr host; - Service::Ptr service; - - tie(host, service) = GetHostService(checkable); - - if (service) { - tableName = "servicestate"; - attrs->Set("service_checksum", objectCheckSum); - attrs->Set("host_checksum", CalculateCheckSumString(host->GetName())); - } else { - tableName = "hoststate"; - attrs->Set("host_checksum", objectCheckSum); - } - - attrs->Set("last_check", checkable->GetLastCheck()); - attrs->Set("next_check", checkable->GetNextCheck()); - - attrs->Set("severity", checkable->GetSeverity()); - -/* - 'host_checksum' => null, - 'command' => null, // JSON, array - 'execution_start' => null, - 'execution_end' => null, - 'schedule_start' => null, - 'schedule_end' => null, - 'exit_status' => null, - 'output' => null, - 'performance_data' => null, // JSON, array - - -10.0.3.12:6379> keys icinga:hoststate.* -1) "icinga:hoststate.~\xf5a\x91+\x03\x97\x99\xb5(\x16 CYm\xb1\xdf\x85\xa2\xcb" -10.0.3.12:6379> get "icinga:hoststate.~\xf5a\x91+\x03\x97\x99\xb5(\x16 CYm\xb1\xdf\x85\xa2\xcb" -"{\"command\":[\"\\/usr\\/lib\\/nagios\\/plugins\\/check_ping\",\"-H\",\"127.0.0.1\",\"-c\",\"5000,100%\",\"-w\",\"3000,80%\"],\"execution_start\":1492007581.7624,\"execution_end\":1492007585.7654,\"schedule_start\":1492007581.7609,\"schedule_end\":1492007585.7655,\"exit_status\":0,\"output\":\"PING OK - Packet loss = 0%, RTA = 0.08 ms\",\"performance_data\":[\"rta=0.076000ms;3000.000000;5000.000000;0.000000\",\"pl=0%;80;100;0\"]}" - -*/ - - CheckResult::Ptr cr = checkable->GetLastCheckResult(); - - if (cr) { - attrs->Set("command", JsonEncode(cr->GetCommand())); - attrs->Set("execution_start", cr->GetExecutionStart()); - attrs->Set("execution_end", cr->GetExecutionEnd()); - attrs->Set("schedule_start", cr->GetScheduleStart()); - attrs->Set("schedule_end", cr->GetScheduleStart()); - attrs->Set("exit_status", cr->GetExitStatus()); - attrs->Set("output", cr->GetOutput()); - attrs->Set("performance_data", JsonEncode(cr->GetPerformanceData())); - } - - String jsonAttrs = JsonEncode(attrs); - String key = "icinga:" + tableName + "." + objectCheckSum; - ExecuteQuery({ "SET", key, jsonAttrs }); - - /* expire in check_interval * attempts + timeout + some more seconds */ - double expireTime = checkable->GetCheckInterval() * checkable->GetMaxCheckAttempts() + 60; - ExecuteQuery({ "EXPIRE", key, String(expireTime) }); - } +// /* Serialize config object attributes */ +// Dictionary::Ptr objectAttrs = SerializeObjectAttrs(object, FAState); +// +// String jsonBody = JsonEncode(objectAttrs); +// +// String objectName = object->GetName(); +// +// ExecuteQuery({ "HSET", "icinga:status:" + typeName, objectName, jsonBody }); +// +// /* Icinga DB part for Icinga Web 2 */ +// Checkable::Ptr checkable = dynamic_pointer_cast(object); +// +// if (checkable) { +// Dictionary::Ptr attrs = new Dictionary(); +// String tableName; +// String objectCheckSum = CalculateCheckSumString(objectName); +// +// Host::Ptr host; +// Service::Ptr service; +// +// tie(host, service) = GetHostService(checkable); +// +// if (service) { +// tableName = "servicestate"; +// attrs->Set("service_checksum", objectCheckSum); +// attrs->Set("host_checksum", CalculateCheckSumString(host->GetName())); +// } else { +// tableName = "hoststate"; +// attrs->Set("host_checksum", objectCheckSum); +// } +// +// attrs->Set("last_check", checkable->GetLastCheck()); +// attrs->Set("next_check", checkable->GetNextCheck()); +// +// attrs->Set("severity", checkable->GetSeverity()); +// +///* +// 'host_checksum' => null, +// 'command' => null, // JSON, array +// 'execution_start' => null, +// 'execution_end' => null, +// 'schedule_start' => null, +// 'schedule_end' => null, +// 'exit_status' => null, +// 'output' => null, +// 'performance_data' => null, // JSON, array +// +// +//10.0.3.12:6379> keys icinga:hoststate.* +//1) "icinga:hoststate.~\xf5a\x91+\x03\x97\x99\xb5(\x16 CYm\xb1\xdf\x85\xa2\xcb" +//10.0.3.12:6379> get "icinga:hoststate.~\xf5a\x91+\x03\x97\x99\xb5(\x16 CYm\xb1\xdf\x85\xa2\xcb" +//"{\"command\":[\"\\/usr\\/lib\\/nagios\\/plugins\\/check_ping\",\"-H\",\"127.0.0.1\",\"-c\",\"5000,100%\",\"-w\",\"3000,80%\"],\"execution_start\":1492007581.7624,\"execution_end\":1492007585.7654,\"schedule_start\":1492007581.7609,\"schedule_end\":1492007585.7655,\"exit_status\":0,\"output\":\"PING OK - Packet loss = 0%, RTA = 0.08 ms\",\"performance_data\":[\"rta=0.076000ms;3000.000000;5000.000000;0.000000\",\"pl=0%;80;100;0\"]}" +// +//*/ +// +// CheckResult::Ptr cr = checkable->GetLastCheckResult(); +// +// if (cr) { +// attrs->Set("command", JsonEncode(cr->GetCommand())); +// attrs->Set("execution_start", cr->GetExecutionStart()); +// attrs->Set("execution_end", cr->GetExecutionEnd()); +// attrs->Set("schedule_start", cr->GetScheduleStart()); +// attrs->Set("schedule_end", cr->GetScheduleStart()); +// attrs->Set("exit_status", cr->GetExitStatus()); +// attrs->Set("output", cr->GetOutput()); +// attrs->Set("performance_data", JsonEncode(cr->GetPerformanceData())); +// } +// +// String jsonAttrs = JsonEncode(attrs); +// String key = "icinga:" + tableName + "." + objectCheckSum; +// ExecuteQuery({ "SET", key, jsonAttrs }); +// +// /* expire in check_interval * attempts + timeout + some more seconds */ +// double expireTime = checkable->GetCheckInterval() * checkable->GetMaxCheckAttempts() + 60; +// ExecuteQuery({ "EXPIRE", key, String(expireTime) }); +// } } void RedisWriter::StateChangedHandler(const ConfigObject::Ptr& object) @@ -260,16 +324,7 @@ void RedisWriter::StateChangedHandler(const ConfigObject::Ptr& object) Type::Ptr type = object->GetReflectionType(); for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { - rw->m_WorkQueue.Enqueue(boost::bind(&RedisWriter::SendStatusUpdate, rw, object, type->GetName())); - } -} - -void RedisWriter::VarsChangedHandler(const ConfigObject::Ptr& object) -{ - Type::Ptr type = object->GetReflectionType(); - - for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { - rw->m_WorkQueue.Enqueue(boost::bind(&RedisWriter::SendConfigUpdate, rw, object, type->GetName(), true)); + rw->m_WorkQueue.Enqueue(boost::bind(&RedisWriter::SendStatusUpdate, rw, object, true)); } } @@ -280,12 +335,12 @@ void RedisWriter::VersionChangedHandler(const ConfigObject::Ptr& object) if (object->IsActive()) { /* Create or update the object config */ for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { - rw->m_WorkQueue.Enqueue(boost::bind(&RedisWriter::SendConfigUpdate, rw.get(), object, type->GetName(), true)); + rw->m_WorkQueue.Enqueue(boost::bind(&RedisWriter::SendConfigUpdate, rw.get(), object, true, true)); } } else if (!object->IsActive() && object->GetExtension("ConfigObjectDeleted")) { /* same as in apilistener-configsync.cpp */ /* Delete object config */ for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { - rw->m_WorkQueue.Enqueue(boost::bind(&RedisWriter::SendConfigDelete, rw.get(), object, type->GetName())); + rw->m_WorkQueue.Enqueue(boost::bind(&RedisWriter::SendConfigDelete, rw.get(), object)); } } } diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 33cd49022..d1aac8c5c 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -86,11 +86,18 @@ String RedisWriter::HashValue(const Value& value) return SHA1(JsonEncode(temp)); } -Dictionary::Ptr RedisWriter::SerializeObjectAttrs(const Object::Ptr& object, int fieldType) +void RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType) { Type::Ptr type = object->GetReflectionType(); - Dictionary::Ptr resultAttrs = new Dictionary(); + String typeName = type->GetName(); + String objectName = object->GetName(); + + std::vector > queries; + + queries.push_back({ "DEL", keyPrefix + object->GetName() }); + + std::vector hmsetCommand({ "HMSET", keyPrefix + typeName + ":" + objectName }); for (int fid = 0; fid < type->GetFieldCount(); fid++) { Field field = type->GetFieldInfo(fid); @@ -108,10 +115,14 @@ Dictionary::Ptr RedisWriter::SerializeObjectAttrs(const Object::Ptr& object, int if (field.Attributes & FANavigation && !(field.Attributes & (FAConfig | FAState))) continue; + hmsetCommand.push_back(field.Name); + Value sval = Serialize(val); - resultAttrs->Set(field.Name, sval); + hmsetCommand.push_back(JsonEncode(sval)); } - return resultAttrs; + queries.push_back(hmsetCommand); + + ExecuteQueries(queries); } diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index ca43a476a..39bd6fc4b 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -64,9 +64,10 @@ private: /* config & status dump */ void UpdateAllConfigObjects(void); - void SendConfigUpdate(const ConfigObject::Ptr& object, const String& typeName, bool runtimeUpdate = false); - void SendConfigDelete(const ConfigObject::Ptr& object, const String& typeName); - void SendStatusUpdate(const ConfigObject::Ptr& object, const String& typeName); + void SendConfigUpdate(const ConfigObject::Ptr& object, bool useTransaction, bool runtimeUpdate = false); + void SendConfigDelete(const ConfigObject::Ptr& object); + void SendStatusUpdate(const ConfigObject::Ptr& object, bool useTransaction); + void UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType); /* utilities */ static String FormatCheckSumBinary(const String& str); @@ -77,10 +78,8 @@ private: static String CalculateCheckSumVars(const CustomVarObject::Ptr& object); static String HashValue(const Value& value); - static Dictionary::Ptr SerializeObjectAttrs(const Object::Ptr& object, int fieldType); static void StateChangedHandler(const ConfigObject::Ptr& object); - static void VarsChangedHandler(const ConfigObject::Ptr& object); static void VersionChangedHandler(const ConfigObject::Ptr& object); void AssertOnWorkQueue(void); From f631bf8cb5ef0aa71f9f2dde6f6bd7bc9767ed9e Mon Sep 17 00:00:00 2001 From: Gunnar Beutner Date: Thu, 12 Oct 2017 11:47:49 +0200 Subject: [PATCH 009/219] Use sets for subscriptions refs #5656 --- lib/redis/rediswriter.cpp | 65 ++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index b89e59bf1..4f7fb8414 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -32,7 +32,9 @@ REGISTER_TYPE(RedisWriter); RedisWriter::RedisWriter(void) : m_Context(NULL) -{ } +{ + m_WorkQueue.SetName("RedisWriter"); +} /** * Starts the component. @@ -134,6 +136,8 @@ void RedisWriter::TryToReconnect(void) if (dbIndex != 0) ExecuteQuery({ "SELECT", Convert::ToString(dbIndex) }); + UpdateSubscriptions(); + /* Config dump */ m_ConfigDumpInProgress = true; @@ -151,16 +155,19 @@ void RedisWriter::UpdateSubscriptions(void) { AssertOnWorkQueue(); + Log(LogInformation, "RedisWriter", "Updating Redis subscriptions"); + + m_Subscriptions.clear(); + if (!m_Context) return; - Log(LogInformation, "RedisWriter", "Updating Redis subscriptions"); - - Array::Ptr keys = new Array(); long long cursor = 0; + String keyPrefix = "icinga:subscription:"; + do { - boost::shared_ptr reply = ExecuteQuery({ "SCAN", Convert::ToString(cursor), "MATCH", "icinga:subscription:*", "COUNT", "1000" }); + boost::shared_ptr reply = ExecuteQuery({ "SCAN", Convert::ToString(cursor), "MATCH", keyPrefix + "*", "COUNT", "1000" }); VERIFY(reply->type == REDIS_REPLY_ARRAY); VERIFY(reply->elements % 2 == 0); @@ -173,36 +180,30 @@ void RedisWriter::UpdateSubscriptions(void) for (size_t i = 0; i < keysReply->elements; i++) { redisReply *keyReply = keysReply->element[i]; VERIFY(keyReply->type == REDIS_REPLY_STRING); - keys->Add(keyReply->str); + + String key = keyReply->str; + + try { + boost::shared_ptr redisReply = ExecuteQuery({ "SMEMBERS", key }); + VERIFY(redisReply->type == REDIS_REPLY_ARRAY); + + RedisSubscriptionInfo rsi; + + for (size_t j = 0; j < redisReply->elements; j++) { + rsi.EventTypes.insert(redisReply->element[j]->str); + } + + Log(LogInformation, "RedisWriter") + << "Subscriber Info - Key: " << key << " Value: " << Value(Array::FromSet(rsi.EventTypes)); + + m_Subscriptions[key.SubStr(keyPrefix.GetLength())] = rsi; + } catch (const std::exception& ex) { + Log(LogWarning, "RedisWriter") + << "Invalid Redis subscriber info for subscriber '" << key << "': " << DiagnosticInformation(ex); + } } } while (cursor != 0); - m_Subscriptions.clear(); - - ObjectLock olock(keys); - for (String key : keys) { - try { - boost::shared_ptr redisReply = ExecuteQuery({ "LRANGE", key, "0", "-1" }); - VERIFY(redisReply->type == REDIS_REPLY_ARRAY); - - RedisSubscriptionInfo rsi; - Array::Ptr printer = new Array(); - for (size_t j = 0; j < redisReply->elements; j++) { - rsi.EventTypes.insert(redisReply->element[j]->str); - printer->Add(redisReply->element[j]->str); - } - Log(LogInformation, "RedisWriter") - << "Subscriber Info - Key: " << key << " Value: " << printer->ToString(); - - m_Subscriptions[key.SubStr(20)] = rsi; - - } catch (const std::exception& ex) { - Log(LogWarning, "RedisWriter") - << "Invalid Redis subscriber info for subscriber '" << key << "': " << DiagnosticInformation(ex); - - continue; - } - } Log(LogInformation, "RedisWriter") << "Current Redis event subscriptions: " << m_Subscriptions.size(); } From c8561676ec454c6ef5d50d300315e31c32189a57 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Thu, 12 Oct 2017 17:46:06 +0200 Subject: [PATCH 010/219] Add limit for subscriptions --- lib/redis/rediswriter.cpp | 59 ++++++++++++++++++++++++++------------- lib/redis/rediswriter.hpp | 1 + 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index 4f7fb8414..6bad770ab 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -22,11 +22,12 @@ #include "remote/eventqueue.hpp" #include "base/json.hpp" #include "base/statsfunction.hpp" +#include using namespace icinga; //TODO Make configurable and figure out a sane default -#define MAX_EVENTS 5000 +#define MAX_EVENTS_DEFAULT 5000 REGISTER_TYPE(RedisWriter); @@ -178,29 +179,20 @@ void RedisWriter::UpdateSubscriptions(void) redisReply *keysReply = reply->element[1]; for (size_t i = 0; i < keysReply->elements; i++) { + if (boost::algorithm::ends_with(keysReply->element[i]->str, ":limit")) + continue; redisReply *keyReply = keysReply->element[i]; VERIFY(keyReply->type == REDIS_REPLY_STRING); - String key = keyReply->str; - - try { - boost::shared_ptr redisReply = ExecuteQuery({ "SMEMBERS", key }); - VERIFY(redisReply->type == REDIS_REPLY_ARRAY); - - RedisSubscriptionInfo rsi; - - for (size_t j = 0; j < redisReply->elements; j++) { - rsi.EventTypes.insert(redisReply->element[j]->str); - } + RedisSubscriptionInfo rsi; + String key = keysReply->element[i]->str; + if (!RedisWriter::GetSubscriptionTypes(key, rsi)) Log(LogInformation, "RedisWriter") - << "Subscriber Info - Key: " << key << " Value: " << Value(Array::FromSet(rsi.EventTypes)); - + << "Subscription \"" << key<< "\" has no types listed."; + else m_Subscriptions[key.SubStr(keyPrefix.GetLength())] = rsi; - } catch (const std::exception& ex) { - Log(LogWarning, "RedisWriter") - << "Invalid Redis subscriber info for subscriber '" << key << "': " << DiagnosticInformation(ex); - } + } } while (cursor != 0); @@ -208,6 +200,25 @@ void RedisWriter::UpdateSubscriptions(void) << "Current Redis event subscriptions: " << m_Subscriptions.size(); } +int RedisWriter::GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi) +{ + try { + boost::shared_ptr redisReply = ExecuteQuery({ "SMEMBERS", key }); + VERIFY(redisReply->type == REDIS_REPLY_ARRAY); + + for (size_t j = 0; j < redisReply->elements; j++) { + rsi.EventTypes.insert(redisReply->element[j]->str); + } + + Log(LogInformation, "RedisWriter") + << "Subscriber Info - Key: " << key << " Value: " << Value(Array::FromSet(rsi.EventTypes)); + + } catch (const std::exception& ex) { + Log(LogWarning, "RedisWriter") + << "Invalid Redis subscriber info for subscriber '" << key << "': " << DiagnosticInformation(ex); + } +} + void RedisWriter::PublishStatsTimerHandler(void) { m_WorkQueue.Enqueue(boost::bind(&RedisWriter::PublishStats, this)); @@ -282,9 +293,19 @@ void RedisWriter::HandleEvent(const Dictionary::Ptr& event) String body = JsonEncode(event); + boost::shared_ptr maxExists = ExecuteQuery({ "EXISTS", "icinga:subscription:" + name + ":limit" }); + long maxEvents = MAX_EVENTS_DEFAULT; + if (maxExists->integer) { + boost::shared_ptr redisReply = ExecuteQuery({ "GET", "icinga:subscription:" + name + ":limit"}); + VERIFY(redisReply->type == REDIS_REPLY_STRING); + Log(LogInformation, "RedisWriter") + << "Got limit " << redisReply->str << " for " << name; + maxEvents = Convert::ToLong(redisReply->str); + } + ExecuteQuery({ "MULTI" }); ExecuteQuery({ "LPUSH", "icinga:event:" + name, body }); - ExecuteQuery({ "LTRIM", "icinga:event:" + name, "0", String(MAX_EVENTS - 1)}); + ExecuteQuery({ "LTRIM", "icinga:event:" + name, "0", String(maxEvents - 1)}); ExecuteQuery({ "EXEC" }); } } diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 39bd6fc4b..24c47f9dd 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -59,6 +59,7 @@ private: void UpdateSubscriptionsTimerHandler(void); void UpdateSubscriptions(void); + int GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi); void PublishStatsTimerHandler(void); void PublishStats(void); From 26a758b941b5e4966a56f4101541e0e934945db2 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Mon, 16 Oct 2017 12:03:16 +0200 Subject: [PATCH 011/219] Update docs --- doc/99-redis.md | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/doc/99-redis.md b/doc/99-redis.md index 5da9cfdb7..eb6b32f8e 100644 --- a/doc/99-redis.md +++ b/doc/99-redis.md @@ -1,4 +1,46 @@ -### Keys +# Icinga2 Redis +## Subscriptions and events + +Using the redis feature allows you to add subscriptions for events which will then be served by Icinga 2 over Redis. + +Possible event types: + +* CheckResult +* StateChange +* Notification +* AcknowledgementSet +* AcknowledgementCleared +* CommentAdded +* CommentRemoved +* DowntimeAdded +* DowntimeRemoved +* DowntimeStarted +* DowntimeTriggered + +### Creating a subscription +A subscription is created by creating a new key `icinga:subscription:` in redis with a set of event types. All +events matching the the type of those listed will then be added to a list at `icinga:event:`. The events are +rotated on a first-in-first-out basis, the default limit is (TODO) and can be overwritten by setting +`icinga:subscription::limit` to the desired ammount. + +The events are saved as json encoded strings, similar to the API. + +It is recommended to use `LPOP` to read from the list and discard read events at the same time. + +Example: + +``` +$ redis-cli +127.0.0.1:6379> SADD icinga:subscription:noma-2 "Notification" "CheckResult" +(integer) 2 +127.0.0.1:6379> SET icinga:subscription:noma-2:limit 500 +OK +... +127.0.0.1:6379> LRANGE icinga:event:noma-2 0 1 +1) "{\"check_result\":{\"active\":true,\"check_source\":\"icinga-1\",\"command\":[\"/usr/lib/nagios/plugins/check_ping\" ... ]}}" +1) "{\"check_result\":{\"active\":true,\"check_source\":\"icinga-2\",\"command\":[\"/usr/lib/nagios/plugins/check_ping\" ... ]}}" +``` + All Keys are prefixed with "icinga:" @@ -10,4 +52,3 @@ config:{type} | Hash | Config | Has all the config with name config:{type}:checksum | Hash | Checksums | Key is name, returns a json encoded string of checksums subscription:{name} | String | sub description | json string describing the subsciption event:{name} | List | sub output | Publish endpoint for subscription of the same name -:trigger::configchange | List | ? | I have no idea what this is or where it comes from From 5eab856673880f54a8a86645c86febb99a07159e Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Mon, 16 Oct 2017 12:03:49 +0200 Subject: [PATCH 012/219] Publish events to one enpoint --- lib/redis/rediswriter.cpp | 21 ++++++++++++++++++++- lib/redis/rediswriter.hpp | 1 + 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index 6bad770ab..c6248475b 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -147,6 +147,7 @@ void RedisWriter::TryToReconnect(void) m_ConfigDumpInProgress = false; } +/* void RedisWriter::UpdateSubscriptionsTimerHandler(void) { m_WorkQueue.Enqueue(boost::bind(&RedisWriter::UpdateSubscriptions, this)); @@ -218,6 +219,7 @@ int RedisWriter::GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi) << "Invalid Redis subscriber info for subscriber '" << key << "': " << DiagnosticInformation(ex); } } +*/ void RedisWriter::PublishStatsTimerHandler(void) { @@ -270,13 +272,14 @@ void RedisWriter::HandleEvents(void) if (!event) continue; - m_WorkQueue.Enqueue(boost::bind(&RedisWriter::HandleEvent, this, event)); + m_WorkQueue.Enqueue(boost::bind(&RedisWriter::SendEvent, this, event)); } queue->RemoveClient(this); EventQueue::UnregisterIfUnused(queueName, queue); } +/* void RedisWriter::HandleEvent(const Dictionary::Ptr& event) { AssertOnWorkQueue(); @@ -309,6 +312,22 @@ void RedisWriter::HandleEvent(const Dictionary::Ptr& event) ExecuteQuery({ "EXEC" }); } } +*/ + +void RedisWriter::HandleEvent(const Dictionary::Ptr& event) +{ + AssertOnWorkQueue(); + + if (!m_Context) + return; + + String body = JsonEncode(event); + + Log(LogInformation, "RedisWriter") + << "Sending event \"" << body << "\""; + ExecuteQuery({ "PUBLISH", "icinga:event:all", body }); + ExecuteQuery({ "PUBLISH", "icinga:event:" + event->Get("type"), body }); +} void RedisWriter::Stop(bool runtimeRemoved) { diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 24c47f9dd..f4aec001a 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -56,6 +56,7 @@ private: void TryToReconnect(void); void HandleEvents(void); void HandleEvent(const Dictionary::Ptr& event); + void SendEvent(const Dictionary::Ptr& event); void UpdateSubscriptionsTimerHandler(void); void UpdateSubscriptions(void); From 61edfcb68d7ec342054bdfb1bf900399c010cfd9 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Tue, 15 May 2018 14:43:01 +0200 Subject: [PATCH 013/219] Apply code style --- lib/redis/rediswriter-status.cpp | 19 +++---- lib/redis/rediswriter.cpp | 86 ++++++++++++++++---------------- lib/redis/rediswriter.hpp | 24 ++++----- 3 files changed, 66 insertions(+), 63 deletions(-) diff --git a/lib/redis/rediswriter-status.cpp b/lib/redis/rediswriter-status.cpp index 6b5ebbb0b..b8562e2f6 100644 --- a/lib/redis/rediswriter-status.cpp +++ b/lib/redis/rediswriter-status.cpp @@ -46,14 +46,14 @@ value: JsonEncode(Serialize(object, FAState)) INITIALIZE_ONCE(&RedisWriter::ConfigStaticInitialize); -void RedisWriter::ConfigStaticInitialize(void) +void RedisWriter::ConfigStaticInitialize() { /* triggered in ProcessCheckResult(), requires UpdateNextCheck() to be called before */ - ConfigObject::OnStateChanged.connect(boost::bind(&RedisWriter::StateChangedHandler, _1)); + ConfigObject::OnStateChanged.connect(std::bind(&RedisWriter::StateChangedHandler, _1)); /* triggered on create, update and delete objects */ - ConfigObject::OnActiveChanged.connect(boost::bind(&RedisWriter::VersionChangedHandler, _1)); - ConfigObject::OnVersionChanged.connect(boost::bind(&RedisWriter::VersionChangedHandler, _1)); + ConfigObject::OnActiveChanged.connect(std::bind(&RedisWriter::VersionChangedHandler, _1)); + ConfigObject::OnVersionChanged.connect(std::bind(&RedisWriter::VersionChangedHandler, _1)); } void RedisWriter::UpdateAllConfigObjects(void) @@ -68,7 +68,7 @@ void RedisWriter::UpdateAllConfigObjects(void) const String keyPrefix = "icinga:config:"; do { - boost::shared_ptr reply = ExecuteQuery({ "SCAN", Convert::ToString(cursor), "MATCH", keyPrefix + "*", "COUNT", "1000" }); + std::shared_ptr reply = ExecuteQuery({ "SCAN", Convert::ToString(cursor), "MATCH", keyPrefix + "*", "COUNT", "1000" }); VERIFY(reply->type == REDIS_REPLY_ARRAY); VERIFY(reply->elements % 2 == 0); @@ -116,6 +116,7 @@ void RedisWriter::UpdateAllConfigObjects(void) for (const Type::Ptr& type : Type::GetAllTypes()) { ConfigType *ctype = dynamic_cast(type.get()); + if (!ctype) continue; @@ -135,7 +136,7 @@ void RedisWriter::UpdateAllConfigObjects(void) } Log(LogInformation, "RedisWriter") - << "Initial config/status dump finished in " << Utility::GetTime() - startTime << " seconds."; + << "Initial config/status dump finished in " << Utility::GetTime() - startTime << " seconds."; } void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTransaction, bool runtimeUpdate) @@ -324,7 +325,7 @@ void RedisWriter::StateChangedHandler(const ConfigObject::Ptr& object) Type::Ptr type = object->GetReflectionType(); for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { - rw->m_WorkQueue.Enqueue(boost::bind(&RedisWriter::SendStatusUpdate, rw, object, true)); + rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendStatusUpdate, rw, object, true)); } } @@ -335,12 +336,12 @@ void RedisWriter::VersionChangedHandler(const ConfigObject::Ptr& object) if (object->IsActive()) { /* Create or update the object config */ for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { - rw->m_WorkQueue.Enqueue(boost::bind(&RedisWriter::SendConfigUpdate, rw.get(), object, true, true)); + rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendConfigUpdate, rw.get(), object, true, true)); } } else if (!object->IsActive() && object->GetExtension("ConfigObjectDeleted")) { /* same as in apilistener-configsync.cpp */ /* Delete object config */ for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { - rw->m_WorkQueue.Enqueue(boost::bind(&RedisWriter::SendConfigDelete, rw.get(), object)); + rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendConfigDelete, rw.get(), object)); } } } diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index c6248475b..b9169a6e7 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -31,8 +31,8 @@ using namespace icinga; REGISTER_TYPE(RedisWriter); -RedisWriter::RedisWriter(void) - : m_Context(NULL) +RedisWriter::RedisWriter() + : m_Context(NULL) { m_WorkQueue.SetName("RedisWriter"); } @@ -45,31 +45,31 @@ void RedisWriter::Start(bool runtimeCreated) ObjectImpl::Start(runtimeCreated); Log(LogInformation, "RedisWriter") - << "'" << GetName() << "' started."; + << "'" << GetName() << "' started."; m_ConfigDumpInProgress = false; - m_WorkQueue.SetExceptionCallback(boost::bind(&RedisWriter::ExceptionHandler, this, _1)); + m_WorkQueue.SetExceptionCallback(std::bind(&RedisWriter::ExceptionHandler, this, _1)); m_ReconnectTimer = new Timer(); m_ReconnectTimer->SetInterval(15); - m_ReconnectTimer->OnTimerExpired.connect(boost::bind(&RedisWriter::ReconnectTimerHandler, this)); + m_ReconnectTimer->OnTimerExpired.connect(std::bind(&RedisWriter::ReconnectTimerHandler, this)); m_ReconnectTimer->Start(); m_ReconnectTimer->Reschedule(0); m_SubscriptionTimer = new Timer(); m_SubscriptionTimer->SetInterval(15); - m_SubscriptionTimer->OnTimerExpired.connect(boost::bind(&RedisWriter::UpdateSubscriptionsTimerHandler, this)); + m_SubscriptionTimer->OnTimerExpired.connect(std::bind(&RedisWriter::UpdateSubscriptionsTimerHandler, this)); m_SubscriptionTimer->Start(); m_StatsTimer = new Timer(); m_StatsTimer->SetInterval(10); - m_StatsTimer->OnTimerExpired.connect(boost::bind(&RedisWriter::PublishStatsTimerHandler, this)); + m_StatsTimer->OnTimerExpired.connect(std::bind(&RedisWriter::PublishStatsTimerHandler, this)); m_StatsTimer->Start(); m_WorkQueue.SetName("RedisWriter"); - boost::thread thread(boost::bind(&RedisWriter::HandleEvents, this)); + boost::thread thread(std::bind(&RedisWriter::HandleEvents, this)); thread.detach(); } @@ -78,7 +78,7 @@ void RedisWriter::ExceptionHandler(boost::exception_ptr exp) Log(LogCritical, "RedisWriter", "Exception during redis query. Verify that Redis is operational."); Log(LogDebug, "RedisWriter") - << "Exception during redis operation: " << DiagnosticInformation(exp); + << "Exception during redis operation: " << DiagnosticInformation(exp); if (m_Context) { redisFree(m_Context); @@ -86,12 +86,12 @@ void RedisWriter::ExceptionHandler(boost::exception_ptr exp) } } -void RedisWriter::ReconnectTimerHandler(void) +void RedisWriter::ReconnectTimerHandler() { - m_WorkQueue.Enqueue(boost::bind(&RedisWriter::TryToReconnect, this)); + m_WorkQueue.Enqueue(std::bind(&RedisWriter::TryToReconnect, this)); } -void RedisWriter::TryToReconnect(void) +void RedisWriter::TryToReconnect() { AssertOnWorkQueue(); @@ -113,7 +113,7 @@ void RedisWriter::TryToReconnect(void) Log(LogWarning, "RedisWriter", "Cannot allocate redis context."); } else { Log(LogWarning, "RedisWriter", "Connection error: ") - << m_Context->errstr; + << m_Context->errstr; } if (m_Context) { @@ -148,12 +148,12 @@ void RedisWriter::TryToReconnect(void) } /* -void RedisWriter::UpdateSubscriptionsTimerHandler(void) +void RedisWriter::UpdateSubscriptionsTimerHandler() { - m_WorkQueue.Enqueue(boost::bind(&RedisWriter::UpdateSubscriptions, this)); + m_WorkQueue.Enqueue(std::bind(&RedisWriter::UpdateSubscriptions, this)); } -void RedisWriter::UpdateSubscriptions(void) +void RedisWriter::UpdateSubscriptions() { AssertOnWorkQueue(); @@ -169,7 +169,7 @@ void RedisWriter::UpdateSubscriptions(void) String keyPrefix = "icinga:subscription:"; do { - boost::shared_ptr reply = ExecuteQuery({ "SCAN", Convert::ToString(cursor), "MATCH", keyPrefix + "*", "COUNT", "1000" }); + std::shared_ptr reply = ExecuteQuery({ "SCAN", Convert::ToString(cursor), "MATCH", keyPrefix + "*", "COUNT", "1000" }); VERIFY(reply->type == REDIS_REPLY_ARRAY); VERIFY(reply->elements % 2 == 0); @@ -190,21 +190,20 @@ void RedisWriter::UpdateSubscriptions(void) if (!RedisWriter::GetSubscriptionTypes(key, rsi)) Log(LogInformation, "RedisWriter") - << "Subscription \"" << key<< "\" has no types listed."; + << "Subscription \"" << key<< "\" has no types listed."; else m_Subscriptions[key.SubStr(keyPrefix.GetLength())] = rsi; - } } while (cursor != 0); Log(LogInformation, "RedisWriter") - << "Current Redis event subscriptions: " << m_Subscriptions.size(); + << "Current Redis event subscriptions: " << m_Subscriptions.size(); } int RedisWriter::GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi) { try { - boost::shared_ptr redisReply = ExecuteQuery({ "SMEMBERS", key }); + std::shared_ptr redisReply = ExecuteQuery({ "SMEMBERS", key }); VERIFY(redisReply->type == REDIS_REPLY_ARRAY); for (size_t j = 0; j < redisReply->elements; j++) { @@ -223,10 +222,10 @@ int RedisWriter::GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi) void RedisWriter::PublishStatsTimerHandler(void) { - m_WorkQueue.Enqueue(boost::bind(&RedisWriter::PublishStats, this)); + m_WorkQueue.Enqueue(std::bind(&RedisWriter::PublishStats, this)); } -void RedisWriter::PublishStats(void) +void RedisWriter::PublishStats() { AssertOnWorkQueue(); @@ -243,7 +242,7 @@ void RedisWriter::PublishStats(void) ExecuteQuery({ "PUBLISH", "icinga:stats", jsonStats }); } -void RedisWriter::HandleEvents(void) +void RedisWriter::HandleEvents() { String queueName = Utility::NewUniqueID(); EventQueue::Ptr queue = new EventQueue(queueName); @@ -272,7 +271,7 @@ void RedisWriter::HandleEvents(void) if (!event) continue; - m_WorkQueue.Enqueue(boost::bind(&RedisWriter::SendEvent, this, event)); + m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendEvent, this, event)); } queue->RemoveClient(this); @@ -296,13 +295,15 @@ void RedisWriter::HandleEvent(const Dictionary::Ptr& event) String body = JsonEncode(event); - boost::shared_ptr maxExists = ExecuteQuery({ "EXISTS", "icinga:subscription:" + name + ":limit" }); + std::shared_ptr maxExists = ExecuteQuery({ "EXISTS", "icinga:subscription:" + name + ":limit" }); long maxEvents = MAX_EVENTS_DEFAULT; if (maxExists->integer) { - boost::shared_ptr redisReply = ExecuteQuery({ "GET", "icinga:subscription:" + name + ":limit"}); + std::shared_ptr redisReply = ExecuteQuery({ "GET", "icinga:subscription:" + name + ":limit"}); VERIFY(redisReply->type == REDIS_REPLY_STRING); + Log(LogInformation, "RedisWriter") << "Got limit " << redisReply->str << " for " << name; + maxEvents = Convert::ToLong(redisReply->str); } @@ -324,7 +325,8 @@ void RedisWriter::HandleEvent(const Dictionary::Ptr& event) String body = JsonEncode(event); Log(LogInformation, "RedisWriter") - << "Sending event \"" << body << "\""; + << "Sending event \"" << body << "\""; + ExecuteQuery({ "PUBLISH", "icinga:event:all", body }); ExecuteQuery({ "PUBLISH", "icinga:event:" + event->Get("type"), body }); } @@ -332,17 +334,17 @@ void RedisWriter::HandleEvent(const Dictionary::Ptr& event) void RedisWriter::Stop(bool runtimeRemoved) { Log(LogInformation, "RedisWriter") - << "'" << GetName() << "' stopped."; + << "'" << GetName() << "' stopped."; ObjectImpl::Stop(runtimeRemoved); } -void RedisWriter::AssertOnWorkQueue(void) +void RedisWriter::AssertOnWorkQueue() { ASSERT(m_WorkQueue.IsWorkerThread()); } -boost::shared_ptr RedisWriter::ExecuteQuery(const std::vector& query) +std::shared_ptr RedisWriter::ExecuteQuery(const std::vector& query) { const char **argv; size_t *argvlen; @@ -362,23 +364,23 @@ boost::shared_ptr RedisWriter::ExecuteQuery(const std::vectortype == REDIS_REPLY_ERROR) { Log(LogCritical, "RedisWriter") - << "Redis query failed: " << reply->str; + << "Redis query failed: " << reply->str; String msg = reply->str; freeReplyObject(reply); BOOST_THROW_EXCEPTION( - redis_error() - << errinfo_message(msg) - << errinfo_redis_query(Utility::Join(Array::FromVector(query), ' ', false)) + redis_error() + << errinfo_message(msg) + << errinfo_redis_query(Utility::Join(Array::FromVector(query), ' ', false)) ); } - return boost::shared_ptr(reply, freeReplyObject); + return std::shared_ptr(reply, freeReplyObject); } -std::vector > RedisWriter::ExecuteQueries(const std::vector >& queries) +std::vector > RedisWriter::ExecuteQueries(const std::vector >& queries) { const char **argv; size_t *argvlen; @@ -398,7 +400,7 @@ std::vector > RedisWriter::ExecuteQueries(const st delete [] argvlen; } - std::vector > replies; + std::vector > replies; for (size_t i = 0; i < queries.size(); i++) { redisReply *rawReply; @@ -420,14 +422,14 @@ std::vector > RedisWriter::ExecuteQueries(const st if (reply->type == REDIS_REPLY_ERROR) { Log(LogCritical, "RedisWriter") - << "Redis query failed: " << reply->str; + << "Redis query failed: " << reply->str; String msg = reply->str; BOOST_THROW_EXCEPTION( - redis_error() - << errinfo_message(msg) - << errinfo_redis_query(Utility::Join(Array::FromVector(query), ' ', false)) + redis_error() + << errinfo_message(msg) + << errinfo_redis_query(Utility::Join(Array::FromVector(query), ' ', false)) ); } } diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index f4aec001a..20a82f0c6 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -44,28 +44,28 @@ public: DECLARE_OBJECT(RedisWriter); DECLARE_OBJECTNAME(RedisWriter); - RedisWriter(void); + RedisWriter(); - static void ConfigStaticInitialize(void); + static void ConfigStaticInitialize(); virtual void Start(bool runtimeCreated) override; virtual void Stop(bool runtimeRemoved) override; private: - void ReconnectTimerHandler(void); - void TryToReconnect(void); - void HandleEvents(void); + void ReconnectTimerHandler(); + void TryToReconnect(); + void HandleEvents(); void HandleEvent(const Dictionary::Ptr& event); void SendEvent(const Dictionary::Ptr& event); - void UpdateSubscriptionsTimerHandler(void); - void UpdateSubscriptions(void); + void UpdateSubscriptionsTimerHandler(); + void UpdateSubscriptions(); int GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi); - void PublishStatsTimerHandler(void); - void PublishStats(void); + void PublishStatsTimerHandler(); + void PublishStats(); /* config & status dump */ - void UpdateAllConfigObjects(void); + void UpdateAllConfigObjects(); void SendConfigUpdate(const ConfigObject::Ptr& object, bool useTransaction, bool runtimeUpdate = false); void SendConfigDelete(const ConfigObject::Ptr& object); void SendStatusUpdate(const ConfigObject::Ptr& object, bool useTransaction); @@ -84,12 +84,12 @@ private: static void StateChangedHandler(const ConfigObject::Ptr& object); static void VersionChangedHandler(const ConfigObject::Ptr& object); - void AssertOnWorkQueue(void); + void AssertOnWorkQueue(); void ExceptionHandler(boost::exception_ptr exp); boost::shared_ptr ExecuteQuery(const std::vector& query); - std::vector > ExecuteQueries(const std::vector >& queries); + std::vector > ExecuteQueries(const std::vector >& queries); Timer::Ptr m_StatsTimer; Timer::Ptr m_ReconnectTimer; From ddf2aea0650a460214a2ee081fa26c0797c9da6f Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Tue, 15 May 2018 17:18:04 +0200 Subject: [PATCH 014/219] Compile redis into the icinga2 binary; apply more C++11 code changes --- CMakeLists.txt | 3 +++ icinga-app/CMakeLists.txt | 4 ++++ lib/redis/CMakeLists.txt | 23 ++++++++--------------- lib/redis/rediswriter-status.cpp | 2 +- lib/redis/rediswriter.cpp | 24 ++++++++++++++---------- lib/redis/rediswriter.hpp | 6 +++--- lib/redis/rediswriter.ti | 4 ++-- third-party/hiredis/CMakeLists.txt | 18 ++---------------- 8 files changed, 37 insertions(+), 47 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 04349e75a..8e1b016d1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -204,6 +204,9 @@ if(HAVE_SYSTEMD) list(APPEND base_DEPS systemd) endif() +if(ICINGA2_WITH_REDIS) + list(APPEND base_OBJS $) +endif() if(EDITLINE_FOUND) list(APPEND base_DEPS ${EDITLINE_LIBRARIES}) diff --git a/icinga-app/CMakeLists.txt b/icinga-app/CMakeLists.txt index ee3443b28..0ba780316 100644 --- a/icinga-app/CMakeLists.txt +++ b/icinga-app/CMakeLists.txt @@ -53,6 +53,10 @@ if(ICINGA2_WITH_PERFDATA) list(APPEND icinga_app_SOURCES $) endif() +if(ICINGA2_WITH_REDIS) + list(APPEND icinga_app_SOURCES $) +endif() + add_executable(icinga-app $ ${base_OBJS} diff --git a/lib/redis/CMakeLists.txt b/lib/redis/CMakeLists.txt index 540fffa06..b12fd0229 100644 --- a/lib/redis/CMakeLists.txt +++ b/lib/redis/CMakeLists.txt @@ -1,5 +1,5 @@ # Icinga 2 -# Copyright (C) 2012-2017 Icinga Development Team (https://www.icinga.com/) +# Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -15,29 +15,27 @@ # along with this program; if not, write to the Free Software Foundation # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. -mkclass_target(rediswriter.ti rediswriter.tcpp rediswriter.thpp) +mkclass_target(rediswriter.ti rediswriter-ti.cpp rediswriter-ti.hpp) set(redis_SOURCES - rediswriter.cpp rediswriter-status.cpp rediswriter-utility.cpp rediswriter.thpp + rediswriter.cpp rediswriter-status.cpp rediswriter-utility.cpp rediswriter-ti.hpp ) if(ICINGA2_UNITY_BUILD) mkunity_target(redis redis redis_SOURCES) endif() -add_library(redis SHARED ${redis_SOURCES}) - -target_link_libraries(redis ${Boost_LIBRARIES} base config icinga remote hiredis) +add_library(redis OBJECT ${redis_SOURCES}) include_directories(${icinga2_SOURCE_DIR}/third-party) + +add_dependencies(redis base config icinga remote) + link_directories(${icinga2_BINARY_DIR}/third-party/hiredis) set_target_properties ( redis PROPERTIES - INSTALL_RPATH ${CMAKE_INSTALL_FULL_LIBDIR}/icinga2 - DEFINE_SYMBOL I2_REDIS_BUILD FOLDER Components - VERSION ${SPEC_VERSION} ) install_if_not_exists( @@ -45,9 +43,4 @@ install_if_not_exists( ${CMAKE_INSTALL_SYSCONFDIR}/icinga2/features-available ) -install( - TARGETS redis - RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}/icinga2 -) - +set(CPACK_NSIS_EXTRA_INSTALL_COMMANDS "${CPACK_NSIS_EXTRA_INSTALL_COMMANDS}" PARENT_SCOPE) diff --git a/lib/redis/rediswriter-status.cpp b/lib/redis/rediswriter-status.cpp index b8562e2f6..a6d44d1e1 100644 --- a/lib/redis/rediswriter-status.cpp +++ b/lib/redis/rediswriter-status.cpp @@ -1,6 +1,6 @@ /****************************************************************************** * Icinga 2 * - * Copyright (C) 2012-2017 Icinga Development Team (https://www.icinga.com/) * + * Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License * diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index b9169a6e7..04dd20f55 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -1,6 +1,6 @@ /****************************************************************************** * Icinga 2 * - * Copyright (C) 2012-2017 Icinga Development Team (https://www.icinga.com/) * + * Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License * @@ -18,7 +18,7 @@ ******************************************************************************/ #include "redis/rediswriter.hpp" -#include "redis/rediswriter.tcpp" +#include "redis/rediswriter-ti.cpp" #include "remote/eventqueue.hpp" #include "base/json.hpp" #include "base/statsfunction.hpp" @@ -147,7 +147,6 @@ void RedisWriter::TryToReconnect() m_ConfigDumpInProgress = false; } -/* void RedisWriter::UpdateSubscriptionsTimerHandler() { m_WorkQueue.Enqueue(std::bind(&RedisWriter::UpdateSubscriptions, this)); @@ -218,7 +217,6 @@ int RedisWriter::GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi) << "Invalid Redis subscriber info for subscriber '" << key << "': " << DiagnosticInformation(ex); } } -*/ void RedisWriter::PublishStatsTimerHandler(void) { @@ -233,10 +231,18 @@ void RedisWriter::PublishStats() return; //TODO: Figure out if more stats can be useful here. - StatsFunction::Ptr func = StatsFunctionRegistry::GetInstance()->GetItem("CIB"); + Dictionary::Ptr statsFunctions = ScriptGlobal::Get("StatsFunctions", &Empty); + + if (!statsFunctions) + return; + + Function::Ptr func = statsFunctions->Get("CIB"); + Dictionary::Ptr status = new Dictionary(); Array::Ptr perfdata = new Array(); - func->Invoke(status, perfdata); + + func->Invoke({ status, perfdata }); + String jsonStats = JsonEncode(status); ExecuteQuery({ "PUBLISH", "icinga:stats", jsonStats }); @@ -278,7 +284,6 @@ void RedisWriter::HandleEvents() EventQueue::UnregisterIfUnused(queueName, queue); } -/* void RedisWriter::HandleEvent(const Dictionary::Ptr& event) { AssertOnWorkQueue(); @@ -313,9 +318,8 @@ void RedisWriter::HandleEvent(const Dictionary::Ptr& event) ExecuteQuery({ "EXEC" }); } } -*/ -void RedisWriter::HandleEvent(const Dictionary::Ptr& event) +void RedisWriter::SendEvent(const Dictionary::Ptr& event) { AssertOnWorkQueue(); @@ -412,7 +416,7 @@ std::vector > RedisWriter::ExecuteQueries(const std: ); } - boost::shared_ptr reply(rawReply, freeReplyObject); + std::shared_ptr reply(rawReply, freeReplyObject); replies.push_back(reply); } diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 20a82f0c6..4124b4b64 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Icinga 2 * - * Copyright (C) 2012-2017 Icinga Development Team (https://www.icinga.com/) * + * Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License * @@ -20,7 +20,7 @@ #ifndef REDISWRITER_H #define REDISWRITER_H -#include "redis/rediswriter.thpp" +#include "redis/rediswriter-ti.hpp" #include "icinga/customvarobject.hpp" #include "remote/messageorigin.hpp" #include "base/timer.hpp" @@ -88,7 +88,7 @@ private: void ExceptionHandler(boost::exception_ptr exp); - boost::shared_ptr ExecuteQuery(const std::vector& query); + std::shared_ptr ExecuteQuery(const std::vector& query); std::vector > ExecuteQueries(const std::vector >& queries); Timer::Ptr m_StatsTimer; diff --git a/lib/redis/rediswriter.ti b/lib/redis/rediswriter.ti index 0aa76c9c0..db3df0ab5 100644 --- a/lib/redis/rediswriter.ti +++ b/lib/redis/rediswriter.ti @@ -1,6 +1,6 @@ /****************************************************************************** * Icinga 2 * - * Copyright (C) 2012-2017 Icinga Development Team (https://www.icinga.com/) * + * Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License * @@ -19,7 +19,7 @@ #include "base/configobject.hpp" -library demo; +library redis; namespace icinga { diff --git a/third-party/hiredis/CMakeLists.txt b/third-party/hiredis/CMakeLists.txt index afda9579f..e1a00a443 100644 --- a/third-party/hiredis/CMakeLists.txt +++ b/third-party/hiredis/CMakeLists.txt @@ -1,5 +1,5 @@ # Icinga 2 -# Copyright (C) 2012-2017 Icinga Development Team (https://www.icinga.com/) +# Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -15,7 +15,7 @@ # along with this program; if not, write to the Free Software Foundation # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. -add_library(hiredis SHARED net.c net.h hiredis.c hiredis.h sds.c sds.h async.c async.h read.c read.h) +add_library(hiredis OBJECT net.c net.h hiredis.c hiredis.h sds.c sds.h async.c async.h read.c read.h) if(HAVE_VISIBILITY_HIDDEN) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=default") @@ -25,18 +25,4 @@ endif() set_target_properties ( hiredis PROPERTIES FOLDER Lib - VERSION ${SPEC_VERSION} ) - -install( - TARGETS hiredis - RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}/icinga2 -) - -if(APPLE) - install( - TARGETS hiredis - LIBRARY DESTINATION ${CMAKE_INSTALL_BINDIR}/icinga-studio.app/Contents - ) -endif() From f715d479e783ae569dba76902213f56a3c1e2106 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Wed, 23 May 2018 16:27:47 +0200 Subject: [PATCH 015/219] Rename source file --- lib/redis/CMakeLists.txt | 2 +- lib/redis/{rediswriter-status.cpp => rediswriter-objects.cpp} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename lib/redis/{rediswriter-status.cpp => rediswriter-objects.cpp} (100%) diff --git a/lib/redis/CMakeLists.txt b/lib/redis/CMakeLists.txt index b12fd0229..dbf707d01 100644 --- a/lib/redis/CMakeLists.txt +++ b/lib/redis/CMakeLists.txt @@ -18,7 +18,7 @@ mkclass_target(rediswriter.ti rediswriter-ti.cpp rediswriter-ti.hpp) set(redis_SOURCES - rediswriter.cpp rediswriter-status.cpp rediswriter-utility.cpp rediswriter-ti.hpp + rediswriter.cpp rediswriter-objects.cpp rediswriter-utility.cpp rediswriter-ti.hpp ) if(ICINGA2_UNITY_BUILD) diff --git a/lib/redis/rediswriter-status.cpp b/lib/redis/rediswriter-objects.cpp similarity index 100% rename from lib/redis/rediswriter-status.cpp rename to lib/redis/rediswriter-objects.cpp From 33a00e4b5a53765cae1518de937c0f5133b3c7bf Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Tue, 29 May 2018 12:54:14 +0200 Subject: [PATCH 016/219] Move UpdateObjectAttrs into the objects scope --- lib/redis/rediswriter-objects.cpp | 40 +++++++++++++++++++++++++++++++ lib/redis/rediswriter-utility.cpp | 40 ------------------------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index a6d44d1e1..fbbf86d5e 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -320,6 +320,46 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, bool useTran // } } +void RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType) +{ + Type::Ptr type = object->GetReflectionType(); + + String typeName = type->GetName().ToLower(); + String objectName = object->GetName(); + + std::vector > queries; + + queries.push_back({ "DEL", keyPrefix + object->GetName() }); + + std::vector hmsetCommand({ "HMSET", keyPrefix + typeName + ":" + object->GetName() }); + + for (int fid = 0; fid < type->GetFieldCount(); fid++) { + Field field = type->GetFieldInfo(fid); + + if ((field.Attributes & fieldType) == 0) + continue; + + Value val = object->GetField(fid); + + /* hide attributes which shouldn't be user-visible */ + if (field.Attributes & FANoUserView) + continue; + + /* hide internal navigation fields */ + if (field.Attributes & FANavigation && !(field.Attributes & (FAConfig | FAState))) + continue; + + hmsetCommand.push_back(field.Name); + + Value sval = Serialize(val); + hmsetCommand.push_back(JsonEncode(sval)); + } + + queries.push_back(hmsetCommand); + + ExecuteQueries(queries); +} + void RedisWriter::StateChangedHandler(const ConfigObject::Ptr& object) { Type::Ptr type = object->GetReflectionType(); diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index d1aac8c5c..4c6b378ad 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -86,43 +86,3 @@ String RedisWriter::HashValue(const Value& value) return SHA1(JsonEncode(temp)); } -void RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType) -{ - Type::Ptr type = object->GetReflectionType(); - - String typeName = type->GetName(); - String objectName = object->GetName(); - - std::vector > queries; - - queries.push_back({ "DEL", keyPrefix + object->GetName() }); - - std::vector hmsetCommand({ "HMSET", keyPrefix + typeName + ":" + objectName }); - - for (int fid = 0; fid < type->GetFieldCount(); fid++) { - Field field = type->GetFieldInfo(fid); - - if ((field.Attributes & fieldType) == 0) - continue; - - Value val = object->GetField(fid); - - /* hide attributes which shouldn't be user-visible */ - if (field.Attributes & FANoUserView) - continue; - - /* hide internal navigation fields */ - if (field.Attributes & FANavigation && !(field.Attributes & (FAConfig | FAState))) - continue; - - hmsetCommand.push_back(field.Name); - - Value sval = Serialize(val); - hmsetCommand.push_back(JsonEncode(sval)); - } - - queries.push_back(hmsetCommand); - - ExecuteQueries(queries); -} - From 8c38c7eddc41418578f3849b887ced0849f42d97 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Tue, 29 May 2018 12:54:44 +0200 Subject: [PATCH 017/219] Ensure that type names are lower cases for Redis lookups --- lib/redis/rediswriter-objects.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index fbbf86d5e..021691b86 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -120,7 +120,7 @@ void RedisWriter::UpdateAllConfigObjects(void) if (!ctype) continue; - String typeName = type->GetName(); + String typeName = type->GetName().ToLower(); /* replace into aka delete insert is faster than a full diff */ ExecuteQuery({ "DEL", "icinga:config:" + typeName, "icinga:config:" + typeName + ":checksum", "icinga:status:" + typeName }); @@ -157,6 +157,10 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran UpdateObjectAttrs("icinga:config:", object, FAConfig); + Type::Ptr type = object->GetReflectionType(); + + String typeName = type->GetName().ToLower(); + // /* Serialize config object attributes */ // Dictionary::Ptr objectAttrs = SerializeObjectAttrs(object, FAConfig); // @@ -199,7 +203,6 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran // ExecuteQuery({ "HSET", "icinga:config:" + typeName + ":checksum", objectName, checkSumBody }); if (runtimeUpdate) { - Type::Ptr type = object->GetReflectionType(); ExecuteQuery({ "PUBLISH", "icinga:config:update", type->GetName() + ":" + object->GetName() }); } @@ -215,7 +218,7 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) if (!m_Context) return; - String typeName = object->GetReflectionType()->GetName(); + String typeName = object->GetReflectionType()->GetName().ToLower(); String objectName = object->GetName(); ExecuteQueries({ From 26a2095426dcb6e9fbe941850fec0de22564b798 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Tue, 29 May 2018 14:19:18 +0200 Subject: [PATCH 018/219] Implement config object check sums, shot one --- lib/redis/rediswriter-objects.cpp | 94 +++++++++++++++---------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 021691b86..f7051ba00 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -155,55 +155,52 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran if (useTransaction) ExecuteQuery({ "MULTI" }); + /* Send all object attributes to redis, no extra checksums involved here. */ UpdateObjectAttrs("icinga:config:", object, FAConfig); + /* Calculate object specific checksums and store them in a different namespace. */ Type::Ptr type = object->GetReflectionType(); String typeName = type->GetName().ToLower(); + String objectKey = CalculateCheckSumString(object->GetName()); -// /* Serialize config object attributes */ -// Dictionary::Ptr objectAttrs = SerializeObjectAttrs(object, FAConfig); -// -// String jsonBody = JsonEncode(objectAttrs); -// -// String objectName = object->GetName(); -// -// ExecuteQuery({ "HSET", "icinga:config:" + typeName, objectName, jsonBody }); -// -// /* check sums */ -// /* hset icinga:config:Host:checksums localhost { "name_checksum": "...", "properties_checksum": "...", "groups_checksum": "...", "vars_checksum": null } */ -// Dictionary::Ptr checkSum = new Dictionary(); -// -// checkSum->Set("name_checksum", CalculateCheckSumString(object->GetName())); -// -// // TODO: move this elsewhere -// Checkable::Ptr checkable = dynamic_pointer_cast(object); -// -// if (checkable) { -// Host::Ptr host; -// Service::Ptr service; -// -// tie(host, service) = GetHostService(checkable); -// -// if (service) -// checkSum->Set("groups_checksum", CalculateCheckSumGroups(service->GetGroups())); -// else -// checkSum->Set("groups_checksum", CalculateCheckSumGroups(host->GetGroups())); -// } -// -// checkSum->Set("properties_checksum", CalculateCheckSumProperties(object)); -// -// CustomVarObject::Ptr customVarObject = dynamic_pointer_cast(object); -// -// if (customVarObject) -// checkSum->Set("vars_checksum", CalculateCheckSumVars(customVarObject)); -// -// String checkSumBody = JsonEncode(checkSum); -// -// ExecuteQuery({ "HSET", "icinga:config:" + typeName + ":checksum", objectName, checkSumBody }); + Dictionary::Ptr checkSums = new Dictionary(); + checkSums->Set("name_checksum", CalculateCheckSumString(object->GetName())); + // TODO: move this elsewhere + Checkable::Ptr checkable = dynamic_pointer_cast(object); + + if (checkable) { + Host::Ptr host; + Service::Ptr service; + + tie(host, service) = GetHostService(checkable); + + if (service) + checkSums->Set("groups_checksum", CalculateCheckSumGroups(service->GetGroups())); + else + checkSums->Set("groups_checksum", CalculateCheckSumGroups(host->GetGroups())); + } + + checkSums->Set("properties_checksum", CalculateCheckSumProperties(object)); + + //TODO: Move this somewhere else. + CustomVarObject::Ptr customVarObject = dynamic_pointer_cast(object); + + if (customVarObject) + checkSums->Set("vars_checksum", CalculateCheckSumVars(customVarObject)); + + String checkSumsBody = JsonEncode(checkSums); + + Log(LogDebug, "RedisWriter") + << "HSET icinga:config:" << typeName << ":checksum " << objectKey << " " << checkSumsBody; + + ExecuteQuery({ "HSET", "icinga:config:" + typeName + ":checksum", objectKey, checkSumsBody }); + + + /* Send an update event to subscribers. */ if (runtimeUpdate) { - ExecuteQuery({ "PUBLISH", "icinga:config:update", type->GetName() + ":" + object->GetName() }); + ExecuteQuery({ "PUBLISH", "icinga:config:update", typeName + ":" + objectKey }); } if (useTransaction) @@ -219,12 +216,12 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) return; String typeName = object->GetReflectionType()->GetName().ToLower(); - String objectName = object->GetName(); + String objectKey = CalculateCheckSumString(object->GetName()); ExecuteQueries({ - { "DEL", "icinga:config:" + typeName, objectName }, - { "DEL", "icinga:status:" + typeName, objectName }, - { "PUBLISH", "icinga:config:delete", typeName + ":" + objectName } + { "DEL", "icinga:config:" + typeName, objectKey }, + { "DEL", "icinga:status:" + typeName, objectKey }, + { "PUBLISH", "icinga:config:delete", typeName + ":" + objectKey } }); } @@ -330,11 +327,14 @@ void RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject: String typeName = type->GetName().ToLower(); String objectName = object->GetName(); + /* Use the name checksum as unique key. */ + String objectKey = CalculateCheckSumString(object->GetName()); + std::vector > queries; - queries.push_back({ "DEL", keyPrefix + object->GetName() }); + queries.push_back({ "DEL", keyPrefix + objectKey }); - std::vector hmsetCommand({ "HMSET", keyPrefix + typeName + ":" + object->GetName() }); + std::vector hmsetCommand({ "HMSET", keyPrefix + typeName + ":" + objectKey }); for (int fid = 0; fid < type->GetFieldCount(); fid++) { Field field = type->GetFieldInfo(fid); From 782486ce9ce008b95141ec2f882de9a16b4b8572 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Tue, 29 May 2018 14:19:32 +0200 Subject: [PATCH 019/219] Ensure that arrays are sorted for checksum calculation --- lib/redis/rediswriter-utility.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 4c6b378ad..0fecd8c22 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -45,10 +45,15 @@ String RedisWriter::CalculateCheckSumGroups(const Array::Ptr& groups) { String output; - { - ObjectLock olock(groups); + /* Ensure that checksums happen in a defined order. */ + Array::Ptr tmpGroups = groups->ShallowClone(); - for (const String& group : groups) { + tmpGroups->Sort(); + + { + ObjectLock olock(tmpGroups); + + for (const String& group : tmpGroups) { output += SHA1(group); } } From 79f06a90d4059d16dfd4164bd45ac53e0bec9045 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Mon, 4 Jun 2018 10:06:41 +0200 Subject: [PATCH 020/219] WIP --- lib/redis/rediswriter-objects.cpp | 29 +++++++++-------------------- lib/redis/rediswriter.cpp | 4 ++-- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index f7051ba00..72465080d 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -30,20 +30,6 @@ using namespace icinga; -/* -- icinga:config: as hash -key: sha1 checksum(name) -value: JsonEncode(Serialize(object, FAConfig)) + config_checksum - -Diff between calculated config_checksum and Redis json config_checksum -Alternative: Replace into. - - -- icinga:status: as hash -key: sha1 checksum(name) -value: JsonEncode(Serialize(object, FAState)) -*/ - INITIALIZE_ONCE(&RedisWriter::ConfigStaticInitialize); void RedisWriter::ConfigStaticInitialize() @@ -162,7 +148,8 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran Type::Ptr type = object->GetReflectionType(); String typeName = type->GetName().ToLower(); - String objectKey = CalculateCheckSumString(object->GetName()); +// String objectKey = CalculateCheckSumString(object->GetName()); + String objectKey = object->GetName(); Dictionary::Ptr checkSums = new Dictionary(); checkSums->Set("name_checksum", CalculateCheckSumString(object->GetName())); @@ -216,11 +203,12 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) return; String typeName = object->GetReflectionType()->GetName().ToLower(); - String objectKey = CalculateCheckSumString(object->GetName()); + //String objectKey = CalculateCheckSumString(object->GetName()); + String objectKey = object->GetName(); ExecuteQueries({ - { "DEL", "icinga:config:" + typeName, objectKey }, - { "DEL", "icinga:status:" + typeName, objectKey }, + { "DEL", "icinga:config:" + typeName + ":" + objectKey }, + { "DEL", "icinga:status:" + typeName + ":" + objectKey }, { "PUBLISH", "icinga:config:delete", typeName + ":" + objectKey } }); @@ -328,11 +316,12 @@ void RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject: String objectName = object->GetName(); /* Use the name checksum as unique key. */ - String objectKey = CalculateCheckSumString(object->GetName()); + //String objectKey = CalculateCheckSumString(object->GetName()); + String objectKey = object->GetName(); std::vector > queries; - queries.push_back({ "DEL", keyPrefix + objectKey }); + queries.push_back({ "DEL", keyPrefix + typeName + ":" + objectKey }); std::vector hmsetCommand({ "HMSET", keyPrefix + typeName + ":" + objectKey }); diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index 04dd20f55..12383c837 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -328,8 +328,8 @@ void RedisWriter::SendEvent(const Dictionary::Ptr& event) String body = JsonEncode(event); - Log(LogInformation, "RedisWriter") - << "Sending event \"" << body << "\""; +// Log(LogInformation, "RedisWriter") +// << "Sending event \"" << body << "\""; ExecuteQuery({ "PUBLISH", "icinga:event:all", body }); ExecuteQuery({ "PUBLISH", "icinga:event:" + event->Get("type"), body }); From 60a5a14aa27372198dea85f06fa21bd8a8721872 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Mon, 4 Jun 2018 15:42:33 +0200 Subject: [PATCH 021/219] Checksums --- lib/redis/rediswriter-objects.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 72465080d..144cfb98e 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -148,8 +148,8 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran Type::Ptr type = object->GetReflectionType(); String typeName = type->GetName().ToLower(); -// String objectKey = CalculateCheckSumString(object->GetName()); - String objectKey = object->GetName(); + String objectKey = CalculateCheckSumString(object->GetName()); + //String objectKey = object->GetName(); Dictionary::Ptr checkSums = new Dictionary(); checkSums->Set("name_checksum", CalculateCheckSumString(object->GetName())); @@ -180,9 +180,9 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran String checkSumsBody = JsonEncode(checkSums); Log(LogDebug, "RedisWriter") - << "HSET icinga:config:" << typeName << ":checksum " << objectKey << " " << checkSumsBody; + << "HSET icinga:config:checksum:" << typeName << " " << objectKey << " " << checkSumsBody; - ExecuteQuery({ "HSET", "icinga:config:" + typeName + ":checksum", objectKey, checkSumsBody }); + ExecuteQuery({ "HSET", "icinga:config:checksum:" + typeName, objectKey, checkSumsBody }); /* Send an update event to subscribers. */ @@ -203,8 +203,8 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) return; String typeName = object->GetReflectionType()->GetName().ToLower(); - //String objectKey = CalculateCheckSumString(object->GetName()); - String objectKey = object->GetName(); + String objectKey = CalculateCheckSumString(object->GetName()); + //String objectKey = object->GetName(); ExecuteQueries({ { "DEL", "icinga:config:" + typeName + ":" + objectKey }, @@ -316,8 +316,8 @@ void RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject: String objectName = object->GetName(); /* Use the name checksum as unique key. */ - //String objectKey = CalculateCheckSumString(object->GetName()); - String objectKey = object->GetName(); + String objectKey = CalculateCheckSumString(object->GetName()); + //String objectKey = object->GetName(); std::vector > queries; From b32b81d95786b8bb46717256da87cd9f2a33f4e6 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 5 Jun 2018 15:01:43 +0200 Subject: [PATCH 022/219] RedisWriter: Pack objects consistently for hashing --- lib/redis/rediswriter-utility.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 0fecd8c22..ec096443c 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -19,7 +19,7 @@ #include "redis/rediswriter.hpp" #include "icinga/customvarobject.hpp" -#include "base/json.hpp" +#include "base/object-packer.hpp" #include "base/logger.hpp" #include "base/serializer.hpp" #include "base/tlsutility.hpp" @@ -88,6 +88,6 @@ String RedisWriter::HashValue(const Value& value) else temp = value; - return SHA1(JsonEncode(temp)); + return SHA1(PackObject(temp)); } From dc73db01b837f8c6d4cdb4db82a5f44c170af025 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 5 Jun 2018 15:30:31 +0200 Subject: [PATCH 023/219] RedisWriter: use one checksum algo for everything --- lib/redis/rediswriter-utility.cpp | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index ec096443c..8f4250537 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -43,22 +43,12 @@ String RedisWriter::CalculateCheckSumString(const String& str) String RedisWriter::CalculateCheckSumGroups(const Array::Ptr& groups) { - String output; - /* Ensure that checksums happen in a defined order. */ Array::Ptr tmpGroups = groups->ShallowClone(); tmpGroups->Sort(); - { - ObjectLock olock(tmpGroups); - - for (const String& group : tmpGroups) { - output += SHA1(group); - } - } - - return SHA1(output); + return SHA1(PackObject(tmpGroups)); } String RedisWriter::CalculateCheckSumProperties(const ConfigObject::Ptr& object) From dcdf6b8a44b70a3ad8f699d051257c4e154334aa Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 5 Jun 2018 16:46:02 +0200 Subject: [PATCH 024/219] RedisWriter: Exclude explicitly checksummed properties from properties_checksum --- lib/redis/rediswriter-objects.cpp | 14 +++++++++--- lib/redis/rediswriter-utility.cpp | 36 +++++++++++++++++++++++++++---- lib/redis/rediswriter.hpp | 3 ++- 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 144cfb98e..6c3642277 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -27,6 +27,7 @@ #include "base/tlsutility.hpp" #include "base/initialize.hpp" #include "base/convert.hpp" +#include using namespace icinga; @@ -151,6 +152,8 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran String objectKey = CalculateCheckSumString(object->GetName()); //String objectKey = object->GetName(); + std::set propertiesBlacklist ({"name"}); + Dictionary::Ptr checkSums = new Dictionary(); checkSums->Set("name_checksum", CalculateCheckSumString(object->GetName())); @@ -158,6 +161,8 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran Checkable::Ptr checkable = dynamic_pointer_cast(object); if (checkable) { + propertiesBlacklist.emplace("groups"); + Host::Ptr host; Service::Ptr service; @@ -169,13 +174,16 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran checkSums->Set("groups_checksum", CalculateCheckSumGroups(host->GetGroups())); } - checkSums->Set("properties_checksum", CalculateCheckSumProperties(object)); - //TODO: Move this somewhere else. CustomVarObject::Ptr customVarObject = dynamic_pointer_cast(object); - if (customVarObject) + if (customVarObject) { + propertiesBlacklist.emplace("vars"); + checkSums->Set("vars_checksum", CalculateCheckSumVars(customVarObject)); + } + + checkSums->Set("properties_checksum", CalculateCheckSumProperties(object, propertiesBlacklist)); String checkSumsBody = JsonEncode(checkSums); diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 8f4250537..897938711 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -24,6 +24,7 @@ #include "base/serializer.hpp" #include "base/tlsutility.hpp" #include "base/initialize.hpp" +#include "base/objectlock.hpp" using namespace icinga; @@ -51,10 +52,10 @@ String RedisWriter::CalculateCheckSumGroups(const Array::Ptr& groups) return SHA1(PackObject(tmpGroups)); } -String RedisWriter::CalculateCheckSumProperties(const ConfigObject::Ptr& object) +String RedisWriter::CalculateCheckSumProperties(const ConfigObject::Ptr& object, const std::set& propertiesBlacklist) { //TODO: consider precision of 6 for double values; use specific config fields for hashing? - return HashValue(object); + return HashValue(object, propertiesBlacklist); } String RedisWriter::CalculateCheckSumVars(const CustomVarObject::Ptr& object) @@ -67,16 +68,43 @@ String RedisWriter::CalculateCheckSumVars(const CustomVarObject::Ptr& object) return HashValue(vars); } +static const std::set propertiesBlacklistEmpty; + String RedisWriter::HashValue(const Value& value) +{ + return HashValue(value, propertiesBlacklistEmpty); +} + +String RedisWriter::HashValue(const Value& value, const std::set& propertiesBlacklist) { Value temp; + bool mutabl; Type::Ptr type = value.GetReflectionType(); - if (ConfigObject::TypeInstance->IsAssignableFrom(type)) + if (ConfigObject::TypeInstance->IsAssignableFrom(type)) { temp = Serialize(value, FAConfig); - else + mutabl = true; + } else { temp = value; + mutabl = false; + } + + if (propertiesBlacklist.size() && temp.IsObject()) { + Dictionary::Ptr dict = dynamic_pointer_cast((Object::Ptr)temp); + + if (dict) { + if (!mutabl) + dict = dict->ShallowClone(); + + ObjectLock olock(dict); + for (auto& property : propertiesBlacklist) + dict->Remove(property); + + if (!mutabl) + temp = dict; + } + } return SHA1(PackObject(temp)); } diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 4124b4b64..98e0dbdb9 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -76,10 +76,11 @@ private: static String CalculateCheckSumString(const String& str); static String CalculateCheckSumGroups(const Array::Ptr& groups); - static String CalculateCheckSumProperties(const ConfigObject::Ptr& object); + static String CalculateCheckSumProperties(const ConfigObject::Ptr& object, const std::set& propertiesBlacklist); static String CalculateCheckSumVars(const CustomVarObject::Ptr& object); static String HashValue(const Value& value); + static String HashValue(const Value& value, const std::set& propertiesBlacklist); static void StateChangedHandler(const ConfigObject::Ptr& object); static void VersionChangedHandler(const ConfigObject::Ptr& object); From e980485e237a516afee82ff7a3631c9d8630875d Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 6 Jun 2018 12:31:41 +0200 Subject: [PATCH 025/219] RedisWriter: Explicitly checksum Zone#endpoints --- lib/redis/rediswriter-objects.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 6c3642277..497c969a1 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -172,6 +172,23 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran checkSums->Set("groups_checksum", CalculateCheckSumGroups(service->GetGroups())); else checkSums->Set("groups_checksum", CalculateCheckSumGroups(host->GetGroups())); + } else { + Zone::Ptr zone = dynamic_pointer_cast(object); + + if (zone) { + propertiesBlacklist.emplace("endpoints"); + + auto endpointObjects = zone->GetEndpoints(); + Array::Ptr endpoints = new Array(); + endpoints->Resize(endpointObjects.size()); + + Array::SizeType i = 0; + for (auto& endpointObject : endpointObjects) { + endpoints->Set(i++, endpointObject->GetName()); + } + + checkSums->Set("endpoints_checksum", CalculateCheckSumGroups(endpoints)); + } } //TODO: Move this somewhere else. From 3ea3b2ba2706b4fe001f68ebab91ebc2246cc4c3 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 7 Jun 2018 12:49:38 +0200 Subject: [PATCH 026/219] RedisWriter: checksum package, source_location and templates separately in metadata_checksum --- lib/redis/rediswriter-objects.cpp | 3 ++- lib/redis/rediswriter-utility.cpp | 27 ++++++++++++++++++++++++--- lib/redis/rediswriter.hpp | 3 ++- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 497c969a1..7dac7287e 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -152,7 +152,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran String objectKey = CalculateCheckSumString(object->GetName()); //String objectKey = object->GetName(); - std::set propertiesBlacklist ({"name"}); + std::set propertiesBlacklist ({"name", "__name", "package", "source_location", "templates"}); Dictionary::Ptr checkSums = new Dictionary(); checkSums->Set("name_checksum", CalculateCheckSumString(object->GetName())); @@ -200,6 +200,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran checkSums->Set("vars_checksum", CalculateCheckSumVars(customVarObject)); } + checkSums->Set("metadata_checksum", CalculateCheckSumMetadata(object)); checkSums->Set("properties_checksum", CalculateCheckSumProperties(object, propertiesBlacklist)); String checkSumsBody = JsonEncode(checkSums); diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 897938711..707fd69dd 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -58,6 +58,13 @@ String RedisWriter::CalculateCheckSumProperties(const ConfigObject::Ptr& object, return HashValue(object, propertiesBlacklist); } +static const std::set metadataWhitelist ({"package", "source_location", "templates"}); + +String RedisWriter::CalculateCheckSumMetadata(const ConfigObject::Ptr& object) +{ + return HashValue(object, metadataWhitelist, true); +} + String RedisWriter::CalculateCheckSumVars(const CustomVarObject::Ptr& object) { Dictionary::Ptr vars = object->GetVars(); @@ -75,7 +82,7 @@ String RedisWriter::HashValue(const Value& value) return HashValue(value, propertiesBlacklistEmpty); } -String RedisWriter::HashValue(const Value& value, const std::set& propertiesBlacklist) +String RedisWriter::HashValue(const Value& value, const std::set& propertiesBlacklist, bool propertiesWhitelist) { Value temp; bool mutabl; @@ -98,8 +105,22 @@ String RedisWriter::HashValue(const Value& value, const std::set& proper dict = dict->ShallowClone(); ObjectLock olock(dict); - for (auto& property : propertiesBlacklist) - dict->Remove(property); + + if (propertiesWhitelist) { + auto current = dict->Begin(); + auto propertiesBlacklistEnd = propertiesBlacklist.end(); + + while (current != dict->End()) { + if (propertiesBlacklist.find(current->first) == propertiesBlacklistEnd) { + dict->Remove(current++); + } else { + ++current; + } + } + } else { + for (auto& property : propertiesBlacklist) + dict->Remove(property); + } if (!mutabl) temp = dict; diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 98e0dbdb9..194dff8ab 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -77,10 +77,11 @@ private: static String CalculateCheckSumString(const String& str); static String CalculateCheckSumGroups(const Array::Ptr& groups); static String CalculateCheckSumProperties(const ConfigObject::Ptr& object, const std::set& propertiesBlacklist); + static String CalculateCheckSumMetadata(const ConfigObject::Ptr& object); static String CalculateCheckSumVars(const CustomVarObject::Ptr& object); static String HashValue(const Value& value); - static String HashValue(const Value& value, const std::set& propertiesBlacklist); + static String HashValue(const Value& value, const std::set& propertiesBlacklist, bool propertiesWhitelist = false); static void StateChangedHandler(const ConfigObject::Ptr& object); static void VersionChangedHandler(const ConfigObject::Ptr& object); From a5aef627f835de9aa92df18817e3c05987416ee5 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 7 Jun 2018 12:53:05 +0200 Subject: [PATCH 027/219] RedisWriter: use __name as object key --- lib/redis/rediswriter-objects.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 7dac7287e..e9652f847 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -149,8 +149,8 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran Type::Ptr type = object->GetReflectionType(); String typeName = type->GetName().ToLower(); - String objectKey = CalculateCheckSumString(object->GetName()); - //String objectKey = object->GetName(); + //String objectKey = CalculateCheckSumString(object->GetName()); + String objectKey = object->GetName(); std::set propertiesBlacklist ({"name", "__name", "package", "source_location", "templates"}); @@ -229,8 +229,8 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) return; String typeName = object->GetReflectionType()->GetName().ToLower(); - String objectKey = CalculateCheckSumString(object->GetName()); - //String objectKey = object->GetName(); + //String objectKey = CalculateCheckSumString(object->GetName()); + String objectKey = object->GetName(); ExecuteQueries({ { "DEL", "icinga:config:" + typeName + ":" + objectKey }, @@ -342,8 +342,8 @@ void RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject: String objectName = object->GetName(); /* Use the name checksum as unique key. */ - String objectKey = CalculateCheckSumString(object->GetName()); - //String objectKey = object->GetName(); + //String objectKey = CalculateCheckSumString(object->GetName()); + String objectKey = object->GetName(); std::vector > queries; From f05a5eecc29fbdf68482b9a38b94ec887c6f9b0f Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 8 Jun 2018 11:46:45 +0200 Subject: [PATCH 028/219] Centralize identifier computation --- lib/redis/rediswriter-objects.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index e9652f847..379a6cb14 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -43,6 +43,11 @@ void RedisWriter::ConfigStaticInitialize() ConfigObject::OnVersionChanged.connect(std::bind(&RedisWriter::VersionChangedHandler, _1)); } +static inline String GetIdentifier(const ConfigObject::Ptr& object) +{ + return object->GetName(); +} + void RedisWriter::UpdateAllConfigObjects(void) { AssertOnWorkQueue(); @@ -149,8 +154,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran Type::Ptr type = object->GetReflectionType(); String typeName = type->GetName().ToLower(); - //String objectKey = CalculateCheckSumString(object->GetName()); - String objectKey = object->GetName(); + String objectKey = GetIdentifier(object); std::set propertiesBlacklist ({"name", "__name", "package", "source_location", "templates"}); @@ -229,8 +233,7 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) return; String typeName = object->GetReflectionType()->GetName().ToLower(); - //String objectKey = CalculateCheckSumString(object->GetName()); - String objectKey = object->GetName(); + String objectKey = GetIdentifier(object); ExecuteQueries({ { "DEL", "icinga:config:" + typeName + ":" + objectKey }, @@ -339,11 +342,9 @@ void RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject: Type::Ptr type = object->GetReflectionType(); String typeName = type->GetName().ToLower(); - String objectName = object->GetName(); /* Use the name checksum as unique key. */ - //String objectKey = CalculateCheckSumString(object->GetName()); - String objectKey = object->GetName(); + String objectKey = GetIdentifier(object); std::vector > queries; From 3229982beed1e335a51044875155ba8cddc8880b Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 8 Jun 2018 12:02:27 +0200 Subject: [PATCH 029/219] RedisWriter: compute name_checksum as expected --- lib/redis/rediswriter-objects.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 379a6cb14..447f9653a 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -159,7 +159,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran std::set propertiesBlacklist ({"name", "__name", "package", "source_location", "templates"}); Dictionary::Ptr checkSums = new Dictionary(); - checkSums->Set("name_checksum", CalculateCheckSumString(object->GetName())); + checkSums->Set("name_checksum", CalculateCheckSumString(object->GetShortName())); // TODO: move this elsewhere Checkable::Ptr checkable = dynamic_pointer_cast(object); From d7f6d7994be18fb014d85d725f6610a41e18a6d2 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Fri, 8 Jun 2018 11:38:36 +0200 Subject: [PATCH 030/219] Move stats into a separate file --- lib/redis/CMakeLists.txt | 2 +- lib/redis/rediswriter-stats.cpp | 48 +++++++++++++++++++++++++++++++++ lib/redis/rediswriter.cpp | 15 +---------- lib/redis/rediswriter.hpp | 3 +++ 4 files changed, 53 insertions(+), 15 deletions(-) create mode 100644 lib/redis/rediswriter-stats.cpp diff --git a/lib/redis/CMakeLists.txt b/lib/redis/CMakeLists.txt index dbf707d01..5ff39f81d 100644 --- a/lib/redis/CMakeLists.txt +++ b/lib/redis/CMakeLists.txt @@ -18,7 +18,7 @@ mkclass_target(rediswriter.ti rediswriter-ti.cpp rediswriter-ti.hpp) set(redis_SOURCES - rediswriter.cpp rediswriter-objects.cpp rediswriter-utility.cpp rediswriter-ti.hpp + rediswriter.cpp rediswriter-objects.cpp rediswriter-stats.cpp rediswriter-utility.cpp rediswriter-ti.hpp ) if(ICINGA2_UNITY_BUILD) diff --git a/lib/redis/rediswriter-stats.cpp b/lib/redis/rediswriter-stats.cpp new file mode 100644 index 000000000..152d0b64b --- /dev/null +++ b/lib/redis/rediswriter-stats.cpp @@ -0,0 +1,48 @@ +/****************************************************************************** + * Icinga 2 * + * Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License * + * as published by the Free Software Foundation; either version 2 * + * of the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software Foundation * + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * + ******************************************************************************/ + +#include "redis/rediswriter.hpp" +#include "base/json.hpp" +#include "base/logger.hpp" +#include "base/serializer.hpp" +#include "base/statsfunction.hpp" +#include "base/convert.hpp" + +using namespace icinga; + +Dictionary::Ptr RedisWriter::GetStats() +{ + //TODO: Figure out if more stats can be useful here. + Dictionary::Ptr statsFunctions = ScriptGlobal::Get("StatsFunctions", &Empty); + + if (!statsFunctions) + Dictionary::Ptr(); + + Function::Ptr func = statsFunctions->Get("CIB"); + + Dictionary::Ptr status = new Dictionary(); + Array::Ptr perfdata = new Array(); + + func->Invoke({ status, perfdata }); + +// String jsonStats = JsonEncode(status); + + return status; +} + diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index 12383c837..aaa4b3629 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -21,7 +21,6 @@ #include "redis/rediswriter-ti.cpp" #include "remote/eventqueue.hpp" #include "base/json.hpp" -#include "base/statsfunction.hpp" #include using namespace icinga; @@ -230,19 +229,7 @@ void RedisWriter::PublishStats() if (!m_Context) return; - //TODO: Figure out if more stats can be useful here. - Dictionary::Ptr statsFunctions = ScriptGlobal::Get("StatsFunctions", &Empty); - - if (!statsFunctions) - return; - - Function::Ptr func = statsFunctions->Get("CIB"); - - Dictionary::Ptr status = new Dictionary(); - Array::Ptr perfdata = new Array(); - - func->Invoke({ status, perfdata }); - + Dictionary::Ptr status = GetStats(); String jsonStats = JsonEncode(status); ExecuteQuery({ "PUBLISH", "icinga:stats", jsonStats }); diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 194dff8ab..9adf7dddb 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -71,6 +71,9 @@ private: void SendStatusUpdate(const ConfigObject::Ptr& object, bool useTransaction); void UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType); + /* Stats */ + Dictionary::Ptr GetStats(); + /* utilities */ static String FormatCheckSumBinary(const String& str); From fae5b88e485cda7e8ddf2444619c7b276079e356 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Fri, 8 Jun 2018 12:47:45 +0200 Subject: [PATCH 031/219] Add all stats similar to /v1/status --- lib/redis/rediswriter-stats.cpp | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/lib/redis/rediswriter-stats.cpp b/lib/redis/rediswriter-stats.cpp index 152d0b64b..1a234b64d 100644 --- a/lib/redis/rediswriter-stats.cpp +++ b/lib/redis/rediswriter-stats.cpp @@ -28,21 +28,33 @@ using namespace icinga; Dictionary::Ptr RedisWriter::GetStats() { + Dictionary::Ptr stats = new Dictionary(); + //TODO: Figure out if more stats can be useful here. Dictionary::Ptr statsFunctions = ScriptGlobal::Get("StatsFunctions", &Empty); if (!statsFunctions) Dictionary::Ptr(); - Function::Ptr func = statsFunctions->Get("CIB"); + ObjectLock olock(statsFunctions); - Dictionary::Ptr status = new Dictionary(); - Array::Ptr perfdata = new Array(); + for (const Dictionary::Pair& kv : statsFunctions) + { + Function::Ptr func = kv.second; - func->Invoke({ status, perfdata }); + if (!func) + BOOST_THROW_EXCEPTION(std::invalid_argument("Invalid status function name.")); -// String jsonStats = JsonEncode(status); + Dictionary::Ptr status = new Dictionary(); + Array::Ptr perfdata = new Array(); + func->Invoke({ status, perfdata }); - return status; + stats->Set(kv.first, new Dictionary({ + { "status", status }, + { "perfdata", Serialize(perfdata, FAState) } + })); + } + + return stats; } From 4eee9572ab77b7acfba3103dbf64469bc30d5fc5 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 22 Jun 2018 15:55:19 +0200 Subject: [PATCH 032/219] RedisWriter: identify config objects by SHA1(PackObject([Environment, __name])) --- lib/redis/rediswriter-objects.cpp | 5 ----- lib/redis/rediswriter-utility.cpp | 9 +++++++++ lib/redis/rediswriter.hpp | 1 + 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 447f9653a..8d889dd5b 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -43,11 +43,6 @@ void RedisWriter::ConfigStaticInitialize() ConfigObject::OnVersionChanged.connect(std::bind(&RedisWriter::VersionChangedHandler, _1)); } -static inline String GetIdentifier(const ConfigObject::Ptr& object) -{ - return object->GetName(); -} - void RedisWriter::UpdateAllConfigObjects(void) { AssertOnWorkQueue(); diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 707fd69dd..ac480274e 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -25,6 +25,8 @@ #include "base/tlsutility.hpp" #include "base/initialize.hpp" #include "base/objectlock.hpp" +#include "base/array.hpp" +#include "base/scriptglobal.hpp" using namespace icinga; @@ -37,6 +39,13 @@ String RedisWriter::FormatCheckSumBinary(const String& str) return output; } +static Value l_DefaultEnv = "production"; + +String RedisWriter::GetIdentifier(const ConfigObject::Ptr& object) +{ + return HashValue((Array::Ptr)new Array({ScriptGlobal::Get("Environment", &l_DefaultEnv), object->GetName()})); +} + String RedisWriter::CalculateCheckSumString(const String& str) { return SHA1(str); diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 9adf7dddb..7be55ace4 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -77,6 +77,7 @@ private: /* utilities */ static String FormatCheckSumBinary(const String& str); + static String GetIdentifier(const ConfigObject::Ptr& object); static String CalculateCheckSumString(const String& str); static String CalculateCheckSumGroups(const Array::Ptr& groups); static String CalculateCheckSumProperties(const ConfigObject::Ptr& object, const std::set& propertiesBlacklist); From 724ce1f172558f79d026b1a95c99b40f121525c6 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 22 Jun 2018 16:18:22 +0200 Subject: [PATCH 033/219] RedisWriter: dump also environment_checksum (per config object) --- lib/redis/rediswriter-objects.cpp | 1 + lib/redis/rediswriter-utility.cpp | 7 ++++++- lib/redis/rediswriter.hpp | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 8d889dd5b..27ad52bba 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -155,6 +155,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran Dictionary::Ptr checkSums = new Dictionary(); checkSums->Set("name_checksum", CalculateCheckSumString(object->GetShortName())); + checkSums->Set("environment_checksum", CalculateCheckSumString(GetEnvironment())); // TODO: move this elsewhere Checkable::Ptr checkable = dynamic_pointer_cast(object); diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index ac480274e..ad3a8b5ba 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -41,9 +41,14 @@ String RedisWriter::FormatCheckSumBinary(const String& str) static Value l_DefaultEnv = "production"; +String RedisWriter::GetEnvironment() +{ + return ScriptGlobal::Get("Environment", &l_DefaultEnv); +} + String RedisWriter::GetIdentifier(const ConfigObject::Ptr& object) { - return HashValue((Array::Ptr)new Array({ScriptGlobal::Get("Environment", &l_DefaultEnv), object->GetName()})); + return HashValue((Array::Ptr)new Array({GetEnvironment(), object->GetName()})); } String RedisWriter::CalculateCheckSumString(const String& str) diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 7be55ace4..cc11a10e1 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -78,6 +78,7 @@ private: static String FormatCheckSumBinary(const String& str); static String GetIdentifier(const ConfigObject::Ptr& object); + static String GetEnvironment(); static String CalculateCheckSumString(const String& str); static String CalculateCheckSumGroups(const Array::Ptr& groups); static String CalculateCheckSumProperties(const ConfigObject::Ptr& object, const std::set& propertiesBlacklist); From c92827fa5eba02aa958993e30ea778433dd6b94c Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 22 Jun 2018 17:19:25 +0200 Subject: [PATCH 034/219] RedisWriter: dump monitored objects' groups' identifiers (group_checksums) --- lib/redis/rediswriter-objects.cpp | 40 +++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 27ad52bba..698205715 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -21,12 +21,15 @@ #include "icinga/customvarobject.hpp" #include "icinga/host.hpp" #include "icinga/service.hpp" +#include "icinga/hostgroup.hpp" +#include "icinga/servicegroup.hpp" #include "base/json.hpp" #include "base/logger.hpp" #include "base/serializer.hpp" #include "base/tlsutility.hpp" #include "base/initialize.hpp" #include "base/convert.hpp" +#include "base/array.hpp" #include using namespace icinga; @@ -126,6 +129,16 @@ void RedisWriter::UpdateAllConfigObjects(void) << "Initial config/status dump finished in " << Utility::GetTime() - startTime << " seconds."; } +static ConfigObject::Ptr GetHostGroup(const String& name) +{ + return ConfigObject::GetObject(name); +} + +static ConfigObject::Ptr GetServiceGroup(const String& name) +{ + return ConfigObject::GetObject(name); +} + void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTransaction, bool runtimeUpdate) { AssertOnWorkQueue(); @@ -168,10 +181,29 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran tie(host, service) = GetHostService(checkable); - if (service) - checkSums->Set("groups_checksum", CalculateCheckSumGroups(service->GetGroups())); - else - checkSums->Set("groups_checksum", CalculateCheckSumGroups(host->GetGroups())); + Array::Ptr groups; + ConfigObject::Ptr (*getGroup)(const String& name); + + if (service) { + groups = service->GetGroups(); + getGroup = &::GetServiceGroup; + } else { + groups = host->GetGroups(); + getGroup = &::GetHostGroup; + } + + checkSums->Set("groups_checksum", CalculateCheckSumGroups(groups)); + + Array::Ptr groupChecksums = new Array(); + + ObjectLock groupsLock (groups); + ObjectLock groupChecksumsLock (groupChecksums); + + for (auto group : groups) { + groupChecksums->Add(GetIdentifier((*getGroup)(group.Get()))); + } + + checkSums->Set("group_checksums", groupChecksums); } else { Zone::Ptr zone = dynamic_pointer_cast(object); From 1ad9c80143dcfe86b4b63285c5a1c15eaa8c4518 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 22 Jun 2018 17:29:47 +0200 Subject: [PATCH 035/219] RedisWriter: sync all config objects atomically --- lib/redis/rediswriter-objects.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 698205715..c8941c519 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -93,14 +93,13 @@ void RedisWriter::UpdateAllConfigObjects(void) if (!ctype) continue; - if (ctype->GetObject(name)) - continue; - deleteQuery.push_back("icinga:config:" + type + ":" + name); deleteQuery.push_back("icinga:status:" + type + ":" + name); } } while (cursor != 0); + ExecuteQuery({ "MULTI" }); + if (deleteQuery.size() > 1) ExecuteQuery(deleteQuery); @@ -125,6 +124,8 @@ void RedisWriter::UpdateAllConfigObjects(void) ExecuteQuery({ "PUBLISH", "icinga:config:dump", typeName }); } + ExecuteQuery({ "EXEC" }); + Log(LogInformation, "RedisWriter") << "Initial config/status dump finished in " << Utility::GetTime() - startTime << " seconds."; } From e7411ea158e7d6263609a04201f29d6b6f3f24d9 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 22 Jun 2018 17:44:27 +0200 Subject: [PATCH 036/219] RedisWriter#UpdateAllConfigObjects(): use one transaction per config object type --- lib/redis/rediswriter-objects.cpp | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index c8941c519..2eb1bcb55 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -30,6 +30,7 @@ #include "base/initialize.hpp" #include "base/convert.hpp" #include "base/array.hpp" +#include #include using namespace icinga; @@ -52,7 +53,7 @@ void RedisWriter::UpdateAllConfigObjects(void) double startTime = Utility::GetTime(); - std::vector deleteQuery({ "DEL" }); + std::map> deleteQueries; long long cursor = 0; const String keyPrefix = "icinga:config:"; @@ -93,22 +94,29 @@ void RedisWriter::UpdateAllConfigObjects(void) if (!ctype) continue; + auto& deleteQuery = deleteQueries[ptype.get()]; + + if (deleteQuery.empty()) + deleteQuery.emplace_back("DEL"); + deleteQuery.push_back("icinga:config:" + type + ":" + name); deleteQuery.push_back("icinga:status:" + type + ":" + name); } } while (cursor != 0); - ExecuteQuery({ "MULTI" }); - - if (deleteQuery.size() > 1) - ExecuteQuery(deleteQuery); - for (const Type::Ptr& type : Type::GetAllTypes()) { ConfigType *ctype = dynamic_cast(type.get()); if (!ctype) continue; + ExecuteQuery({ "MULTI" }); + + auto& deleteQuery = deleteQueries[type.get()]; + + if (deleteQuery.size() > 1) + ExecuteQuery(deleteQuery); + String typeName = type->GetName().ToLower(); /* replace into aka delete insert is faster than a full diff */ @@ -122,9 +130,9 @@ void RedisWriter::UpdateAllConfigObjects(void) /* publish config type dump finished */ ExecuteQuery({ "PUBLISH", "icinga:config:dump", typeName }); - } - ExecuteQuery({ "EXEC" }); + ExecuteQuery({ "EXEC" }); + } Log(LogInformation, "RedisWriter") << "Initial config/status dump finished in " << Utility::GetTime() - startTime << " seconds."; From d15769fa34e69254c240a1dfc076977c4975f1fd Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 22 Jun 2018 17:57:36 +0200 Subject: [PATCH 037/219] RedisWriter: fix disappeared config objects not being deleted --- lib/redis/rediswriter-objects.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 2eb1bcb55..e826f4dad 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -58,6 +58,11 @@ void RedisWriter::UpdateAllConfigObjects(void) const String keyPrefix = "icinga:config:"; + std::map lcTypes; + for (const Type::Ptr& type : Type::GetAllTypes()) { + lcTypes.emplace(type->GetName().ToLower(), type->GetName()); + } + do { std::shared_ptr reply = ExecuteQuery({ "SCAN", Convert::ToString(cursor), "MATCH", keyPrefix + "*", "COUNT", "1000" }); @@ -84,16 +89,12 @@ void RedisWriter::UpdateAllConfigObjects(void) String type = namePair.SubStr(0, pos); String name = namePair.SubStr(pos + 1); - Type::Ptr ptype = Type::GetByName(type); + auto actualTypeName = lcTypes.find(type); - if (!ptype) - continue; - - ConfigType *ctype = dynamic_cast(ptype.get()); - - if (!ctype) + if (actualTypeName == lcTypes.end()) continue; + Type::Ptr ptype = Type::GetByName(actualTypeName->second); auto& deleteQuery = deleteQueries[ptype.get()]; if (deleteQuery.empty()) From 550e385b9e9ad6afa448fa04203923f2a3be927f Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 25 Jun 2018 09:58:08 +0200 Subject: [PATCH 038/219] RedisWriter: fix checksums of disappeared config objects not being deleted --- lib/redis/rediswriter-objects.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index e826f4dad..be06bbba2 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -97,8 +97,10 @@ void RedisWriter::UpdateAllConfigObjects(void) Type::Ptr ptype = Type::GetByName(actualTypeName->second); auto& deleteQuery = deleteQueries[ptype.get()]; - if (deleteQuery.empty()) + if (deleteQuery.empty()) { deleteQuery.emplace_back("DEL"); + deleteQuery.emplace_back("icinga:config:checksum:" + type); + } deleteQuery.push_back("icinga:config:" + type + ":" + name); deleteQuery.push_back("icinga:status:" + type + ":" + name); From 50d83e850f02f5072dff4bf2174f8335a23a5951 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 25 Jun 2018 17:13:13 +0200 Subject: [PATCH 039/219] RedisWriter: dump endpoints' zones' identifiers (zone_checksum) --- lib/redis/rediswriter-objects.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index be06bbba2..4be06c5e6 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -232,6 +232,13 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran } checkSums->Set("endpoints_checksum", CalculateCheckSumGroups(endpoints)); + } else { + Endpoint::Ptr endpoint = dynamic_pointer_cast(object); + + if (endpoint) { + ConfigObject::Ptr zone = endpoint->GetZone(); + checkSums->Set("zone_checksum", GetIdentifier(zone)); + } } } From c5e913dfd45555caec74d35bc9e6eb88e52ab775 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Thu, 28 Jun 2018 17:38:32 +0200 Subject: [PATCH 040/219] Fix error handling for GetSubscriptionTypes() --- lib/redis/rediswriter.cpp | 16 ++++++++++++---- lib/redis/rediswriter.hpp | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index aaa4b3629..a72b4764a 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -186,11 +186,12 @@ void RedisWriter::UpdateSubscriptions() RedisSubscriptionInfo rsi; String key = keysReply->element[i]->str; - if (!RedisWriter::GetSubscriptionTypes(key, rsi)) + if (!RedisWriter::GetSubscriptionTypes(key, rsi)) { Log(LogInformation, "RedisWriter") - << "Subscription \"" << key<< "\" has no types listed."; - else + << "Subscription \"" << key << "\" has no types listed."; + } else { m_Subscriptions[key.SubStr(keyPrefix.GetLength())] = rsi; + } } } while (cursor != 0); @@ -198,12 +199,15 @@ void RedisWriter::UpdateSubscriptions() << "Current Redis event subscriptions: " << m_Subscriptions.size(); } -int RedisWriter::GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi) +bool RedisWriter::GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi) { try { std::shared_ptr redisReply = ExecuteQuery({ "SMEMBERS", key }); VERIFY(redisReply->type == REDIS_REPLY_ARRAY); + if (redisReply->elements == 0) + return false; + for (size_t j = 0; j < redisReply->elements; j++) { rsi.EventTypes.insert(redisReply->element[j]->str); } @@ -214,7 +218,11 @@ int RedisWriter::GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi) } catch (const std::exception& ex) { Log(LogWarning, "RedisWriter") << "Invalid Redis subscriber info for subscriber '" << key << "': " << DiagnosticInformation(ex); + + return false; } + + return true; } void RedisWriter::PublishStatsTimerHandler(void) diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index cc11a10e1..0856ca9f9 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -60,7 +60,7 @@ private: void UpdateSubscriptionsTimerHandler(); void UpdateSubscriptions(); - int GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi); + bool GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi); void PublishStatsTimerHandler(); void PublishStats(); From 85f621cb902810f49e25f9b34da0c26d0076d541 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Mon, 2 Jul 2018 09:37:23 +0200 Subject: [PATCH 041/219] Add zone checksum for redis to checkables --- lib/redis/rediswriter-objects.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 4be06c5e6..0b93bcd34 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -216,6 +216,8 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran } checkSums->Set("group_checksums", groupChecksums); + + checkSums->Set("zone_checksum", GetIdentifier(checkable->GetZone())); } else { Zone::Ptr zone = dynamic_pointer_cast(object); From a4b0f9125e4929ff5ef66e6aed162014dc2c62b5 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Tue, 3 Jul 2018 13:30:10 +0200 Subject: [PATCH 042/219] Only sync zone_checksum when object in zone --- lib/redis/rediswriter-objects.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 0b93bcd34..570a0efd3 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -217,7 +217,8 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran checkSums->Set("group_checksums", groupChecksums); - checkSums->Set("zone_checksum", GetIdentifier(checkable->GetZone())); + if (checkable->GetZone()) + checkSums->Set("zone_checksum", GetIdentifier(checkable->GetZone())); } else { Zone::Ptr zone = dynamic_pointer_cast(object); From e4355f27c908c72092af40d6a9d373ee21f1e2ac Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Tue, 3 Jul 2018 14:57:09 +0200 Subject: [PATCH 043/219] Send zone_checksum globally, if 'zone' attribute is set --- lib/redis/rediswriter-objects.cpp | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 570a0efd3..7923ff5ec 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -179,10 +179,17 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran std::set propertiesBlacklist ({"name", "__name", "package", "source_location", "templates"}); Dictionary::Ptr checkSums = new Dictionary(); + checkSums->Set("name_checksum", CalculateCheckSumString(object->GetShortName())); checkSums->Set("environment_checksum", CalculateCheckSumString(GetEnvironment())); - // TODO: move this elsewhere + /* 'zone' is available for all config objects, therefore calculate the checksum. */ + Zone::Ptr zone = static_pointer_cast(object->GetZone()); + + if (zone) + checkSums->Set("zone_checksum", GetIdentifier(zone)); + + /* Calculate checkable checksums. */ Checkable::Ptr checkable = dynamic_pointer_cast(object); if (checkable) { @@ -216,9 +223,6 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran } checkSums->Set("group_checksums", groupChecksums); - - if (checkable->GetZone()) - checkSums->Set("zone_checksum", GetIdentifier(checkable->GetZone())); } else { Zone::Ptr zone = dynamic_pointer_cast(object); @@ -235,17 +239,11 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran } checkSums->Set("endpoints_checksum", CalculateCheckSumGroups(endpoints)); - } else { - Endpoint::Ptr endpoint = dynamic_pointer_cast(object); - - if (endpoint) { - ConfigObject::Ptr zone = endpoint->GetZone(); - checkSums->Set("zone_checksum", GetIdentifier(zone)); - } } + /* zone_checksum for endpoints already is calculated above. */ } - //TODO: Move this somewhere else. + /* Custom var checksums. */ CustomVarObject::Ptr customVarObject = dynamic_pointer_cast(object); if (customVarObject) { From 413e35180595a9ff217b5a63ee4f200b7d984577 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 3 Jul 2018 15:52:35 +0200 Subject: [PATCH 044/219] RedisWriter: dump custom vars verbosely --- lib/redis/rediswriter-objects.cpp | 10 ++ lib/redis/rediswriter-utility.cpp | 168 ++++++++++++++++++++++++++++++ lib/redis/rediswriter.hpp | 1 + 3 files changed, 179 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 7923ff5ec..cbbee47b3 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -250,6 +250,16 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran propertiesBlacklist.emplace("vars"); checkSums->Set("vars_checksum", CalculateCheckSumVars(customVarObject)); + + auto vars (SerializeVars(customVarObject)); + if (vars) { + auto varsJson (JsonEncode(vars)); + + Log(LogDebug, "RedisWriter") + << "HSET icinga:config:customvars:" << typeName << " " << objectKey << " " << varsJson; + + ExecuteQuery({ "HSET", "icinga:config:customvars:" + typeName, objectKey, varsJson }); + } } checkSums->Set("metadata_checksum", CalculateCheckSumMetadata(object)); diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index ad3a8b5ba..6553da679 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -27,6 +27,9 @@ #include "base/objectlock.hpp" #include "base/array.hpp" #include "base/scriptglobal.hpp" +#include +#include +#include using namespace icinga; @@ -89,6 +92,171 @@ String RedisWriter::CalculateCheckSumVars(const CustomVarObject::Ptr& object) return HashValue(vars); } +/** + * Collect the leaves of haystack in needles + * + * haystack = { + * "disks": { + * "disk": {}, + * "disk /": { + * "disk_partitions": "/" + * } + * } + * } + * + * path = [] + * + * needles = { + * "disks": [ + * [ + * ["disks", "disk /", "disk_partitions"], + * "/" + * ] + * ] + * } + * + * @param haystack A config object's custom vars as returned by {@link CustomVarObject#GetVars()} + * @param path Used for buffering only, shall be empty + * @param needles The result, a mapping from the top-level custom var key to a list of leaves with full path and value + */ +static void CollectScalarVars(Value haystack, std::vector& path, std::map>>& needles) +{ + switch (haystack.GetType()) { + case ValueObject: + { + const Object::Ptr& obj = haystack.Get(); + + Dictionary::Ptr dict = dynamic_pointer_cast(obj); + if (dict) { + for (auto& kv : dict) { + path.emplace_back(kv.first); + CollectScalarVars(kv.second, path, needles); + path.pop_back(); + } + break; + } + + Array::Ptr arr = dynamic_pointer_cast(obj); + if (arr) { + double index = 0.0; + for (auto& v : arr) { + path.emplace_back(index); + CollectScalarVars(v, path, needles); + path.pop_back(); + + index += 1.0; + } + break; + } + } + + haystack = Empty; + + case ValueString: + case ValueNumber: + case ValueBoolean: + case ValueEmpty: + needles[path[0].Get()].emplace_back(Array::FromVector(path), haystack); + break; + + default: + VERIFY(!"Invalid variant type."); + } +} + +/** + * Prepare object's custom vars for being written to Redis + * + * object.vars = { + * "disks": { + * "disk": {}, + * "disk /": { + * "disk_partitions": "/" + * } + * } + * } + * + * return { + * SHA1(PackObject([ + * Environment, + * "disks", + * { + * "disk": {}, + * "disk /": { + * "disk_partitions": "/" + * } + * } + * ])): { + * "env_checksum": SHA1(Environment), + * "name_checksum": SHA1("disks"), + * "name": "disks", + * "value": { + * "disk": {}, + * "disk /": { + * "disk_partitions": "/" + * } + * }, + * "flat": { + * SHA1(PackObject(["disks", "disk /", "disk_partitions"])): { + * "name": ["disks", "disk /", "disk_partitions"], + * "value": "/" + * } + * } + * } + * } + * + * @param object Config object with custom vars + * + * @return JSON-like data structure for Redis + */ +Dictionary::Ptr RedisWriter::SerializeVars(const CustomVarObject::Ptr& object) +{ + Dictionary::Ptr vars = object->GetVars(); + + if (!vars) + return nullptr; + + std::map>> scalarVars; + + { + std::vector pathBuf; + CollectScalarVars(vars, pathBuf, scalarVars); + } + + Dictionary::Ptr res = new Dictionary(); + auto env (GetEnvironment()); + auto envChecksum (SHA1(env)); + + for (auto& kv : vars) { + Dictionary::Ptr flatVars = new Dictionary(); + + { + auto it (scalarVars.find(kv.first)); + if (it != scalarVars.end()) { + for (auto& scalarVar : it->second) { + flatVars->Set(SHA1(PackObject(scalarVar.first)), (Dictionary::Ptr)new Dictionary({ + {"name", scalarVar.first}, + {"value", scalarVar.second} + })); + } + } + } + + res->Set( + SHA1(PackObject((Array::Ptr)new Array({env, kv.first, kv.second}))), + (Dictionary::Ptr)new Dictionary({ + {"env_checksum", envChecksum}, + {"name_checksum", SHA1(kv.first)}, + {"name", kv.first}, + {"value", kv.second}, + {"flat", flatVars} + }) + ); + } + + return res; +} + static const std::set propertiesBlacklistEmpty; String RedisWriter::HashValue(const Value& value) diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 0856ca9f9..a8720e63e 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -84,6 +84,7 @@ private: static String CalculateCheckSumProperties(const ConfigObject::Ptr& object, const std::set& propertiesBlacklist); static String CalculateCheckSumMetadata(const ConfigObject::Ptr& object); static String CalculateCheckSumVars(const CustomVarObject::Ptr& object); + static Dictionary::Ptr SerializeVars(const CustomVarObject::Ptr& object); static String HashValue(const Value& value); static String HashValue(const Value& value, const std::set& propertiesBlacklist, bool propertiesWhitelist = false); From 3ce8f5599ecd922214d64fa2f0316f6b1223aac2 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Wed, 4 Jul 2018 16:44:44 +0200 Subject: [PATCH 045/219] Fix missing object locks --- lib/redis/rediswriter-utility.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 6553da679..adbc4402a 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -127,18 +127,26 @@ static void CollectScalarVars(Value haystack, std::vector& path, std::map const Object::Ptr& obj = haystack.Get(); Dictionary::Ptr dict = dynamic_pointer_cast(obj); + if (dict) { + ObjectLock olock(dict); + for (auto& kv : dict) { path.emplace_back(kv.first); CollectScalarVars(kv.second, path, needles); path.pop_back(); } + break; } Array::Ptr arr = dynamic_pointer_cast(obj); + if (arr) { double index = 0.0; + + ObjectLock xlock(arr); + for (auto& v : arr) { path.emplace_back(index); CollectScalarVars(v, path, needles); @@ -146,6 +154,7 @@ static void CollectScalarVars(Value haystack, std::vector& path, std::map index += 1.0; } + break; } } @@ -227,6 +236,8 @@ Dictionary::Ptr RedisWriter::SerializeVars(const CustomVarObject::Ptr& object) auto env (GetEnvironment()); auto envChecksum (SHA1(env)); + ObjectLock olock(vars); + for (auto& kv : vars) { Dictionary::Ptr flatVars = new Dictionary(); From e31679b2de6355032cb7b4a60a597f487cdb13aa Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Tue, 10 Jul 2018 11:09:37 +0200 Subject: [PATCH 046/219] Update Redis prefix namespaces and move their declaration into the class icinga:config:object:: icinga:config:checksum: icinga:config:customvar: icinga:status:object:: This avoids multiple definitions all over the code. Maybe we want to make the schema configurable at some point. --- lib/redis/rediswriter-objects.cpp | 33 +++++++++++++++---------------- lib/redis/rediswriter.cpp | 5 +++++ lib/redis/rediswriter.hpp | 6 ++++++ 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index cbbee47b3..38caedd78 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -56,15 +56,13 @@ void RedisWriter::UpdateAllConfigObjects(void) std::map> deleteQueries; long long cursor = 0; - const String keyPrefix = "icinga:config:"; - std::map lcTypes; for (const Type::Ptr& type : Type::GetAllTypes()) { lcTypes.emplace(type->GetName().ToLower(), type->GetName()); } do { - std::shared_ptr reply = ExecuteQuery({ "SCAN", Convert::ToString(cursor), "MATCH", keyPrefix + "*", "COUNT", "1000" }); + std::shared_ptr reply = ExecuteQuery({ "SCAN", Convert::ToString(cursor), "MATCH", m_PrefixConfigObject + "*", "COUNT", "1000" }); VERIFY(reply->type == REDIS_REPLY_ARRAY); VERIFY(reply->elements % 2 == 0); @@ -79,7 +77,7 @@ void RedisWriter::UpdateAllConfigObjects(void) VERIFY(keyReply->type == REDIS_REPLY_STRING); String key = keyReply->str; - String namePair = key.SubStr(keyPrefix.GetLength()); + String namePair = key.SubStr(m_PrefixConfigObject.GetLength()); String::SizeType pos = namePair.FindFirstOf(":"); @@ -99,11 +97,11 @@ void RedisWriter::UpdateAllConfigObjects(void) if (deleteQuery.empty()) { deleteQuery.emplace_back("DEL"); - deleteQuery.emplace_back("icinga:config:checksum:" + type); + deleteQuery.emplace_back(m_PrefixConfigCheckSum + type); } - deleteQuery.push_back("icinga:config:" + type + ":" + name); - deleteQuery.push_back("icinga:status:" + type + ":" + name); + deleteQuery.push_back(m_PrefixConfigObject + type + ":" + name); + deleteQuery.push_back(m_PrefixStatusObject + type + ":" + name); } } while (cursor != 0); @@ -123,7 +121,7 @@ void RedisWriter::UpdateAllConfigObjects(void) String typeName = type->GetName().ToLower(); /* replace into aka delete insert is faster than a full diff */ - ExecuteQuery({ "DEL", "icinga:config:" + typeName, "icinga:config:" + typeName + ":checksum", "icinga:status:" + typeName }); + ExecuteQuery({ "DEL", m_PrefixConfigObject + typeName, m_PrefixConfigCheckSum + typeName, m_PrefixStatusObject + typeName }); /* fetch all objects and dump them */ for (const ConfigObject::Ptr& object : ctype->GetObjects()) { @@ -167,8 +165,8 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran if (useTransaction) ExecuteQuery({ "MULTI" }); - /* Send all object attributes to redis, no extra checksums involved here. */ - UpdateObjectAttrs("icinga:config:", object, FAConfig); + /* Send all object attributes to Redis, no extra checksums involved here. */ + UpdateObjectAttrs(m_PrefixConfigObject, object, FAConfig); /* Calculate object specific checksums and store them in a different namespace. */ Type::Ptr type = object->GetReflectionType(); @@ -252,13 +250,14 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran checkSums->Set("vars_checksum", CalculateCheckSumVars(customVarObject)); auto vars (SerializeVars(customVarObject)); + if (vars) { auto varsJson (JsonEncode(vars)); Log(LogDebug, "RedisWriter") - << "HSET icinga:config:customvars:" << typeName << " " << objectKey << " " << varsJson; + << "HSET " << m_PrefixConfigCustomVar + typeName << " " << objectKey << " " << varsJson; - ExecuteQuery({ "HSET", "icinga:config:customvars:" + typeName, objectKey, varsJson }); + ExecuteQuery({ "HSET", m_PrefixConfigCustomVar + typeName, objectKey, varsJson }); } } @@ -268,9 +267,9 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran String checkSumsBody = JsonEncode(checkSums); Log(LogDebug, "RedisWriter") - << "HSET icinga:config:checksum:" << typeName << " " << objectKey << " " << checkSumsBody; + << "HSET " << m_PrefixConfigCheckSum + typeName << " " << objectKey << " " << checkSumsBody; - ExecuteQuery({ "HSET", "icinga:config:checksum:" + typeName, objectKey, checkSumsBody }); + ExecuteQuery({ "HSET", m_PrefixConfigCheckSum + typeName, objectKey, checkSumsBody }); /* Send an update event to subscribers. */ @@ -294,8 +293,8 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) String objectKey = GetIdentifier(object); ExecuteQueries({ - { "DEL", "icinga:config:" + typeName + ":" + objectKey }, - { "DEL", "icinga:status:" + typeName + ":" + objectKey }, + { "DEL", m_PrefixConfigObject + typeName + ":" + objectKey }, + { "DEL", m_PrefixStatusObject + typeName + ":" + objectKey }, { "PUBLISH", "icinga:config:delete", typeName + ":" + objectKey } }); @@ -312,7 +311,7 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, bool useTran if (useTransaction) ExecuteQuery({ "MULTI" }); - UpdateObjectAttrs("icinga:status:", object, FAState); + UpdateObjectAttrs(m_PrefixStatusObject, object, FAState); if (useTransaction) ExecuteQuery({ "EXEC" }); diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index a72b4764a..8c4c74149 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -34,6 +34,11 @@ RedisWriter::RedisWriter() : m_Context(NULL) { m_WorkQueue.SetName("RedisWriter"); + + m_PrefixConfigObject = "icinga:config:object:"; + m_PrefixConfigCheckSum = "icinga:config:checksum:"; + m_PrefixConfigCustomVar = "icinga:config:customvar:"; + m_PrefixStatusObject = "icinga:status:object:"; } /** diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index a8720e63e..b3cb3192b 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -105,6 +105,12 @@ private: WorkQueue m_WorkQueue; redisContext *m_Context; std::map m_Subscriptions; + + String m_PrefixConfigObject; + String m_PrefixConfigCheckSum; + String m_PrefixConfigCustomVar; + String m_PrefixStatusObject; + bool m_ConfigDumpInProgress; }; From 48f1ce215bc08fb081700dcd955faf58bbca86de Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Tue, 10 Jul 2018 11:53:05 +0200 Subject: [PATCH 047/219] Drop obsolete delete query on config dump --- lib/redis/rediswriter-objects.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 38caedd78..4bde3c60f 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -113,6 +113,7 @@ void RedisWriter::UpdateAllConfigObjects(void) ExecuteQuery({ "MULTI" }); + /* Delete obsolete object keys first. */ auto& deleteQuery = deleteQueries[type.get()]; if (deleteQuery.size() > 1) @@ -120,9 +121,6 @@ void RedisWriter::UpdateAllConfigObjects(void) String typeName = type->GetName().ToLower(); - /* replace into aka delete insert is faster than a full diff */ - ExecuteQuery({ "DEL", m_PrefixConfigObject + typeName, m_PrefixConfigCheckSum + typeName, m_PrefixStatusObject + typeName }); - /* fetch all objects and dump them */ for (const ConfigObject::Ptr& object : ctype->GetObjects()) { SendConfigUpdate(object, false); From 4fb73a44bdf57f79542e9eb7f4fc0d31319446c0 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Tue, 10 Jul 2018 15:14:28 +0200 Subject: [PATCH 048/219] Add 'host_checksum' for services --- lib/redis/rediswriter-objects.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 4bde3c60f..62694f3c1 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -189,6 +189,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran Checkable::Ptr checkable = dynamic_pointer_cast(object); if (checkable) { + /* groups_checksum, group_checksums */ propertiesBlacklist.emplace("groups"); Host::Ptr host; @@ -202,6 +203,9 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran if (service) { groups = service->GetGroups(); getGroup = &::GetServiceGroup; + + /* Calculate the host_checksum */ + checkSums->Set("host_checksum", GetIdentifier(service->GetHost())); } else { groups = host->GetGroups(); getGroup = &::GetHostGroup; From bb2e7854fd91669fe733aa6ce48603da34276735 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Tue, 10 Jul 2018 15:15:57 +0200 Subject: [PATCH 049/219] Add command_endpoint, event/check_command, action/notes_url, icon_image checksums for host/service objects --- lib/redis/rediswriter-objects.cpp | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 62694f3c1..41fefab87 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -23,6 +23,10 @@ #include "icinga/service.hpp" #include "icinga/hostgroup.hpp" #include "icinga/servicegroup.hpp" +#include "icinga/checkcommand.hpp" +#include "icinga/eventcommand.hpp" +#include "icinga/notificationcommand.hpp" +#include "remote/zone.hpp" #include "base/json.hpp" #include "base/logger.hpp" #include "base/serializer.hpp" @@ -223,6 +227,32 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran } checkSums->Set("group_checksums", groupChecksums); + + /* command_endpoint_checksum / node_checksum */ + Endpoint::Ptr commandEndpoint = checkable->GetCommandEndpoint(); + + if (commandEndpoint) + checkSums->Set("command_endpoint_checksum", GetIdentifier(commandEndpoint)); + + /* *_command_checksum */ + checkSums->Set("check_command_checksum", GetIdentifier(checkable->GetCheckCommand())); + + EventCommand::Ptr eventCommand = checkable->GetEventCommand(); + + if (eventCommand) + checkSums->Set("event_command_checksum", GetIdentifier(eventCommand)); + + /* *_url_checksum, icon_image_checksum */ + String actionUrl = checkable->GetActionUrl(); + String notesUrl = checkable->GetNotesUrl(); + String iconImage = checkable->GetIconImage(); + + if (!actionUrl.IsEmpty()) + checkSums->Set("action_url_checksum", CalculateCheckSumString(actionUrl)); + if (!notesUrl.IsEmpty()) + checkSums->Set("notes_url_checksum", CalculateCheckSumString(notesUrl)); + if (!iconImage.IsEmpty()) + checkSums->Set("icon_image_checksum", CalculateCheckSumString(iconImage)); } else { Zone::Ptr zone = dynamic_pointer_cast(object); From 876d98a13f97941d334f08367e11b0cbaf431d9b Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Tue, 10 Jul 2018 15:16:04 +0200 Subject: [PATCH 050/219] Add 'parent_checksum' for zone objects --- lib/redis/rediswriter-objects.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 41fefab87..e5a3096b6 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -269,6 +269,11 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran } checkSums->Set("endpoints_checksum", CalculateCheckSumGroups(endpoints)); + + Zone::Ptr parentZone = zone->GetParent(); + + if (parentZone) + checkSums->Set("parent_checksum", GetIdentifier(parentZone)); } /* zone_checksum for endpoints already is calculated above. */ } From d09a435fe34412c728dbb45893af534522eb5e61 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Thu, 12 Jul 2018 15:05:10 +0200 Subject: [PATCH 051/219] Export usergroup(s)_checksum(s) Warning: ugly code --- lib/redis/rediswriter-objects.cpp | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index e5a3096b6..b5d68fa60 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -23,6 +23,7 @@ #include "icinga/service.hpp" #include "icinga/hostgroup.hpp" #include "icinga/servicegroup.hpp" +#include "icinga/usergroup.hpp" #include "icinga/checkcommand.hpp" #include "icinga/eventcommand.hpp" #include "icinga/notificationcommand.hpp" @@ -151,6 +152,11 @@ static ConfigObject::Ptr GetServiceGroup(const String& name) return ConfigObject::GetObject(name); } +static ConfigObject::Ptr GetUserGroup(const String& name) +{ + return ConfigObject::GetObject(name); +} + void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTransaction, bool runtimeUpdate) { AssertOnWorkQueue(); @@ -189,6 +195,31 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran if (zone) checkSums->Set("zone_checksum", GetIdentifier(zone)); + User::Ptr user = dynamic_pointer_cast(object); + + if (user) { + propertiesBlacklist.emplace("groups"); + + Array::Ptr groups; + ConfigObject::Ptr (*getGroup)(const String& name); + + groups = user->GetGroups(); + getGroup = &::GetUserGroup; + + checkSums->Set("groups_checksum", CalculateCheckSumGroups(groups)); + + Array::Ptr groupChecksums = new Array(); + + ObjectLock groupsLock (groups); + ObjectLock groupChecksumsLock (groupChecksums); + + for (auto group : groups) { + groupChecksums->Add(GetIdentifier((*getGroup)(group.Get()))); + } + + checkSums->Set("group_checksums", groupChecksums); + } + /* Calculate checkable checksums. */ Checkable::Ptr checkable = dynamic_pointer_cast(object); From d65fb529ebab91b974c85a10fbb4b3508e553be5 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Tue, 10 Jul 2018 16:35:32 +0200 Subject: [PATCH 052/219] Simplify host_checksum --- lib/redis/rediswriter-objects.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index b5d68fa60..ed4549d98 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -240,7 +240,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran getGroup = &::GetServiceGroup; /* Calculate the host_checksum */ - checkSums->Set("host_checksum", GetIdentifier(service->GetHost())); + checkSums->Set("host_checksum", GetIdentifier(host)); } else { groups = host->GetGroups(); getGroup = &::GetHostGroup; From 1b81b85582b8492ccbac25f05628433da5e55e2e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 19 Jul 2018 18:34:40 +0200 Subject: [PATCH 053/219] RedisWriter: dump zone_checksum for endpoints refs #11 --- lib/redis/rediswriter-objects.cpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index ed4549d98..3f0cdf001 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -189,11 +189,21 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran checkSums->Set("name_checksum", CalculateCheckSumString(object->GetShortName())); checkSums->Set("environment_checksum", CalculateCheckSumString(GetEnvironment())); - /* 'zone' is available for all config objects, therefore calculate the checksum. */ - Zone::Ptr zone = static_pointer_cast(object->GetZone()); + auto endpoint (dynamic_pointer_cast(object)); - if (zone) - checkSums->Set("zone_checksum", GetIdentifier(zone)); + if (endpoint) { + auto endpointZone (endpoint->GetZone()); + + if (endpointZone) { + checkSums->Set("zone_checksum", GetIdentifier(endpointZone)); + } + } else { + /* 'zone' is available for all config objects, therefore calculate the checksum. */ + auto zone (dynamic_pointer_cast(object->GetZone())); + + if (zone) + checkSums->Set("zone_checksum", GetIdentifier(zone)); + } User::Ptr user = dynamic_pointer_cast(object); From a908f847503961cd94778fcbae32c97a7b18fe00 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 23 Jul 2018 17:52:29 +0200 Subject: [PATCH 054/219] Revert unneccessary dynamic_pointer_cast --- lib/redis/rediswriter-objects.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 3f0cdf001..68f8ab360 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -199,7 +199,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran } } else { /* 'zone' is available for all config objects, therefore calculate the checksum. */ - auto zone (dynamic_pointer_cast(object->GetZone())); + auto zone (static_pointer_cast(object->GetZone())); if (zone) checkSums->Set("zone_checksum", GetIdentifier(zone)); From b04b1d954ef6e84ae119bb1704c4460056aec94e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 26 Jul 2018 13:46:36 +0200 Subject: [PATCH 055/219] RedisWriter: compute checksums of command arguments and env vars refs #14 --- lib/redis/rediswriter-objects.cpp | 38 ++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 68f8ab360..80289488a 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -18,6 +18,7 @@ ******************************************************************************/ #include "redis/rediswriter.hpp" +#include "icinga/command.hpp" #include "icinga/customvarobject.hpp" #include "icinga/host.hpp" #include "icinga/service.hpp" @@ -315,8 +316,43 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran if (parentZone) checkSums->Set("parent_checksum", GetIdentifier(parentZone)); + } else { + /* zone_checksum for endpoints already is calculated above. */ + + auto command (dynamic_pointer_cast(object)); + + if (command) { + Dictionary::Ptr arguments = command->GetArguments(); + Dictionary::Ptr argumentChecksums = new Dictionary; + + if (arguments) { + ObjectLock argumentsLock (arguments); + + for (auto& kv : arguments) { + argumentChecksums->Set(kv.first, HashValue(kv.second)); + } + } + + checkSums->Set("arguments_checksum", HashValue(arguments)); + checkSums->Set("argument_checksums", argumentChecksums); + propertiesBlacklist.emplace("arguments"); + + Dictionary::Ptr envvars = command->GetEnv(); + Dictionary::Ptr envvarChecksums = new Dictionary; + + if (envvars) { + ObjectLock argumentsLock (envvars); + + for (auto& kv : envvars) { + envvarChecksums->Set(kv.first, HashValue(kv.second)); + } + } + + checkSums->Set("envvars_checksum", HashValue(envvars)); + checkSums->Set("envvar_checksums", envvarChecksums); + propertiesBlacklist.emplace("env"); + } } - /* zone_checksum for endpoints already is calculated above. */ } /* Custom var checksums. */ From 3e0a2f59e2fa2dc62585519e58724bfbe5d53afb Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 25 Jul 2018 10:36:33 +0200 Subject: [PATCH 056/219] RedisWriter: make config object identifiers type-aware SHA1(PackObject([Environment, Type, __name])) --- lib/redis/rediswriter-utility.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index adbc4402a..750cbe249 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -51,7 +51,7 @@ String RedisWriter::GetEnvironment() String RedisWriter::GetIdentifier(const ConfigObject::Ptr& object) { - return HashValue((Array::Ptr)new Array({GetEnvironment(), object->GetName()})); + return HashValue((Array::Ptr)new Array({GetEnvironment(), object->GetReflectionType()->GetName(), object->GetName()})); } String RedisWriter::CalculateCheckSumString(const String& str) From c70316a8359de41c1ec8c2404b0f0efc0155cf14 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Thu, 26 Jul 2018 15:55:50 +0200 Subject: [PATCH 057/219] Sync ranges_checksum for timeperiods Signed-off-by: Michael Friedrich --- lib/redis/rediswriter-objects.cpp | 61 +++++++++++++++++++++++++++++-- lib/redis/rediswriter-utility.cpp | 8 ++-- lib/redis/rediswriter.hpp | 2 +- 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 80289488a..4ca17e6ad 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -28,6 +28,7 @@ #include "icinga/checkcommand.hpp" #include "icinga/eventcommand.hpp" #include "icinga/notificationcommand.hpp" +#include "icinga/timeperiod.hpp" #include "remote/zone.hpp" #include "base/json.hpp" #include "base/logger.hpp" @@ -158,6 +159,11 @@ static ConfigObject::Ptr GetUserGroup(const String& name) return ConfigObject::GetObject(name); } +static ConfigObject::Ptr GetClude(const String& name) +{ + return ConfigObject::GetObject(name); +} + void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTransaction, bool runtimeUpdate) { AssertOnWorkQueue(); @@ -217,7 +223,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran groups = user->GetGroups(); getGroup = &::GetUserGroup; - checkSums->Set("groups_checksum", CalculateCheckSumGroups(groups)); + checkSums->Set("groups_checksum", CalculateCheckSumArray(groups)); Array::Ptr groupChecksums = new Array(); @@ -257,7 +263,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran getGroup = &::GetHostGroup; } - checkSums->Set("groups_checksum", CalculateCheckSumGroups(groups)); + checkSums->Set("groups_checksum", CalculateCheckSumArray(groups)); Array::Ptr groupChecksums = new Array(); @@ -310,7 +316,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran endpoints->Set(i++, endpointObject->GetName()); } - checkSums->Set("endpoints_checksum", CalculateCheckSumGroups(endpoints)); + checkSums->Set("endpoints_checksum", CalculateCheckSumArray(endpoints)); Zone::Ptr parentZone = zone->GetParent(); @@ -351,6 +357,55 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran checkSums->Set("envvars_checksum", HashValue(envvars)); checkSums->Set("envvar_checksums", envvarChecksums); propertiesBlacklist.emplace("env"); + } else { + auto timeperiod (dynamic_pointer_cast(object)); + + if (timeperiod) { + Dictionary::Ptr ranges = timeperiod->GetRanges(); + + checkSums->Set("ranges_checksum", HashValue(ranges)); + propertiesBlacklist.emplace("ranges"); + + // Compute checksums for Includes (like groups) + Array::Ptr includes; + ConfigObject::Ptr (*getInclude)(const String& name); + + includes = timeperiod->GetIncludes(); + getInclude = &::GetClude; + + checkSums->Set("includes_checksum", CalculateCheckSumArray(includes)); + + Array::Ptr includeChecksums = new Array(); + + ObjectLock includesLock (includes); + ObjectLock includeChecksumsLock (includeChecksums); + + for (auto include : includes) { + includeChecksums->Add(GetIdentifier((*getInclude)(include.Get()))); + } + + checkSums->Set("include_checksums", includeChecksums); + + // Compute checksums for Excludes (like groups) + Array::Ptr excludes; + ConfigObject::Ptr (*getExclude)(const String& name); + + excludes = timeperiod->GetExcludes(); + getExclude = &::GetClude; + + checkSums->Set("excludes_checksum", CalculateCheckSumArray(excludes)); + + Array::Ptr excludeChecksums = new Array(); + + ObjectLock excludesLock (excludes); + ObjectLock excludeChecksumsLock (excludeChecksums); + + for (auto exclude : excludes) { + excludeChecksums->Add(GetIdentifier((*getExclude)(exclude.Get()))); + } + + checkSums->Set("exclude_checksums", excludeChecksums); + } } } } diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 750cbe249..d6f2ab523 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -59,14 +59,14 @@ String RedisWriter::CalculateCheckSumString(const String& str) return SHA1(str); } -String RedisWriter::CalculateCheckSumGroups(const Array::Ptr& groups) +String RedisWriter::CalculateCheckSumArray(const Array::Ptr& arr) { /* Ensure that checksums happen in a defined order. */ - Array::Ptr tmpGroups = groups->ShallowClone(); + Array::Ptr tmpArr = arr->ShallowClone(); - tmpGroups->Sort(); + tmpArr->Sort(); - return SHA1(PackObject(tmpGroups)); + return SHA1(PackObject(tmpArr)); } String RedisWriter::CalculateCheckSumProperties(const ConfigObject::Ptr& object, const std::set& propertiesBlacklist) diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index b3cb3192b..601281d78 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -80,7 +80,7 @@ private: static String GetIdentifier(const ConfigObject::Ptr& object); static String GetEnvironment(); static String CalculateCheckSumString(const String& str); - static String CalculateCheckSumGroups(const Array::Ptr& groups); + static String CalculateCheckSumArray(const Array::Ptr& arr); static String CalculateCheckSumProperties(const ConfigObject::Ptr& object, const std::set& propertiesBlacklist); static String CalculateCheckSumMetadata(const ConfigObject::Ptr& object); static String CalculateCheckSumVars(const CustomVarObject::Ptr& object); From de99d68351b74431c23467182a81e06cb9eabeee Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Mon, 6 Aug 2018 10:55:24 +0200 Subject: [PATCH 058/219] Only use the type for commands in the object identifier checksum --- lib/redis/rediswriter-objects.cpp | 28 ++++++++++++++-------------- lib/redis/rediswriter-utility.cpp | 12 ++++++++++-- lib/redis/rediswriter.hpp | 2 +- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 4ca17e6ad..3d747b8f5 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -187,7 +187,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran Type::Ptr type = object->GetReflectionType(); String typeName = type->GetName().ToLower(); - String objectKey = GetIdentifier(object); + String objectKey = GetObjectIdentifier(object); std::set propertiesBlacklist ({"name", "__name", "package", "source_location", "templates"}); @@ -202,14 +202,14 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran auto endpointZone (endpoint->GetZone()); if (endpointZone) { - checkSums->Set("zone_checksum", GetIdentifier(endpointZone)); + checkSums->Set("zone_checksum", GetObjectIdentifier(endpointZone)); } } else { /* 'zone' is available for all config objects, therefore calculate the checksum. */ auto zone (static_pointer_cast(object->GetZone())); if (zone) - checkSums->Set("zone_checksum", GetIdentifier(zone)); + checkSums->Set("zone_checksum", GetObjectIdentifier(zone)); } User::Ptr user = dynamic_pointer_cast(object); @@ -231,7 +231,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran ObjectLock groupChecksumsLock (groupChecksums); for (auto group : groups) { - groupChecksums->Add(GetIdentifier((*getGroup)(group.Get()))); + groupChecksums->Add(GetObjectIdentifier((*getGroup)(group.Get()))); } checkSums->Set("group_checksums", groupChecksums); @@ -257,7 +257,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran getGroup = &::GetServiceGroup; /* Calculate the host_checksum */ - checkSums->Set("host_checksum", GetIdentifier(host)); + checkSums->Set("host_checksum", GetObjectIdentifier(host)); } else { groups = host->GetGroups(); getGroup = &::GetHostGroup; @@ -271,7 +271,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran ObjectLock groupChecksumsLock (groupChecksums); for (auto group : groups) { - groupChecksums->Add(GetIdentifier((*getGroup)(group.Get()))); + groupChecksums->Add(GetObjectIdentifier((*getGroup)(group.Get()))); } checkSums->Set("group_checksums", groupChecksums); @@ -280,15 +280,15 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran Endpoint::Ptr commandEndpoint = checkable->GetCommandEndpoint(); if (commandEndpoint) - checkSums->Set("command_endpoint_checksum", GetIdentifier(commandEndpoint)); + checkSums->Set("command_endpoint_checksum", GetObjectIdentifier(commandEndpoint)); /* *_command_checksum */ - checkSums->Set("check_command_checksum", GetIdentifier(checkable->GetCheckCommand())); + checkSums->Set("check_command_checksum", GetObjectIdentifier(checkable->GetCheckCommand())); EventCommand::Ptr eventCommand = checkable->GetEventCommand(); if (eventCommand) - checkSums->Set("event_command_checksum", GetIdentifier(eventCommand)); + checkSums->Set("event_command_checksum", GetObjectIdentifier(eventCommand)); /* *_url_checksum, icon_image_checksum */ String actionUrl = checkable->GetActionUrl(); @@ -321,7 +321,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran Zone::Ptr parentZone = zone->GetParent(); if (parentZone) - checkSums->Set("parent_checksum", GetIdentifier(parentZone)); + checkSums->Set("parent_checksum", GetObjectIdentifier(parentZone)); } else { /* zone_checksum for endpoints already is calculated above. */ @@ -381,7 +381,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran ObjectLock includeChecksumsLock (includeChecksums); for (auto include : includes) { - includeChecksums->Add(GetIdentifier((*getInclude)(include.Get()))); + includeChecksums->Add(GetObjectIdentifier((*getInclude)(include.Get()))); } checkSums->Set("include_checksums", includeChecksums); @@ -401,7 +401,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran ObjectLock excludeChecksumsLock (excludeChecksums); for (auto exclude : excludes) { - excludeChecksums->Add(GetIdentifier((*getExclude)(exclude.Get()))); + excludeChecksums->Add(GetObjectIdentifier((*getExclude)(exclude.Get()))); } checkSums->Set("exclude_checksums", excludeChecksums); @@ -459,7 +459,7 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) return; String typeName = object->GetReflectionType()->GetName().ToLower(); - String objectKey = GetIdentifier(object); + String objectKey = GetObjectIdentifier(object); ExecuteQueries({ { "DEL", m_PrefixConfigObject + typeName + ":" + objectKey }, @@ -570,7 +570,7 @@ void RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject: String typeName = type->GetName().ToLower(); /* Use the name checksum as unique key. */ - String objectKey = GetIdentifier(object); + String objectKey = GetObjectIdentifier(object); std::vector > queries; diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index d6f2ab523..4251a30a5 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -19,6 +19,9 @@ #include "redis/rediswriter.hpp" #include "icinga/customvarobject.hpp" +#include "icinga/checkcommand.hpp" +#include "icinga/notificationcommand.hpp" +#include "icinga/eventcommand.hpp" #include "base/object-packer.hpp" #include "base/logger.hpp" #include "base/serializer.hpp" @@ -49,9 +52,14 @@ String RedisWriter::GetEnvironment() return ScriptGlobal::Get("Environment", &l_DefaultEnv); } -String RedisWriter::GetIdentifier(const ConfigObject::Ptr& object) +String RedisWriter::GetObjectIdentifier(const ConfigObject::Ptr& object) { - return HashValue((Array::Ptr)new Array({GetEnvironment(), object->GetReflectionType()->GetName(), object->GetName()})); + Type::Ptr type = object->GetReflectionType(); + + if (type == CheckCommand::TypeInstance || type == NotificationCommand::TypeInstance || type == EventCommand::TypeInstance) + return HashValue((Array::Ptr)new Array({GetEnvironment(), type->GetName(), object->GetName()})); + else + return HashValue((Array::Ptr)new Array({GetEnvironment(), object->GetName()})); } String RedisWriter::CalculateCheckSumString(const String& str) diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 601281d78..5ed2891e4 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -77,7 +77,7 @@ private: /* utilities */ static String FormatCheckSumBinary(const String& str); - static String GetIdentifier(const ConfigObject::Ptr& object); + static String GetObjectIdentifier(const ConfigObject::Ptr& object); static String GetEnvironment(); static String CalculateCheckSumString(const String& str); static String CalculateCheckSumArray(const Array::Ptr& arr); From cd0524062417e6e22157c9f070e84deecf401d3b Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 7 Aug 2018 14:16:09 +0200 Subject: [PATCH 059/219] RedisWriter: dump period_checksum for users --- lib/redis/rediswriter-objects.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 3d747b8f5..e7e49e525 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -235,6 +235,11 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran } checkSums->Set("group_checksums", groupChecksums); + + auto period (user->GetPeriod()); + + if (period) + checkSums->Set("period_checksum", GetObjectIdentifier(period)); } /* Calculate checkable checksums. */ From 4d402932178af8b3dfd128e4d0377019597e4f90 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 8 Aug 2018 14:26:41 +0200 Subject: [PATCH 060/219] RedisWriter: dump all_parents_checksum and all_parents_checksums for zones --- lib/redis/rediswriter-objects.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index e7e49e525..69f808bbd 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -327,6 +327,15 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran if (parentZone) checkSums->Set("parent_checksum", GetObjectIdentifier(parentZone)); + + Array::Ptr parents (new Array); + + for (auto& parent : zone->GetAllParentsRaw()) { + parents->Add(GetObjectIdentifier(parent)); + } + + checkSums->Set("all_parents_checksums", parents); + checkSums->Set("all_parents_checksum", HashValue(zone->GetAllParents())); } else { /* zone_checksum for endpoints already is calculated above. */ From 3a8cf8d74d0315471357157d6197b4c6dd4c0eff Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 9 Aug 2018 10:49:36 +0200 Subject: [PATCH 061/219] RedisWriter: dump missing checksums for notifications refs #18 --- lib/redis/rediswriter-objects.cpp | 43 +++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 69f808bbd..3471cae17 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -242,6 +242,49 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran checkSums->Set("period_checksum", GetObjectIdentifier(period)); } + Notification::Ptr notification = dynamic_pointer_cast(object); + + if (notification) { + Host::Ptr host; + Service::Ptr service; + auto users (notification->GetUsers()); + Array::Ptr userChecksums = new Array(); + Array::Ptr userNames = new Array(); + auto usergroups (notification->GetUserGroups()); + Array::Ptr usergroupChecksums = new Array(); + Array::Ptr usergroupNames = new Array(); + + tie(host, service) = GetHostService(notification->GetCheckable()); + + checkSums->Set("host_checksum", GetObjectIdentifier(host)); + checkSums->Set("command_checksum", GetObjectIdentifier(notification->GetCommand())); + + if (service) + checkSums->Set("service_checksum", GetObjectIdentifier(service)); + + userChecksums->Reserve(users.size()); + userNames->Reserve(users.size()); + + for (auto& user : users) { + userChecksums->Add(GetObjectIdentifier(user)); + userNames->Add(user->GetName()); + } + + checkSums->Set("user_checksums", userChecksums); + checkSums->Set("users_checksum", CalculateCheckSumArray(userNames)); + + usergroupChecksums->Reserve(usergroups.size()); + usergroupNames->Reserve(usergroups.size()); + + for (auto& usergroup : usergroups) { + usergroupChecksums->Add(GetObjectIdentifier(usergroup)); + usergroupNames->Add(usergroup->GetName()); + } + + checkSums->Set("usergroup_checksums", usergroupChecksums); + checkSums->Set("usergroups_checksum", CalculateCheckSumArray(usergroupNames)); + } + /* Calculate checkable checksums. */ Checkable::Ptr checkable = dynamic_pointer_cast(object); From 65c38a3958fd391f7c4f4fece9840b3374beb0c3 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Fri, 10 Aug 2018 10:22:30 +0200 Subject: [PATCH 062/219] ignore user/usergroup for property checksum --- lib/redis/rediswriter-objects.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 3471cae17..5821ef9e0 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -262,6 +262,8 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran if (service) checkSums->Set("service_checksum", GetObjectIdentifier(service)); + propertiesBlacklist.emplace("users"); + userChecksums->Reserve(users.size()); userNames->Reserve(users.size()); @@ -273,6 +275,8 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran checkSums->Set("user_checksums", userChecksums); checkSums->Set("users_checksum", CalculateCheckSumArray(userNames)); + propertiesBlacklist.emplace("user_groups"); + usergroupChecksums->Reserve(usergroups.size()); usergroupNames->Reserve(usergroups.size()); From a47e16fb5b72c5447a257e84e5bf589879201a8c Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Thu, 9 Aug 2018 14:28:19 +0200 Subject: [PATCH 063/219] Rename command_checksums --- lib/redis/rediswriter-objects.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 5821ef9e0..69c0cb9ab 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -335,12 +335,12 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran checkSums->Set("command_endpoint_checksum", GetObjectIdentifier(commandEndpoint)); /* *_command_checksum */ - checkSums->Set("check_command_checksum", GetObjectIdentifier(checkable->GetCheckCommand())); + checkSums->Set("checkcommand_checksum", GetObjectIdentifier(checkable->GetCheckCommand())); EventCommand::Ptr eventCommand = checkable->GetEventCommand(); if (eventCommand) - checkSums->Set("event_command_checksum", GetObjectIdentifier(eventCommand)); + checkSums->Set("eventcommand_checksum", GetObjectIdentifier(eventCommand)); /* *_url_checksum, icon_image_checksum */ String actionUrl = checkable->GetActionUrl(); From c4962559ff6e13e7d9c38c11bc4195d9d80ab08b Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Fri, 10 Aug 2018 13:37:42 +0200 Subject: [PATCH 064/219] Remove parent_checksum parent_checksum is always the first element in all_parents_checksums --- lib/redis/rediswriter-objects.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 69c0cb9ab..baaa56d0d 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -370,11 +370,6 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran checkSums->Set("endpoints_checksum", CalculateCheckSumArray(endpoints)); - Zone::Ptr parentZone = zone->GetParent(); - - if (parentZone) - checkSums->Set("parent_checksum", GetObjectIdentifier(parentZone)); - Array::Ptr parents (new Array); for (auto& parent : zone->GetAllParentsRaw()) { From d0a45521a9c9622d99bc329680b9c6d2a9077c08 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 12 Sep 2018 17:15:20 +0200 Subject: [PATCH 065/219] Unify schema structure refs #15 --- lib/redis/rediswriter-objects.cpp | 89 ++++--------------------------- 1 file changed, 11 insertions(+), 78 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index baaa56d0d..0f2c83350 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -19,6 +19,8 @@ #include "redis/rediswriter.hpp" #include "icinga/command.hpp" +#include "base/configtype.hpp" +#include "base/configobject.hpp" #include "icinga/customvarobject.hpp" #include "icinga/host.hpp" #include "icinga/service.hpp" @@ -60,73 +62,18 @@ void RedisWriter::UpdateAllConfigObjects(void) double startTime = Utility::GetTime(); - std::map> deleteQueries; - long long cursor = 0; - - std::map lcTypes; - for (const Type::Ptr& type : Type::GetAllTypes()) { - lcTypes.emplace(type->GetName().ToLower(), type->GetName()); - } - - do { - std::shared_ptr reply = ExecuteQuery({ "SCAN", Convert::ToString(cursor), "MATCH", m_PrefixConfigObject + "*", "COUNT", "1000" }); - - VERIFY(reply->type == REDIS_REPLY_ARRAY); - VERIFY(reply->elements % 2 == 0); - - redisReply *cursorReply = reply->element[0]; - cursor = Convert::ToLong(cursorReply->str); - - redisReply *keysReply = reply->element[1]; - - for (size_t i = 0; i < keysReply->elements; i++) { - redisReply *keyReply = keysReply->element[i]; - VERIFY(keyReply->type == REDIS_REPLY_STRING); - - String key = keyReply->str; - String namePair = key.SubStr(m_PrefixConfigObject.GetLength()); - - String::SizeType pos = namePair.FindFirstOf(":"); - - if (pos == String::NPos) - continue; - - String type = namePair.SubStr(0, pos); - String name = namePair.SubStr(pos + 1); - - auto actualTypeName = lcTypes.find(type); - - if (actualTypeName == lcTypes.end()) - continue; - - Type::Ptr ptype = Type::GetByName(actualTypeName->second); - auto& deleteQuery = deleteQueries[ptype.get()]; - - if (deleteQuery.empty()) { - deleteQuery.emplace_back("DEL"); - deleteQuery.emplace_back(m_PrefixConfigCheckSum + type); - } - - deleteQuery.push_back(m_PrefixConfigObject + type + ":" + name); - deleteQuery.push_back(m_PrefixStatusObject + type + ":" + name); - } - } while (cursor != 0); - for (const Type::Ptr& type : Type::GetAllTypes()) { ConfigType *ctype = dynamic_cast(type.get()); if (!ctype) continue; + auto lcType (type->GetName().ToLower()); + ExecuteQuery({ "MULTI" }); /* Delete obsolete object keys first. */ - auto& deleteQuery = deleteQueries[type.get()]; - - if (deleteQuery.size() > 1) - ExecuteQuery(deleteQuery); - - String typeName = type->GetName().ToLower(); + ExecuteQuery({"DEL", m_PrefixConfigCheckSum + lcType, m_PrefixConfigObject + lcType, m_PrefixStatusObject + lcType}); /* fetch all objects and dump them */ for (const ConfigObject::Ptr& object : ctype->GetObjects()) { @@ -135,7 +82,7 @@ void RedisWriter::UpdateAllConfigObjects(void) } /* publish config type dump finished */ - ExecuteQuery({ "PUBLISH", "icinga:config:dump", typeName }); + ExecuteQuery({ "PUBLISH", "icinga:config:dump", lcType }); ExecuteQuery({ "EXEC" }); } @@ -518,7 +465,7 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) String objectKey = GetObjectIdentifier(object); ExecuteQueries({ - { "DEL", m_PrefixConfigObject + typeName + ":" + objectKey }, + { "HDEL", m_PrefixConfigObject + typeName, objectKey }, { "DEL", m_PrefixStatusObject + typeName + ":" + objectKey }, { "PUBLISH", "icinga:config:delete", typeName + ":" + objectKey } }); @@ -622,17 +569,7 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, bool useTran void RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType) { Type::Ptr type = object->GetReflectionType(); - - String typeName = type->GetName().ToLower(); - - /* Use the name checksum as unique key. */ - String objectKey = GetObjectIdentifier(object); - - std::vector > queries; - - queries.push_back({ "DEL", keyPrefix + typeName + ":" + objectKey }); - - std::vector hmsetCommand({ "HMSET", keyPrefix + typeName + ":" + objectKey }); + Dictionary::Ptr attrs (new Dictionary); for (int fid = 0; fid < type->GetFieldCount(); fid++) { Field field = type->GetFieldInfo(fid); @@ -650,15 +587,11 @@ void RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject: if (field.Attributes & FANavigation && !(field.Attributes & (FAConfig | FAState))) continue; - hmsetCommand.push_back(field.Name); - - Value sval = Serialize(val); - hmsetCommand.push_back(JsonEncode(sval)); + attrs->Set(field.Name, Serialize(val)); } - queries.push_back(hmsetCommand); - - ExecuteQueries(queries); + /* Use the name checksum as unique key. */ + ExecuteQuery({"HSET", keyPrefix + type->GetName().ToLower(), GetObjectIdentifier(object), JsonEncode(attrs)}); } void RedisWriter::StateChangedHandler(const ConfigObject::Ptr& object) From 315e9ffad8cdc871794a7aac9089eef1f7fea10e Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Thu, 13 Sep 2018 15:40:27 +0200 Subject: [PATCH 066/219] Stringify flat values --- lib/redis/rediswriter-utility.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 4251a30a5..626506f65 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -30,6 +30,7 @@ #include "base/objectlock.hpp" #include "base/array.hpp" #include "base/scriptglobal.hpp" +#include "base/convert.hpp" #include #include #include @@ -253,9 +254,13 @@ Dictionary::Ptr RedisWriter::SerializeVars(const CustomVarObject::Ptr& object) auto it (scalarVars.find(kv.first)); if (it != scalarVars.end()) { for (auto& scalarVar : it->second) { + String strVal = Convert::ToString(scalarVar.second); + if (scalarVar.second.GetType() == ValueEmpty) + strVal = "NULL"; + flatVars->Set(SHA1(PackObject(scalarVar.first)), (Dictionary::Ptr)new Dictionary({ {"name", scalarVar.first}, - {"value", scalarVar.second} + {"value", strVal} })); } } From ed3db5b4918a5c80b573e52a56cceb196c339894 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 18 Sep 2018 15:47:44 +0200 Subject: [PATCH 067/219] Fix elephant oversight --- lib/redis/rediswriter-stats.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/redis/rediswriter-stats.cpp b/lib/redis/rediswriter-stats.cpp index 1a234b64d..8ed024cf2 100644 --- a/lib/redis/rediswriter-stats.cpp +++ b/lib/redis/rediswriter-stats.cpp @@ -31,16 +31,16 @@ Dictionary::Ptr RedisWriter::GetStats() Dictionary::Ptr stats = new Dictionary(); //TODO: Figure out if more stats can be useful here. - Dictionary::Ptr statsFunctions = ScriptGlobal::Get("StatsFunctions", &Empty); + Namespace::Ptr statsFunctions = ScriptGlobal::Get("StatsFunctions", &Empty); if (!statsFunctions) Dictionary::Ptr(); ObjectLock olock(statsFunctions); - for (const Dictionary::Pair& kv : statsFunctions) + for (auto& kv : statsFunctions) { - Function::Ptr func = kv.second; + Function::Ptr func = kv.second->Get(); if (!func) BOOST_THROW_EXCEPTION(std::invalid_argument("Invalid status function name.")); From 8f411c7475e7e6fa065f9250c30cf4ace07ece2f Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Thu, 27 Sep 2018 16:54:52 +0200 Subject: [PATCH 068/219] Support comments for icingadb This splits comments into Host and Service comments --- lib/redis/rediswriter-objects.cpp | 35 +++++++++++++++++++++++++------ lib/redis/rediswriter.hpp | 2 +- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 0f2c83350..b6d72868c 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -127,9 +127,6 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran if (useTransaction) ExecuteQuery({ "MULTI" }); - /* Send all object attributes to Redis, no extra checksums involved here. */ - UpdateObjectAttrs(m_PrefixConfigObject, object, FAConfig); - /* Calculate object specific checksums and store them in a different namespace. */ Type::Ptr type = object->GetReflectionType(); @@ -408,11 +405,32 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran } checkSums->Set("exclude_checksums", excludeChecksums); + } else { + icinga::Comment::Ptr comment = dynamic_pointer_cast(object); + if (comment) { + propertiesBlacklist.emplace("name"); + propertiesBlacklist.emplace("host_name"); + + Host::Ptr host; + Service::Ptr service; + tie(host, service) = GetHostService(comment->GetCheckable()); + if (service) { + propertiesBlacklist.emplace("service_name"); + checkSums->Set("service_checksum", GetObjectIdentifier(service)); + typeName = "servicecomment"; + } else { + checkSums->Set("host_checksum", GetObjectIdentifier(host)); + typeName = "hostcomment"; + } + } } } } } + /* Send all object attributes to Redis, no extra checksums involved here. */ + UpdateObjectAttrs(m_PrefixConfigObject, object, FAConfig, typeName); + /* Custom var checksums. */ CustomVarObject::Ptr customVarObject = dynamic_pointer_cast(object); @@ -483,7 +501,8 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, bool useTran if (useTransaction) ExecuteQuery({ "MULTI" }); - UpdateObjectAttrs(m_PrefixStatusObject, object, FAState); + //TODO: Manage type names + UpdateObjectAttrs(m_PrefixStatusObject, object, FAState, ""); if (useTransaction) ExecuteQuery({ "EXEC" }); @@ -566,7 +585,7 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, bool useTran // } } -void RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType) +void RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride) { Type::Ptr type = object->GetReflectionType(); Dictionary::Ptr attrs (new Dictionary); @@ -591,7 +610,11 @@ void RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject: } /* Use the name checksum as unique key. */ - ExecuteQuery({"HSET", keyPrefix + type->GetName().ToLower(), GetObjectIdentifier(object), JsonEncode(attrs)}); + String typeName = type->GetName().ToLower(); + if (!typeNameOverride.IsEmpty()) + typeName = typeNameOverride.ToLower(); + + ExecuteQuery({"HSET", keyPrefix + typeName, GetObjectIdentifier(object), JsonEncode(attrs)}); } void RedisWriter::StateChangedHandler(const ConfigObject::Ptr& object) diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 5ed2891e4..a47cfd9ab 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -69,7 +69,7 @@ private: void SendConfigUpdate(const ConfigObject::Ptr& object, bool useTransaction, bool runtimeUpdate = false); void SendConfigDelete(const ConfigObject::Ptr& object); void SendStatusUpdate(const ConfigObject::Ptr& object, bool useTransaction); - void UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType); + void UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride); /* Stats */ Dictionary::Ptr GetStats(); From 0456298c58807093dea90c2eaf9c1aa7143c16e2 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Mon, 1 Oct 2018 10:18:49 +0200 Subject: [PATCH 069/219] Reformat code CLion is now pretty close at what we use with Icinga, other changes do make sense to me. --- lib/redis/rediswriter-objects.cpp | 117 +++++++++++++++--------------- 1 file changed, 60 insertions(+), 57 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index b6d72868c..7578bd150 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -68,12 +68,13 @@ void RedisWriter::UpdateAllConfigObjects(void) if (!ctype) continue; - auto lcType (type->GetName().ToLower()); + auto lcType(type->GetName().ToLower()); - ExecuteQuery({ "MULTI" }); + ExecuteQuery({"MULTI"}); /* Delete obsolete object keys first. */ - ExecuteQuery({"DEL", m_PrefixConfigCheckSum + lcType, m_PrefixConfigObject + lcType, m_PrefixStatusObject + lcType}); + ExecuteQuery( + {"DEL", m_PrefixConfigCheckSum + lcType, m_PrefixConfigObject + lcType, m_PrefixStatusObject + lcType}); /* fetch all objects and dump them */ for (const ConfigObject::Ptr& object : ctype->GetObjects()) { @@ -82,13 +83,13 @@ void RedisWriter::UpdateAllConfigObjects(void) } /* publish config type dump finished */ - ExecuteQuery({ "PUBLISH", "icinga:config:dump", lcType }); + ExecuteQuery({"PUBLISH", "icinga:config:dump", lcType}); - ExecuteQuery({ "EXEC" }); + ExecuteQuery({"EXEC"}); } Log(LogInformation, "RedisWriter") - << "Initial config/status dump finished in " << Utility::GetTime() - startTime << " seconds."; + << "Initial config/status dump finished in " << Utility::GetTime() - startTime << " seconds."; } static ConfigObject::Ptr GetHostGroup(const String& name) @@ -125,7 +126,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran */ if (useTransaction) - ExecuteQuery({ "MULTI" }); + ExecuteQuery({"MULTI"}); /* Calculate object specific checksums and store them in a different namespace. */ Type::Ptr type = object->GetReflectionType(); @@ -133,24 +134,24 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran String typeName = type->GetName().ToLower(); String objectKey = GetObjectIdentifier(object); - std::set propertiesBlacklist ({"name", "__name", "package", "source_location", "templates"}); + std::set propertiesBlacklist({"name", "__name", "package", "source_location", "templates"}); Dictionary::Ptr checkSums = new Dictionary(); checkSums->Set("name_checksum", CalculateCheckSumString(object->GetShortName())); checkSums->Set("environment_checksum", CalculateCheckSumString(GetEnvironment())); - auto endpoint (dynamic_pointer_cast(object)); + auto endpoint(dynamic_pointer_cast(object)); if (endpoint) { - auto endpointZone (endpoint->GetZone()); + auto endpointZone(endpoint->GetZone()); if (endpointZone) { checkSums->Set("zone_checksum", GetObjectIdentifier(endpointZone)); } } else { /* 'zone' is available for all config objects, therefore calculate the checksum. */ - auto zone (static_pointer_cast(object->GetZone())); + auto zone(static_pointer_cast(object->GetZone())); if (zone) checkSums->Set("zone_checksum", GetObjectIdentifier(zone)); @@ -171,8 +172,8 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran Array::Ptr groupChecksums = new Array(); - ObjectLock groupsLock (groups); - ObjectLock groupChecksumsLock (groupChecksums); + ObjectLock groupsLock(groups); + ObjectLock groupChecksumsLock(groupChecksums); for (auto group : groups) { groupChecksums->Add(GetObjectIdentifier((*getGroup)(group.Get()))); @@ -180,7 +181,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran checkSums->Set("group_checksums", groupChecksums); - auto period (user->GetPeriod()); + auto period(user->GetPeriod()); if (period) checkSums->Set("period_checksum", GetObjectIdentifier(period)); @@ -191,10 +192,10 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran if (notification) { Host::Ptr host; Service::Ptr service; - auto users (notification->GetUsers()); + auto users(notification->GetUsers()); Array::Ptr userChecksums = new Array(); Array::Ptr userNames = new Array(); - auto usergroups (notification->GetUserGroups()); + auto usergroups(notification->GetUserGroups()); Array::Ptr usergroupChecksums = new Array(); Array::Ptr usergroupNames = new Array(); @@ -263,8 +264,8 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran Array::Ptr groupChecksums = new Array(); - ObjectLock groupsLock (groups); - ObjectLock groupChecksumsLock (groupChecksums); + ObjectLock groupsLock(groups); + ObjectLock groupChecksumsLock(groupChecksums); for (auto group : groups) { groupChecksums->Add(GetObjectIdentifier((*getGroup)(group.Get()))); @@ -314,7 +315,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran checkSums->Set("endpoints_checksum", CalculateCheckSumArray(endpoints)); - Array::Ptr parents (new Array); + Array::Ptr parents(new Array); for (auto& parent : zone->GetAllParentsRaw()) { parents->Add(GetObjectIdentifier(parent)); @@ -325,14 +326,14 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran } else { /* zone_checksum for endpoints already is calculated above. */ - auto command (dynamic_pointer_cast(object)); + auto command(dynamic_pointer_cast(object)); if (command) { Dictionary::Ptr arguments = command->GetArguments(); Dictionary::Ptr argumentChecksums = new Dictionary; if (arguments) { - ObjectLock argumentsLock (arguments); + ObjectLock argumentsLock(arguments); for (auto& kv : arguments) { argumentChecksums->Set(kv.first, HashValue(kv.second)); @@ -347,7 +348,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran Dictionary::Ptr envvarChecksums = new Dictionary; if (envvars) { - ObjectLock argumentsLock (envvars); + ObjectLock argumentsLock(envvars); for (auto& kv : envvars) { envvarChecksums->Set(kv.first, HashValue(kv.second)); @@ -358,7 +359,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran checkSums->Set("envvar_checksums", envvarChecksums); propertiesBlacklist.emplace("env"); } else { - auto timeperiod (dynamic_pointer_cast(object)); + auto timeperiod(dynamic_pointer_cast(object)); if (timeperiod) { Dictionary::Ptr ranges = timeperiod->GetRanges(); @@ -377,8 +378,8 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran Array::Ptr includeChecksums = new Array(); - ObjectLock includesLock (includes); - ObjectLock includeChecksumsLock (includeChecksums); + ObjectLock includesLock(includes); + ObjectLock includeChecksumsLock(includeChecksums); for (auto include : includes) { includeChecksums->Add(GetObjectIdentifier((*getInclude)(include.Get()))); @@ -397,8 +398,8 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran Array::Ptr excludeChecksums = new Array(); - ObjectLock excludesLock (excludes); - ObjectLock excludeChecksumsLock (excludeChecksums); + ObjectLock excludesLock(excludes); + ObjectLock excludeChecksumsLock(excludeChecksums); for (auto exclude : excludes) { excludeChecksums->Add(GetObjectIdentifier((*getExclude)(exclude.Get()))); @@ -406,23 +407,23 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran checkSums->Set("exclude_checksums", excludeChecksums); } else { - icinga::Comment::Ptr comment = dynamic_pointer_cast(object); - if (comment) { - propertiesBlacklist.emplace("name"); - propertiesBlacklist.emplace("host_name"); + icinga::Comment::Ptr comment = dynamic_pointer_cast(object); + if (comment) { + propertiesBlacklist.emplace("name"); + propertiesBlacklist.emplace("host_name"); Host::Ptr host; - Service::Ptr service; - tie(host, service) = GetHostService(comment->GetCheckable()); - if (service) { + Service::Ptr service; + tie(host, service) = GetHostService(comment->GetCheckable()); + if (service) { propertiesBlacklist.emplace("service_name"); checkSums->Set("service_checksum", GetObjectIdentifier(service)); - typeName = "servicecomment"; - } else { - checkSums->Set("host_checksum", GetObjectIdentifier(host)); - typeName = "hostcomment"; - } - } + typeName = "servicecomment"; + } else { + checkSums->Set("host_checksum", GetObjectIdentifier(host)); + typeName = "hostcomment"; + } + } } } } @@ -439,15 +440,15 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran checkSums->Set("vars_checksum", CalculateCheckSumVars(customVarObject)); - auto vars (SerializeVars(customVarObject)); + auto vars(SerializeVars(customVarObject)); if (vars) { - auto varsJson (JsonEncode(vars)); + auto varsJson(JsonEncode(vars)); Log(LogDebug, "RedisWriter") - << "HSET " << m_PrefixConfigCustomVar + typeName << " " << objectKey << " " << varsJson; + << "HSET " << m_PrefixConfigCustomVar + typeName << " " << objectKey << " " << varsJson; - ExecuteQuery({ "HSET", m_PrefixConfigCustomVar + typeName, objectKey, varsJson }); + ExecuteQuery({"HSET", m_PrefixConfigCustomVar + typeName, objectKey, varsJson}); } } @@ -457,18 +458,18 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran String checkSumsBody = JsonEncode(checkSums); Log(LogDebug, "RedisWriter") - << "HSET " << m_PrefixConfigCheckSum + typeName << " " << objectKey << " " << checkSumsBody; + << "HSET " << m_PrefixConfigCheckSum + typeName << " " << objectKey << " " << checkSumsBody; - ExecuteQuery({ "HSET", m_PrefixConfigCheckSum + typeName, objectKey, checkSumsBody }); + ExecuteQuery({"HSET", m_PrefixConfigCheckSum + typeName, objectKey, checkSumsBody}); /* Send an update event to subscribers. */ if (runtimeUpdate) { - ExecuteQuery({ "PUBLISH", "icinga:config:update", typeName + ":" + objectKey }); + ExecuteQuery({"PUBLISH", "icinga:config:update", typeName + ":" + objectKey}); } if (useTransaction) - ExecuteQuery({ "EXEC" }); + ExecuteQuery({"EXEC"}); } void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) @@ -483,10 +484,10 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) String objectKey = GetObjectIdentifier(object); ExecuteQueries({ - { "HDEL", m_PrefixConfigObject + typeName, objectKey }, - { "DEL", m_PrefixStatusObject + typeName + ":" + objectKey }, - { "PUBLISH", "icinga:config:delete", typeName + ":" + objectKey } - }); + {"HDEL", m_PrefixConfigObject + typeName, objectKey}, + {"DEL", m_PrefixStatusObject + typeName + ":" + objectKey}, + {"PUBLISH", "icinga:config:delete", typeName + ":" + objectKey} + }); } @@ -499,13 +500,13 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, bool useTran return; if (useTransaction) - ExecuteQuery({ "MULTI" }); + ExecuteQuery({"MULTI"}); //TODO: Manage type names UpdateObjectAttrs(m_PrefixStatusObject, object, FAState, ""); if (useTransaction) - ExecuteQuery({ "EXEC" }); + ExecuteQuery({"EXEC"}); // /* Serialize config object attributes */ // Dictionary::Ptr objectAttrs = SerializeObjectAttrs(object, FAState); @@ -585,10 +586,11 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, bool useTran // } } -void RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride) +void RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType, + const String& typeNameOverride) { Type::Ptr type = object->GetReflectionType(); - Dictionary::Ptr attrs (new Dictionary); + Dictionary::Ptr attrs(new Dictionary); for (int fid = 0; fid < type->GetFieldCount(); fid++) { Field field = type->GetFieldInfo(fid); @@ -635,7 +637,8 @@ void RedisWriter::VersionChangedHandler(const ConfigObject::Ptr& object) for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendConfigUpdate, rw.get(), object, true, true)); } - } else if (!object->IsActive() && object->GetExtension("ConfigObjectDeleted")) { /* same as in apilistener-configsync.cpp */ + } else if (!object->IsActive() && + object->GetExtension("ConfigObjectDeleted")) { /* same as in apilistener-configsync.cpp */ /* Delete object config */ for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendConfigDelete, rw.get(), object)); From d96dcf869eec4071fb32f3ebb8ef39fe5ef1691a Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Tue, 16 Oct 2018 14:21:07 +0200 Subject: [PATCH 070/219] Add RedisConnection object This works using an async Redis connection. Multiple connections are not possible. The connection is fed from a workqueue filled with RedisCommands in the way of a vector of strings. --- lib/redis/CMakeLists.txt | 2 +- lib/redis/redisconnection.cpp | 138 ++++++++++++++++++++++++++++++++++ lib/redis/redisconnection.hpp | 65 ++++++++++++++++ 3 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 lib/redis/redisconnection.cpp create mode 100644 lib/redis/redisconnection.hpp diff --git a/lib/redis/CMakeLists.txt b/lib/redis/CMakeLists.txt index 5ff39f81d..5e953ad7e 100644 --- a/lib/redis/CMakeLists.txt +++ b/lib/redis/CMakeLists.txt @@ -18,7 +18,7 @@ mkclass_target(rediswriter.ti rediswriter-ti.cpp rediswriter-ti.hpp) set(redis_SOURCES - rediswriter.cpp rediswriter-objects.cpp rediswriter-stats.cpp rediswriter-utility.cpp rediswriter-ti.hpp + rediswriter.cpp rediswriter-objects.cpp rediswriter-stats.cpp rediswriter-utility.cpp redisconnection.cpp rediswriter-ti.hpp ) if(ICINGA2_UNITY_BUILD) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp new file mode 100644 index 000000000..f4a0ee93d --- /dev/null +++ b/lib/redis/redisconnection.cpp @@ -0,0 +1,138 @@ +/****************************************************************************** + * Icinga 2 * + * Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License * + * as published by the Free Software Foundation; either version 2 * + * of the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software Foundation * + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * + ******************************************************************************/ + +#include "base/object.hpp" +#include "redis/redisconnection.hpp" +#include "base/workqueue.hpp" +#include +#include +#include "base/utility.hpp" + +using namespace icinga; +/* +struct redis_error : virtual std::exception, virtual boost::exception { }; +struct errinfo_redis_query_; +typedef boost::error_info errinfo_redis_query; +*/ +RedisConnection::RedisConnection(const String host, const int port, const String path) : +m_Host(host), m_Port(port), m_Path(path) { + m_RedisConnectionWorkQueue.SetName("RedisConnection"); +} + +void RedisConnection::StaticInitialize() +{ + +} + +void RedisConnection::Start() +{ + RedisConnection::Connect(); +} + +void RedisConnection::AssertOnWorkQueue() +{ + ASSERT(m_RedisConnectionWorkQueue.IsWorkerThread()); +} + +void RedisConnection::Connect() { + if (m_Context) + return; + + Log(LogInformation, "RedisWriter", "Trying to connect to redis server Async"); + + if (m_Path.IsEmpty()) + m_Context = redisAsyncConnect(m_Host.CStr(), m_Port); + else + m_Context = redisAsyncConnectUnix(m_Path.CStr()); + + if (!m_Context || m_Context->err) { + if (!m_Context) { + Log(LogWarning, "RedisWriter", "Cannot allocate redis context."); + } else { + Log(LogWarning, "RedisWriter", "Connection error: ") + << m_Context->errstr; + } + + if (m_Context) { + redisAsyncFree(m_Context); + m_Context = NULL; + } + } + + redisAsyncSetDisconnectCallback(m_Context, &DisconnectCallback); + + //TODO: Authentication, DB selection, error handling +} + +void RedisConnection::DisconnectCallback(const redisAsyncContext *c, int status) { + if (status == REDIS_OK) + Log(LogCritical, "RedisWriter") << "Redis disconnected by user"; + else + Log(LogCritical, "Rediswriter") << "Redis disconnected for reasons"; + +} +void RedisConnection::ExecuteQuery(const std::vector& query, redisCallbackFn *fn, void *privdata) +{ + m_RedisConnectionWorkQueue.Enqueue(std::bind(&RedisConnection::SendMessageInternal, this, query, fn, privdata)); +} + +void RedisConnection::ExecuteQueries(const std::vector >& queries, redisCallbackFn *fn, void *privdata) +{ + for (const auto& query : queries) { + m_RedisConnectionWorkQueue.Enqueue(std::bind(&RedisConnection::SendMessageInternal, this, query, fn, privdata)); + } +} + +void RedisConnection::SendMessageInternal(const std::vector& query, redisCallbackFn *fn, void *privdata) +{ + AssertOnWorkQueue(); + + if (!m_Context) { + Log(LogCritical, "RedisWriter") + << "Connection lost"; + return; + } + + const char **argv; + size_t *argvlen; + + argv = new const char *[query.size()]; + argvlen = new size_t[query.size()]; + + for (std::vector::size_type i = 0; i < query.size(); i++) { + argv[i] = query[i].CStr(); + argvlen[i] = query[i].GetLength(); + } + + int r = redisAsyncCommandArgv(m_Context, fn, privdata, query.size(), argv, argvlen); + + delete [] argv; + delete [] argvlen; + + if (r == REDIS_REPLY_ERROR) { + Log(LogCritical, "RedisWriter") + << "Redis Async query failed"; + + BOOST_THROW_EXCEPTION( + redis_error() + << errinfo_redis_query("FUCK") + << errinfo_redis_query(Utility::Join(Array::FromVector(query), ' ', false)) + ); + } +} \ No newline at end of file diff --git a/lib/redis/redisconnection.hpp b/lib/redis/redisconnection.hpp new file mode 100644 index 000000000..bcab12f5d --- /dev/null +++ b/lib/redis/redisconnection.hpp @@ -0,0 +1,65 @@ +/****************************************************************************** + * Icinga 2 * + * Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License * + * as published by the Free Software Foundation; either version 2 * + * of the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software Foundation * + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * + ******************************************************************************/ + +#ifndef REDISCONNECTION_H +#define REDISCONNECTION_H + +#include +#include "base/object.hpp" +#include "base/workqueue.hpp" + +namespace icinga { +/** + * An Async Redis connection. + * + * @ingroup redis + */ +class RedisConnection final : public Object +{ +public: + DECLARE_PTR_TYPEDEFS(RedisConnection); + + RedisConnection(const String host, const int port, const String path); + + void Start(); + + void Connect(); + + void ExecuteQuery(const std::vector& query, redisCallbackFn *fn = nullptr, void *privdata = nullptr); + void ExecuteQueries(const std::vector >& queries, redisCallbackFn *fn = nullptr, void *privdata = nullptr); + + +private: + static void StaticInitialize(); + void SendMessageInternal(const std::vector& query, redisCallbackFn *fn, void *privdata); + void AssertOnWorkQueue(); + static void DisconnectCallback(const redisAsyncContext *c, int status); + + WorkQueue m_RedisConnectionWorkQueue{100000}; + + redisAsyncContext *m_Context; + + String m_Path; + String m_Host; + int m_Port; +}; + +} + +#endif //REDISCONNECTION_H From 64515b81e39a63bdbb5f5f6db9253f18d2b9412a Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Wed, 17 Oct 2018 14:33:57 +0200 Subject: [PATCH 071/219] Add disconnect --- lib/redis/redisconnection.cpp | 8 +++++++- lib/redis/redisconnection.hpp | 5 +++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index f4a0ee93d..721433339 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -23,6 +23,7 @@ #include #include #include "base/utility.hpp" +#include "redis/rediswriter.hpp" using namespace icinga; /* @@ -80,9 +81,14 @@ void RedisConnection::Connect() { //TODO: Authentication, DB selection, error handling } +void RedisConnection::Disconnect() +{ + redisAsyncDisconnect(m_Context); +} + void RedisConnection::DisconnectCallback(const redisAsyncContext *c, int status) { if (status == REDIS_OK) - Log(LogCritical, "RedisWriter") << "Redis disconnected by user"; + Log(LogInformation, "RedisWriter") << "Redis disconnected by us"; else Log(LogCritical, "Rediswriter") << "Redis disconnected for reasons"; diff --git a/lib/redis/redisconnection.hpp b/lib/redis/redisconnection.hpp index bcab12f5d..f23753235 100644 --- a/lib/redis/redisconnection.hpp +++ b/lib/redis/redisconnection.hpp @@ -40,9 +40,10 @@ public: void Start(); void Connect(); + void Disconnect(); - void ExecuteQuery(const std::vector& query, redisCallbackFn *fn = nullptr, void *privdata = nullptr); - void ExecuteQueries(const std::vector >& queries, redisCallbackFn *fn = nullptr, void *privdata = nullptr); + void ExecuteQuery(const std::vector& query, redisCallbackFn *fn = NULL, void *privdata = NULL); + void ExecuteQueries(const std::vector >& queries, redisCallbackFn *fn = NULL, void *privdata = NULL); private: From 6aaa35a0a6c7118d19e592d975e0517d2b3ef654 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Thu, 18 Oct 2018 14:39:59 +0200 Subject: [PATCH 072/219] Rewrite the RedisWriter There should be more atomic commits but the whole thing was a mess. This commit changes the synchrounous Redis connection into an asynchronous one in its own class RedisConnection. The RedisConnection uses a Workqueue with commands to fire against the Redis server. When a response is required a callback must be supplied, refer to RedisWriter::RedisGet(). Known Issues: - Authentication has no error handling and can break the connection - Error handling in general is iffy due to the nature of the async redis connection - Getting a reply out of RedisConnection is not trivial - HandleRW... sunt dracones --- lib/redis/redisconnection.cpp | 88 ++++-- lib/redis/redisconnection.hpp | 63 +++-- lib/redis/rediswriter-objects.cpp | 437 ++++++++++++++++-------------- lib/redis/rediswriter-utility.cpp | 30 ++ lib/redis/rediswriter.cpp | 214 +++++---------- lib/redis/rediswriter.hpp | 25 +- 6 files changed, 447 insertions(+), 410 deletions(-) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index 721433339..db5fb2742 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -20,30 +20,31 @@ #include "base/object.hpp" #include "redis/redisconnection.hpp" #include "base/workqueue.hpp" -#include -#include +#include "base/logger.hpp" +#include "base/convert.hpp" #include "base/utility.hpp" #include "redis/rediswriter.hpp" +#include "hiredis/hiredis.h" + using namespace icinga; -/* -struct redis_error : virtual std::exception, virtual boost::exception { }; -struct errinfo_redis_query_; -typedef boost::error_info errinfo_redis_query; -*/ -RedisConnection::RedisConnection(const String host, const int port, const String path) : -m_Host(host), m_Port(port), m_Path(path) { + +RedisConnection::RedisConnection(const String host, const int port, const String path, const String password, const int db) : + m_Host(host), m_Port(port), m_Path(path), m_Password(password), m_DbIndex(db), m_Context(NULL) +{ m_RedisConnectionWorkQueue.SetName("RedisConnection"); } void RedisConnection::StaticInitialize() { - } void RedisConnection::Start() { RedisConnection::Connect(); + + std::thread thread(std::bind(&RedisConnection::HandleRW, this)); + thread.detach(); } void RedisConnection::AssertOnWorkQueue() @@ -51,11 +52,31 @@ void RedisConnection::AssertOnWorkQueue() ASSERT(m_RedisConnectionWorkQueue.IsWorkerThread()); } -void RedisConnection::Connect() { +void RedisConnection::HandleRW() +{ + Utility::SetThreadName("RedisConnection Handler"); + + for (;;) { + try { + { + boost::mutex::scoped_lock lock(m_CMutex); + redisAsyncHandleWrite(m_Context); + redisAsyncHandleRead(m_Context); + } + Utility::Sleep(0.1); + } catch (const std::exception&) { + Log(LogCritical, "RedisWriter", "Internal Redis Error"); + } + } +} + +void RedisConnection::Connect() +{ if (m_Context) return; Log(LogInformation, "RedisWriter", "Trying to connect to redis server Async"); + boost::mutex::scoped_lock lock(m_CMutex); if (m_Path.IsEmpty()) m_Context = redisAsyncConnect(m_Host.CStr(), m_Port); @@ -78,7 +99,16 @@ void RedisConnection::Connect() { redisAsyncSetDisconnectCallback(m_Context, &DisconnectCallback); - //TODO: Authentication, DB selection, error handling + /* TODO: This currently does not work properly: + * In case of error the connection is broken, yet the Context is not set to faulty. May be a bug with hiredis. + * Error case: Password does not match, or even: "Client sent AUTH, but no password is set" which also results in an error. + */ + if (!m_Password.IsEmpty()) { + ExecuteQuery({"AUTH", m_Password}); + } + + if (m_DbIndex != 0) + ExecuteQuery({"SELECT", Convert::ToString(m_DbIndex)}); } void RedisConnection::Disconnect() @@ -86,19 +116,31 @@ void RedisConnection::Disconnect() redisAsyncDisconnect(m_Context); } -void RedisConnection::DisconnectCallback(const redisAsyncContext *c, int status) { +void RedisConnection::DisconnectCallback(const redisAsyncContext *c, int status) +{ if (status == REDIS_OK) Log(LogInformation, "RedisWriter") << "Redis disconnected by us"; - else - Log(LogCritical, "Rediswriter") << "Redis disconnected for reasons"; + else { + if (c->err != 0) + Log(LogCritical, "RedisWriter") << "Redis disconnected by server. Reason: " << c->errstr; + else + Log(LogCritical, "RedisWriter") << "Redis disconnected by server"; + } } + +bool RedisConnection::IsConnected() +{ + return (REDIS_CONNECTED & m_Context->c.flags) == REDIS_CONNECTED; +} + void RedisConnection::ExecuteQuery(const std::vector& query, redisCallbackFn *fn, void *privdata) { m_RedisConnectionWorkQueue.Enqueue(std::bind(&RedisConnection::SendMessageInternal, this, query, fn, privdata)); } -void RedisConnection::ExecuteQueries(const std::vector >& queries, redisCallbackFn *fn, void *privdata) +void +RedisConnection::ExecuteQueries(const std::vector >& queries, redisCallbackFn *fn, void *privdata) { for (const auto& query : queries) { m_RedisConnectionWorkQueue.Enqueue(std::bind(&RedisConnection::SendMessageInternal, this, query, fn, privdata)); @@ -109,6 +151,8 @@ void RedisConnection::SendMessageInternal(const std::vector& query, redi { AssertOnWorkQueue(); + boost::mutex::scoped_lock lock(m_CMutex); + if (!m_Context) { Log(LogCritical, "RedisWriter") << "Connection lost"; @@ -120,16 +164,21 @@ void RedisConnection::SendMessageInternal(const std::vector& query, redi argv = new const char *[query.size()]; argvlen = new size_t[query.size()]; + String debugstr; for (std::vector::size_type i = 0; i < query.size(); i++) { argv[i] = query[i].CStr(); argvlen[i] = query[i].GetLength(); + debugstr += argv[i]; + debugstr += " "; } + Log(LogDebug, "RedisWriter, Connection") + << "Sending Command: " << debugstr; int r = redisAsyncCommandArgv(m_Context, fn, privdata, query.size(), argv, argvlen); - delete [] argv; - delete [] argvlen; + delete[] argv; + delete[] argvlen; if (r == REDIS_REPLY_ERROR) { Log(LogCritical, "RedisWriter") @@ -137,8 +186,7 @@ void RedisConnection::SendMessageInternal(const std::vector& query, redi BOOST_THROW_EXCEPTION( redis_error() - << errinfo_redis_query("FUCK") << errinfo_redis_query(Utility::Join(Array::FromVector(query), ' ', false)) ); } -} \ No newline at end of file +} diff --git a/lib/redis/redisconnection.hpp b/lib/redis/redisconnection.hpp index f23753235..c968c15af 100644 --- a/lib/redis/redisconnection.hpp +++ b/lib/redis/redisconnection.hpp @@ -24,43 +24,62 @@ #include "base/object.hpp" #include "base/workqueue.hpp" -namespace icinga { +namespace icinga +{ /** * An Async Redis connection. * * @ingroup redis */ -class RedisConnection final : public Object -{ -public: - DECLARE_PTR_TYPEDEFS(RedisConnection); + class RedisConnection final : public Object + { + public: + DECLARE_PTR_TYPEDEFS(RedisConnection); - RedisConnection(const String host, const int port, const String path); + RedisConnection(const String host, const int port, const String path, const String password = "", const int db = 0); - void Start(); + void Start(); - void Connect(); - void Disconnect(); + void Connect(); - void ExecuteQuery(const std::vector& query, redisCallbackFn *fn = NULL, void *privdata = NULL); - void ExecuteQueries(const std::vector >& queries, redisCallbackFn *fn = NULL, void *privdata = NULL); + void Disconnect(); + bool IsConnected(); -private: - static void StaticInitialize(); - void SendMessageInternal(const std::vector& query, redisCallbackFn *fn, void *privdata); - void AssertOnWorkQueue(); - static void DisconnectCallback(const redisAsyncContext *c, int status); + void ExecuteQuery(const std::vector& query, redisCallbackFn *fn = NULL, void *privdata = NULL); - WorkQueue m_RedisConnectionWorkQueue{100000}; + void ExecuteQueries(const std::vector >& queries, redisCallbackFn *fn = NULL, + void *privdata = NULL); - redisAsyncContext *m_Context; + private: + static void StaticInitialize(); - String m_Path; - String m_Host; - int m_Port; -}; + void SendMessageInternal(const std::vector& query, redisCallbackFn *fn, void *privdata); + void AssertOnWorkQueue(); + + void HandleRW(); + + static void DisconnectCallback(const redisAsyncContext *c, int status); + + WorkQueue m_RedisConnectionWorkQueue{100000}; + Timer::Ptr m_EventLoop; + + redisAsyncContext *m_Context; + + String m_Path; + String m_Host; + int m_Port; + String m_Password; + int m_DbIndex; + + boost::mutex m_CMutex; + }; + + struct redis_error : virtual std::exception, virtual boost::exception { }; + + struct errinfo_redis_query_; + typedef boost::error_info errinfo_redis_query; } #endif //REDISCONNECTION_H diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 7578bd150..9e36dcbff 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -18,6 +18,7 @@ ******************************************************************************/ #include "redis/rediswriter.hpp" +#include "redis/redisconnection.hpp" #include "icinga/command.hpp" #include "base/configtype.hpp" #include "base/configobject.hpp" @@ -39,6 +40,7 @@ #include "base/initialize.hpp" #include "base/convert.hpp" #include "base/array.hpp" +#include "base/exception.hpp" #include #include @@ -56,77 +58,126 @@ void RedisWriter::ConfigStaticInitialize() ConfigObject::OnVersionChanged.connect(std::bind(&RedisWriter::VersionChangedHandler, _1)); } -void RedisWriter::UpdateAllConfigObjects(void) +void RedisWriter::UpdateAllConfigObjects() { - AssertOnWorkQueue(); - double startTime = Utility::GetTime(); - for (const Type::Ptr& type : Type::GetAllTypes()) { - ConfigType *ctype = dynamic_cast(type.get()); + m_Rcon->ExecuteQuery({"flushall"}); + // Use a Workqueue to pack objects in parallel + WorkQueue upq(25000, Configuration::Concurrency); + upq.SetName("RedisWriter:ConfigDump"); + + typedef std::pair TypePair; + std::vector types; + + for (const Type::Ptr& type : Type::GetAllTypes()) { + ConfigType *ctype = dynamic_cast(type.get()); if (!ctype) continue; - auto lcType(type->GetName().ToLower()); + String lcType (type->GetName().ToLower()); + types.emplace_back(ctype, lcType); + m_Rcon->ExecuteQuery({"DEL", m_PrefixConfigCheckSum + lcType, m_PrefixConfigObject + lcType, m_PrefixStatusObject + lcType}); + } - ExecuteQuery({"MULTI"}); + upq.ParallelFor(types, [this](const TypePair& type) { + size_t bulkCounter = 0; + auto attributes = new std::vector(); + attributes->emplace_back("HMSET"); + attributes->emplace_back(m_PrefixConfigObject + type.second); + auto customVars = new std::vector(); + customVars->emplace_back("HMSET"); + customVars->emplace_back(m_PrefixConfigCustomVar + type.second); + auto checksums = new std::vector(); + checksums->emplace_back("HMSET"); + checksums->emplace_back(m_PrefixConfigCheckSum + type.second); - /* Delete obsolete object keys first. */ - ExecuteQuery( - {"DEL", m_PrefixConfigCheckSum + lcType, m_PrefixConfigObject + lcType, m_PrefixStatusObject + lcType}); - /* fetch all objects and dump them */ - for (const ConfigObject::Ptr& object : ctype->GetObjects()) { - SendConfigUpdate(object, false); - SendStatusUpdate(object, false); + for (const ConfigObject::Ptr& object : type.first->GetObjects()) { + CreateConfigUpdate(object, *attributes, *customVars, *checksums, false); + SendStatusUpdate(object); + bulkCounter ++; + if (!bulkCounter % 100) { + if (attributes->size() > 2) { + m_Rcon->ExecuteQuery(*attributes); + attributes->erase(attributes->begin() + 2, attributes->end()); + } + if (customVars->size() > 2) { + m_Rcon->ExecuteQuery(*customVars); + customVars->erase(customVars->begin() + 2, customVars->end()); + } + if (checksums->size() > 2) { + m_Rcon->ExecuteQuery(*checksums); + checksums->erase(checksums->begin() + 2, checksums->end()); + } + } } + if (attributes->size() > 2) + m_Rcon->ExecuteQuery(*attributes); + if (customVars->size() > 2) + m_Rcon->ExecuteQuery(*customVars); + if (checksums->size() > 2) + m_Rcon->ExecuteQuery(*checksums); - /* publish config type dump finished */ - ExecuteQuery({"PUBLISH", "icinga:config:dump", lcType}); + Log(LogNotice, "RedisWriter") + << "Dumped " << bulkCounter << " objects of type " << type.second; + }); - ExecuteQuery({"EXEC"}); + upq.Join(); + + if (upq.HasExceptions()) { + for (auto exc : upq.GetExceptions()) { + Log(LogCritical, "RedisWriter") + << "Exception during ConfigDump: " << exc; + } } Log(LogInformation, "RedisWriter") << "Initial config/status dump finished in " << Utility::GetTime() - startTime << " seconds."; } -static ConfigObject::Ptr GetHostGroup(const String& name) +template +static ConfigObject::Ptr GetObjectByName(const String& name) { - return ConfigObject::GetObject(name); + return ConfigObject::GetObject(name); } -static ConfigObject::Ptr GetServiceGroup(const String& name) +// Used to update a single object, used for runtime updates +void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate) { - return ConfigObject::GetObject(name); + String typeName = object->GetReflectionType()->GetName().ToLower(); + auto attributes = new std::vector(); + attributes->emplace_back("HSET"); + attributes->emplace_back(m_PrefixConfigObject +typeName); + auto customVars = new std::vector(); + customVars->emplace_back("HSET"); + customVars->emplace_back(m_PrefixConfigCustomVar + typeName); + auto checksums = new std::vector(); + checksums->emplace_back("HSET"); + checksums->emplace_back(m_PrefixConfigCheckSum +typeName); + + CreateConfigUpdate(object, *attributes, *customVars, *checksums, runtimeUpdate); + + m_Rcon->ExecuteQuery(*attributes); + m_Rcon->ExecuteQuery(*customVars); + m_Rcon->ExecuteQuery(*checksums); } -static ConfigObject::Ptr GetUserGroup(const String& name) +/* Creates a config update with computed checksums etc. + * Writes attributes, customVars and checksums into the respective supplied vectors. Adds two values to each vector + * (if applicable), first the key then the value. To use in a Redis command the command (e.g. HSET) and the key (e.g. + * icinga:config:object:downtime) need to be prepended. There is nothing to indicate success or failure. + */ +void RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, std::vector& attributes, std::vector& customVars, std::vector& checksums, bool runtimeUpdate) { - return ConfigObject::GetObject(name); -} - -static ConfigObject::Ptr GetClude(const String& name) -{ - return ConfigObject::GetObject(name); -} - -void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTransaction, bool runtimeUpdate) -{ - AssertOnWorkQueue(); - - /* during startup we might send duplicated object config, ignore them without any connection */ - if (!m_Context) - return; - /* TODO: This isn't essentially correct as we don't keep track of config objects ourselves. This would avoid duplicated config updates at startup. if (!runtimeUpdate && m_ConfigDumpInProgress) return; */ - if (useTransaction) - ExecuteQuery({"MULTI"}); + if (m_Rcon == nullptr) + return; /* Calculate object specific checksums and store them in a different namespace. */ Type::Ptr type = object->GetReflectionType(); @@ -158,7 +209,6 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran } User::Ptr user = dynamic_pointer_cast(object); - if (user) { propertiesBlacklist.emplace("groups"); @@ -166,7 +216,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran ConfigObject::Ptr (*getGroup)(const String& name); groups = user->GetGroups(); - getGroup = &::GetUserGroup; + getGroup = &::GetObjectByName; checkSums->Set("groups_checksum", CalculateCheckSumArray(groups)); @@ -188,7 +238,6 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran } Notification::Ptr notification = dynamic_pointer_cast(object); - if (notification) { Host::Ptr host; Service::Ptr service; @@ -236,7 +285,6 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran /* Calculate checkable checksums. */ Checkable::Ptr checkable = dynamic_pointer_cast(object); - if (checkable) { /* groups_checksum, group_checksums */ propertiesBlacklist.emplace("groups"); @@ -251,13 +299,13 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran if (service) { groups = service->GetGroups(); - getGroup = &::GetServiceGroup; + getGroup = &::GetObjectByName; /* Calculate the host_checksum */ checkSums->Set("host_checksum", GetObjectIdentifier(host)); } else { groups = host->GetGroups(); - getGroup = &::GetHostGroup; + getGroup = &::GetObjectByName; } checkSums->Set("groups_checksum", CalculateCheckSumArray(groups)); @@ -298,139 +346,137 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran checkSums->Set("notes_url_checksum", CalculateCheckSumString(notesUrl)); if (!iconImage.IsEmpty()) checkSums->Set("icon_image_checksum", CalculateCheckSumString(iconImage)); - } else { - Zone::Ptr zone = dynamic_pointer_cast(object); + } - if (zone) { - propertiesBlacklist.emplace("endpoints"); + Zone::Ptr zone = dynamic_pointer_cast(object); + if (zone) { + propertiesBlacklist.emplace("endpoints"); - auto endpointObjects = zone->GetEndpoints(); - Array::Ptr endpoints = new Array(); - endpoints->Resize(endpointObjects.size()); + auto endpointObjects = zone->GetEndpoints(); + Array::Ptr endpoints = new Array(); + endpoints->Resize(endpointObjects.size()); - Array::SizeType i = 0; - for (auto& endpointObject : endpointObjects) { - endpoints->Set(i++, endpointObject->GetName()); + Array::SizeType i = 0; + for (auto& endpointObject : endpointObjects) { + endpoints->Set(i++, endpointObject->GetName()); + } + + checkSums->Set("endpoints_checksum", CalculateCheckSumArray(endpoints)); + + Array::Ptr parents(new Array); + + for (auto& parent : zone->GetAllParentsRaw()) { + parents->Add(GetObjectIdentifier(parent)); + } + + checkSums->Set("all_parents_checksums", parents); + checkSums->Set("all_parents_checksum", HashValue(zone->GetAllParents())); + } + + /* zone_checksum for endpoints already is calculated above. */ + + auto command(dynamic_pointer_cast(object)); + if (command) { + Dictionary::Ptr arguments = command->GetArguments(); + Dictionary::Ptr argumentChecksums = new Dictionary; + + if (arguments) { + ObjectLock argumentsLock(arguments); + + for (auto& kv : arguments) { + argumentChecksums->Set(kv.first, HashValue(kv.second)); } + } - checkSums->Set("endpoints_checksum", CalculateCheckSumArray(endpoints)); + checkSums->Set("arguments_checksum", HashValue(arguments)); + checkSums->Set("argument_checksums", argumentChecksums); + propertiesBlacklist.emplace("arguments"); - Array::Ptr parents(new Array); + Dictionary::Ptr envvars = command->GetEnv(); + Dictionary::Ptr envvarChecksums = new Dictionary; - for (auto& parent : zone->GetAllParentsRaw()) { - parents->Add(GetObjectIdentifier(parent)); + if (envvars) { + ObjectLock argumentsLock(envvars); + + for (auto& kv : envvars) { + envvarChecksums->Set(kv.first, HashValue(kv.second)); } + } - checkSums->Set("all_parents_checksums", parents); - checkSums->Set("all_parents_checksum", HashValue(zone->GetAllParents())); + checkSums->Set("envvars_checksum", HashValue(envvars)); + checkSums->Set("envvar_checksums", envvarChecksums); + propertiesBlacklist.emplace("env"); + } + + auto timeperiod(dynamic_pointer_cast(object)); + if (timeperiod) { + Dictionary::Ptr ranges = timeperiod->GetRanges(); + + checkSums->Set("ranges_checksum", HashValue(ranges)); + propertiesBlacklist.emplace("ranges"); + + // Compute checksums for Includes (like groups) + Array::Ptr includes; + ConfigObject::Ptr (*getInclude)(const String& name); + + includes = timeperiod->GetIncludes(); + getInclude = &::GetObjectByName; + + checkSums->Set("includes_checksum", CalculateCheckSumArray(includes)); + + Array::Ptr includeChecksums = new Array(); + + ObjectLock includesLock(includes); + ObjectLock includeChecksumsLock(includeChecksums); + + for (auto include : includes) { + includeChecksums->Add(GetObjectIdentifier((*getInclude)(include.Get()))); + } + + checkSums->Set("include_checksums", includeChecksums); + + // Compute checksums for Excludes (like groups) + Array::Ptr excludes; + ConfigObject::Ptr (*getExclude)(const String& name); + + excludes = timeperiod->GetExcludes(); + getExclude = &::GetObjectByName; + + checkSums->Set("excludes_checksum", CalculateCheckSumArray(excludes)); + + Array::Ptr excludeChecksums = new Array(); + + ObjectLock excludesLock(excludes); + ObjectLock excludeChecksumsLock(excludeChecksums); + + for (auto exclude : excludes) { + excludeChecksums->Add(GetObjectIdentifier((*getExclude)(exclude.Get()))); + } + + checkSums->Set("exclude_checksums", excludeChecksums); + } + + icinga::Comment::Ptr comment = dynamic_pointer_cast(object); + if (comment) { + propertiesBlacklist.emplace("name"); + propertiesBlacklist.emplace("host_name"); + + Host::Ptr host; + Service::Ptr service; + tie(host, service) = GetHostService(comment->GetCheckable()); + if (service) { + propertiesBlacklist.emplace("service_name"); + checkSums->Set("service_checksum", GetObjectIdentifier(service)); + typeName = "servicecomment"; } else { - /* zone_checksum for endpoints already is calculated above. */ - - auto command(dynamic_pointer_cast(object)); - - if (command) { - Dictionary::Ptr arguments = command->GetArguments(); - Dictionary::Ptr argumentChecksums = new Dictionary; - - if (arguments) { - ObjectLock argumentsLock(arguments); - - for (auto& kv : arguments) { - argumentChecksums->Set(kv.first, HashValue(kv.second)); - } - } - - checkSums->Set("arguments_checksum", HashValue(arguments)); - checkSums->Set("argument_checksums", argumentChecksums); - propertiesBlacklist.emplace("arguments"); - - Dictionary::Ptr envvars = command->GetEnv(); - Dictionary::Ptr envvarChecksums = new Dictionary; - - if (envvars) { - ObjectLock argumentsLock(envvars); - - for (auto& kv : envvars) { - envvarChecksums->Set(kv.first, HashValue(kv.second)); - } - } - - checkSums->Set("envvars_checksum", HashValue(envvars)); - checkSums->Set("envvar_checksums", envvarChecksums); - propertiesBlacklist.emplace("env"); - } else { - auto timeperiod(dynamic_pointer_cast(object)); - - if (timeperiod) { - Dictionary::Ptr ranges = timeperiod->GetRanges(); - - checkSums->Set("ranges_checksum", HashValue(ranges)); - propertiesBlacklist.emplace("ranges"); - - // Compute checksums for Includes (like groups) - Array::Ptr includes; - ConfigObject::Ptr (*getInclude)(const String& name); - - includes = timeperiod->GetIncludes(); - getInclude = &::GetClude; - - checkSums->Set("includes_checksum", CalculateCheckSumArray(includes)); - - Array::Ptr includeChecksums = new Array(); - - ObjectLock includesLock(includes); - ObjectLock includeChecksumsLock(includeChecksums); - - for (auto include : includes) { - includeChecksums->Add(GetObjectIdentifier((*getInclude)(include.Get()))); - } - - checkSums->Set("include_checksums", includeChecksums); - - // Compute checksums for Excludes (like groups) - Array::Ptr excludes; - ConfigObject::Ptr (*getExclude)(const String& name); - - excludes = timeperiod->GetExcludes(); - getExclude = &::GetClude; - - checkSums->Set("excludes_checksum", CalculateCheckSumArray(excludes)); - - Array::Ptr excludeChecksums = new Array(); - - ObjectLock excludesLock(excludes); - ObjectLock excludeChecksumsLock(excludeChecksums); - - for (auto exclude : excludes) { - excludeChecksums->Add(GetObjectIdentifier((*getExclude)(exclude.Get()))); - } - - checkSums->Set("exclude_checksums", excludeChecksums); - } else { - icinga::Comment::Ptr comment = dynamic_pointer_cast(object); - if (comment) { - propertiesBlacklist.emplace("name"); - propertiesBlacklist.emplace("host_name"); - - Host::Ptr host; - Service::Ptr service; - tie(host, service) = GetHostService(comment->GetCheckable()); - if (service) { - propertiesBlacklist.emplace("service_name"); - checkSums->Set("service_checksum", GetObjectIdentifier(service)); - typeName = "servicecomment"; - } else { - checkSums->Set("host_checksum", GetObjectIdentifier(host)); - typeName = "hostcomment"; - } - } - } - } + checkSums->Set("host_checksum", GetObjectIdentifier(host)); + typeName = "hostcomment"; } } /* Send all object attributes to Redis, no extra checksums involved here. */ - UpdateObjectAttrs(m_PrefixConfigObject, object, FAConfig, typeName); + auto tempAttrs = (UpdateObjectAttrs(m_PrefixConfigObject, object, FAConfig, typeName)); + attributes.insert(attributes.end(), std::begin(tempAttrs), std::end(tempAttrs)); /* Custom var checksums. */ CustomVarObject::Ptr customVarObject = dynamic_pointer_cast(object); @@ -445,10 +491,8 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran if (vars) { auto varsJson(JsonEncode(vars)); - Log(LogDebug, "RedisWriter") - << "HSET " << m_PrefixConfigCustomVar + typeName << " " << objectKey << " " << varsJson; - - ExecuteQuery({"HSET", m_PrefixConfigCustomVar + typeName, objectKey, varsJson}); + customVars.emplace_back(objectKey); + customVars.emplace_back(varsJson); } } @@ -457,56 +501,34 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool useTran String checkSumsBody = JsonEncode(checkSums); - Log(LogDebug, "RedisWriter") - << "HSET " << m_PrefixConfigCheckSum + typeName << " " << objectKey << " " << checkSumsBody; - - ExecuteQuery({"HSET", m_PrefixConfigCheckSum + typeName, objectKey, checkSumsBody}); + checksums.emplace_back(objectKey); + checksums.emplace_back(checkSumsBody); /* Send an update event to subscribers. */ if (runtimeUpdate) { - ExecuteQuery({"PUBLISH", "icinga:config:update", typeName + ":" + objectKey}); + m_Rcon->ExecuteQuery({"PUBLISH", "icinga:config:update", typeName + ":" + objectKey}); } - - if (useTransaction) - ExecuteQuery({"EXEC"}); } void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) { - AssertOnWorkQueue(); - - /* during startup we might send duplicated object config, ignore them without any connection */ - if (!m_Context) - return; - String typeName = object->GetReflectionType()->GetName().ToLower(); String objectKey = GetObjectIdentifier(object); - ExecuteQueries({ + m_Rcon->ExecuteQueries({ {"HDEL", m_PrefixConfigObject + typeName, objectKey}, {"DEL", m_PrefixStatusObject + typeName + ":" + objectKey}, - {"PUBLISH", "icinga:config:delete", typeName + ":" + objectKey} + {"PUBLISH", "icinga:config:delete", typeName + ":" + objectKey} }); - } -void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, bool useTransaction) +void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) { - AssertOnWorkQueue(); - - /* during startup we might receive check results, ignore them without any connection */ - if (!m_Context) - return; - - if (useTransaction) - ExecuteQuery({"MULTI"}); - //TODO: Manage type names - UpdateObjectAttrs(m_PrefixStatusObject, object, FAState, ""); + //TODO: Figure out what we need when we implement the history and state sync +// UpdateObjectAttrs(m_PrefixStatusObject, object, FAState, ""); - if (useTransaction) - ExecuteQuery({"EXEC"}); // /* Serialize config object attributes */ // Dictionary::Ptr objectAttrs = SerializeObjectAttrs(object, FAState); @@ -586,7 +608,7 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, bool useTran // } } -void RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType, +std::vector RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride) { Type::Ptr type = object->GetReflectionType(); @@ -616,7 +638,8 @@ void RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject: if (!typeNameOverride.IsEmpty()) typeName = typeNameOverride.ToLower(); - ExecuteQuery({"HSET", keyPrefix + typeName, GetObjectIdentifier(object), JsonEncode(attrs)}); + return {GetObjectIdentifier(object), JsonEncode(attrs)}; + //m_Rcon->ExecuteQuery({"HSET", keyPrefix + typeName, GetObjectIdentifier(object), JsonEncode(attrs)}); } void RedisWriter::StateChangedHandler(const ConfigObject::Ptr& object) @@ -624,7 +647,7 @@ void RedisWriter::StateChangedHandler(const ConfigObject::Ptr& object) Type::Ptr type = object->GetReflectionType(); for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { - rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendStatusUpdate, rw, object, true)); + rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendStatusUpdate, rw, object)); } } @@ -633,15 +656,15 @@ void RedisWriter::VersionChangedHandler(const ConfigObject::Ptr& object) Type::Ptr type = object->GetReflectionType(); if (object->IsActive()) { - /* Create or update the object config */ + // Create or update the object config for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { - rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendConfigUpdate, rw.get(), object, true, true)); +// rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendConfigUpdate, rw, object, true)); } } else if (!object->IsActive() && - object->GetExtension("ConfigObjectDeleted")) { /* same as in apilistener-configsync.cpp */ - /* Delete object config */ + object->GetExtension("ConfigObjectDeleted")) { // same as in apilistener-configsync.cpp + // Delete object config for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { - rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendConfigDelete, rw.get(), object)); + rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendConfigDelete, rw, object)); } } } diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 626506f65..2592828a4 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -336,3 +336,33 @@ String RedisWriter::HashValue(const Value& value, const std::set& proper return SHA1(PackObject(temp)); } +//Used to duplicate a redisReply, needed as redisReplies are freed when the async callback finishes +redisReply* RedisWriter::dupReplyObject(redisReply* reply) +{ + redisReply* r = (redisReply*)calloc(1, sizeof(*r)); + memcpy(r, reply, sizeof(*r)); + if(REDIS_REPLY_ERROR==reply->type || REDIS_REPLY_STRING==reply->type || REDIS_REPLY_STATUS==reply->type) //copy str + { + r->str = (char*)malloc(reply->len+1); + memcpy(r->str, reply->str, reply->len); + r->str[reply->len] = '\0'; + } + else if(REDIS_REPLY_ARRAY==reply->type) //copy array + { + r->element = (redisReply**)calloc(reply->elements, sizeof(redisReply*)); + memset(r->element, 0, r->elements*sizeof(redisReply*)); + for(uint32_t i=0; ielements; ++i) + { + if(NULL!=reply->element[i]) + { + if( NULL == (r->element[i] = dupReplyObject(reply->element[i])) ) + { + //clone child failed, free current reply, and return NULL + freeReplyObject(r); + return NULL; + } + } + } + } + return r; +} diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index 8c4c74149..d93b602a7 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -19,8 +19,11 @@ #include "redis/rediswriter.hpp" #include "redis/rediswriter-ti.cpp" +#include "redis/redisconnection.hpp" #include "remote/eventqueue.hpp" #include "base/json.hpp" +#include "rediswriter.hpp" + #include using namespace icinga; @@ -31,8 +34,10 @@ using namespace icinga; REGISTER_TYPE(RedisWriter); RedisWriter::RedisWriter() - : m_Context(NULL) +: m_Rcon(nullptr) { + m_Rcon = nullptr; + m_WorkQueue.SetName("RedisWriter"); m_PrefixConfigObject = "icinga:config:object:"; @@ -52,6 +57,10 @@ void RedisWriter::Start(bool runtimeCreated) << "'" << GetName() << "' started."; m_ConfigDumpInProgress = false; + m_ConfigDumpDone = false; + + m_Rcon = new RedisConnection(GetHost(), GetPort(), GetPath(), GetPassword(), GetDbIndex()); + m_Rcon->Start(); m_WorkQueue.SetExceptionCallback(std::bind(&RedisWriter::ExceptionHandler, this, _1)); @@ -75,6 +84,7 @@ void RedisWriter::Start(bool runtimeCreated) boost::thread thread(std::bind(&RedisWriter::HandleEvents, this)); thread.detach(); + } void RedisWriter::ExceptionHandler(boost::exception_ptr exp) @@ -83,11 +93,6 @@ void RedisWriter::ExceptionHandler(boost::exception_ptr exp) Log(LogDebug, "RedisWriter") << "Exception during redis operation: " << DiagnosticInformation(exp); - - if (m_Context) { - redisFree(m_Context); - m_Context = NULL; - } } void RedisWriter::ReconnectTimerHandler() @@ -99,55 +104,23 @@ void RedisWriter::TryToReconnect() { AssertOnWorkQueue(); - if (m_Context) + if (m_ConfigDumpDone && m_Rcon->IsConnected()) return; - - String path = GetPath(); - String host = GetHost(); - - Log(LogInformation, "RedisWriter", "Trying to connect to redis server"); - - if (path.IsEmpty()) - m_Context = redisConnect(host.CStr(), GetPort()); - else - m_Context = redisConnectUnix(path.CStr()); - - if (!m_Context || m_Context->err) { - if (!m_Context) { - Log(LogWarning, "RedisWriter", "Cannot allocate redis context."); - } else { - Log(LogWarning, "RedisWriter", "Connection error: ") - << m_Context->errstr; - } - - if (m_Context) { - redisFree(m_Context); - m_Context = NULL; - } - - return; - } - - String password = GetPassword(); - - /* TODO: exception is fired but terminates reconnect silently. - * Error case: Password does not match, or even: "Client sent AUTH, but no password is set" which also results in an error. - */ - if (!password.IsEmpty()) - ExecuteQuery({ "AUTH", password }); - - int dbIndex = GetDbIndex(); - - if (dbIndex != 0) - ExecuteQuery({ "SELECT", Convert::ToString(dbIndex) }); + else if (!m_Rcon->IsConnected()) + m_Rcon->Start(); UpdateSubscriptions(); + if (m_ConfigDumpInProgress || m_ConfigDumpDone) + return; + /* Config dump */ m_ConfigDumpInProgress = true; UpdateAllConfigObjects(); + m_ConfigDumpDone = true; + m_ConfigDumpInProgress = false; } @@ -164,15 +137,12 @@ void RedisWriter::UpdateSubscriptions() m_Subscriptions.clear(); - if (!m_Context) - return; - long long cursor = 0; String keyPrefix = "icinga:subscription:"; do { - std::shared_ptr reply = ExecuteQuery({ "SCAN", Convert::ToString(cursor), "MATCH", keyPrefix + "*", "COUNT", "1000" }); + auto reply = RedisGet({ "SCAN", Convert::ToString(cursor), "MATCH", keyPrefix + "*", "COUNT", "1000" }); VERIFY(reply->type == REDIS_REPLY_ARRAY); VERIFY(reply->elements % 2 == 0); @@ -207,7 +177,7 @@ void RedisWriter::UpdateSubscriptions() bool RedisWriter::GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi) { try { - std::shared_ptr redisReply = ExecuteQuery({ "SMEMBERS", key }); + redisReply *redisReply = RedisGet({ "SMEMBERS", key }); VERIFY(redisReply->type == REDIS_REPLY_ARRAY); if (redisReply->elements == 0) @@ -239,13 +209,10 @@ void RedisWriter::PublishStats() { AssertOnWorkQueue(); - if (!m_Context) - return; - Dictionary::Ptr status = GetStats(); String jsonStats = JsonEncode(status); - ExecuteQuery({ "PUBLISH", "icinga:stats", jsonStats }); + m_Rcon->ExecuteQuery({ "PUBLISH", "icinga:stats", jsonStats }); } void RedisWriter::HandleEvents() @@ -288,9 +255,6 @@ void RedisWriter::HandleEvent(const Dictionary::Ptr& event) { AssertOnWorkQueue(); - if (!m_Context) - return; - for (const std::pair& kv : m_Subscriptions) { const auto& name = kv.first; const auto& rsi = kv.second; @@ -300,10 +264,11 @@ void RedisWriter::HandleEvent(const Dictionary::Ptr& event) String body = JsonEncode(event); - std::shared_ptr maxExists = ExecuteQuery({ "EXISTS", "icinga:subscription:" + name + ":limit" }); + redisReply *maxExists = RedisGet({ "EXISTS", "icinga:subscription:" + name + ":limit" }); + long maxEvents = MAX_EVENTS_DEFAULT; if (maxExists->integer) { - std::shared_ptr redisReply = ExecuteQuery({ "GET", "icinga:subscription:" + name + ":limit"}); + redisReply *redisReply =RedisGet({ "GET", "icinga:subscription:" + name + ":limit"}); VERIFY(redisReply->type == REDIS_REPLY_STRING); Log(LogInformation, "RedisWriter") @@ -312,10 +277,11 @@ void RedisWriter::HandleEvent(const Dictionary::Ptr& event) maxEvents = Convert::ToLong(redisReply->str); } - ExecuteQuery({ "MULTI" }); - ExecuteQuery({ "LPUSH", "icinga:event:" + name, body }); - ExecuteQuery({ "LTRIM", "icinga:event:" + name, "0", String(maxEvents - 1)}); - ExecuteQuery({ "EXEC" }); + m_Rcon->ExecuteQueries({ + { "MULTI" }, + { "LPUSH", "icinga:event:" + name, body }, + { "LTRIM", "icinga:event:" + name, "0", String(maxEvents - 1)}, + { "EXEC" }}); } } @@ -323,16 +289,14 @@ void RedisWriter::SendEvent(const Dictionary::Ptr& event) { AssertOnWorkQueue(); - if (!m_Context) - return; - String body = JsonEncode(event); // Log(LogInformation, "RedisWriter") // << "Sending event \"" << body << "\""; - ExecuteQuery({ "PUBLISH", "icinga:event:all", body }); - ExecuteQuery({ "PUBLISH", "icinga:event:" + event->Get("type"), body }); + m_Rcon->ExecuteQueries({ + { "PUBLISH", "icinga:event:all", body }, + { "PUBLISH", "icinga:event:" + event->Get("type"), body }}); } void RedisWriter::Stop(bool runtimeRemoved) @@ -348,95 +312,47 @@ void RedisWriter::AssertOnWorkQueue() ASSERT(m_WorkQueue.IsWorkerThread()); } -std::shared_ptr RedisWriter::ExecuteQuery(const std::vector& query) -{ - const char **argv; - size_t *argvlen; - argv = new const char *[query.size()]; - argvlen = new size_t[query.size()]; +/* + * This whole spiel is required as we mostly use a "fire and forget" approach with the Redis Connection. To wait for a + * reply from Redis we have to wait for the callback to finish, this is done with the help of this struct. ready, cv + * and mtx are used for making sure we have the redisReply when we return. + */ +struct synchronousWait { + bool ready; + boost::condition_variable cv; + boost::mutex mtx; + redisReply* reply; +}; - for (std::vector::size_type i = 0; i < query.size(); i++) { - argv[i] = query[i].CStr(); - argvlen[i] = query[i].GetLength(); - } +void RedisWriter::RedisQueryCallback(redisAsyncContext *c, void *r, void *p) { + auto wait = (struct synchronousWait*) p; + auto rp = reinterpret_cast(r); - redisReply *reply = reinterpret_cast(redisCommandArgv(m_Context, query.size(), argv, argvlen)); - delete [] argv; - delete [] argvlen; + if (r == NULL) + wait->reply = nullptr; + else + wait->reply = RedisWriter::dupReplyObject(rp); - if (reply->type == REDIS_REPLY_ERROR) { - Log(LogCritical, "RedisWriter") - << "Redis query failed: " << reply->str; - - String msg = reply->str; - - freeReplyObject(reply); - - BOOST_THROW_EXCEPTION( - redis_error() - << errinfo_message(msg) - << errinfo_redis_query(Utility::Join(Array::FromVector(query), ' ', false)) - ); - } - - return std::shared_ptr(reply, freeReplyObject); + boost::mutex::scoped_lock lock(wait->mtx); + wait->ready = true; + wait->cv.notify_all(); } -std::vector > RedisWriter::ExecuteQueries(const std::vector >& queries) -{ - const char **argv; - size_t *argvlen; - for (const auto& query : queries) { - argv = new const char *[query.size()]; - argvlen = new size_t[query.size()]; +redisReply* RedisWriter::RedisGet(const std::vector& query) { + auto *wait = new synchronousWait; + wait->ready = false; - for (std::vector::size_type i = 0; i < query.size(); i++) { - argv[i] = query[i].CStr(); - argvlen[i] = query[i].GetLength(); - } + m_Rcon->ExecuteQuery(query, RedisQueryCallback, wait); - redisAppendCommandArgv(m_Context, query.size(), argv, argvlen); - - delete [] argv; - delete [] argvlen; + boost::mutex::scoped_lock lock(wait->mtx); + while (!wait->ready) { + wait->cv.timed_wait(lock, boost::posix_time::milliseconds(long(15 * 1000))); + if (!wait->ready) + wait->ready = true; } - std::vector > replies; - - for (size_t i = 0; i < queries.size(); i++) { - redisReply *rawReply; - - if (redisGetReply(m_Context, reinterpret_cast(&rawReply)) == REDIS_ERR) { - BOOST_THROW_EXCEPTION( - redis_error() - << errinfo_message("redisGetReply() failed") - ); - } - - std::shared_ptr reply(rawReply, freeReplyObject); - replies.push_back(reply); - } - - for (size_t i = 0; i < queries.size(); i++) { - const auto& query = queries[i]; - const auto& reply = replies[i]; - - if (reply->type == REDIS_REPLY_ERROR) { - Log(LogCritical, "RedisWriter") - << "Redis query failed: " << reply->str; - - String msg = reply->str; - - BOOST_THROW_EXCEPTION( - redis_error() - << errinfo_message(msg) - << errinfo_redis_query(Utility::Join(Array::FromVector(query), ' ', false)) - ); - } - } - - return replies; -} + return wait->reply; +} \ No newline at end of file diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index a47cfd9ab..34ffbee9d 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -25,6 +25,7 @@ #include "remote/messageorigin.hpp" #include "base/timer.hpp" #include "base/workqueue.hpp" +#include "redis/redisconnection.hpp" #include namespace icinga @@ -66,10 +67,11 @@ private: /* config & status dump */ void UpdateAllConfigObjects(); - void SendConfigUpdate(const ConfigObject::Ptr& object, bool useTransaction, bool runtimeUpdate = false); + void SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate); + void CreateConfigUpdate(const ConfigObject::Ptr& object, std::vector& attributes, std::vector& customVars, std::vector& checksums, bool runtimeUpdate); void SendConfigDelete(const ConfigObject::Ptr& object); - void SendStatusUpdate(const ConfigObject::Ptr& object, bool useTransaction); - void UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride); + void SendStatusUpdate(const ConfigObject::Ptr& object); + std::vector UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride); /* Stats */ Dictionary::Ptr GetStats(); @@ -96,14 +98,16 @@ private: void ExceptionHandler(boost::exception_ptr exp); - std::shared_ptr ExecuteQuery(const std::vector& query); - std::vector > ExecuteQueries(const std::vector >& queries); + //Used to get a reply from the asyncronous connection + redisReply* RedisGet(const std::vector& query); + static void RedisQueryCallback(redisAsyncContext *c, void *r, void *p); + static redisReply* dupReplyObject(redisReply* reply); + Timer::Ptr m_StatsTimer; Timer::Ptr m_ReconnectTimer; Timer::Ptr m_SubscriptionTimer; WorkQueue m_WorkQueue; - redisContext *m_Context; std::map m_Subscriptions; String m_PrefixConfigObject; @@ -112,13 +116,10 @@ private: String m_PrefixStatusObject; bool m_ConfigDumpInProgress; + bool m_ConfigDumpDone; + + RedisConnection::Ptr m_Rcon; }; - -struct redis_error : virtual std::exception, virtual boost::exception { }; - -struct errinfo_redis_query_; -typedef boost::error_info errinfo_redis_query; - } #endif /* REDISWRITER_H */ From b59189b8d3cbabb6ef2a0b1ac382a8bb4ae4b14a Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Tue, 23 Oct 2018 16:32:20 +0200 Subject: [PATCH 073/219] Simplify vector usage Also leak less memory ;) --- lib/redis/rediswriter-objects.cpp | 66 ++++++++++++------------------- 1 file changed, 26 insertions(+), 40 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 9e36dcbff..9edc4cc36 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -83,42 +83,35 @@ void RedisWriter::UpdateAllConfigObjects() upq.ParallelFor(types, [this](const TypePair& type) { size_t bulkCounter = 0; - auto attributes = new std::vector(); - attributes->emplace_back("HMSET"); - attributes->emplace_back(m_PrefixConfigObject + type.second); - auto customVars = new std::vector(); - customVars->emplace_back("HMSET"); - customVars->emplace_back(m_PrefixConfigCustomVar + type.second); - auto checksums = new std::vector(); - checksums->emplace_back("HMSET"); - checksums->emplace_back(m_PrefixConfigCheckSum + type.second); - + std::vector attributes = std::vector({"HMSET", m_PrefixConfigObject + type.second}); + std::vector customVars = std::vector({"HMSET", m_PrefixConfigCustomVar + type.second}); + std::vector checksums = std::vector({"HMSET", m_PrefixConfigCheckSum + type.second}); for (const ConfigObject::Ptr& object : type.first->GetObjects()) { - CreateConfigUpdate(object, *attributes, *customVars, *checksums, false); + CreateConfigUpdate(object, attributes, customVars, checksums, false); SendStatusUpdate(object); bulkCounter ++; if (!bulkCounter % 100) { - if (attributes->size() > 2) { - m_Rcon->ExecuteQuery(*attributes); - attributes->erase(attributes->begin() + 2, attributes->end()); + if (attributes.size() > 2) { + m_Rcon->ExecuteQuery(attributes); + attributes.erase(attributes.begin() + 2, attributes.end()); } - if (customVars->size() > 2) { - m_Rcon->ExecuteQuery(*customVars); - customVars->erase(customVars->begin() + 2, customVars->end()); + if (customVars.size() > 2) { + m_Rcon->ExecuteQuery(customVars); + customVars.erase(customVars.begin() + 2, customVars.end()); } - if (checksums->size() > 2) { - m_Rcon->ExecuteQuery(*checksums); - checksums->erase(checksums->begin() + 2, checksums->end()); + if (checksums.size() > 2) { + m_Rcon->ExecuteQuery(checksums); + checksums.erase(checksums.begin() + 2, checksums.end()); } } } - if (attributes->size() > 2) - m_Rcon->ExecuteQuery(*attributes); - if (customVars->size() > 2) - m_Rcon->ExecuteQuery(*customVars); - if (checksums->size() > 2) - m_Rcon->ExecuteQuery(*checksums); + if (attributes.size() > 2) + m_Rcon->ExecuteQuery(attributes); + if (customVars.size() > 2) + m_Rcon->ExecuteQuery(customVars); + if (checksums.size() > 2) + m_Rcon->ExecuteQuery(checksums); Log(LogNotice, "RedisWriter") << "Dumped " << bulkCounter << " objects of type " << type.second; @@ -147,21 +140,15 @@ static ConfigObject::Ptr GetObjectByName(const String& name) void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate) { String typeName = object->GetReflectionType()->GetName().ToLower(); - auto attributes = new std::vector(); - attributes->emplace_back("HSET"); - attributes->emplace_back(m_PrefixConfigObject +typeName); - auto customVars = new std::vector(); - customVars->emplace_back("HSET"); - customVars->emplace_back(m_PrefixConfigCustomVar + typeName); - auto checksums = new std::vector(); - checksums->emplace_back("HSET"); - checksums->emplace_back(m_PrefixConfigCheckSum +typeName); + std::vector attributes = std::vector({"HMSET", m_PrefixConfigObject + typeName}); + std::vector customVars = std::vector({"HMSET", m_PrefixConfigCustomVar + typeName}); + std::vector checksums = std::vector({"HMSET", m_PrefixConfigCheckSum + typeName}); - CreateConfigUpdate(object, *attributes, *customVars, *checksums, runtimeUpdate); + CreateConfigUpdate(object, attributes, customVars, checksums, runtimeUpdate); - m_Rcon->ExecuteQuery(*attributes); - m_Rcon->ExecuteQuery(*customVars); - m_Rcon->ExecuteQuery(*checksums); + m_Rcon->ExecuteQuery(attributes); + m_Rcon->ExecuteQuery(customVars); + m_Rcon->ExecuteQuery(checksums); } /* Creates a config update with computed checksums etc. @@ -504,7 +491,6 @@ void RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, std::vecto checksums.emplace_back(objectKey); checksums.emplace_back(checkSumsBody); - /* Send an update event to subscribers. */ if (runtimeUpdate) { m_Rcon->ExecuteQuery({"PUBLISH", "icinga:config:update", typeName + ":" + objectKey}); From f89e649871eb0d338ce9b3398a68899a7cd68fa3 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Thu, 25 Oct 2018 17:48:32 +0200 Subject: [PATCH 074/219] Add recursive callback to manage auth and db select --- lib/redis/redisconnection.cpp | 100 +++++++++++++++++++++++----------- lib/redis/redisconnection.hpp | 21 +++++++ lib/redis/rediswriter.cpp | 15 +++-- 3 files changed, 98 insertions(+), 38 deletions(-) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index db5fb2742..7171fc638 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -30,7 +30,7 @@ using namespace icinga; RedisConnection::RedisConnection(const String host, const int port, const String path, const String password, const int db) : - m_Host(host), m_Port(port), m_Path(path), m_Password(password), m_DbIndex(db), m_Context(NULL) + m_Host(host), m_Port(port), m_Path(path), m_Password(password), m_DbIndex(db), m_Context(NULL), m_Connected(false) { m_RedisConnectionWorkQueue.SetName("RedisConnection"); } @@ -70,45 +70,84 @@ void RedisConnection::HandleRW() } } + +void RedisConnection::RedisInitialCallback(redisAsyncContext *c, void *r, void *p) +{ + auto *state = (ConnectionState *) p; + if (r != nullptr) { + redisReply *rep = (redisReply *) r; + if (rep->type == REDIS_REPLY_ERROR) { + Log(LogCritical, "RedisConnection") + << "Failed to connect to Redis: " << rep->str; + state->conn->m_Connected = false; + return; + } + } + + if (state->state != Starting && (!r || c->err)) { + Log(LogCritical, "RedisConnection") << c->errstr; + state->conn->m_Connected = false; + return; + } + + if (state->state == Starting) { + state->state = Auth; + if (!state->conn->m_Password.IsEmpty()) { + boost::mutex::scoped_lock lock(state->conn->m_CMutex); + redisAsyncCommand(c, &RedisInitialCallback, p, "AUTH %s", state->conn->m_Password.CStr()); + return; + } + } + if (state->state == Auth) + { + state->state = DBSelect; + if (state->conn->m_DbIndex != 0) { + boost::mutex::scoped_lock lock(state->conn->m_CMutex); + redisAsyncCommand(c, &RedisInitialCallback, p, "SELECT %d", state->conn->m_DbIndex); + return; + } + } + if (state->state == DBSelect) + state->conn->m_Connected = true; +} +bool RedisConnection::IsConnected() { + return m_Connected; +} + + void RedisConnection::Connect() { if (m_Context) return; Log(LogInformation, "RedisWriter", "Trying to connect to redis server Async"); - boost::mutex::scoped_lock lock(m_CMutex); + { + boost::mutex::scoped_lock lock(m_CMutex); - if (m_Path.IsEmpty()) - m_Context = redisAsyncConnect(m_Host.CStr(), m_Port); - else - m_Context = redisAsyncConnectUnix(m_Path.CStr()); + if (m_Path.IsEmpty()) + m_Context = redisAsyncConnect(m_Host.CStr(), m_Port); + else + m_Context = redisAsyncConnectUnix(m_Path.CStr()); - if (!m_Context || m_Context->err) { - if (!m_Context) { - Log(LogWarning, "RedisWriter", "Cannot allocate redis context."); - } else { - Log(LogWarning, "RedisWriter", "Connection error: ") - << m_Context->errstr; + if (!m_Context || m_Context->err) { + if (!m_Context) { + Log(LogWarning, "RedisWriter", "Cannot allocate redis context."); + } else { + Log(LogWarning, "RedisWriter", "Connection error: ") + << m_Context->errstr; + } + + if (m_Context) { + redisAsyncFree(m_Context); + m_Context = NULL; + } } - if (m_Context) { - redisAsyncFree(m_Context); - m_Context = NULL; - } + redisAsyncSetDisconnectCallback(m_Context, &DisconnectCallback); } - redisAsyncSetDisconnectCallback(m_Context, &DisconnectCallback); - - /* TODO: This currently does not work properly: - * In case of error the connection is broken, yet the Context is not set to faulty. May be a bug with hiredis. - * Error case: Password does not match, or even: "Client sent AUTH, but no password is set" which also results in an error. - */ - if (!m_Password.IsEmpty()) { - ExecuteQuery({"AUTH", m_Password}); - } - - if (m_DbIndex != 0) - ExecuteQuery({"SELECT", Convert::ToString(m_DbIndex)}); + m_State = ConnectionState{Starting, this}; + RedisInitialCallback(m_Context, nullptr, (void*)&m_State); } void RedisConnection::Disconnect() @@ -129,11 +168,6 @@ void RedisConnection::DisconnectCallback(const redisAsyncContext *c, int status) } -bool RedisConnection::IsConnected() -{ - return (REDIS_CONNECTED & m_Context->c.flags) == REDIS_CONNECTED; -} - void RedisConnection::ExecuteQuery(const std::vector& query, redisCallbackFn *fn, void *privdata) { m_RedisConnectionWorkQueue.Enqueue(std::bind(&RedisConnection::SendMessageInternal, this, query, fn, privdata)); diff --git a/lib/redis/redisconnection.hpp b/lib/redis/redisconnection.hpp index c968c15af..6a4afe285 100644 --- a/lib/redis/redisconnection.hpp +++ b/lib/redis/redisconnection.hpp @@ -31,6 +31,20 @@ namespace icinga * * @ingroup redis */ + + enum conn_state{ + Starting, + Auth, + DBSelect, + Done, + }; + + class RedisConnection; + struct ConnectionState { + conn_state state; + RedisConnection *conn; + }; + class RedisConnection final : public Object { public: @@ -62,6 +76,9 @@ namespace icinga static void DisconnectCallback(const redisAsyncContext *c, int status); + static void RedisInitialCallback(redisAsyncContext *c, void *r, void *p); + + WorkQueue m_RedisConnectionWorkQueue{100000}; Timer::Ptr m_EventLoop; @@ -72,14 +89,18 @@ namespace icinga int m_Port; String m_Password; int m_DbIndex; + bool m_Connected; boost::mutex m_CMutex; + ConnectionState m_State; + }; struct redis_error : virtual std::exception, virtual boost::exception { }; struct errinfo_redis_query_; typedef boost::error_info errinfo_redis_query; + } #endif //REDISCONNECTION_H diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index d93b602a7..ca8099315 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -104,11 +104,13 @@ void RedisWriter::TryToReconnect() { AssertOnWorkQueue(); - if (m_ConfigDumpDone && m_Rcon->IsConnected()) + if (m_ConfigDumpDone) return; - else if (!m_Rcon->IsConnected()) + else m_Rcon->Start(); + if (!m_Rcon->IsConnected()) + return; UpdateSubscriptions(); if (m_ConfigDumpInProgress || m_ConfigDumpDone) @@ -135,8 +137,11 @@ void RedisWriter::UpdateSubscriptions() Log(LogInformation, "RedisWriter", "Updating Redis subscriptions"); - m_Subscriptions.clear(); - + if (!m_Rcon->IsConnected()) { + Log(LogCritical, "DEBUG, Redis") + << "NO CONNECT CHIEF"; + return; + } long long cursor = 0; String keyPrefix = "icinga:subscription:"; @@ -349,7 +354,7 @@ redisReply* RedisWriter::RedisGet(const std::vector& query) { boost::mutex::scoped_lock lock(wait->mtx); while (!wait->ready) { - wait->cv.timed_wait(lock, boost::posix_time::milliseconds(long(15 * 1000))); + wait->cv.wait(lock); if (!wait->ready) wait->ready = true; } From 0634e27d6de1384b03f2e3103a9bec0fd3a93e4c Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Fri, 26 Oct 2018 14:07:07 +0200 Subject: [PATCH 075/219] Add new connection handling --- lib/redis/redisconnection.cpp | 63 ++++++++++++++++++------------- lib/redis/redisconnection.hpp | 1 + lib/redis/rediswriter-objects.cpp | 2 - lib/redis/rediswriter.cpp | 7 ++++ 4 files changed, 45 insertions(+), 28 deletions(-) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index 7171fc638..4391a68f7 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -60,6 +60,8 @@ void RedisConnection::HandleRW() try { { boost::mutex::scoped_lock lock(m_CMutex); + if (!m_Connected) + return; redisAsyncHandleWrite(m_Context); redisAsyncHandleRead(m_Context); } @@ -74,7 +76,11 @@ void RedisConnection::HandleRW() void RedisConnection::RedisInitialCallback(redisAsyncContext *c, void *r, void *p) { auto *state = (ConnectionState *) p; - if (r != nullptr) { + if (state->state != Starting && !r) { + Log(LogCritical, "RedisConnection") + << "No answer from Redis during initial connection, is the Redis server running?"; + return; + } else if (r != nullptr) { redisReply *rep = (redisReply *) r; if (rep->type == REDIS_REPLY_ERROR) { Log(LogCritical, "RedisConnection") @@ -84,12 +90,6 @@ void RedisConnection::RedisInitialCallback(redisAsyncContext *c, void *r, void * } } - if (state->state != Starting && (!r || c->err)) { - Log(LogCritical, "RedisConnection") << c->errstr; - state->conn->m_Connected = false; - return; - } - if (state->state == Starting) { state->state = Auth; if (!state->conn->m_Password.IsEmpty()) { @@ -117,7 +117,7 @@ bool RedisConnection::IsConnected() { void RedisConnection::Connect() { - if (m_Context) + if (m_Connected) return; Log(LogInformation, "RedisWriter", "Trying to connect to redis server Async"); @@ -129,20 +129,9 @@ void RedisConnection::Connect() else m_Context = redisAsyncConnectUnix(m_Path.CStr()); - if (!m_Context || m_Context->err) { - if (!m_Context) { - Log(LogWarning, "RedisWriter", "Cannot allocate redis context."); - } else { - Log(LogWarning, "RedisWriter", "Connection error: ") - << m_Context->errstr; - } - - if (m_Context) { - redisAsyncFree(m_Context); - m_Context = NULL; - } - } + m_Context->data = (void*) this; + redisAsyncSetConnectCallback(m_Context, &ConnectCallback); redisAsyncSetDisconnectCallback(m_Context, &DisconnectCallback); } @@ -155,17 +144,39 @@ void RedisConnection::Disconnect() redisAsyncDisconnect(m_Context); } +void RedisConnection::ConnectCallback(const redisAsyncContext *c, int status) +{ + auto *rc = (RedisConnection* ) const_cast(c)->data; + if (status != REDIS_OK) { + if (c->err != 0) { + Log(LogCritical, "RedisConnection") + << "Redis connection failure: " << c->errstr; + } else { + Log(LogCritical, "RedisConnection") + << "Redis connection failure"; + } + rc->m_Connected = false; + } else { + Log(LogInformation, "RedisConnection") + << "Redis Connection: O N L I N E"; + } +} + +// It's unfortunate we can not pass any user data here. All we get to do is log a message and hope for the best void RedisConnection::DisconnectCallback(const redisAsyncContext *c, int status) { + auto *rc = (RedisConnection* ) const_cast(c)->data; + boost::mutex::scoped_lock lock(rc->m_CMutex); if (status == REDIS_OK) - Log(LogInformation, "RedisWriter") << "Redis disconnected by us"; + Log(LogInformation, "RedisConnection") << "Redis disconnected by us"; else { if (c->err != 0) - Log(LogCritical, "RedisWriter") << "Redis disconnected by server. Reason: " << c->errstr; + Log(LogCritical, "RedisConnection") << "Redis disconnected by server. Reason: " << c->errstr; else - Log(LogCritical, "RedisWriter") << "Redis disconnected by server"; + Log(LogCritical, "RedisConnection") << "Redis disconnected by server"; } + rc->m_Connected = false; } void RedisConnection::ExecuteQuery(const std::vector& query, redisCallbackFn *fn, void *privdata) @@ -187,9 +198,9 @@ void RedisConnection::SendMessageInternal(const std::vector& query, redi boost::mutex::scoped_lock lock(m_CMutex); - if (!m_Context) { + if (!m_Context || !m_Connected) { Log(LogCritical, "RedisWriter") - << "Connection lost"; + << "Not connected to Redis"; return; } diff --git a/lib/redis/redisconnection.hpp b/lib/redis/redisconnection.hpp index 6a4afe285..3dee85e93 100644 --- a/lib/redis/redisconnection.hpp +++ b/lib/redis/redisconnection.hpp @@ -75,6 +75,7 @@ namespace icinga void HandleRW(); static void DisconnectCallback(const redisAsyncContext *c, int status); + static void ConnectCallback(const redisAsyncContext *c, int status); static void RedisInitialCallback(redisAsyncContext *c, void *r, void *p); diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 9edc4cc36..07c772642 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -62,8 +62,6 @@ void RedisWriter::UpdateAllConfigObjects() { double startTime = Utility::GetTime(); - m_Rcon->ExecuteQuery({"flushall"}); - // Use a Workqueue to pack objects in parallel WorkQueue upq(25000, Configuration::Concurrency); upq.SetName("RedisWriter:ConfigDump"); diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index ca8099315..5760d91d7 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -111,6 +111,7 @@ void RedisWriter::TryToReconnect() if (!m_Rcon->IsConnected()) return; + UpdateSubscriptions(); if (m_ConfigDumpInProgress || m_ConfigDumpDone) @@ -214,6 +215,9 @@ void RedisWriter::PublishStats() { AssertOnWorkQueue(); + if (!m_Rcon->IsConnected()) + return; + Dictionary::Ptr status = GetStats(); String jsonStats = JsonEncode(status); @@ -294,6 +298,9 @@ void RedisWriter::SendEvent(const Dictionary::Ptr& event) { AssertOnWorkQueue(); + if (!m_Rcon->IsConnected()) + return; + String body = JsonEncode(event); // Log(LogInformation, "RedisWriter") From 1d21626b302a82f6a6d3dc89c9ccd93fbdea1517 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Fri, 26 Oct 2018 15:01:05 +0200 Subject: [PATCH 076/219] Code style --- lib/redis/redisconnection.cpp | 2 +- lib/redis/rediswriter-objects.cpp | 12 ++++++------ lib/redis/rediswriter.cpp | 11 +++++++---- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index 4391a68f7..9f889ab2e 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -75,7 +75,7 @@ void RedisConnection::HandleRW() void RedisConnection::RedisInitialCallback(redisAsyncContext *c, void *r, void *p) { - auto *state = (ConnectionState *) p; + auto state = (ConnectionState *) p; if (state->state != Starting && !r) { Log(LogCritical, "RedisConnection") << "No answer from Redis during initial connection, is the Redis server running?"; diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 07c772642..a7f68a51e 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -81,9 +81,9 @@ void RedisWriter::UpdateAllConfigObjects() upq.ParallelFor(types, [this](const TypePair& type) { size_t bulkCounter = 0; - std::vector attributes = std::vector({"HMSET", m_PrefixConfigObject + type.second}); - std::vector customVars = std::vector({"HMSET", m_PrefixConfigCustomVar + type.second}); - std::vector checksums = std::vector({"HMSET", m_PrefixConfigCheckSum + type.second}); + auto attributes = std::vector({"HMSET", m_PrefixConfigObject + type.second}); + auto customVars = std::vector({"HMSET", m_PrefixConfigCustomVar + type.second}); + auto checksums = std::vector({"HMSET", m_PrefixConfigCheckSum + type.second}); for (const ConfigObject::Ptr& object : type.first->GetObjects()) { CreateConfigUpdate(object, attributes, customVars, checksums, false); @@ -138,9 +138,9 @@ static ConfigObject::Ptr GetObjectByName(const String& name) void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate) { String typeName = object->GetReflectionType()->GetName().ToLower(); - std::vector attributes = std::vector({"HMSET", m_PrefixConfigObject + typeName}); - std::vector customVars = std::vector({"HMSET", m_PrefixConfigCustomVar + typeName}); - std::vector checksums = std::vector({"HMSET", m_PrefixConfigCheckSum + typeName}); + auto attributes = std::vector({"HMSET", m_PrefixConfigObject + typeName}); + auto customVars = std::vector({"HMSET", m_PrefixConfigCustomVar + typeName}); + auto checksums = std::vector({"HMSET", m_PrefixConfigCheckSum + typeName}); CreateConfigUpdate(object, attributes, customVars, checksums, runtimeUpdate); diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index 5760d91d7..3ca657f90 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -138,11 +138,14 @@ void RedisWriter::UpdateSubscriptions() Log(LogInformation, "RedisWriter", "Updating Redis subscriptions"); - if (!m_Rcon->IsConnected()) { - Log(LogCritical, "DEBUG, Redis") - << "NO CONNECT CHIEF"; + /* TODO: + * Silently return in this case. Usually the RedisConnection checks for connectivity and logs in failure case. + * But since we expect and answer here and break Icinga in case of receiving no answer/an unexpected one we opt for + * better safe than sorry here. Future implementation needs to include an improved error handling and answer verification. + */ + if (!m_Rcon->IsConnected()) return; - } + long long cursor = 0; String keyPrefix = "icinga:subscription:"; From 36588ce31ae6cb26ef54d5aef0341af6200a439e Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Fri, 26 Oct 2018 16:33:22 +0200 Subject: [PATCH 077/219] Code formatting --- lib/redis/rediswriter-objects.cpp | 35 ++++++++++++++++++------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index a7f68a51e..5059a42f3 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -66,20 +66,22 @@ void RedisWriter::UpdateAllConfigObjects() WorkQueue upq(25000, Configuration::Concurrency); upq.SetName("RedisWriter:ConfigDump"); - typedef std::pair TypePair; + typedef std::pair TypePair; std::vector types; - for (const Type::Ptr& type : Type::GetAllTypes()) { + for (const Type::Ptr& type : Type::GetAllTypes()) { ConfigType *ctype = dynamic_cast(type.get()); if (!ctype) continue; - String lcType (type->GetName().ToLower()); + String lcType(type->GetName().ToLower()); types.emplace_back(ctype, lcType); - m_Rcon->ExecuteQuery({"DEL", m_PrefixConfigCheckSum + lcType, m_PrefixConfigObject + lcType, m_PrefixStatusObject + lcType}); + m_Rcon->ExecuteQuery( + {"DEL", m_PrefixConfigCheckSum + lcType, m_PrefixConfigObject + lcType, m_PrefixStatusObject + lcType}); } - upq.ParallelFor(types, [this](const TypePair& type) { + upq.ParallelFor(types, [this](const TypePair& type) + { size_t bulkCounter = 0; auto attributes = std::vector({"HMSET", m_PrefixConfigObject + type.second}); auto customVars = std::vector({"HMSET", m_PrefixConfigCustomVar + type.second}); @@ -88,7 +90,7 @@ void RedisWriter::UpdateAllConfigObjects() for (const ConfigObject::Ptr& object : type.first->GetObjects()) { CreateConfigUpdate(object, attributes, customVars, checksums, false); SendStatusUpdate(object); - bulkCounter ++; + bulkCounter++; if (!bulkCounter % 100) { if (attributes.size() > 2) { m_Rcon->ExecuteQuery(attributes); @@ -112,7 +114,7 @@ void RedisWriter::UpdateAllConfigObjects() m_Rcon->ExecuteQuery(checksums); Log(LogNotice, "RedisWriter") - << "Dumped " << bulkCounter << " objects of type " << type.second; + << "Dumped " << bulkCounter << " objects of type " << type.second; }); upq.Join(); @@ -128,7 +130,7 @@ void RedisWriter::UpdateAllConfigObjects() << "Initial config/status dump finished in " << Utility::GetTime() - startTime << " seconds."; } -template +template static ConfigObject::Ptr GetObjectByName(const String& name) { return ConfigObject::GetObject(name); @@ -154,7 +156,9 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtime * (if applicable), first the key then the value. To use in a Redis command the command (e.g. HSET) and the key (e.g. * icinga:config:object:downtime) need to be prepended. There is nothing to indicate success or failure. */ -void RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, std::vector& attributes, std::vector& customVars, std::vector& checksums, bool runtimeUpdate) +void RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, std::vector& attributes, + std::vector& customVars, std::vector& checksums, + bool runtimeUpdate) { /* TODO: This isn't essentially correct as we don't keep track of config objects ourselves. This would avoid duplicated config updates at startup. if (!runtimeUpdate && m_ConfigDumpInProgress) @@ -501,10 +505,10 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) String objectKey = GetObjectIdentifier(object); m_Rcon->ExecuteQueries({ - {"HDEL", m_PrefixConfigObject + typeName, objectKey}, - {"DEL", m_PrefixStatusObject + typeName + ":" + objectKey}, - {"PUBLISH", "icinga:config:delete", typeName + ":" + objectKey} - }); + {"HDEL", m_PrefixConfigObject + typeName, objectKey}, + {"DEL", m_PrefixStatusObject + typeName + ":" + objectKey}, + {"PUBLISH", "icinga:config:delete", typeName + ":" + objectKey} + }); } void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) @@ -592,8 +596,9 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) // } } -std::vector RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType, - const String& typeNameOverride) +std::vector +RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType, + const String& typeNameOverride) { Type::Ptr type = object->GetReflectionType(); Dictionary::Ptr attrs(new Dictionary); From 77ce8f67bd33131b9f2475e2a392dd5cc4d69108 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Fri, 26 Oct 2018 16:33:39 +0200 Subject: [PATCH 078/219] Add downtimes sync --- lib/redis/rediswriter-objects.cpp | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 5059a42f3..4a13dfa78 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -463,6 +463,24 @@ void RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, std::vecto } } + icinga::Downtime::Ptr downtime = dynamic_pointer_cast(object); + if (downtime) { + propertiesBlacklist.emplace("name"); + propertiesBlacklist.emplace("host_name"); + Host::Ptr host; + Service::Ptr service; + tie(host, service) = GetHostService(downtime->GetCheckable()); + + if (service) { + propertiesBlacklist.emplace("service_name"); + checkSums->Set("service_checksum", GetObjectIdentifier(service)); + typeName = "servicedowntime"; + } else { + checkSums->Set("host_checksum", GetObjectIdentifier(host)); + typeName = "hostdowntime"; + } + + } /* Send all object attributes to Redis, no extra checksums involved here. */ auto tempAttrs = (UpdateObjectAttrs(m_PrefixConfigObject, object, FAConfig, typeName)); attributes.insert(attributes.end(), std::begin(tempAttrs), std::end(tempAttrs)); @@ -486,6 +504,8 @@ void RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, std::vecto } checkSums->Set("metadata_checksum", CalculateCheckSumMetadata(object)); + + /* TODO: Problem: This does not account for `is_in_effect`, `trigger_time` of downtimes. */ checkSums->Set("properties_checksum", CalculateCheckSumProperties(object, propertiesBlacklist)); String checkSumsBody = JsonEncode(checkSums); @@ -622,6 +642,14 @@ RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& attrs->Set(field.Name, Serialize(val)); } + /* Downtimes require in_effect, which is not an attribute */ + Downtime::Ptr downtime = dynamic_pointer_cast(object); + if (downtime) { + attrs->Set("in_effect", Serialize(downtime->IsInEffect())); + attrs->Set("trigger_time", Serialize(downtime->GetTriggerTime())); + } + + /* Use the name checksum as unique key. */ String typeName = type->GetName().ToLower(); if (!typeNameOverride.IsEmpty()) From 3c651b4c99d23fbe92f0eea7b940aa47f96add7a Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Mon, 29 Oct 2018 13:48:15 +0100 Subject: [PATCH 079/219] Fix icingadb specific type names --- lib/redis/rediswriter-objects.cpp | 124 ++++++++++++++++++------------ lib/redis/rediswriter-utility.cpp | 27 +++++++ lib/redis/rediswriter.hpp | 10 ++- 3 files changed, 108 insertions(+), 53 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 4a13dfa78..52709c02a 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -75,20 +75,30 @@ void RedisWriter::UpdateAllConfigObjects() continue; String lcType(type->GetName().ToLower()); - types.emplace_back(ctype, lcType); - m_Rcon->ExecuteQuery( - {"DEL", m_PrefixConfigCheckSum + lcType, m_PrefixConfigObject + lcType, m_PrefixStatusObject + lcType}); + + if (lcType == "downtime") { + types.emplace_back(ctype, "hostdowntime"); + types.emplace_back(ctype, "servicedowntime"); + } else if (lcType == "comment") { + types.emplace_back(ctype, "hostcomment"); + types.emplace_back(ctype, "servicecomment"); + } else { + types.emplace_back(ctype, lcType); + } } upq.ParallelFor(types, [this](const TypePair& type) { + String lcType = type.second; + m_Rcon->ExecuteQuery( + {"DEL", m_PrefixConfigCheckSum + lcType, m_PrefixConfigObject + lcType, m_PrefixStatusObject + lcType}); size_t bulkCounter = 0; - auto attributes = std::vector({"HMSET", m_PrefixConfigObject + type.second}); - auto customVars = std::vector({"HMSET", m_PrefixConfigCustomVar + type.second}); - auto checksums = std::vector({"HMSET", m_PrefixConfigCheckSum + type.second}); + auto attributes = std::vector({"HMSET", m_PrefixConfigObject + lcType}); + auto customVars = std::vector({"HMSET", m_PrefixConfigCustomVar + lcType}); + auto checksums = std::vector({"HMSET", m_PrefixConfigCheckSum + lcType}); for (const ConfigObject::Ptr& object : type.first->GetObjects()) { - CreateConfigUpdate(object, attributes, customVars, checksums, false); + CreateConfigUpdate(object, lcType, attributes, customVars, checksums, false); SendStatusUpdate(object); bulkCounter++; if (!bulkCounter % 100) { @@ -139,49 +149,22 @@ static ConfigObject::Ptr GetObjectByName(const String& name) // Used to update a single object, used for runtime updates void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate) { - String typeName = object->GetReflectionType()->GetName().ToLower(); + String typeName = GetLowerCaseTypeNameDB(object); + auto attributes = std::vector({"HMSET", m_PrefixConfigObject + typeName}); auto customVars = std::vector({"HMSET", m_PrefixConfigCustomVar + typeName}); auto checksums = std::vector({"HMSET", m_PrefixConfigCheckSum + typeName}); - CreateConfigUpdate(object, attributes, customVars, checksums, runtimeUpdate); + CreateConfigUpdate(object, typeName, attributes, customVars, checksums, runtimeUpdate); m_Rcon->ExecuteQuery(attributes); m_Rcon->ExecuteQuery(customVars); m_Rcon->ExecuteQuery(checksums); } -/* Creates a config update with computed checksums etc. - * Writes attributes, customVars and checksums into the respective supplied vectors. Adds two values to each vector - * (if applicable), first the key then the value. To use in a Redis command the command (e.g. HSET) and the key (e.g. - * icinga:config:object:downtime) need to be prepended. There is nothing to indicate success or failure. - */ -void RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, std::vector& attributes, - std::vector& customVars, std::vector& checksums, - bool runtimeUpdate) +void RedisWriter::MakeTypeChecksums(const ConfigObject::Ptr& object, std::set& propertiesBlacklist, Dictionary::Ptr& checkSums) { - /* TODO: This isn't essentially correct as we don't keep track of config objects ourselves. This would avoid duplicated config updates at startup. - if (!runtimeUpdate && m_ConfigDumpInProgress) - return; - */ - - if (m_Rcon == nullptr) - return; - - /* Calculate object specific checksums and store them in a different namespace. */ - Type::Ptr type = object->GetReflectionType(); - - String typeName = type->GetName().ToLower(); - String objectKey = GetObjectIdentifier(object); - - std::set propertiesBlacklist({"name", "__name", "package", "source_location", "templates"}); - - Dictionary::Ptr checkSums = new Dictionary(); - - checkSums->Set("name_checksum", CalculateCheckSumString(object->GetShortName())); - checkSums->Set("environment_checksum", CalculateCheckSumString(GetEnvironment())); - - auto endpoint(dynamic_pointer_cast(object)); + Endpoint::Ptr endpoint = dynamic_pointer_cast(object); if (endpoint) { auto endpointZone(endpoint->GetZone()); @@ -189,6 +172,8 @@ void RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, std::vecto if (endpointZone) { checkSums->Set("zone_checksum", GetObjectIdentifier(endpointZone)); } + + return; } else { /* 'zone' is available for all config objects, therefore calculate the checksum. */ auto zone(static_pointer_cast(object->GetZone())); @@ -224,6 +209,8 @@ void RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, std::vecto if (period) checkSums->Set("period_checksum", GetObjectIdentifier(period)); + + return; } Notification::Ptr notification = dynamic_pointer_cast(object); @@ -270,6 +257,7 @@ void RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, std::vecto checkSums->Set("usergroup_checksums", usergroupChecksums); checkSums->Set("usergroups_checksum", CalculateCheckSumArray(usergroupNames)); + return; } /* Calculate checkable checksums. */ @@ -335,6 +323,8 @@ void RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, std::vecto checkSums->Set("notes_url_checksum", CalculateCheckSumString(notesUrl)); if (!iconImage.IsEmpty()) checkSums->Set("icon_image_checksum", CalculateCheckSumString(iconImage)); + + return; } Zone::Ptr zone = dynamic_pointer_cast(object); @@ -360,11 +350,12 @@ void RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, std::vecto checkSums->Set("all_parents_checksums", parents); checkSums->Set("all_parents_checksum", HashValue(zone->GetAllParents())); + return; } /* zone_checksum for endpoints already is calculated above. */ - auto command(dynamic_pointer_cast(object)); + Command::Ptr command = dynamic_pointer_cast(object); if (command) { Dictionary::Ptr arguments = command->GetArguments(); Dictionary::Ptr argumentChecksums = new Dictionary; @@ -395,9 +386,11 @@ void RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, std::vecto checkSums->Set("envvars_checksum", HashValue(envvars)); checkSums->Set("envvar_checksums", envvarChecksums); propertiesBlacklist.emplace("env"); + + return; } - auto timeperiod(dynamic_pointer_cast(object)); + TimePeriod::Ptr timeperiod = dynamic_pointer_cast(object); if (timeperiod) { Dictionary::Ptr ranges = timeperiod->GetRanges(); @@ -443,9 +436,11 @@ void RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, std::vecto } checkSums->Set("exclude_checksums", excludeChecksums); + + return; } - icinga::Comment::Ptr comment = dynamic_pointer_cast(object); + Comment::Ptr comment = dynamic_pointer_cast(object); if (comment) { propertiesBlacklist.emplace("name"); propertiesBlacklist.emplace("host_name"); @@ -456,14 +451,13 @@ void RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, std::vecto if (service) { propertiesBlacklist.emplace("service_name"); checkSums->Set("service_checksum", GetObjectIdentifier(service)); - typeName = "servicecomment"; - } else { + } else checkSums->Set("host_checksum", GetObjectIdentifier(host)); - typeName = "hostcomment"; - } + + return; } - icinga::Downtime::Ptr downtime = dynamic_pointer_cast(object); + Downtime::Ptr downtime = dynamic_pointer_cast(object); if (downtime) { propertiesBlacklist.emplace("name"); propertiesBlacklist.emplace("host_name"); @@ -474,13 +468,41 @@ void RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, std::vecto if (service) { propertiesBlacklist.emplace("service_name"); checkSums->Set("service_checksum", GetObjectIdentifier(service)); - typeName = "servicedowntime"; - } else { + } else checkSums->Set("host_checksum", GetObjectIdentifier(host)); - typeName = "hostdowntime"; - } + return; } +} + +/* Creates a config update with computed checksums etc. + * Writes attributes, customVars and checksums into the respective supplied vectors. Adds two values to each vector + * (if applicable), first the key then the value. To use in a Redis command the command (e.g. HSET) and the key (e.g. + * icinga:config:object:downtime) need to be prepended. There is nothing to indicate success or failure. + */ +void RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, const String typeName, std::vector& attributes, + std::vector& customVars, std::vector& checksums, + bool runtimeUpdate) +{ + /* TODO: This isn't essentially correct as we don't keep track of config objects ourselves. This would avoid duplicated config updates at startup. + if (!runtimeUpdate && m_ConfigDumpInProgress) + return; + */ + + if (m_Rcon == nullptr) + return; + + String objectKey = GetObjectIdentifier(object); + + std::set propertiesBlacklist({"name", "__name", "package", "source_location", "templates"}); + + Dictionary::Ptr checkSums = new Dictionary(); + + checkSums->Set("name_checksum", CalculateCheckSumString(object->GetShortName())); + checkSums->Set("environment_checksum", CalculateCheckSumString(GetEnvironment())); + + MakeTypeChecksums(object, propertiesBlacklist, checkSums); + /* Send all object attributes to Redis, no extra checksums involved here. */ auto tempAttrs = (UpdateObjectAttrs(m_PrefixConfigObject, object, FAConfig, typeName)); attributes.insert(attributes.end(), std::begin(tempAttrs), std::end(tempAttrs)); diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 2592828a4..d5064f723 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -22,6 +22,7 @@ #include "icinga/checkcommand.hpp" #include "icinga/notificationcommand.hpp" #include "icinga/eventcommand.hpp" +#include "base/configtype.hpp" #include "base/object-packer.hpp" #include "base/logger.hpp" #include "base/serializer.hpp" @@ -336,6 +337,32 @@ String RedisWriter::HashValue(const Value& value, const std::set& proper return SHA1(PackObject(temp)); } +String RedisWriter::GetLowerCaseTypeNameDB(const ConfigObject::Ptr& obj) +{ + String typeName = obj->GetReflectionType()->GetName().ToLower(); + if (typeName == "downtime") { + Downtime::Ptr downtime = dynamic_pointer_cast(obj); + Host::Ptr host; + Service::Ptr service; + tie(host, service) = GetHostService(downtime->GetCheckable()); + if (service) + typeName = "servicedowntime"; + else + typeName = "hostdowntime"; + } else if (typeName == "comment") { + Comment::Ptr comment = dynamic_pointer_cast(obj); + Host::Ptr host; + Service::Ptr service; + tie(host, service) = GetHostService(comment->GetCheckable()); + if (service) + typeName = "servicecomment"; + else + typeName = "hostcomment"; + } + + return typeName; +} + //Used to duplicate a redisReply, needed as redisReplies are freed when the async callback finishes redisReply* RedisWriter::dupReplyObject(redisReply* reply) { diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 34ffbee9d..d2b045c83 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -68,10 +68,12 @@ private: /* config & status dump */ void UpdateAllConfigObjects(); void SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate); - void CreateConfigUpdate(const ConfigObject::Ptr& object, std::vector& attributes, std::vector& customVars, std::vector& checksums, bool runtimeUpdate); + void CreateConfigUpdate(const ConfigObject::Ptr& object, const String type, std::vector& attributes, + std::vector& customVars, std::vector& checksums, bool runtimeUpdate); void SendConfigDelete(const ConfigObject::Ptr& object); void SendStatusUpdate(const ConfigObject::Ptr& object); - std::vector UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride); + std::vector UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType, + const String& typeNameOverride); /* Stats */ Dictionary::Ptr GetStats(); @@ -91,6 +93,10 @@ private: static String HashValue(const Value& value); static String HashValue(const Value& value, const std::set& propertiesBlacklist, bool propertiesWhitelist = false); + static String GetLowerCaseTypeNameDB(const ConfigObject::Ptr& obj); + static void MakeTypeChecksums(const ConfigObject::Ptr& object, std::set& propertiesBlacklist, Dictionary::Ptr& checkSums); + + static void StateChangedHandler(const ConfigObject::Ptr& object); static void VersionChangedHandler(const ConfigObject::Ptr& object); From 5e3e114a1612dfad63dd27e7ba860fd1454e4a52 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Mon, 29 Oct 2018 16:27:25 +0100 Subject: [PATCH 080/219] Handle icingadb- / icinga-type mismatches fixes servicedowntimes being written to hostdowntimes (and vice versa) --- lib/redis/rediswriter-objects.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 52709c02a..920941950 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -98,6 +98,8 @@ void RedisWriter::UpdateAllConfigObjects() auto checksums = std::vector({"HMSET", m_PrefixConfigCheckSum + lcType}); for (const ConfigObject::Ptr& object : type.first->GetObjects()) { + if (lcType != GetLowerCaseTypeNameDB(object)) + continue; CreateConfigUpdate(object, lcType, attributes, customVars, checksums, false); SendStatusUpdate(object); bulkCounter++; @@ -165,7 +167,6 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtime void RedisWriter::MakeTypeChecksums(const ConfigObject::Ptr& object, std::set& propertiesBlacklist, Dictionary::Ptr& checkSums) { Endpoint::Ptr endpoint = dynamic_pointer_cast(object); - if (endpoint) { auto endpointZone(endpoint->GetZone()); From e889de966edf3262039eb59e5fa3981d1ff9fc92 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Mon, 29 Oct 2018 16:56:24 +0100 Subject: [PATCH 081/219] Publish dump event fixes #34 --- lib/redis/rediswriter-objects.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 920941950..d02b83c93 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -125,6 +125,8 @@ void RedisWriter::UpdateAllConfigObjects() if (checksums.size() > 2) m_Rcon->ExecuteQuery(checksums); + m_Rcon->ExecuteQuery({"PUBLISH", "icinga:config:dump", lcType}); + Log(LogNotice, "RedisWriter") << "Dumped " << bulkCounter << " objects of type " << type.second; }); From 47905a25e86c31585cbd9a0d59130f9e7cc6a25d Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Tue, 30 Oct 2018 16:00:41 +0100 Subject: [PATCH 082/219] Improve event payload Adds host/service_id Acknowledgement events. AcknowledgementSet events also send the comment_id. fixes #29 --- lib/redis/rediswriter.cpp | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index 3ca657f90..b9843da45 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -22,10 +22,12 @@ #include "redis/redisconnection.hpp" #include "remote/eventqueue.hpp" #include "base/json.hpp" -#include "rediswriter.hpp" +#include "icinga/checkable.hpp" +#include "icinga/host.hpp" #include + using namespace icinga; //TODO Make configurable and figure out a sane default @@ -280,7 +282,7 @@ void RedisWriter::HandleEvent(const Dictionary::Ptr& event) long maxEvents = MAX_EVENTS_DEFAULT; if (maxExists->integer) { - redisReply *redisReply =RedisGet({ "GET", "icinga:subscription:" + name + ":limit"}); + redisReply *redisReply = RedisGet({ "GET", "icinga:subscription:" + name + ":limit"}); VERIFY(redisReply->type == REDIS_REPLY_STRING); Log(LogInformation, "RedisWriter") @@ -304,6 +306,33 @@ void RedisWriter::SendEvent(const Dictionary::Ptr& event) if (!m_Rcon->IsConnected()) return; + String type = event->Get("type"); + if (type.Contains("Acknowledgement")) { + Checkable::Ptr checkable; + + if (event->Contains("service")) { + checkable = Service::GetByNamePair(event->Get("host"), event->Get("service")); + event->Set("service_id", GetObjectIdentifier(checkable)); + } else { + checkable = Host::GetByName(event->Get("host")); + event->Set("host_id", GetObjectIdentifier(checkable)); + } + + if (type == "AcknowledgementSet") { + Timestamp entry = 0; + Comment::Ptr AckComment; + for (const Comment::Ptr& c : checkable->GetComments()) { + if (c->GetEntryType() == CommentAcknowledgement) { + if (c->GetEntryTime() > entry) { + entry = c->GetEntryTime(); + AckComment = c; + } + } + } + event->Set("comment_id", GetObjectIdentifier(AckComment)); + } + } + String body = JsonEncode(event); // Log(LogInformation, "RedisWriter") From f4145579cae175ed8f2c160af9275348e09daedd Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 9 Nov 2018 15:27:19 +0100 Subject: [PATCH 083/219] Add missing include --- lib/redis/rediswriter-utility.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index d5064f723..87146622a 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -22,6 +22,7 @@ #include "icinga/checkcommand.hpp" #include "icinga/notificationcommand.hpp" #include "icinga/eventcommand.hpp" +#include "icinga/host.hpp" #include "base/configtype.hpp" #include "base/object-packer.hpp" #include "base/logger.hpp" From 67208c4f074f211f18441e35a4a15e8bc9ce0698 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Thu, 15 Nov 2018 10:34:06 +0100 Subject: [PATCH 084/219] Handle runtime config updates fixes #37 --- lib/redis/rediswriter-objects.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index d02b83c93..c85568ff4 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -700,7 +700,7 @@ void RedisWriter::VersionChangedHandler(const ConfigObject::Ptr& object) if (object->IsActive()) { // Create or update the object config for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { -// rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendConfigUpdate, rw, object, true)); + rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendConfigUpdate, rw, object, true)); } } else if (!object->IsActive() && object->GetExtension("ConfigObjectDeleted")) { // same as in apilistener-configsync.cpp From eec1928a0b426f6e5cbca29abd3695f27ea0068f Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Fri, 16 Nov 2018 10:15:00 +0100 Subject: [PATCH 085/219] Fix segfault on startup --- lib/redis/rediswriter-objects.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index c85568ff4..5c80d1ad2 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -153,6 +153,9 @@ static ConfigObject::Ptr GetObjectByName(const String& name) // Used to update a single object, used for runtime updates void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate) { + if (!m_Rcon->IsConnected()) + return; + String typeName = GetLowerCaseTypeNameDB(object); auto attributes = std::vector({"HMSET", m_PrefixConfigObject + typeName}); From a056e1483480117ee925992c1dda43d0885496b2 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Fri, 16 Nov 2018 15:42:36 +0100 Subject: [PATCH 086/219] Fix another RedisWriter crash Same as the other fix, this makes us lose events --- lib/redis/rediswriter-objects.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 5c80d1ad2..8b0e972d4 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -703,13 +703,15 @@ void RedisWriter::VersionChangedHandler(const ConfigObject::Ptr& object) if (object->IsActive()) { // Create or update the object config for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { - rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendConfigUpdate, rw, object, true)); + if (rw) + rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendConfigUpdate, rw, object, true)); } } else if (!object->IsActive() && object->GetExtension("ConfigObjectDeleted")) { // same as in apilistener-configsync.cpp // Delete object config for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { - rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendConfigDelete, rw, object)); + if (rw) + rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendConfigDelete, rw, object)); } } } From 5f64d809d7b69d3f1bca9474d0544a13bdeb9a7f Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Mon, 26 Nov 2018 13:34:05 +0100 Subject: [PATCH 087/219] Check if RedisConnection exists before checking connection --- lib/redis/rediswriter-objects.cpp | 2 +- lib/redis/rediswriter.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 8b0e972d4..3673973ff 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -153,7 +153,7 @@ static ConfigObject::Ptr GetObjectByName(const String& name) // Used to update a single object, used for runtime updates void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate) { - if (!m_Rcon->IsConnected()) + if (!m_Rcon || !m_Rcon->IsConnected()) return; String typeName = GetLowerCaseTypeNameDB(object); diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index b9843da45..41fe0d048 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -111,7 +111,7 @@ void RedisWriter::TryToReconnect() else m_Rcon->Start(); - if (!m_Rcon->IsConnected()) + if (!m_Rcon || !m_Rcon->IsConnected()) return; UpdateSubscriptions(); @@ -145,7 +145,7 @@ void RedisWriter::UpdateSubscriptions() * But since we expect and answer here and break Icinga in case of receiving no answer/an unexpected one we opt for * better safe than sorry here. Future implementation needs to include an improved error handling and answer verification. */ - if (!m_Rcon->IsConnected()) + if (!m_Rcon || !m_Rcon->IsConnected()) return; long long cursor = 0; @@ -220,7 +220,7 @@ void RedisWriter::PublishStats() { AssertOnWorkQueue(); - if (!m_Rcon->IsConnected()) + if (!m_Rcon || !m_Rcon->IsConnected()) return; Dictionary::Ptr status = GetStats(); @@ -303,7 +303,7 @@ void RedisWriter::SendEvent(const Dictionary::Ptr& event) { AssertOnWorkQueue(); - if (!m_Rcon->IsConnected()) + if (!m_Rcon || !m_Rcon->IsConnected()) return; String type = event->Get("type"); From c825a3f4b54ed72ba0564ffe26bf93c862cb0e07 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Mon, 12 Nov 2018 17:45:32 +0100 Subject: [PATCH 088/219] Remove unnecessary parameter --- lib/redis/rediswriter-objects.cpp | 4 ++-- lib/redis/rediswriter.hpp | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 3673973ff..90b771fcd 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -510,7 +510,7 @@ void RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, const Stri MakeTypeChecksums(object, propertiesBlacklist, checkSums); /* Send all object attributes to Redis, no extra checksums involved here. */ - auto tempAttrs = (UpdateObjectAttrs(m_PrefixConfigObject, object, FAConfig, typeName)); + auto tempAttrs = (UpdateObjectAttrs(object, FAConfig, typeName)); attributes.insert(attributes.end(), std::begin(tempAttrs), std::end(tempAttrs)); /* Custom var checksums. */ @@ -645,7 +645,7 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) } std::vector -RedisWriter::UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType, +RedisWriter::UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride) { Type::Ptr type = object->GetReflectionType(); diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index d2b045c83..2583f0c6c 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -72,8 +72,7 @@ private: std::vector& customVars, std::vector& checksums, bool runtimeUpdate); void SendConfigDelete(const ConfigObject::Ptr& object); void SendStatusUpdate(const ConfigObject::Ptr& object); - std::vector UpdateObjectAttrs(const String& keyPrefix, const ConfigObject::Ptr& object, int fieldType, - const String& typeNameOverride); + std::vector UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride); /* Stats */ Dictionary::Ptr GetStats(); From 2215e9e16fcc728e5b5d669fdbb3cd264ebaaf03 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Mon, 12 Nov 2018 17:46:35 +0100 Subject: [PATCH 089/219] Implement minimal state sync --- lib/redis/rediswriter-objects.cpp | 58 +++++++++++++++++++++++++------ lib/redis/rediswriter.hpp | 1 + 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 90b771fcd..c08314dfc 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -561,18 +561,25 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) { - //TODO: Manage type names - //TODO: Figure out what we need when we implement the history and state sync -// UpdateObjectAttrs(m_PrefixStatusObject, object, FAState, ""); + //TODO: This is probably uncesessary? + Checkable::Ptr checkable = dynamic_pointer_cast(object); + if (!checkable) + return; + String streamname = "hoststatus"; + Host::Ptr host = dynamic_pointer_cast(object); + if (!host) + streamname = "servicestatus"; -// /* Serialize config object attributes */ -// Dictionary::Ptr objectAttrs = SerializeObjectAttrs(object, FAState); -// -// String jsonBody = JsonEncode(objectAttrs); -// -// String objectName = object->GetName(); -// + /* Serialize config object attributes + * TODO: Flatten last check result + */ + auto objectAttrs = SerializeState(object); + + std::vector streamadd({"XADD", streamname, "*", "id", GetObjectIdentifier(object)}); + streamadd.insert(streamadd.end(), std::begin(objectAttrs), std::end(objectAttrs)); + + m_Rcon->ExecuteQuery(streamadd); // ExecuteQuery({ "HSET", "icinga:status:" + typeName, objectName, jsonBody }); // // /* Icinga DB part for Icinga Web 2 */ @@ -644,6 +651,37 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) // } } +std::vector RedisWriter::SerializeState(const Object::Ptr& object) +{ + Type::Ptr type = object->GetReflectionType(); + + std::vector resultAttrs = std::vector(); + + for (int fid = 0; fid < type->GetFieldCount(); fid++) { + Field field = type->GetFieldInfo(fid); + + if ((field.Attributes & FAState) == 0) + continue; + + Value val = object->GetField(fid); + + /* hide attributes which shouldn't be user-visible */ + if (field.Attributes & FANoUserView) + continue; + + /* hide internal navigation fields */ + if (field.Attributes & FANavigation && !(field.Attributes & (FAConfig | FAState))) + continue; + + Value sval = Serialize(val); + resultAttrs.emplace_back(field.Name); + resultAttrs.emplace_back(sval); + } + + return resultAttrs; +} + + std::vector RedisWriter::UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride) diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 2583f0c6c..be52545e6 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -73,6 +73,7 @@ private: void SendConfigDelete(const ConfigObject::Ptr& object); void SendStatusUpdate(const ConfigObject::Ptr& object); std::vector UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride); + std::vector SerializeState(const Object::Ptr& object); /* Stats */ Dictionary::Ptr GetStats(); From 3da4cef1e64a3d9b7127614aa5bdfc7cb78d6f0c Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Tue, 13 Nov 2018 15:22:31 +0100 Subject: [PATCH 090/219] Update state sync --- lib/redis/rediswriter-objects.cpp | 113 ++++++++++++++++++++++++++++-- 1 file changed, 106 insertions(+), 7 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index c08314dfc..5502c9875 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -563,23 +563,122 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) { //TODO: This is probably uncesessary? Checkable::Ptr checkable = dynamic_pointer_cast(object); - if (!checkable) + if (!checkable) { + Log(LogCritical, "DEBUG") << "THIS IS NOT A CHECKABLE, IT'S A " << object->GetReflectionType()->GetName(); return; + } + + bool isHost; + String streamname; + streamname = "servicestatus"; + + Host::Ptr host; + Service::Ptr service; + + tie(host, service) = GetHostService(checkable); + + if (service) { + streamname = "servicestate"; + isHost = false; + } else { + streamname = "hoststate"; + isHost = true; + } - String streamname = "hoststatus"; - Host::Ptr host = dynamic_pointer_cast(object); - if (!host) - streamname = "servicestatus"; /* Serialize config object attributes * TODO: Flatten last check result */ auto objectAttrs = SerializeState(object); - std::vector streamadd({"XADD", streamname, "*", "id", GetObjectIdentifier(object)}); - streamadd.insert(streamadd.end(), std::begin(objectAttrs), std::end(objectAttrs)); + std::vector streamadd({"XADD", streamname, "*"}); + + streamadd.emplace_back("id"); + streamadd.emplace_back(GetObjectIdentifier(object)); + streamadd.emplace_back("env_id"); + streamadd.emplace_back(CalculateCheckSumString(GetEnvironment())); + streamadd.emplace_back("state_type"); + streamadd.emplace_back(checkable->GetStateType()); + + streamadd.emplace_back("state"); + if (isHost) { + streamadd.emplace_back(host->GetState()); + } else { + streamadd.emplace_back(service->GetState()); + } + + streamadd.emplace_back("last_hard_state"); + if (isHost) { + streamadd.emplace_back(host->GetLastHardState()); + } else { + streamadd.emplace_back(service->GetLastHardState()); + } + + streamadd.emplace_back("check_attempt"); + streamadd.emplace_back(checkable->GetCheckAttempt()); + + //streamadd.emplace_back("severity") + //streamadd.emplace_back(checkable->GetSeverity()); + + streamadd.emplace_back("is_active"); + streamadd.emplace_back(checkable->IsActive()); + + // TODO: Is it possible there is no last checkresult? + CheckResult::Ptr cr = checkable->GetLastCheckResult(); + + streamadd.emplace_back("output"); + streamadd.emplace_back(JsonEncode(cr->GetOutput())); + //streamadd.emplace_back("long_output", ) + streamadd.emplace_back("performance_data"); + streamadd.emplace_back(JsonEncode(cr->GetPerformanceData())); + streamadd.emplace_back("command"); + streamadd.emplace_back(JsonEncode(cr->GetCommand())); + //streamadd.emplace_back("is_problem", !checkable->IsReachable() && !checkable->IsAcknowledged()); + streamadd.emplace_back("is_handled"); + streamadd.emplace_back(!checkable->IsAcknowledged()); + streamadd.emplace_back("is_flapping"); + streamadd.emplace_back(checkable->IsFlapping()); + + streamadd.emplace_back("is_acknowledged"); + streamadd.emplace_back(checkable->IsAcknowledged()); + if (checkable->IsAcknowledged()) { + Timestamp entry = 0; + Comment::Ptr AckComment; + for (const Comment::Ptr& c : checkable->GetComments()) { + if (c->GetEntryType() == CommentAcknowledgement) { + if (c->GetEntryTime() > entry) { + entry = c->GetEntryTime(); + AckComment = c; + } + } + } + streamadd.emplace_back("acknowledgement_comment_id"); + streamadd.emplace_back(GetObjectIdentifier(AckComment)); + } + + streamadd.emplace_back("in_downtime"); + streamadd.emplace_back(checkable->IsInDowntime()); + /* + if (checkable->IsInDowntime()) + streamadd.emplace_back("downtime_id", checkable->GetDowntimes()); + */ + + streamadd.emplace_back("execution_time"); + streamadd.emplace_back(cr->CalculateExecutionTime()); + //streamadd.emplace_back("latency", TODO: What); + streamadd.emplace_back("check_timeout"); + streamadd.emplace_back(checkable->GetCheckTimeout()); + + //streamadd.emplace_back("last_update", TODO: What?); + streamadd.emplace_back("last_state_change"); + streamadd.emplace_back(checkable->GetLastStateChange()); + //streamadd.emplace_back("last_soft_state", TODO: We want "previous"); + //streamadd.emplace_back("last_hard_state", TODO: We want "previous"); + streamadd.emplace_back("next_check"); + streamadd.emplace_back(checkable->GetNextCheck()); m_Rcon->ExecuteQuery(streamadd); + // ExecuteQuery({ "HSET", "icinga:status:" + typeName, objectName, jsonBody }); // // /* Icinga DB part for Icinga Web 2 */ From fcb375297353dc2f42d9916061936176851a3481 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Tue, 13 Nov 2018 16:43:50 +0100 Subject: [PATCH 091/219] Remove debug log message --- lib/redis/rediswriter-objects.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 5502c9875..afd96f91e 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -561,12 +561,9 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) { - //TODO: This is probably uncesessary? Checkable::Ptr checkable = dynamic_pointer_cast(object); - if (!checkable) { - Log(LogCritical, "DEBUG") << "THIS IS NOT A CHECKABLE, IT'S A " << object->GetReflectionType()->GetName(); + if (!checkable) return; - } bool isHost; String streamname; From 3def496b621235549b196f9fac1d521d858dd148 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Wed, 14 Nov 2018 14:30:31 +0100 Subject: [PATCH 092/219] Update state sync Makes state serialization its own function to be used also when writing out the initial state --- lib/redis/rediswriter-objects.cpp | 250 ++++++++++++++++-------------- lib/redis/rediswriter.hpp | 2 +- 2 files changed, 133 insertions(+), 119 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index afd96f91e..1fe6d49a4 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -117,6 +117,12 @@ void RedisWriter::UpdateAllConfigObjects() checksums.erase(checksums.begin() + 2, checksums.end()); } } + + if (lcType == "host" || lcType == "service") { + Dictionary::Ptr objectAttrs = SerializeState(object); + + m_Rcon->ExecuteQuery({"HSET", "icinga:status:object:" + lcType, GetObjectIdentifier(object), JsonEncode(objectAttrs)}); + } } if (attributes.size() > 2) m_Rcon->ExecuteQuery(attributes); @@ -565,115 +571,24 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) if (!checkable) return; - bool isHost; - String streamname; - streamname = "servicestatus"; - Host::Ptr host; Service::Ptr service; tie(host, service) = GetHostService(checkable); - if (service) { + String streamname; + if (service) streamname = "servicestate"; - isHost = false; - } else { + else streamname = "hoststate"; - isHost = true; - } - - /* Serialize config object attributes - * TODO: Flatten last check result - */ - auto objectAttrs = SerializeState(object); + Dictionary::Ptr objectAttrs = SerializeState(object); std::vector streamadd({"XADD", streamname, "*"}); - - streamadd.emplace_back("id"); - streamadd.emplace_back(GetObjectIdentifier(object)); - streamadd.emplace_back("env_id"); - streamadd.emplace_back(CalculateCheckSumString(GetEnvironment())); - streamadd.emplace_back("state_type"); - streamadd.emplace_back(checkable->GetStateType()); - - streamadd.emplace_back("state"); - if (isHost) { - streamadd.emplace_back(host->GetState()); - } else { - streamadd.emplace_back(service->GetState()); + for (const Dictionary::Pair& kv : objectAttrs) { + streamadd.emplace_back(kv.first); + streamadd.emplace_back(kv.second); } - - streamadd.emplace_back("last_hard_state"); - if (isHost) { - streamadd.emplace_back(host->GetLastHardState()); - } else { - streamadd.emplace_back(service->GetLastHardState()); - } - - streamadd.emplace_back("check_attempt"); - streamadd.emplace_back(checkable->GetCheckAttempt()); - - //streamadd.emplace_back("severity") - //streamadd.emplace_back(checkable->GetSeverity()); - - streamadd.emplace_back("is_active"); - streamadd.emplace_back(checkable->IsActive()); - - // TODO: Is it possible there is no last checkresult? - CheckResult::Ptr cr = checkable->GetLastCheckResult(); - - streamadd.emplace_back("output"); - streamadd.emplace_back(JsonEncode(cr->GetOutput())); - //streamadd.emplace_back("long_output", ) - streamadd.emplace_back("performance_data"); - streamadd.emplace_back(JsonEncode(cr->GetPerformanceData())); - streamadd.emplace_back("command"); - streamadd.emplace_back(JsonEncode(cr->GetCommand())); - //streamadd.emplace_back("is_problem", !checkable->IsReachable() && !checkable->IsAcknowledged()); - streamadd.emplace_back("is_handled"); - streamadd.emplace_back(!checkable->IsAcknowledged()); - streamadd.emplace_back("is_flapping"); - streamadd.emplace_back(checkable->IsFlapping()); - - streamadd.emplace_back("is_acknowledged"); - streamadd.emplace_back(checkable->IsAcknowledged()); - if (checkable->IsAcknowledged()) { - Timestamp entry = 0; - Comment::Ptr AckComment; - for (const Comment::Ptr& c : checkable->GetComments()) { - if (c->GetEntryType() == CommentAcknowledgement) { - if (c->GetEntryTime() > entry) { - entry = c->GetEntryTime(); - AckComment = c; - } - } - } - streamadd.emplace_back("acknowledgement_comment_id"); - streamadd.emplace_back(GetObjectIdentifier(AckComment)); - } - - streamadd.emplace_back("in_downtime"); - streamadd.emplace_back(checkable->IsInDowntime()); - /* - if (checkable->IsInDowntime()) - streamadd.emplace_back("downtime_id", checkable->GetDowntimes()); - */ - - streamadd.emplace_back("execution_time"); - streamadd.emplace_back(cr->CalculateExecutionTime()); - //streamadd.emplace_back("latency", TODO: What); - streamadd.emplace_back("check_timeout"); - streamadd.emplace_back(checkable->GetCheckTimeout()); - - //streamadd.emplace_back("last_update", TODO: What?); - streamadd.emplace_back("last_state_change"); - streamadd.emplace_back(checkable->GetLastStateChange()); - //streamadd.emplace_back("last_soft_state", TODO: We want "previous"); - //streamadd.emplace_back("last_hard_state", TODO: We want "previous"); - streamadd.emplace_back("next_check"); - streamadd.emplace_back(checkable->GetNextCheck()); - m_Rcon->ExecuteQuery(streamadd); // ExecuteQuery({ "HSET", "icinga:status:" + typeName, objectName, jsonBody }); @@ -747,34 +662,133 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) // } } -std::vector RedisWriter::SerializeState(const Object::Ptr& object) +Dictionary::Ptr RedisWriter::SerializeState(const Object::Ptr& object) { - Type::Ptr type = object->GetReflectionType(); + std::vector state = std::vector(); + Dictionary::Ptr attrs = new Dictionary(); - std::vector resultAttrs = std::vector(); + Checkable::Ptr checkable = dynamic_pointer_cast(object); + if (!checkable) + return nullptr; - for (int fid = 0; fid < type->GetFieldCount(); fid++) { - Field field = type->GetFieldInfo(fid); + bool isHost; - if ((field.Attributes & FAState) == 0) - continue; + Host::Ptr host; + Service::Ptr service; - Value val = object->GetField(fid); + tie(host, service) = GetHostService(checkable); - /* hide attributes which shouldn't be user-visible */ - if (field.Attributes & FANoUserView) - continue; - - /* hide internal navigation fields */ - if (field.Attributes & FANavigation && !(field.Attributes & (FAConfig | FAState))) - continue; - - Value sval = Serialize(val); - resultAttrs.emplace_back(field.Name); - resultAttrs.emplace_back(sval); + if (service) { + isHost = false; + } else { + isHost = true; } - return resultAttrs; + attrs->Set("id", GetObjectIdentifier(checkable)); + state.emplace_back("id"); + state.emplace_back(GetObjectIdentifier(checkable)); + attrs->Set("env_id", CalculateCheckSumString(GetEnvironment())); + state.emplace_back("env_id"); + state.emplace_back(CalculateCheckSumString(GetEnvironment())); + attrs->Set("state_type", checkable->GetStateType()); + state.emplace_back("state_type"); + state.emplace_back(checkable->GetStateType()); + + state.emplace_back("state"); + if (isHost) { + state.emplace_back(host->GetState()); + attrs->Set("state", host->GetState()); + } else { + state.emplace_back(service->GetState()); + attrs->Set("state", service->GetState()); + } + + state.emplace_back("last_hard_state"); + if (isHost) { + state.emplace_back(host->GetLastHardState()); + attrs->Set("last_hard_state", host->GetLastHardState()); + } else { + state.emplace_back(service->GetLastHardState()); + attrs->Set("last_hard_state", service->GetLastHardState()); + + } + + attrs->Set("check_attempt", checkable->GetCheckAttempt()); + state.emplace_back("check_attempt"); + state.emplace_back(checkable->GetCheckAttempt()); + + //streamadd.emplace_back("severity") + //streamadd.emplace_back(checkable->GetSeverity()); + + attrs->Set("is_active", checkable->IsActive()); + state.emplace_back("is_active"); + state.emplace_back(checkable->IsActive()); + + // TODO: Is it possible there is no last checkresult? + CheckResult::Ptr cr = checkable->GetLastCheckResult(); + + attrs->Set("output", JsonEncode(cr->GetOutput())); + state.emplace_back("output"); + state.emplace_back(JsonEncode(cr->GetOutput())); + //streamadd.emplace_back("long_output", ) + attrs->Set("performance_data", JsonEncode(cr->GetOutput())); + state.emplace_back("performance_data"); + state.emplace_back(JsonEncode(cr->GetPerformanceData())); + attrs->Set("command", JsonEncode(cr->GetCommand())); + state.emplace_back("command"); + state.emplace_back(JsonEncode(cr->GetCommand())); + //streamadd.emplace_back("is_problem", !checkable->IsReachable() && !checkable->IsAcknowledged()); + //streamadd.emplace_back("is_handled"); + attrs->Set("is_flapping", checkable->IsFlapping()); + state.emplace_back("is_flapping"); + state.emplace_back(checkable->IsFlapping()); + + attrs->Set("is_acknowledged", checkable->IsAcknowledged()); + state.emplace_back("is_acknowledged"); + state.emplace_back(checkable->IsAcknowledged()); + if (checkable->IsAcknowledged()) { + Timestamp entry = 0; + Comment::Ptr AckComment; + for (const Comment::Ptr& c : checkable->GetComments()) { + if (c->GetEntryType() == CommentAcknowledgement) { + if (c->GetEntryTime() > entry) { + entry = c->GetEntryTime(); + AckComment = c; + } + } + } + attrs->Set("acknowledgement_comment_id", GetObjectIdentifier(AckComment)); + state.emplace_back("acknowledgement_comment_id"); + state.emplace_back(GetObjectIdentifier(AckComment)); + } + + attrs->Set("in_downtime", checkable->IsInDowntime()); + state.emplace_back("in_downtime"); + state.emplace_back(checkable->IsInDowntime()); + /* + if (checkable->IsInDowntime()) + streamadd.emplace_back("downtime_id", checkable->GetDowntimes()); + */ + + attrs->Set("execution_time", cr->CalculateExecutionTime()); + state.emplace_back("execution_time"); + state.emplace_back(cr->CalculateExecutionTime()); + //streamadd.emplace_back("latency", TODO: What); + attrs->Set("check_timeout", checkable->GetCheckTimeout()); + state.emplace_back("check_timeout"); + state.emplace_back(checkable->GetCheckTimeout()); + + //streamadd.emplace_back("last_update", TODO: What?); + attrs->Set("last_state_change", checkable->GetLastStateChange()); + state.emplace_back("last_state_change"); + state.emplace_back(checkable->GetLastStateChange()); + //streamadd.emplace_back("last_soft_state", TODO: We want "previous"); + //streamadd.emplace_back("last_hard_state", TODO: We want "previous"); + attrs->Set("next_check", checkable->GetNextCheck()); + state.emplace_back("next_check"); + state.emplace_back(checkable->GetNextCheck()); + + return attrs; } diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index be52545e6..bcaf3ce97 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -73,7 +73,7 @@ private: void SendConfigDelete(const ConfigObject::Ptr& object); void SendStatusUpdate(const ConfigObject::Ptr& object); std::vector UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride); - std::vector SerializeState(const Object::Ptr& object); + Dictionary::Ptr SerializeState(const Object::Ptr& object); /* Stats */ Dictionary::Ptr GetStats(); From f68ad634b2954f432dbb08f149b8b1a4ea73042a Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Wed, 14 Nov 2018 14:32:57 +0100 Subject: [PATCH 093/219] Rename state stream keys --- lib/redis/rediswriter-objects.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 1fe6d49a4..445efd849 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -578,9 +578,9 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) String streamname; if (service) - streamname = "servicestate"; + streamname = "icinga:state:stream:service"; else - streamname = "hoststate"; + streamname = "icinga:state:stream:service"; Dictionary::Ptr objectAttrs = SerializeState(object); From 223ca6c6a077c69320a4045cbd3dc59ca0941da3 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Wed, 14 Nov 2018 14:41:11 +0100 Subject: [PATCH 094/219] Clean up code --- lib/redis/rediswriter-objects.cpp | 173 ++++++------------------------ 1 file changed, 32 insertions(+), 141 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 445efd849..261e4a825 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -121,7 +121,8 @@ void RedisWriter::UpdateAllConfigObjects() if (lcType == "host" || lcType == "service") { Dictionary::Ptr objectAttrs = SerializeState(object); - m_Rcon->ExecuteQuery({"HSET", "icinga:status:object:" + lcType, GetObjectIdentifier(object), JsonEncode(objectAttrs)}); + m_Rcon->ExecuteQuery({"HSET", "icinga:status:object:" + lcType, GetObjectIdentifier(object), + JsonEncode(objectAttrs)}); } } if (attributes.size() > 2) @@ -175,7 +176,8 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtime m_Rcon->ExecuteQuery(checksums); } -void RedisWriter::MakeTypeChecksums(const ConfigObject::Ptr& object, std::set& propertiesBlacklist, Dictionary::Ptr& checkSums) +void RedisWriter::MakeTypeChecksums(const ConfigObject::Ptr& object, std::set& propertiesBlacklist, + Dictionary::Ptr& checkSums) { Endpoint::Ptr endpoint = dynamic_pointer_cast(object); if (endpoint) { @@ -492,9 +494,10 @@ void RedisWriter::MakeTypeChecksums(const ConfigObject::Ptr& object, std::set& attributes, - std::vector& customVars, std::vector& checksums, - bool runtimeUpdate) +void +RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, const String typeName, std::vector& attributes, + std::vector& customVars, std::vector& checksums, + bool runtimeUpdate) { /* TODO: This isn't essentially correct as we don't keep track of config objects ourselves. This would avoid duplicated config updates at startup. if (!runtimeUpdate && m_ConfigDumpInProgress) @@ -589,82 +592,12 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) streamadd.emplace_back(kv.first); streamadd.emplace_back(kv.second); } - m_Rcon->ExecuteQuery(streamadd); -// ExecuteQuery({ "HSET", "icinga:status:" + typeName, objectName, jsonBody }); -// -// /* Icinga DB part for Icinga Web 2 */ -// Checkable::Ptr checkable = dynamic_pointer_cast(object); -// -// if (checkable) { -// Dictionary::Ptr attrs = new Dictionary(); -// String tableName; -// String objectCheckSum = CalculateCheckSumString(objectName); -// -// Host::Ptr host; -// Service::Ptr service; -// -// tie(host, service) = GetHostService(checkable); -// -// if (service) { -// tableName = "servicestate"; -// attrs->Set("service_checksum", objectCheckSum); -// attrs->Set("host_checksum", CalculateCheckSumString(host->GetName())); -// } else { -// tableName = "hoststate"; -// attrs->Set("host_checksum", objectCheckSum); -// } -// -// attrs->Set("last_check", checkable->GetLastCheck()); -// attrs->Set("next_check", checkable->GetNextCheck()); -// -// attrs->Set("severity", checkable->GetSeverity()); -// -///* -// 'host_checksum' => null, -// 'command' => null, // JSON, array -// 'execution_start' => null, -// 'execution_end' => null, -// 'schedule_start' => null, -// 'schedule_end' => null, -// 'exit_status' => null, -// 'output' => null, -// 'performance_data' => null, // JSON, array -// -// -//10.0.3.12:6379> keys icinga:hoststate.* -//1) "icinga:hoststate.~\xf5a\x91+\x03\x97\x99\xb5(\x16 CYm\xb1\xdf\x85\xa2\xcb" -//10.0.3.12:6379> get "icinga:hoststate.~\xf5a\x91+\x03\x97\x99\xb5(\x16 CYm\xb1\xdf\x85\xa2\xcb" -//"{\"command\":[\"\\/usr\\/lib\\/nagios\\/plugins\\/check_ping\",\"-H\",\"127.0.0.1\",\"-c\",\"5000,100%\",\"-w\",\"3000,80%\"],\"execution_start\":1492007581.7624,\"execution_end\":1492007585.7654,\"schedule_start\":1492007581.7609,\"schedule_end\":1492007585.7655,\"exit_status\":0,\"output\":\"PING OK - Packet loss = 0%, RTA = 0.08 ms\",\"performance_data\":[\"rta=0.076000ms;3000.000000;5000.000000;0.000000\",\"pl=0%;80;100;0\"]}" -// -//*/ -// -// CheckResult::Ptr cr = checkable->GetLastCheckResult(); -// -// if (cr) { -// attrs->Set("command", JsonEncode(cr->GetCommand())); -// attrs->Set("execution_start", cr->GetExecutionStart()); -// attrs->Set("execution_end", cr->GetExecutionEnd()); -// attrs->Set("schedule_start", cr->GetScheduleStart()); -// attrs->Set("schedule_end", cr->GetScheduleStart()); -// attrs->Set("exit_status", cr->GetExitStatus()); -// attrs->Set("output", cr->GetOutput()); -// attrs->Set("performance_data", JsonEncode(cr->GetPerformanceData())); -// } -// -// String jsonAttrs = JsonEncode(attrs); -// String key = "icinga:" + tableName + "." + objectCheckSum; -// ExecuteQuery({ "SET", key, jsonAttrs }); -// -// /* expire in check_interval * attempts + timeout + some more seconds */ -// double expireTime = checkable->GetCheckInterval() * checkable->GetMaxCheckAttempts() + 60; -// ExecuteQuery({ "EXPIRE", key, String(expireTime) }); -// } + m_Rcon->ExecuteQuery(streamadd); } Dictionary::Ptr RedisWriter::SerializeState(const Object::Ptr& object) { - std::vector state = std::vector(); Dictionary::Ptr attrs = new Dictionary(); Checkable::Ptr checkable = dynamic_pointer_cast(object); @@ -678,74 +611,44 @@ Dictionary::Ptr RedisWriter::SerializeState(const Object::Ptr& object) tie(host, service) = GetHostService(checkable); - if (service) { + if (service) isHost = false; - } else { + else isHost = true; - } - attrs->Set("id", GetObjectIdentifier(checkable)); - state.emplace_back("id"); - state.emplace_back(GetObjectIdentifier(checkable)); + attrs->Set("id", GetObjectIdentifier(checkable));; attrs->Set("env_id", CalculateCheckSumString(GetEnvironment())); - state.emplace_back("env_id"); - state.emplace_back(CalculateCheckSumString(GetEnvironment())); attrs->Set("state_type", checkable->GetStateType()); - state.emplace_back("state_type"); - state.emplace_back(checkable->GetStateType()); - state.emplace_back("state"); - if (isHost) { - state.emplace_back(host->GetState()); + if (isHost) attrs->Set("state", host->GetState()); - } else { - state.emplace_back(service->GetState()); + else attrs->Set("state", service->GetState()); - } - state.emplace_back("last_hard_state"); - if (isHost) { - state.emplace_back(host->GetLastHardState()); + if (isHost) attrs->Set("last_hard_state", host->GetLastHardState()); - } else { - state.emplace_back(service->GetLastHardState()); + else attrs->Set("last_hard_state", service->GetLastHardState()); - } - attrs->Set("check_attempt", checkable->GetCheckAttempt()); - state.emplace_back("check_attempt"); - state.emplace_back(checkable->GetCheckAttempt()); - //streamadd.emplace_back("severity") - //streamadd.emplace_back(checkable->GetSeverity()); + //attrs->Set("severity") + //attrs->Set(checkable->GetSeverity()); attrs->Set("is_active", checkable->IsActive()); - state.emplace_back("is_active"); - state.emplace_back(checkable->IsActive()); // TODO: Is it possible there is no last checkresult? CheckResult::Ptr cr = checkable->GetLastCheckResult(); attrs->Set("output", JsonEncode(cr->GetOutput())); - state.emplace_back("output"); - state.emplace_back(JsonEncode(cr->GetOutput())); - //streamadd.emplace_back("long_output", ) + //attrs->Set("long_output", ) TODO attrs->Set("performance_data", JsonEncode(cr->GetOutput())); - state.emplace_back("performance_data"); - state.emplace_back(JsonEncode(cr->GetPerformanceData())); attrs->Set("command", JsonEncode(cr->GetCommand())); - state.emplace_back("command"); - state.emplace_back(JsonEncode(cr->GetCommand())); - //streamadd.emplace_back("is_problem", !checkable->IsReachable() && !checkable->IsAcknowledged()); - //streamadd.emplace_back("is_handled"); + //attrs->Set("is_problem", !checkable->IsReachable() && !checkable->IsAcknowledged()); TODO + //attrs->Set("is_handled"); TODO attrs->Set("is_flapping", checkable->IsFlapping()); - state.emplace_back("is_flapping"); - state.emplace_back(checkable->IsFlapping()); attrs->Set("is_acknowledged", checkable->IsAcknowledged()); - state.emplace_back("is_acknowledged"); - state.emplace_back(checkable->IsAcknowledged()); if (checkable->IsAcknowledged()) { Timestamp entry = 0; Comment::Ptr AckComment; @@ -758,35 +661,23 @@ Dictionary::Ptr RedisWriter::SerializeState(const Object::Ptr& object) } } attrs->Set("acknowledgement_comment_id", GetObjectIdentifier(AckComment)); - state.emplace_back("acknowledgement_comment_id"); - state.emplace_back(GetObjectIdentifier(AckComment)); } attrs->Set("in_downtime", checkable->IsInDowntime()); - state.emplace_back("in_downtime"); - state.emplace_back(checkable->IsInDowntime()); /* if (checkable->IsInDowntime()) - streamadd.emplace_back("downtime_id", checkable->GetDowntimes()); + attrs->Set("downtime_id", checkable->GetDowntimes()); TODO */ attrs->Set("execution_time", cr->CalculateExecutionTime()); - state.emplace_back("execution_time"); - state.emplace_back(cr->CalculateExecutionTime()); - //streamadd.emplace_back("latency", TODO: What); + //attrs->Set("latency", TODO: What); attrs->Set("check_timeout", checkable->GetCheckTimeout()); - state.emplace_back("check_timeout"); - state.emplace_back(checkable->GetCheckTimeout()); - //streamadd.emplace_back("last_update", TODO: What?); + //sattrs->Set("last_update", TODO: What?); attrs->Set("last_state_change", checkable->GetLastStateChange()); - state.emplace_back("last_state_change"); - state.emplace_back(checkable->GetLastStateChange()); - //streamadd.emplace_back("last_soft_state", TODO: We want "previous"); - //streamadd.emplace_back("last_hard_state", TODO: We want "previous"); + //attrs->Set("last_soft_state", TODO: We want "previous"); + //attrs->Set("last_hard_state", TODO: We want "previous"); attrs->Set("next_check", checkable->GetNextCheck()); - state.emplace_back("next_check"); - state.emplace_back(checkable->GetNextCheck()); return attrs; } @@ -818,12 +709,12 @@ RedisWriter::UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, attrs->Set(field.Name, Serialize(val)); } - /* Downtimes require in_effect, which is not an attribute */ - Downtime::Ptr downtime = dynamic_pointer_cast(object); - if (downtime) { - attrs->Set("in_effect", Serialize(downtime->IsInEffect())); - attrs->Set("trigger_time", Serialize(downtime->GetTriggerTime())); - } + /* Downtimes require in_effect, which is not an attribute */ + Downtime::Ptr downtime = dynamic_pointer_cast(object); + if (downtime) { + attrs->Set("in_effect", Serialize(downtime->IsInEffect())); + attrs->Set("trigger_time", Serialize(downtime->GetTriggerTime())); + } /* Use the name checksum as unique key. */ From d8737b238ab5239dd4d350fa28d6d0c08b3179f6 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Wed, 14 Nov 2018 16:42:48 +0100 Subject: [PATCH 095/219] Update state sync --- lib/redis/rediswriter-objects.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 261e4a825..ab77c97ef 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -583,7 +583,7 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) if (service) streamname = "icinga:state:stream:service"; else - streamname = "icinga:state:stream:service"; + streamname = "icinga:state:stream:host"; Dictionary::Ptr objectAttrs = SerializeState(object); @@ -671,7 +671,9 @@ Dictionary::Ptr RedisWriter::SerializeState(const Object::Ptr& object) attrs->Set("execution_time", cr->CalculateExecutionTime()); //attrs->Set("latency", TODO: What); - attrs->Set("check_timeout", checkable->GetCheckTimeout()); + + if (checkable->GetCheckTimeout()) + attrs->Set("check_timeout", checkable->GetCheckTimeout()); //sattrs->Set("last_update", TODO: What?); attrs->Set("last_state_change", checkable->GetLastStateChange()); From 9b9cf6d5b0e4780a1f6172092ba12895dd4a5394 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Thu, 15 Nov 2018 10:30:41 +0100 Subject: [PATCH 096/219] Update sate sync fix a crash, possibly two --- lib/redis/rediswriter-objects.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index ab77c97ef..869f93e75 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -588,6 +588,7 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) Dictionary::Ptr objectAttrs = SerializeState(object); std::vector streamadd({"XADD", streamname, "*"}); + ObjectLock olock(objectAttrs); for (const Dictionary::Pair& kv : objectAttrs) { streamadd.emplace_back(kv.first); streamadd.emplace_back(kv.second); @@ -640,10 +641,12 @@ Dictionary::Ptr RedisWriter::SerializeState(const Object::Ptr& object) // TODO: Is it possible there is no last checkresult? CheckResult::Ptr cr = checkable->GetLastCheckResult(); - attrs->Set("output", JsonEncode(cr->GetOutput())); - //attrs->Set("long_output", ) TODO - attrs->Set("performance_data", JsonEncode(cr->GetOutput())); - attrs->Set("command", JsonEncode(cr->GetCommand())); + if (cr) { + attrs->Set("output", JsonEncode(cr->GetOutput())); + //attrs->Set("long_output", ) TODO + attrs->Set("performance_data", JsonEncode(cr->GetOutput())); + attrs->Set("command", JsonEncode(cr->GetCommand())); + } //attrs->Set("is_problem", !checkable->IsReachable() && !checkable->IsAcknowledged()); TODO //attrs->Set("is_handled"); TODO attrs->Set("is_flapping", checkable->IsFlapping()); From a7e615fbc0ebb29b10e4856fe786478cd5edd003 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Thu, 15 Nov 2018 10:56:36 +0100 Subject: [PATCH 097/219] Fix one unchecked checkresult access --- lib/redis/rediswriter-objects.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 869f93e75..10127eb4f 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -646,6 +646,7 @@ Dictionary::Ptr RedisWriter::SerializeState(const Object::Ptr& object) //attrs->Set("long_output", ) TODO attrs->Set("performance_data", JsonEncode(cr->GetOutput())); attrs->Set("command", JsonEncode(cr->GetCommand())); + attrs->Set("execution_time", cr->CalculateExecutionTime()); } //attrs->Set("is_problem", !checkable->IsReachable() && !checkable->IsAcknowledged()); TODO //attrs->Set("is_handled"); TODO @@ -672,7 +673,6 @@ Dictionary::Ptr RedisWriter::SerializeState(const Object::Ptr& object) attrs->Set("downtime_id", checkable->GetDowntimes()); TODO */ - attrs->Set("execution_time", cr->CalculateExecutionTime()); //attrs->Set("latency", TODO: What); if (checkable->GetCheckTimeout()) From 02ecc3e44588e0fd04b22ddfbde105295c93e83c Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Thu, 15 Nov 2018 15:47:28 +0100 Subject: [PATCH 098/219] Add additional fields --- lib/icinga/compatutility.cpp | 4 ++-- lib/redis/rediswriter-objects.cpp | 30 +++++++++++++++++------------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/lib/icinga/compatutility.cpp b/lib/icinga/compatutility.cpp index 305d3b247..5945ebfaf 100644 --- a/lib/icinga/compatutility.cpp +++ b/lib/icinga/compatutility.cpp @@ -249,7 +249,7 @@ std::set CompatUtility::GetCheckableNotificationUserGroups(const return usergroups; } -/* Used in DB IDO, StatusDataWriter, Livestatus, CompatLogger, GelfWriter. */ +/* Used in DB IDO, StatusDataWriter, Livestatus, CompatLogger, GelfWriter, RedisWriter. */ String CompatUtility::GetCheckResultOutput(const CheckResult::Ptr& cr) { if (!cr) @@ -264,7 +264,7 @@ String CompatUtility::GetCheckResultOutput(const CheckResult::Ptr& cr) return raw_output.SubStr(0, line_end); } -/* Used in DB IDO, StatusDataWriter and Livestatus. */ +/* Used in DB IDO, StatusDataWriter and Livestatus, RedisWriter. */ String CompatUtility::GetCheckResultLongOutput(const CheckResult::Ptr& cr) { if (!cr) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 10127eb4f..c428e1472 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -20,6 +20,7 @@ #include "redis/rediswriter.hpp" #include "redis/redisconnection.hpp" #include "icinga/command.hpp" +#include "icinga/compatutility.hpp" #include "base/configtype.hpp" #include "base/configobject.hpp" #include "icinga/customvarobject.hpp" @@ -638,18 +639,27 @@ Dictionary::Ptr RedisWriter::SerializeState(const Object::Ptr& object) attrs->Set("is_active", checkable->IsActive()); - // TODO: Is it possible there is no last checkresult? CheckResult::Ptr cr = checkable->GetLastCheckResult(); if (cr) { - attrs->Set("output", JsonEncode(cr->GetOutput())); - //attrs->Set("long_output", ) TODO - attrs->Set("performance_data", JsonEncode(cr->GetOutput())); + // TODO: Long Output did not work in my test cases. Need to investigate + attrs->Set("output", JsonEncode(CompatUtility::GetCheckResultOutput(cr))); + attrs->Set("long_output", CompatUtility::GetCheckResultLongOutput(cr)); + attrs->Set("performance_data", JsonEncode(cr->GetPerformanceData())); attrs->Set("command", JsonEncode(cr->GetCommand())); attrs->Set("execution_time", cr->CalculateExecutionTime()); + attrs->Set("latency", cr->CalculateLatency()); } - //attrs->Set("is_problem", !checkable->IsReachable() && !checkable->IsAcknowledged()); TODO - //attrs->Set("is_handled"); TODO + + bool isProblem = !checkable->IsStateOK(checkable->GetStateRaw()); + attrs->Set("is_problem", isProblem); + + bool isHandledNoDependency = isProblem && checkable->IsInDowntime() && checkable->IsAcknowledged(); + if (isHost) + attrs->Set("is_handled", isHandledNoDependency); + else + attrs->Set("is_handled", isHandledNoDependency && !checkable->IsStateOK(service->GetHost()->GetStateRaw())); + attrs->Set("is_flapping", checkable->IsFlapping()); attrs->Set("is_acknowledged", checkable->IsAcknowledged()); @@ -668,17 +678,11 @@ Dictionary::Ptr RedisWriter::SerializeState(const Object::Ptr& object) } attrs->Set("in_downtime", checkable->IsInDowntime()); - /* - if (checkable->IsInDowntime()) - attrs->Set("downtime_id", checkable->GetDowntimes()); TODO - */ - - //attrs->Set("latency", TODO: What); if (checkable->GetCheckTimeout()) attrs->Set("check_timeout", checkable->GetCheckTimeout()); - //sattrs->Set("last_update", TODO: What?); + attrs->Set("last_update", Utility::GetTime()); attrs->Set("last_state_change", checkable->GetLastStateChange()); //attrs->Set("last_soft_state", TODO: We want "previous"); //attrs->Set("last_hard_state", TODO: We want "previous"); From 9aec47efb3c0b01b8129f60fb0ffc7da26cf3ab6 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Thu, 15 Nov 2018 15:51:56 +0100 Subject: [PATCH 099/219] Use OnStateChange instead of OnStateChanged --- lib/redis/rediswriter-objects.cpp | 4 ++-- lib/redis/rediswriter.hpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index c428e1472..58876ca23 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -52,7 +52,7 @@ INITIALIZE_ONCE(&RedisWriter::ConfigStaticInitialize); void RedisWriter::ConfigStaticInitialize() { /* triggered in ProcessCheckResult(), requires UpdateNextCheck() to be called before */ - ConfigObject::OnStateChanged.connect(std::bind(&RedisWriter::StateChangedHandler, _1)); + Checkable::OnStateChange.connect(std::bind(&RedisWriter::StateChangeHandler, _1)); /* triggered on create, update and delete objects */ ConfigObject::OnActiveChanged.connect(std::bind(&RedisWriter::VersionChangedHandler, _1)); @@ -735,7 +735,7 @@ RedisWriter::UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, //m_Rcon->ExecuteQuery({"HSET", keyPrefix + typeName, GetObjectIdentifier(object), JsonEncode(attrs)}); } -void RedisWriter::StateChangedHandler(const ConfigObject::Ptr& object) +void RedisWriter::StateChangeHandler(const ConfigObject::Ptr &object) { Type::Ptr type = object->GetReflectionType(); diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index bcaf3ce97..5f07c652e 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -97,7 +97,7 @@ private: static void MakeTypeChecksums(const ConfigObject::Ptr& object, std::set& propertiesBlacklist, Dictionary::Ptr& checkSums); - static void StateChangedHandler(const ConfigObject::Ptr& object); + static void StateChangeHandler(const ConfigObject::Ptr &object); static void VersionChangedHandler(const ConfigObject::Ptr& object); void AssertOnWorkQueue(); From 916bfd20fbf5c5eda182829ec9dad9da8949f616 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Thu, 15 Nov 2018 17:04:03 +0100 Subject: [PATCH 100/219] Update state sync --- lib/redis/rediswriter-objects.cpp | 97 +++++++++++++++++-------------- lib/redis/rediswriter.cpp | 14 ++++- lib/redis/rediswriter.hpp | 6 +- 3 files changed, 71 insertions(+), 46 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 58876ca23..b810029d6 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -92,17 +92,25 @@ void RedisWriter::UpdateAllConfigObjects() { String lcType = type.second; m_Rcon->ExecuteQuery( - {"DEL", m_PrefixConfigCheckSum + lcType, m_PrefixConfigObject + lcType, m_PrefixStatusObject + lcType}); + {"DEL", m_PrefixConfigCheckSum + lcType, m_PrefixConfigObject + lcType, m_PrefixStateObject + lcType}); size_t bulkCounter = 0; - auto attributes = std::vector({"HMSET", m_PrefixConfigObject + lcType}); - auto customVars = std::vector({"HMSET", m_PrefixConfigCustomVar + lcType}); - auto checksums = std::vector({"HMSET", m_PrefixConfigCheckSum + lcType}); + std::vector attributes = {"HMSET", m_PrefixConfigObject + lcType}; + std::vector customVars = {"HMSET", m_PrefixConfigCustomVar + lcType}; + std::vector checksums = {"HMSET", m_PrefixConfigCheckSum + lcType}; + std::vector states = {"HMSET", m_PrefixStateObject + lcType }; + bool dumpState = (lcType == "host" || lcType == "service"); for (const ConfigObject::Ptr& object : type.first->GetObjects()) { if (lcType != GetLowerCaseTypeNameDB(object)) continue; CreateConfigUpdate(object, lcType, attributes, customVars, checksums, false); - SendStatusUpdate(object); + + // Write out inital state for checkables + if (dumpState) { + states.emplace_back(GetObjectIdentifier(object)); + states.emplace_back(JsonEncode(SerializeState(dynamic_pointer_cast(object)))); + } + bulkCounter++; if (!bulkCounter % 100) { if (attributes.size() > 2) { @@ -117,13 +125,10 @@ void RedisWriter::UpdateAllConfigObjects() m_Rcon->ExecuteQuery(checksums); checksums.erase(checksums.begin() + 2, checksums.end()); } - } - - if (lcType == "host" || lcType == "service") { - Dictionary::Ptr objectAttrs = SerializeState(object); - - m_Rcon->ExecuteQuery({"HSET", "icinga:status:object:" + lcType, GetObjectIdentifier(object), - JsonEncode(objectAttrs)}); + if (states.size() > 2) { + m_Rcon->ExecuteQuery(states); + states.erase(states.begin() + 2, states.end()); + } } } if (attributes.size() > 2) @@ -132,6 +137,8 @@ void RedisWriter::UpdateAllConfigObjects() m_Rcon->ExecuteQuery(customVars); if (checksums.size() > 2) m_Rcon->ExecuteQuery(checksums); + if (states.size() > 2) + m_Rcon->ExecuteQuery(states); m_Rcon->ExecuteQuery({"PUBLISH", "icinga:config:dump", lcType}); @@ -152,6 +159,12 @@ void RedisWriter::UpdateAllConfigObjects() << "Initial config/status dump finished in " << Utility::GetTime() - startTime << " seconds."; } +void RedisWriter::UpdateState(const Checkable::Ptr& checkable) { + Dictionary::Ptr stateAttrs = SerializeState(checkable); + + m_Rcon->ExecuteQuery({"HSET", m_PrefixStateObject + GetLowerCaseTypeNameDB(checkable), GetObjectIdentifier(checkable), JsonEncode(stateAttrs)}); +} + template static ConfigObject::Ptr GetObjectByName(const String& name) { @@ -166,15 +179,23 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtime String typeName = GetLowerCaseTypeNameDB(object); - auto attributes = std::vector({"HMSET", m_PrefixConfigObject + typeName}); - auto customVars = std::vector({"HMSET", m_PrefixConfigCustomVar + typeName}); - auto checksums = std::vector({"HMSET", m_PrefixConfigCheckSum + typeName}); + std::vector attribute = {"HSET", m_PrefixConfigObject + typeName}; + std::vector customVar = {"HSET", m_PrefixConfigCustomVar + typeName}; + std::vector checksum = {"HSET", m_PrefixConfigCheckSum + typeName}; + std::vector state = {"HSET", m_PrefixStateObject + typeName}; - CreateConfigUpdate(object, typeName, attributes, customVars, checksums, runtimeUpdate); - m_Rcon->ExecuteQuery(attributes); - m_Rcon->ExecuteQuery(customVars); - m_Rcon->ExecuteQuery(checksums); + CreateConfigUpdate(object, typeName, attribute, customVar, checksum, runtimeUpdate); + Checkable::Ptr checkable = dynamic_pointer_cast(object); + if (checkable) { + m_Rcon->ExecuteQuery({"HSET", m_PrefixStateObject + typeName, + GetObjectIdentifier(checkable), JsonEncode(SerializeState(checkable))}); + } + + m_Rcon->ExecuteQuery(attribute); + m_Rcon->ExecuteQuery(checksum); + if (customVar.size() > 2) + m_Rcon->ExecuteQuery(customVar); } void RedisWriter::MakeTypeChecksums(const ConfigObject::Ptr& object, std::set& propertiesBlacklist, @@ -564,8 +585,8 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) m_Rcon->ExecuteQueries({ {"HDEL", m_PrefixConfigObject + typeName, objectKey}, - {"DEL", m_PrefixStatusObject + typeName + ":" + objectKey}, - {"PUBLISH", "icinga:config:delete", typeName + ":" + objectKey} + {"DEL", m_PrefixStateObject + typeName + ":" + objectKey}, + {"PUBLISH", "icinga:config:delete", typeName + ":" + objectKey} }); } @@ -586,7 +607,7 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) else streamname = "icinga:state:stream:host"; - Dictionary::Ptr objectAttrs = SerializeState(object); + Dictionary::Ptr objectAttrs = SerializeState(checkable); std::vector streamadd({"XADD", streamname, "*"}); ObjectLock olock(objectAttrs); @@ -598,39 +619,28 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) m_Rcon->ExecuteQuery(streamadd); } -Dictionary::Ptr RedisWriter::SerializeState(const Object::Ptr& object) +Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) { Dictionary::Ptr attrs = new Dictionary(); - Checkable::Ptr checkable = dynamic_pointer_cast(object); - if (!checkable) - return nullptr; - - bool isHost; - Host::Ptr host; Service::Ptr service; tie(host, service) = GetHostService(checkable); - if (service) - isHost = false; - else - isHost = true; - attrs->Set("id", GetObjectIdentifier(checkable));; attrs->Set("env_id", CalculateCheckSumString(GetEnvironment())); attrs->Set("state_type", checkable->GetStateType()); - if (isHost) - attrs->Set("state", host->GetState()); - else + if (service) attrs->Set("state", service->GetState()); - - if (isHost) - attrs->Set("last_hard_state", host->GetLastHardState()); else + attrs->Set("state", host->GetState()); + + if (service) attrs->Set("last_hard_state", service->GetLastHardState()); + else + attrs->Set("last_hard_state", host->GetLastHardState()); attrs->Set("check_attempt", checkable->GetCheckAttempt()); @@ -655,11 +665,12 @@ Dictionary::Ptr RedisWriter::SerializeState(const Object::Ptr& object) attrs->Set("is_problem", isProblem); bool isHandledNoDependency = isProblem && checkable->IsInDowntime() && checkable->IsAcknowledged(); - if (isHost) - attrs->Set("is_handled", isHandledNoDependency); - else + if (service) attrs->Set("is_handled", isHandledNoDependency && !checkable->IsStateOK(service->GetHost()->GetStateRaw())); + else + attrs->Set("is_handled", isHandledNoDependency); + attrs->Set("is_reachable", checkable->IsReachable()); attrs->Set("is_flapping", checkable->IsFlapping()); attrs->Set("is_acknowledged", checkable->IsAcknowledged()); diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index 41fe0d048..b9a6cc891 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -45,7 +45,7 @@ RedisWriter::RedisWriter() m_PrefixConfigObject = "icinga:config:object:"; m_PrefixConfigCheckSum = "icinga:config:checksum:"; m_PrefixConfigCustomVar = "icinga:config:customvar:"; - m_PrefixStatusObject = "icinga:status:object:"; + m_PrefixStateObject = "icinga:state:object:"; } /** @@ -307,6 +307,18 @@ void RedisWriter::SendEvent(const Dictionary::Ptr& event) return; String type = event->Get("type"); + + if (type == "CheckResult") { + Checkable::Ptr checkable; + if (event->Contains("service")) { + checkable = Service::GetByNamePair(event->Get("host"), event->Get("service")); + } else { + checkable = Host::GetByName(event->Get("host")); + } + // Update State for icingaweb + m_WorkQueue.Enqueue(std::bind(&RedisWriter::UpdateState, this, checkable)); + } + if (type.Contains("Acknowledgement")) { Checkable::Ptr checkable; diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 5f07c652e..01e7dd35b 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -26,6 +26,7 @@ #include "base/timer.hpp" #include "base/workqueue.hpp" #include "redis/redisconnection.hpp" +#include "icinga/checkable.hpp" #include namespace icinga @@ -67,13 +68,14 @@ private: /* config & status dump */ void UpdateAllConfigObjects(); + void UpdateState(const Checkable::Ptr& checkable); void SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate); void CreateConfigUpdate(const ConfigObject::Ptr& object, const String type, std::vector& attributes, std::vector& customVars, std::vector& checksums, bool runtimeUpdate); void SendConfigDelete(const ConfigObject::Ptr& object); void SendStatusUpdate(const ConfigObject::Ptr& object); std::vector UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride); - Dictionary::Ptr SerializeState(const Object::Ptr& object); + Dictionary::Ptr SerializeState(const Checkable::Ptr& checkable); /* Stats */ Dictionary::Ptr GetStats(); @@ -119,7 +121,7 @@ private: String m_PrefixConfigObject; String m_PrefixConfigCheckSum; String m_PrefixConfigCustomVar; - String m_PrefixStatusObject; + String m_PrefixStateObject; bool m_ConfigDumpInProgress; bool m_ConfigDumpDone; From 328adf1fc450481c376ce8a5f5d4ec864883df26 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 26 Nov 2018 15:03:11 +0100 Subject: [PATCH 101/219] Add host/service state sync to acknowledgement --- lib/redis/rediswriter-objects.cpp | 4 +++- lib/redis/rediswriter.cpp | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index b810029d6..280641215 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -685,7 +685,9 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) } } } - attrs->Set("acknowledgement_comment_id", GetObjectIdentifier(AckComment)); + if (AckComment != nullptr) { + attrs->Set("acknowledgement_comment_id", GetObjectIdentifier(AckComment)); + } } attrs->Set("in_downtime", checkable->IsInDowntime()); diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index b9a6cc891..076ad86ff 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -338,6 +338,7 @@ void RedisWriter::SendEvent(const Dictionary::Ptr& event) if (c->GetEntryTime() > entry) { entry = c->GetEntryTime(); AckComment = c; + StateChangeHandler(checkable); } } } From e2caa47f182b057b93bf80b050dca99ff0b61a30 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Tue, 27 Nov 2018 11:03:35 +0100 Subject: [PATCH 102/219] Fix is_handled --- lib/redis/rediswriter-objects.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 280641215..514e1b1ab 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -663,13 +663,7 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) bool isProblem = !checkable->IsStateOK(checkable->GetStateRaw()); attrs->Set("is_problem", isProblem); - - bool isHandledNoDependency = isProblem && checkable->IsInDowntime() && checkable->IsAcknowledged(); - if (service) - attrs->Set("is_handled", isHandledNoDependency && !checkable->IsStateOK(service->GetHost()->GetStateRaw())); - else - attrs->Set("is_handled", isHandledNoDependency); - + attrs->Set("is_handled", isProblem && (checkable->IsInDowntime() || checkable->IsAcknowledged())); attrs->Set("is_reachable", checkable->IsReachable()); attrs->Set("is_flapping", checkable->IsFlapping()); From 6ce4612cd720360a1bde9b5b9bad98c713f466cf Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Thu, 29 Nov 2018 15:42:46 +0100 Subject: [PATCH 103/219] Export last hard/soft state dummies --- lib/redis/rediswriter-objects.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 514e1b1ab..436834857 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -632,15 +632,16 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) attrs->Set("env_id", CalculateCheckSumString(GetEnvironment())); attrs->Set("state_type", checkable->GetStateType()); - if (service) + // TODO: last_hard/soft_state should be "previous". + if (service) { attrs->Set("state", service->GetState()); - else - attrs->Set("state", host->GetState()); - - if (service) + attrs->Set("last_soft_state", service->GetState()); attrs->Set("last_hard_state", service->GetLastHardState()); - else + } else { + attrs->Set("state", host->GetState()); + attrs->Set("last_soft_state", host->GetState()); attrs->Set("last_hard_state", host->GetLastHardState()); + } attrs->Set("check_attempt", checkable->GetCheckAttempt()); @@ -691,8 +692,6 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) attrs->Set("last_update", Utility::GetTime()); attrs->Set("last_state_change", checkable->GetLastStateChange()); - //attrs->Set("last_soft_state", TODO: We want "previous"); - //attrs->Set("last_hard_state", TODO: We want "previous"); attrs->Set("next_check", checkable->GetNextCheck()); return attrs; From d104eb4ac1d57885c0f3665bc1ebf3cbc3308e26 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Thu, 29 Nov 2018 15:46:29 +0100 Subject: [PATCH 104/219] Fix timeout --- lib/redis/rediswriter-objects.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 436834857..7ceaf9069 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -687,7 +687,9 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) attrs->Set("in_downtime", checkable->IsInDowntime()); - if (checkable->GetCheckTimeout()) + if (checkable->GetCheckTimeout().IsEmpty()) + attrs->Set("check_timeout",checkable->GetCheckCommand()->GetTimeout()); + else attrs->Set("check_timeout", checkable->GetCheckTimeout()); attrs->Set("last_update", Utility::GetTime()); From af05a354fa3eb35aa69a52e30a7834659fa2cc0c Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Thu, 29 Nov 2018 16:44:43 +0100 Subject: [PATCH 105/219] Stop json-encoding everything --- lib/redis/rediswriter-objects.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 7ceaf9069..e0421f31e 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -654,10 +654,10 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) if (cr) { // TODO: Long Output did not work in my test cases. Need to investigate - attrs->Set("output", JsonEncode(CompatUtility::GetCheckResultOutput(cr))); + attrs->Set("output", CompatUtility::GetCheckResultOutput(cr)); attrs->Set("long_output", CompatUtility::GetCheckResultLongOutput(cr)); - attrs->Set("performance_data", JsonEncode(cr->GetPerformanceData())); - attrs->Set("command", JsonEncode(cr->GetCommand())); + attrs->Set("performance_data", cr->GetPerformanceData()); + attrs->Set("command", cr->GetCommand()); attrs->Set("execution_time", cr->CalculateExecutionTime()); attrs->Set("latency", cr->CalculateLatency()); } From 2202bde56c95101f92636c0c2bd61ba402c1ffcb Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 30 Nov 2018 09:28:20 +0100 Subject: [PATCH 106/219] Add downtime handlers to state sync --- lib/redis/rediswriter-objects.cpp | 13 +++++++++++++ lib/redis/rediswriter.hpp | 2 ++ 2 files changed, 15 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index e0421f31e..6b797dc5f 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -57,6 +57,13 @@ void RedisWriter::ConfigStaticInitialize() /* triggered on create, update and delete objects */ ConfigObject::OnActiveChanged.connect(std::bind(&RedisWriter::VersionChangedHandler, _1)); ConfigObject::OnVersionChanged.connect(std::bind(&RedisWriter::VersionChangedHandler, _1)); + + /* fixed downtime start */ + Downtime::OnDowntimeStarted.connect(std::bind(&RedisWriter::DowntimeChangedHandler, _1)); + /* flexible downtime start */ + Downtime::OnDowntimeTriggered.connect(std::bind(&RedisWriter::DowntimeChangedHandler, _1)); + /* fixed/flexible downtime end */ + Downtime::OnDowntimeRemoved.connect(std::bind(&RedisWriter::DowntimeChangedHandler, _1)); } void RedisWriter::UpdateAllConfigObjects() @@ -771,3 +778,9 @@ void RedisWriter::VersionChangedHandler(const ConfigObject::Ptr& object) } } } + +void RedisWriter::DowntimeChangedHandler(const Downtime::Ptr& downtime) +{ + Log(LogCritical, "Downtime", "Downtime sync got triggered"); + StateChangeHandler(downtime->GetCheckable()); +} diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 01e7dd35b..5f7031634 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -27,6 +27,7 @@ #include "base/workqueue.hpp" #include "redis/redisconnection.hpp" #include "icinga/checkable.hpp" +#include "icinga/downtime.hpp" #include namespace icinga @@ -101,6 +102,7 @@ private: static void StateChangeHandler(const ConfigObject::Ptr &object); static void VersionChangedHandler(const ConfigObject::Ptr& object); + static void DowntimeChangedHandler(const Downtime::Ptr& downtime); void AssertOnWorkQueue(); From 3fb3e8bb6d2a21348c487da7f0cec59d566d082b Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 30 Nov 2018 10:54:44 +0100 Subject: [PATCH 107/219] Remove dev debug log --- lib/redis/rediswriter-objects.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 6b797dc5f..10455be1d 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -781,6 +781,5 @@ void RedisWriter::VersionChangedHandler(const ConfigObject::Ptr& object) void RedisWriter::DowntimeChangedHandler(const Downtime::Ptr& downtime) { - Log(LogCritical, "Downtime", "Downtime sync got triggered"); StateChangeHandler(downtime->GetCheckable()); } From 933de15c36c62efcd4f254c3b2c53af358ac515b Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Fri, 30 Nov 2018 10:14:39 +0100 Subject: [PATCH 108/219] Remove TODO --- lib/redis/rediswriter-objects.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 10455be1d..9a54cdaae 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -660,7 +660,6 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) CheckResult::Ptr cr = checkable->GetLastCheckResult(); if (cr) { - // TODO: Long Output did not work in my test cases. Need to investigate attrs->Set("output", CompatUtility::GetCheckResultOutput(cr)); attrs->Set("long_output", CompatUtility::GetCheckResultLongOutput(cr)); attrs->Set("performance_data", cr->GetPerformanceData()); From 08bfbbc07046a894fcd6cefb9e26d6f7cc5e5dd3 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Fri, 30 Nov 2018 15:40:31 +0100 Subject: [PATCH 109/219] Encode perfdata --- lib/redis/rediswriter-objects.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 9a54cdaae..f2ddba57a 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -662,7 +662,7 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) if (cr) { attrs->Set("output", CompatUtility::GetCheckResultOutput(cr)); attrs->Set("long_output", CompatUtility::GetCheckResultLongOutput(cr)); - attrs->Set("performance_data", cr->GetPerformanceData()); + attrs->Set("performance_data", JsonEncode(cr->GetPerformanceData())); attrs->Set("command", cr->GetCommand()); attrs->Set("execution_time", cr->CalculateExecutionTime()); attrs->Set("latency", cr->CalculateLatency()); From 992727707d103a9ab86f43fbe45fa88b57e65f07 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 3 Dec 2018 08:14:53 +0100 Subject: [PATCH 110/219] Redis: Encode check command --- lib/redis/rediswriter-objects.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index f2ddba57a..6f73cf1a8 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -663,7 +663,7 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) attrs->Set("output", CompatUtility::GetCheckResultOutput(cr)); attrs->Set("long_output", CompatUtility::GetCheckResultLongOutput(cr)); attrs->Set("performance_data", JsonEncode(cr->GetPerformanceData())); - attrs->Set("command", cr->GetCommand()); + attrs->Set("command", JsonEncode(cr->GetCommand())); attrs->Set("execution_time", cr->CalculateExecutionTime()); attrs->Set("latency", cr->CalculateLatency()); } From 3439ea755f843f6d83f4481618cae5118086acee Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 3 Dec 2018 14:23:07 +0100 Subject: [PATCH 111/219] Fix that state does not get synced on acknowledgement removal --- lib/redis/rediswriter-objects.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 6f73cf1a8..1b0dab1e5 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -53,6 +53,8 @@ void RedisWriter::ConfigStaticInitialize() { /* triggered in ProcessCheckResult(), requires UpdateNextCheck() to be called before */ Checkable::OnStateChange.connect(std::bind(&RedisWriter::StateChangeHandler, _1)); + /* triggered when acknowledged host/service goes back to ok and when the acknowledgement gets deleted */ + Checkable::OnAcknowledgementCleared.connect(std::bind(&RedisWriter::StateChangeHandler, _1)); /* triggered on create, update and delete objects */ ConfigObject::OnActiveChanged.connect(std::bind(&RedisWriter::VersionChangedHandler, _1)); From 553f6f6ba97b4298f4abdb01384f2bf6a3bde8d1 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Mon, 3 Dec 2018 15:47:05 +0100 Subject: [PATCH 112/219] Change commandline and performancedata --- lib/redis/rediswriter-objects.cpp | 8 +++++--- lib/redis/rediswriter-utility.cpp | 29 +++++++++++++++++++++++++++++ lib/redis/rediswriter.hpp | 2 +- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 1b0dab1e5..cff760c80 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -33,6 +33,7 @@ #include "icinga/eventcommand.hpp" #include "icinga/notificationcommand.hpp" #include "icinga/timeperiod.hpp" +#include "icinga/pluginutility.hpp" #include "remote/zone.hpp" #include "base/json.hpp" #include "base/logger.hpp" @@ -664,8 +665,10 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) if (cr) { attrs->Set("output", CompatUtility::GetCheckResultOutput(cr)); attrs->Set("long_output", CompatUtility::GetCheckResultLongOutput(cr)); - attrs->Set("performance_data", JsonEncode(cr->GetPerformanceData())); - attrs->Set("command", JsonEncode(cr->GetCommand())); + if (cr->GetPerformanceData()) + attrs->Set("performance_data", PluginUtility::FormatPerfdata(cr->GetPerformanceData())); + if (!cr->GetCommand().IsEmpty()) + attrs->Set("commandline", FormatCommandLine(cr->GetCommand())); attrs->Set("execution_time", cr->CalculateExecutionTime()); attrs->Set("latency", cr->CalculateLatency()); } @@ -707,7 +710,6 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) return attrs; } - std::vector RedisWriter::UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride) diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 87146622a..62b5e35f2 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -36,6 +36,8 @@ #include #include #include +#include + using namespace icinga; @@ -48,6 +50,33 @@ String RedisWriter::FormatCheckSumBinary(const String& str) return output; } + +String RedisWriter::FormatCommandLine(const Value& commandLine) +{ + String result; + if (commandLine.IsObjectType()) { + Array::Ptr args = commandLine; + bool first = true; + + ObjectLock olock(args); + for (const Value& arg : args) { + String token = "'" + Convert::ToString(arg) + "'"; + + if (first) + first = false; + else + result += String(1, ' '); + + result += token; + } + } else if (!commandLine.IsEmpty()) { + result = commandLine; + boost::algorithm::replace_all(result, "\'", "\\'"); + } + + return result; +} + static Value l_DefaultEnv = "production"; String RedisWriter::GetEnvironment() diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 5f7031634..3e5df9e84 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -83,6 +83,7 @@ private: /* utilities */ static String FormatCheckSumBinary(const String& str); + static String FormatCommandLine(const Value& commandLine); static String GetObjectIdentifier(const ConfigObject::Ptr& object); static String GetEnvironment(); @@ -99,7 +100,6 @@ private: static String GetLowerCaseTypeNameDB(const ConfigObject::Ptr& obj); static void MakeTypeChecksums(const ConfigObject::Ptr& object, std::set& propertiesBlacklist, Dictionary::Ptr& checkSums); - static void StateChangeHandler(const ConfigObject::Ptr &object); static void VersionChangedHandler(const ConfigObject::Ptr& object); static void DowntimeChangedHandler(const Downtime::Ptr& downtime); From 50aa0eb1d39981c294d5ad5831181d58125177fa Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Mon, 3 Dec 2018 16:31:06 +0100 Subject: [PATCH 113/219] Fix command line string --- lib/redis/rediswriter-utility.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 62b5e35f2..77440730b 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -72,6 +72,7 @@ String RedisWriter::FormatCommandLine(const Value& commandLine) } else if (!commandLine.IsEmpty()) { result = commandLine; boost::algorithm::replace_all(result, "\'", "\\'"); + result = "'" + result + "'"; } return result; From 446796dadb827d8ff46d508961fab4a8e2c8fcb6 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Mon, 3 Dec 2018 17:09:58 +0100 Subject: [PATCH 114/219] Fix output and longOutput --- lib/redis/rediswriter-objects.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index cff760c80..d516eda26 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -663,8 +663,20 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) CheckResult::Ptr cr = checkable->GetLastCheckResult(); if (cr) { - attrs->Set("output", CompatUtility::GetCheckResultOutput(cr)); - attrs->Set("long_output", CompatUtility::GetCheckResultLongOutput(cr)); + String rawOutput = cr->GetOutput(); + if (!rawOutput.IsEmpty()) { + size_t lineBreak = rawOutput.Find("\n"); + String output = rawOutput.SubStr(0, lineBreak); + if (!output.IsEmpty()) + attrs->Set("output", rawOutput.SubStr(0, lineBreak)); + + if (lineBreak > 0 && lineBreak != String::NPos) { + String longOutput = rawOutput.SubStr(lineBreak+1, rawOutput.GetLength()); + if (!longOutput.IsEmpty()) + attrs->Set("long_output", longOutput); + } + } + if (cr->GetPerformanceData()) attrs->Set("performance_data", PluginUtility::FormatPerfdata(cr->GetPerformanceData())); if (!cr->GetCommand().IsEmpty()) From e90fb43b919fe53400a84258585632da3bf271ab Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Mon, 3 Dec 2018 17:23:40 +0100 Subject: [PATCH 115/219] Fix empty perf data strings --- lib/redis/rediswriter-objects.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index d516eda26..633aa1fb5 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -677,8 +677,10 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) } } - if (cr->GetPerformanceData()) - attrs->Set("performance_data", PluginUtility::FormatPerfdata(cr->GetPerformanceData())); + String perfData = PluginUtility::FormatPerfdata(cr->GetPerformanceData()); + if (!perfData.IsEmpty()) + attrs->Set("performance_data", perfData); + if (!cr->GetCommand().IsEmpty()) attrs->Set("commandline", FormatCommandLine(cr->GetCommand())); attrs->Set("execution_time", cr->CalculateExecutionTime()); From 5cfd3c1ab0fe88c0079727becc92571e6a541530 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Wed, 28 Nov 2018 11:30:56 +0100 Subject: [PATCH 116/219] Rewrite config dump --- lib/redis/rediswriter-objects.cpp | 544 +++++++++++++++++------------- lib/redis/rediswriter.hpp | 4 +- 2 files changed, 312 insertions(+), 236 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 633aa1fb5..7de0e5252 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -208,249 +208,196 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtime m_Rcon->ExecuteQuery(customVar); } -void RedisWriter::MakeTypeChecksums(const ConfigObject::Ptr& object, std::set& propertiesBlacklist, - Dictionary::Ptr& checkSums) +// Takes object and collects IcingaDB relevant attributes and computes checksums +void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& attributes, Dictionary::Ptr& checksums) { + checksums->Set("name_checksum", CalculateCheckSumString(object->GetName())); + checksums->Set("environment_id", CalculateCheckSumString(GetEnvironment())); + attributes->Set("name", object->GetName()); + + Zone::Ptr ObjectsZone = static_pointer_cast(object->GetZone()); + if (ObjectsZone) { + checksums->Set("zone_id", GetObjectIdentifier(ObjectsZone)); + attributes->Set("zone", ObjectsZone->GetName()); + } + Endpoint::Ptr endpoint = dynamic_pointer_cast(object); if (endpoint) { - auto endpointZone(endpoint->GetZone()); - - if (endpointZone) { - checkSums->Set("zone_checksum", GetObjectIdentifier(endpointZone)); - } - - return; - } else { - /* 'zone' is available for all config objects, therefore calculate the checksum. */ - auto zone(static_pointer_cast(object->GetZone())); - - if (zone) - checkSums->Set("zone_checksum", GetObjectIdentifier(zone)); - } - - User::Ptr user = dynamic_pointer_cast(object); - if (user) { - propertiesBlacklist.emplace("groups"); - - Array::Ptr groups; - ConfigObject::Ptr (*getGroup)(const String& name); - - groups = user->GetGroups(); - getGroup = &::GetObjectByName; - - checkSums->Set("groups_checksum", CalculateCheckSumArray(groups)); - - Array::Ptr groupChecksums = new Array(); - - ObjectLock groupsLock(groups); - ObjectLock groupChecksumsLock(groupChecksums); - - for (auto group : groups) { - groupChecksums->Add(GetObjectIdentifier((*getGroup)(group.Get()))); - } - - checkSums->Set("group_checksums", groupChecksums); - - auto period(user->GetPeriod()); - - if (period) - checkSums->Set("period_checksum", GetObjectIdentifier(period)); - - return; - } - - Notification::Ptr notification = dynamic_pointer_cast(object); - if (notification) { - Host::Ptr host; - Service::Ptr service; - auto users(notification->GetUsers()); - Array::Ptr userChecksums = new Array(); - Array::Ptr userNames = new Array(); - auto usergroups(notification->GetUserGroups()); - Array::Ptr usergroupChecksums = new Array(); - Array::Ptr usergroupNames = new Array(); - - tie(host, service) = GetHostService(notification->GetCheckable()); - - checkSums->Set("host_checksum", GetObjectIdentifier(host)); - checkSums->Set("command_checksum", GetObjectIdentifier(notification->GetCommand())); - - if (service) - checkSums->Set("service_checksum", GetObjectIdentifier(service)); - - propertiesBlacklist.emplace("users"); - - userChecksums->Reserve(users.size()); - userNames->Reserve(users.size()); - - for (auto& user : users) { - userChecksums->Add(GetObjectIdentifier(user)); - userNames->Add(user->GetName()); - } - - checkSums->Set("user_checksums", userChecksums); - checkSums->Set("users_checksum", CalculateCheckSumArray(userNames)); - - propertiesBlacklist.emplace("user_groups"); - - usergroupChecksums->Reserve(usergroups.size()); - usergroupNames->Reserve(usergroups.size()); - - for (auto& usergroup : usergroups) { - usergroupChecksums->Add(GetObjectIdentifier(usergroup)); - usergroupNames->Add(usergroup->GetName()); - } - - checkSums->Set("usergroup_checksums", usergroupChecksums); - checkSums->Set("usergroups_checksum", CalculateCheckSumArray(usergroupNames)); - return; - } - - /* Calculate checkable checksums. */ - Checkable::Ptr checkable = dynamic_pointer_cast(object); - if (checkable) { - /* groups_checksum, group_checksums */ - propertiesBlacklist.emplace("groups"); - - Host::Ptr host; - Service::Ptr service; - - tie(host, service) = GetHostService(checkable); - - Array::Ptr groups; - ConfigObject::Ptr (*getGroup)(const String& name); - - if (service) { - groups = service->GetGroups(); - getGroup = &::GetObjectByName; - - /* Calculate the host_checksum */ - checkSums->Set("host_checksum", GetObjectIdentifier(host)); - } else { - groups = host->GetGroups(); - getGroup = &::GetObjectByName; - } - - checkSums->Set("groups_checksum", CalculateCheckSumArray(groups)); - - Array::Ptr groupChecksums = new Array(); - - ObjectLock groupsLock(groups); - ObjectLock groupChecksumsLock(groupChecksums); - - for (auto group : groups) { - groupChecksums->Add(GetObjectIdentifier((*getGroup)(group.Get()))); - } - - checkSums->Set("group_checksums", groupChecksums); - - /* command_endpoint_checksum / node_checksum */ - Endpoint::Ptr commandEndpoint = checkable->GetCommandEndpoint(); - - if (commandEndpoint) - checkSums->Set("command_endpoint_checksum", GetObjectIdentifier(commandEndpoint)); - - /* *_command_checksum */ - checkSums->Set("checkcommand_checksum", GetObjectIdentifier(checkable->GetCheckCommand())); - - EventCommand::Ptr eventCommand = checkable->GetEventCommand(); - - if (eventCommand) - checkSums->Set("eventcommand_checksum", GetObjectIdentifier(eventCommand)); - - /* *_url_checksum, icon_image_checksum */ - String actionUrl = checkable->GetActionUrl(); - String notesUrl = checkable->GetNotesUrl(); - String iconImage = checkable->GetIconImage(); - - if (!actionUrl.IsEmpty()) - checkSums->Set("action_url_checksum", CalculateCheckSumString(actionUrl)); - if (!notesUrl.IsEmpty()) - checkSums->Set("notes_url_checksum", CalculateCheckSumString(notesUrl)); - if (!iconImage.IsEmpty()) - checkSums->Set("icon_image_checksum", CalculateCheckSumString(iconImage)); + checksums->Set("properties_checksum", HashValue(attributes)); return; } Zone::Ptr zone = dynamic_pointer_cast(object); if (zone) { - propertiesBlacklist.emplace("endpoints"); + attributes->Set("is_global", zone->GetGlobal()); + + checksums->Set("properties_checksum", HashValue(attributes)); - auto endpointObjects = zone->GetEndpoints(); Array::Ptr endpoints = new Array(); - endpoints->Resize(endpointObjects.size()); + endpoints->Resize(zone->GetEndpoints().size()); Array::SizeType i = 0; - for (auto& endpointObject : endpointObjects) { + for (auto& endpointObject : zone->GetEndpoints()) { endpoints->Set(i++, endpointObject->GetName()); } - checkSums->Set("endpoints_checksum", CalculateCheckSumArray(endpoints)); - Array::Ptr parents(new Array); for (auto& parent : zone->GetAllParentsRaw()) { parents->Add(GetObjectIdentifier(parent)); } - checkSums->Set("all_parents_checksums", parents); - checkSums->Set("all_parents_checksum", HashValue(zone->GetAllParents())); + checksums->Set("parent_ids", parents); + checksums->Set("parents_checksum", HashValue(zone->GetAllParents())); + return; } - /* zone_checksum for endpoints already is calculated above. */ + Checkable::Ptr checkable = dynamic_pointer_cast(object); + if (checkable) { + attributes->Set("checkcommand", checkable->GetCheckCommand()->GetName()); + attributes->Set("max_check_attempts", checkable->GetMaxCheckAttempts()); + attributes->Set("check_timeout", checkable->GetCheckTimeout()); + attributes->Set("check_interval", checkable->GetCheckInterval()); + attributes->Set("check_retry_interval", checkable->GetRetryInterval()); + attributes->Set("active_checks_enabled", checkable->GetEnableActiveChecks()); + attributes->Set("passive_checks_enabled", checkable->GetEnablePassiveChecks()); + attributes->Set("event_handler_enabled", checkable->GetEnableEventHandler()); + attributes->Set("notifications_enabled", checkable->GetEnableNotifications()); + attributes->Set("flapping_enabled", checkable->GetEnableFlapping()); + attributes->Set("flapping_threshold_low", checkable->GetFlappingThresholdLow()); + attributes->Set("flapping_threshold_high", checkable->GetFlappingThresholdHigh()); + attributes->Set("perfdata_enabled", checkable->GetEnablePerfdata()); + attributes->Set("is_volatile", checkable->GetVolatile()); + attributes->Set("notes", checkable->GetNotes()); + attributes->Set("icon_image_alt", checkable->GetIconImageAlt()); - Command::Ptr command = dynamic_pointer_cast(object); - if (command) { - Dictionary::Ptr arguments = command->GetArguments(); - Dictionary::Ptr argumentChecksums = new Dictionary; + checksums->Set("checkcommand_id", GetObjectIdentifier(checkable->GetCheckCommand())); - if (arguments) { - ObjectLock argumentsLock(arguments); - - for (auto& kv : arguments) { - argumentChecksums->Set(kv.first, HashValue(kv.second)); - } + Endpoint::Ptr commandEndpoint = checkable->GetCommandEndpoint(); + if (commandEndpoint) { + checksums->Set("command_endpoint_id", GetObjectIdentifier(commandEndpoint)); + attributes->Set("command_endpoint", commandEndpoint->GetName()); } - checkSums->Set("arguments_checksum", HashValue(arguments)); - checkSums->Set("argument_checksums", argumentChecksums); - propertiesBlacklist.emplace("arguments"); - - Dictionary::Ptr envvars = command->GetEnv(); - Dictionary::Ptr envvarChecksums = new Dictionary; - - if (envvars) { - ObjectLock argumentsLock(envvars); - - for (auto& kv : envvars) { - envvarChecksums->Set(kv.first, HashValue(kv.second)); - } + TimePeriod::Ptr timePeriod = checkable->GetCheckPeriod(); + if (timePeriod) { + checksums->Set("check_period_id", GetObjectIdentifier(timePeriod)); + attributes->Set("check_period", timePeriod->GetName()); } - checkSums->Set("envvars_checksum", HashValue(envvars)); - checkSums->Set("envvar_checksums", envvarChecksums); - propertiesBlacklist.emplace("env"); + EventCommand::Ptr eventCommand = checkable->GetEventCommand(); + if (eventCommand) { + checksums->Set("eventcommand_id", GetObjectIdentifier(eventCommand)); + attributes->Set("eventcommand", eventCommand->GetName()); + } + + String actionUrl = checkable->GetActionUrl(); + String notesUrl = checkable->GetNotesUrl(); + String iconImage = checkable->GetIconImage(); + if (!actionUrl.IsEmpty()) + checksums->Set("action_url_id", CalculateCheckSumArray(new Array({GetEnvironment(), actionUrl}))); + if (!notesUrl.IsEmpty()) + checksums->Set("notes_url_id", CalculateCheckSumArray(new Array({GetEnvironment(), notesUrl}))); + if (!iconImage.IsEmpty()) + checksums->Set("icon_image_id", CalculateCheckSumArray(new Array({GetEnvironment(), iconImage}))); + + + Host::Ptr host; + Service::Ptr service; + tie(host, service) = GetHostService(checkable); + + Array::Ptr groups; + ConfigObject::Ptr (*getGroup)(const String& name); + + if (service) { + checksums->Set("host_id", GetObjectIdentifier(service->GetHost())); + attributes->Set("display_name", service->GetDisplayName()); + + groups = service->GetGroups(); + getGroup = &::GetObjectByName; + } else { + attributes->Set("display_name", host->GetDisplayName()); + attributes->Set("address", host->GetAddress()); + attributes->Set("address6", host->GetAddress6()); + + groups = host->GetGroups(); + getGroup = &::GetObjectByName; + } + + checksums->Set("groups_checksum", CalculateCheckSumArray(groups)); + + Array::Ptr groupChecksums = new Array(); + + ObjectLock groupsLock(groups); + ObjectLock groupChecksumsLock(groupChecksums); + + for (auto group : groups) { + groupChecksums->Add(GetObjectIdentifier((*getGroup)(group.Get()))); + } + + checksums->Set("group_ids", groupChecksums); + + checksums->Set("properties_checksum", HashValue(attributes)); + + return; + } + + User::Ptr user = dynamic_pointer_cast(object); + if (user) { + attributes->Set("display_name", user->GetDisplayName()); + attributes->Set("email", user->GetEmail()); + attributes->Set("pager", user->GetPager()); + attributes->Set("notifications_enabled", user->GetEnableNotifications()); + attributes->Set("states", user->GetStates()); + attributes->Set("types", user->GetTypes()); + + Array::Ptr groups; + ConfigObject::Ptr (*getGroup)(const String& name); + + groups = user->GetGroups(); + getGroup = &::GetObjectByName; + + checksums->Set("groups_checksum", CalculateCheckSumArray(groups)); + + Array::Ptr groupChecksums = new Array(); + + ObjectLock groupsLock(groups); + ObjectLock groupChecksumsLock(groupChecksums); + + for (auto group : groups) { + groupChecksums->Add(GetObjectIdentifier((*getGroup)(group.Get()))); + } + + checksums->Set("group_ids", groupChecksums); + + if (user->GetPeriod()) + checksums->Set("period_id", GetObjectIdentifier(user->GetPeriod())); + + checksums->Set("properties_checksum", HashValue(attributes)); return; } TimePeriod::Ptr timeperiod = dynamic_pointer_cast(object); if (timeperiod) { + attributes->Set("display_name", timeperiod->GetDisplayName()); + attributes->Set("prefer_includes", timeperiod->GetPreferIncludes()); + + checksums->Set("properties_checksum", HashValue(attributes)); Dictionary::Ptr ranges = timeperiod->GetRanges(); - checkSums->Set("ranges_checksum", HashValue(ranges)); - propertiesBlacklist.emplace("ranges"); + attributes->Set("ranges", ranges); + checksums->Set("ranges_checksum", HashValue(ranges)); // Compute checksums for Includes (like groups) Array::Ptr includes; ConfigObject::Ptr (*getInclude)(const String& name); - includes = timeperiod->GetIncludes(); getInclude = &::GetObjectByName; - checkSums->Set("includes_checksum", CalculateCheckSumArray(includes)); + checksums->Set("includes_checksum", CalculateCheckSumArray(includes)); Array::Ptr includeChecksums = new Array(); @@ -461,7 +408,7 @@ void RedisWriter::MakeTypeChecksums(const ConfigObject::Ptr& object, std::setAdd(GetObjectIdentifier((*getInclude)(include.Get()))); } - checkSums->Set("include_checksums", includeChecksums); + checksums->Set("include_ids", includeChecksums); // Compute checksums for Excludes (like groups) Array::Ptr excludes; @@ -470,7 +417,7 @@ void RedisWriter::MakeTypeChecksums(const ConfigObject::Ptr& object, std::setGetExcludes(); getExclude = &::GetObjectByName; - checkSums->Set("excludes_checksum", CalculateCheckSumArray(excludes)); + checksums->Set("excludes_checksum", CalculateCheckSumArray(excludes)); Array::Ptr excludeChecksums = new Array(); @@ -481,44 +428,188 @@ void RedisWriter::MakeTypeChecksums(const ConfigObject::Ptr& object, std::setAdd(GetObjectIdentifier((*getExclude)(exclude.Get()))); } - checkSums->Set("exclude_checksums", excludeChecksums); + checksums->Set("exclude_ids", excludeChecksums); + + return; + } + + Notification::Ptr notification = dynamic_pointer_cast(object); + if (notification) { + Host::Ptr host; + Service::Ptr service; + std::set users = notification->GetUsers(); + Array::Ptr userChecksums = new Array(); + Array::Ptr userNames = new Array(); + auto usergroups(notification->GetUserGroups()); + Array::Ptr usergroupChecksums = new Array(); + Array::Ptr usergroupNames = new Array(); + + tie(host, service) = GetHostService(notification->GetCheckable()); + + checksums->Set("properties_checksum", HashValue(attributes)); + checksums->Set("host_id", GetObjectIdentifier(host)); + checksums->Set("command_id", GetObjectIdentifier(notification->GetCommand())); + + TimePeriod::Ptr timeperiod = notification->GetPeriod(); + if (timeperiod) + checksums->Set("period_id", GetObjectIdentifier(timeperiod)); + + if (service) + checksums->Set("service_id", GetObjectIdentifier(service)); + + userChecksums->Reserve(users.size()); + userNames->Reserve(users.size()); + + for (auto& user : users) { + userChecksums->Add(GetObjectIdentifier(user)); + userNames->Add(user->GetName()); + } + + checksums->Set("user_ids", userChecksums); + checksums->Set("users_checksum", CalculateCheckSumArray(userNames)); + + usergroupChecksums->Reserve(usergroups.size()); + usergroupNames->Reserve(usergroups.size()); + + for (auto& usergroup : usergroups) { + usergroupChecksums->Add(GetObjectIdentifier(usergroup)); + usergroupNames->Add(usergroup->GetName()); + } + + checksums->Set("usergroup_ids", usergroupChecksums); + checksums->Set("usergroups_checksum", CalculateCheckSumArray(usergroupNames)); + + if (notification->GetTimes()) { + attributes->Set("times_begin", notification->GetTimes()->Get("begin")); + attributes->Set("times_end",notification->GetTimes()->Get("end")); + } return; } Comment::Ptr comment = dynamic_pointer_cast(object); if (comment) { - propertiesBlacklist.emplace("name"); - propertiesBlacklist.emplace("host_name"); + attributes->Set("author", comment->GetAuthor()); + attributes->Set("text", comment->GetText()); + attributes->Set("entry_type", comment->GetEntryType()); + attributes->Set("is_persistent", comment->GetPersistent()); + attributes->Set("expire_time", comment->GetExpireTime()); Host::Ptr host; Service::Ptr service; tie(host, service) = GetHostService(comment->GetCheckable()); if (service) { - propertiesBlacklist.emplace("service_name"); - checkSums->Set("service_checksum", GetObjectIdentifier(service)); + checksums->Set("service_id", GetObjectIdentifier(service)); } else - checkSums->Set("host_checksum", GetObjectIdentifier(host)); + checksums->Set("host_id", GetObjectIdentifier(host)); + + checksums->Set("properties_checksum", HashValue(attributes)); return; } Downtime::Ptr downtime = dynamic_pointer_cast(object); if (downtime) { - propertiesBlacklist.emplace("name"); - propertiesBlacklist.emplace("host_name"); + attributes->Set("author", downtime->GetAuthor()); + attributes->Set("comment", downtime->GetComment()); + attributes->Set("entry_time", downtime->GetEntryTime()); + attributes->Set("scheduled_start_time", downtime->GetStartTime()); + attributes->Set("scheduled_end_time", downtime->GetEndTime()); + attributes->Set("duration", downtime->GetDuration()); + attributes->Set("is_fixed", downtime->GetFixed()); + attributes->Set("is_in_effect", downtime->IsInEffect()); + if (downtime->IsInEffect()) + attributes->Set("actual_start_time", downtime->GetTriggerTime()); + Host::Ptr host; Service::Ptr service; tie(host, service) = GetHostService(downtime->GetCheckable()); if (service) { - propertiesBlacklist.emplace("service_name"); - checkSums->Set("service_checksum", GetObjectIdentifier(service)); + checksums->Set("service_id", GetObjectIdentifier(service)); } else - checkSums->Set("host_checksum", GetObjectIdentifier(host)); + checksums->Set("host_id", GetObjectIdentifier(host)); + + checksums->Set("properties_checksum", HashValue(attributes)); + return; + } + + UserGroup::Ptr userGroup = dynamic_pointer_cast(object); + if (userGroup) { + attributes->Set("display_name", userGroup->GetDisplayName()); + + checksums->Set("properties_checksum", HashValue(attributes)); return; } + + HostGroup::Ptr hostGroup = dynamic_pointer_cast(object); + if (hostGroup) { + attributes->Set("display_name", hostGroup->GetDisplayName()); + + checksums->Set("properties_checksum", HashValue(attributes)); + + return; + } + + ServiceGroup::Ptr serviceGroup = dynamic_pointer_cast(object); + if (serviceGroup) { + attributes->Set("display_name", serviceGroup->GetDisplayName()); + + checksums->Set("properties_checksum", HashValue(attributes)); + + return; + } + + Command::Ptr command = dynamic_pointer_cast(object); + if (command) { + if (dynamic_pointer_cast(object)) + attributes->Set("type", "CheckCommand"); + else if (dynamic_pointer_cast(object)) + attributes->Set("type", "EventCommand"); + else + attributes->Set("type", "NotificationCommand"); + + attributes->Set("command", command->GetCommandLine()); + attributes->Set("timeout", command->GetTimeout()); + + checksums->Set("properties_checksum", HashValue(attributes)); + + Dictionary::Ptr arguments = command->GetArguments(); + Dictionary::Ptr argumentChecksums = new Dictionary; + + if (arguments) { + ObjectLock argumentsLock(arguments); + + for (auto& kv : arguments) { + argumentChecksums->Set(kv.first, HashValue(kv.second)); + } + + attributes->Set("arguments", arguments); + } + + checksums->Set("arguments_checksum", HashValue(arguments)); + checksums->Set("argument_ids", argumentChecksums); + + Dictionary::Ptr envvars = command->GetEnv(); + Dictionary::Ptr envvarChecksums = new Dictionary; + + if (envvars) { + ObjectLock argumentsLock(envvars); + + for (auto& kv : envvars) { + envvarChecksums->Set(kv.first, HashValue(kv.second)); + } + + attributes->Set("envvars", envvars); + } + + checksums->Set("envvars_checksum", HashValue(envvars)); + checksums->Set("envvar_ids", envvarChecksums); + + return; + } + } /* Creates a config update with computed checksums etc. @@ -539,28 +630,20 @@ RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, const String ty if (m_Rcon == nullptr) return; + Dictionary::Ptr attr = new Dictionary; + Dictionary::Ptr chksm = new Dictionary; + + PrepareObject(object, attr, chksm); + String objectKey = GetObjectIdentifier(object); - std::set propertiesBlacklist({"name", "__name", "package", "source_location", "templates"}); + attributes.emplace_back(objectKey); + attributes.emplace_back(JsonEncode(attr)); - Dictionary::Ptr checkSums = new Dictionary(); - - checkSums->Set("name_checksum", CalculateCheckSumString(object->GetShortName())); - checkSums->Set("environment_checksum", CalculateCheckSumString(GetEnvironment())); - - MakeTypeChecksums(object, propertiesBlacklist, checkSums); - - /* Send all object attributes to Redis, no extra checksums involved here. */ - auto tempAttrs = (UpdateObjectAttrs(object, FAConfig, typeName)); - attributes.insert(attributes.end(), std::begin(tempAttrs), std::end(tempAttrs)); - - /* Custom var checksums. */ CustomVarObject::Ptr customVarObject = dynamic_pointer_cast(object); if (customVarObject) { - propertiesBlacklist.emplace("vars"); - - checkSums->Set("vars_checksum", CalculateCheckSumVars(customVarObject)); + chksm->Set("customvars_checksum", CalculateCheckSumVars(customVarObject)); auto vars(SerializeVars(customVarObject)); @@ -572,15 +655,8 @@ RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, const String ty } } - checkSums->Set("metadata_checksum", CalculateCheckSumMetadata(object)); - - /* TODO: Problem: This does not account for `is_in_effect`, `trigger_time` of downtimes. */ - checkSums->Set("properties_checksum", CalculateCheckSumProperties(object, propertiesBlacklist)); - - String checkSumsBody = JsonEncode(checkSums); - checksums.emplace_back(objectKey); - checksums.emplace_back(checkSumsBody); + checksums.emplace_back(JsonEncode(chksm)); /* Send an update event to subscribers. */ if (runtimeUpdate) { diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 3e5df9e84..c2edc697d 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -98,9 +98,9 @@ private: static String HashValue(const Value& value, const std::set& propertiesBlacklist, bool propertiesWhitelist = false); static String GetLowerCaseTypeNameDB(const ConfigObject::Ptr& obj); - static void MakeTypeChecksums(const ConfigObject::Ptr& object, std::set& propertiesBlacklist, Dictionary::Ptr& checkSums); + static void PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& attributes, Dictionary::Ptr& checkSums); - static void StateChangeHandler(const ConfigObject::Ptr &object); + static void StateChangeHandler(const ConfigObject::Ptr& object); static void VersionChangedHandler(const ConfigObject::Ptr& object); static void DowntimeChangedHandler(const Downtime::Ptr& downtime); From dc5dc463645d1a823bc2b67d70381c259897a89d Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Fri, 30 Nov 2018 14:57:51 +0100 Subject: [PATCH 117/219] Use ReflectionType instead of dynamic casts --- lib/redis/rediswriter-objects.cpp | 63 +++++++++++++++++++------------ 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 7de0e5252..cd2dbd423 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -221,17 +221,20 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr attributes->Set("zone", ObjectsZone->GetName()); } - Endpoint::Ptr endpoint = dynamic_pointer_cast(object); - if (endpoint) { + Type::Ptr type = object->GetReflectionType(); + + if (type == Endpoint::TypeInstance) { + Endpoint::Ptr endpoint = static_pointer_cast(object); + checksums->Set("properties_checksum", HashValue(attributes)); return; } - Zone::Ptr zone = dynamic_pointer_cast(object); - if (zone) { - attributes->Set("is_global", zone->GetGlobal()); + if (type == Zone::TypeInstance) { + Zone::Ptr zone = static_pointer_cast(object); + attributes->Set("is_global", zone->GetGlobal()); checksums->Set("properties_checksum", HashValue(attributes)); Array::Ptr endpoints = new Array(); @@ -254,8 +257,9 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr return; } - Checkable::Ptr checkable = dynamic_pointer_cast(object); - if (checkable) { + if (type == Host::TypeInstance || type == Service::TypeInstance) { + Checkable::Ptr checkable = static_pointer_cast(object); + attributes->Set("checkcommand", checkable->GetCheckCommand()->GetName()); attributes->Set("max_check_attempts", checkable->GetMaxCheckAttempts()); attributes->Set("check_timeout", checkable->GetCheckTimeout()); @@ -344,8 +348,9 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr return; } - User::Ptr user = dynamic_pointer_cast(object); - if (user) { + if (type == User::TypeInstance) { + User::Ptr user = static_pointer_cast(object); + attributes->Set("display_name", user->GetDisplayName()); attributes->Set("email", user->GetEmail()); attributes->Set("pager", user->GetPager()); @@ -380,8 +385,9 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr return; } - TimePeriod::Ptr timeperiod = dynamic_pointer_cast(object); - if (timeperiod) { + if (type == TimePeriod::TypeInstance) { + TimePeriod::Ptr timeperiod = static_pointer_cast(object); + attributes->Set("display_name", timeperiod->GetDisplayName()); attributes->Set("prefer_includes", timeperiod->GetPreferIncludes()); @@ -433,8 +439,9 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr return; } - Notification::Ptr notification = dynamic_pointer_cast(object); - if (notification) { + if (type == Notification::TypeInstance) { + Notification::Ptr notification = static_pointer_cast(object); + Host::Ptr host; Service::Ptr service; std::set users = notification->GetUsers(); @@ -487,8 +494,9 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr return; } - Comment::Ptr comment = dynamic_pointer_cast(object); - if (comment) { + if (type == Comment::TypeInstance) { + Comment::Ptr comment = static_pointer_cast(object); + attributes->Set("author", comment->GetAuthor()); attributes->Set("text", comment->GetText()); attributes->Set("entry_type", comment->GetEntryType()); @@ -508,8 +516,9 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr return; } - Downtime::Ptr downtime = dynamic_pointer_cast(object); - if (downtime) { + if (type == Downtime::TypeInstance) { + Downtime::Ptr downtime = static_pointer_cast(object); + attributes->Set("author", downtime->GetAuthor()); attributes->Set("comment", downtime->GetComment()); attributes->Set("entry_time", downtime->GetEntryTime()); @@ -534,8 +543,9 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr return; } - UserGroup::Ptr userGroup = dynamic_pointer_cast(object); - if (userGroup) { + if (type == UserGroup::TypeInstance) { + UserGroup::Ptr userGroup = static_pointer_cast(object); + attributes->Set("display_name", userGroup->GetDisplayName()); checksums->Set("properties_checksum", HashValue(attributes)); @@ -543,8 +553,9 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr return; } - HostGroup::Ptr hostGroup = dynamic_pointer_cast(object); - if (hostGroup) { + if (type == HostGroup::TypeInstance) { + HostGroup::Ptr hostGroup = static_pointer_cast(object); + attributes->Set("display_name", hostGroup->GetDisplayName()); checksums->Set("properties_checksum", HashValue(attributes)); @@ -552,8 +563,9 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr return; } - ServiceGroup::Ptr serviceGroup = dynamic_pointer_cast(object); - if (serviceGroup) { + if (type == ServiceGroup::TypeInstance) { + ServiceGroup::Ptr serviceGroup = static_pointer_cast(object); + attributes->Set("display_name", serviceGroup->GetDisplayName()); checksums->Set("properties_checksum", HashValue(attributes)); @@ -561,8 +573,9 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr return; } - Command::Ptr command = dynamic_pointer_cast(object); - if (command) { + if (type == CheckCommand::TypeInstance || type == NotificationCommand::TypeInstance || type == EventCommand::TypeInstance) { + Command::Ptr command = static_pointer_cast(object); + if (dynamic_pointer_cast(object)) attributes->Set("type", "CheckCommand"); else if (dynamic_pointer_cast(object)) From accadb5dfbbc4f56710912c8523e175f3c025696 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Fri, 30 Nov 2018 17:01:42 +0100 Subject: [PATCH 118/219] Only dump relevant object types --- lib/redis/rediswriter-objects.cpp | 33 +++++++++++++++++-------------- lib/redis/rediswriter.hpp | 2 +- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index cd2dbd423..c0a46df97 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -208,8 +208,9 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtime m_Rcon->ExecuteQuery(customVar); } -// Takes object and collects IcingaDB relevant attributes and computes checksums -void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& attributes, Dictionary::Ptr& checksums) +// Takes object and collects IcingaDB relevant attributes and computes checksums. Returns whether the object is relevant +// for IcingaDB. +bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& attributes, Dictionary::Ptr& checksums) { checksums->Set("name_checksum", CalculateCheckSumString(object->GetName())); checksums->Set("environment_id", CalculateCheckSumString(GetEnvironment())); @@ -228,7 +229,7 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr checksums->Set("properties_checksum", HashValue(attributes)); - return; + return true; } if (type == Zone::TypeInstance) { @@ -254,7 +255,7 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr checksums->Set("parent_ids", parents); checksums->Set("parents_checksum", HashValue(zone->GetAllParents())); - return; + return true; } if (type == Host::TypeInstance || type == Service::TypeInstance) { @@ -345,7 +346,7 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr checksums->Set("properties_checksum", HashValue(attributes)); - return; + return true; } if (type == User::TypeInstance) { @@ -382,7 +383,7 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr checksums->Set("properties_checksum", HashValue(attributes)); - return; + return true; } if (type == TimePeriod::TypeInstance) { @@ -436,7 +437,7 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr checksums->Set("exclude_ids", excludeChecksums); - return; + return true; } if (type == Notification::TypeInstance) { @@ -491,7 +492,7 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr attributes->Set("times_end",notification->GetTimes()->Get("end")); } - return; + return true; } if (type == Comment::TypeInstance) { @@ -513,7 +514,7 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr checksums->Set("properties_checksum", HashValue(attributes)); - return; + return true; } if (type == Downtime::TypeInstance) { @@ -540,7 +541,7 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr checksums->Set("host_id", GetObjectIdentifier(host)); checksums->Set("properties_checksum", HashValue(attributes)); - return; + return true; } if (type == UserGroup::TypeInstance) { @@ -550,7 +551,7 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr checksums->Set("properties_checksum", HashValue(attributes)); - return; + return true; } if (type == HostGroup::TypeInstance) { @@ -560,7 +561,7 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr checksums->Set("properties_checksum", HashValue(attributes)); - return; + return true; } if (type == ServiceGroup::TypeInstance) { @@ -570,7 +571,7 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr checksums->Set("properties_checksum", HashValue(attributes)); - return; + return true; } if (type == CheckCommand::TypeInstance || type == NotificationCommand::TypeInstance || type == EventCommand::TypeInstance) { @@ -620,9 +621,10 @@ void RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr checksums->Set("envvars_checksum", HashValue(envvars)); checksums->Set("envvar_ids", envvarChecksums); - return; + return true; } + return false; } /* Creates a config update with computed checksums etc. @@ -646,7 +648,8 @@ RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, const String ty Dictionary::Ptr attr = new Dictionary; Dictionary::Ptr chksm = new Dictionary; - PrepareObject(object, attr, chksm); + if (!PrepareObject(object, attr, chksm)) + return; String objectKey = GetObjectIdentifier(object); diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index c2edc697d..a4e8fdc6a 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -98,7 +98,7 @@ private: static String HashValue(const Value& value, const std::set& propertiesBlacklist, bool propertiesWhitelist = false); static String GetLowerCaseTypeNameDB(const ConfigObject::Ptr& obj); - static void PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& attributes, Dictionary::Ptr& checkSums); + static bool PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& attributes, Dictionary::Ptr& checkSums); static void StateChangeHandler(const ConfigObject::Ptr& object); static void VersionChangedHandler(const ConfigObject::Ptr& object); From 6705158d31ee02926689043e79318dde2881b22e Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Tue, 4 Dec 2018 13:05:29 +0100 Subject: [PATCH 119/219] Remove unused variable --- lib/redis/rediswriter-objects.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index c0a46df97..8cb9a6f79 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -225,8 +225,6 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr Type::Ptr type = object->GetReflectionType(); if (type == Endpoint::TypeInstance) { - Endpoint::Ptr endpoint = static_pointer_cast(object); - checksums->Set("properties_checksum", HashValue(attributes)); return true; From 57b9fc94c3c831a26cec709427b068cd75803bb8 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Fri, 7 Dec 2018 10:58:57 +0100 Subject: [PATCH 120/219] Remove is_active_check --- lib/redis/rediswriter-objects.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 8cb9a6f79..cf3994e93 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -747,9 +747,7 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) //attrs->Set("severity") //attrs->Set(checkable->GetSeverity()); - - attrs->Set("is_active", checkable->IsActive()); - + CheckResult::Ptr cr = checkable->GetLastCheckResult(); if (cr) { From 037357aea5a856498acea91efa419667f13d2dd5 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Fri, 7 Dec 2018 16:33:10 +0100 Subject: [PATCH 121/219] Add severity --- lib/icinga/checkable.hpp | 17 ---------- lib/icinga/host.cpp | 37 +++++++++++--------- lib/icinga/service.cpp | 56 ++++++++++++++++++++----------- lib/redis/rediswriter-objects.cpp | 7 ++-- lib/redis/rediswriter-utility.cpp | 1 - lib/redis/rediswriter.hpp | 1 + 6 files changed, 64 insertions(+), 55 deletions(-) diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index 032910adb..498cb1d8f 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -39,23 +39,6 @@ enum CheckableType CheckableService }; -/** - * Severity Flags - * - * @ingroup icinga - */ -enum SeverityFlag -{ - SeverityFlagDowntime = 1, - SeverityFlagAcknowledgement = 2, - SeverityFlagHostDown = 4, - SeverityFlagUnhandled = 8, - SeverityFlagPending = 16, - SeverityFlagWarning = 32, - SeverityFlagUnknown = 64, - SeverityFlagCritical = 128, -}; - class CheckCommand; class EventCommand; class Dependency; diff --git a/lib/icinga/host.cpp b/lib/icinga/host.cpp index 9744eed46..7bb1c434d 100644 --- a/lib/icinga/host.cpp +++ b/lib/icinga/host.cpp @@ -164,32 +164,39 @@ HostState Host::GetLastHardState() const return CalculateState(GetLastHardStateRaw()); } -/* keep in sync with Service::GetSeverity() */ +/* keep in sync with Service::GetSeverity() + * One could think it may be smart to use an enum and some bitmask math here. + * But the only thing the consuming icingaweb2 cares about is being able to + * sort by severity. It is therefore easier to keep them seperated here. */ int Host::GetSeverity() const { int severity = 0; ObjectLock olock(this); - ServiceState state = GetStateRaw(); + HostState state = GetState(); - /* OK/Warning = Up, Critical/Unknownb = Down */ - if (!HasBeenChecked()) - severity |= SeverityFlagPending; - else if (state == ServiceUnknown) - severity |= SeverityFlagCritical; - else if (state == ServiceCritical) - severity |= SeverityFlagCritical; + if (!HasBeenChecked()) { + severity = 16; + } else if (state == HostUp) { + severity = 0; + } else { + if (IsReachable()) + severity = 64; + else + severity = 32; - if (IsInDowntime()) - severity |= SeverityFlagDowntime; - else if (IsAcknowledged()) - severity |= SeverityFlagAcknowledgement; - else - severity |= SeverityFlagUnhandled; + if (IsAcknowledged()) + severity += 512; + else if (IsInDowntime()) + severity += 256; + else + severity += 2048; + } olock.Unlock(); return severity; + } bool Host::IsStateOK(ServiceState state) const diff --git a/lib/icinga/service.cpp b/lib/icinga/service.cpp index e420b64c3..ec80aa6dc 100644 --- a/lib/icinga/service.cpp +++ b/lib/icinga/service.cpp @@ -103,32 +103,50 @@ Host::Ptr Service::GetHost() const return m_Host; } -/* keep in sync with Host::GetSeverity() */ +/* keep in sync with Host::GetSeverity() + * One could think it may be smart to use an enum and some bitmask math here. + * But the only thing the consuming icingaweb2 cares about is being able to + * sort by severity. It is therefore easier to keep them seperated here. */ int Service::GetSeverity() const { - int severity = 0; + int severity; ObjectLock olock(this); ServiceState state = GetStateRaw(); - if (!HasBeenChecked()) - severity |= SeverityFlagPending; - else if (state == ServiceWarning) - severity |= SeverityFlagWarning; - else if (state == ServiceUnknown) - severity |= SeverityFlagUnknown; - else if (state == ServiceCritical) - severity |= SeverityFlagCritical; + if (!HasBeenChecked()) { + severity = 16; + } else if (state == ServiceOK) { + severity = 0; + } else { + switch (state) { + case ServiceWarning: + severity = 32; + break; + case ServiceUnknown: + severity = 64; + break; + case ServiceCritical: + severity = 128; + break; + default: + severity = 256; + } - /* TODO: Add host reachability and handled */ - if (IsInDowntime()) - severity |= SeverityFlagDowntime; - else if (IsAcknowledged()) - severity |= SeverityFlagAcknowledgement; - else if (m_Host && m_Host->GetProblem()) - severity |= SeverityFlagHostDown; - else - severity |= SeverityFlagUnhandled; + Host::Ptr host = GetHost(); + ObjectLock hlock (host); + if (host->GetState() != HostUp || !host->IsReachable()) { + severity += 1024; + } else { + if (IsAcknowledged()) + severity += 512; + else if (IsInDowntime()) + severity += 256; + else + severity += 2048; + } + hlock.Unlock(); + } olock.Unlock(); diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index cf3994e93..898e3aee1 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -737,17 +737,18 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) attrs->Set("state", service->GetState()); attrs->Set("last_soft_state", service->GetState()); attrs->Set("last_hard_state", service->GetLastHardState()); + attrs->Set("severity", service->GetSeverity()); } else { attrs->Set("state", host->GetState()); attrs->Set("last_soft_state", host->GetState()); attrs->Set("last_hard_state", host->GetLastHardState()); + attrs->Set("severity", host->GetSeverity()); } attrs->Set("check_attempt", checkable->GetCheckAttempt()); - //attrs->Set("severity") - //attrs->Set(checkable->GetSeverity()); - + attrs->Set("is_active", checkable->IsActive()); + CheckResult::Ptr cr = checkable->GetLastCheckResult(); if (cr) { diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 77440730b..7eb80fd45 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -50,7 +50,6 @@ String RedisWriter::FormatCheckSumBinary(const String& str) return output; } - String RedisWriter::FormatCommandLine(const Value& commandLine) { String result; diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index a4e8fdc6a..aa5cfe007 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -27,6 +27,7 @@ #include "base/workqueue.hpp" #include "redis/redisconnection.hpp" #include "icinga/checkable.hpp" +#include "icinga/service.hpp" #include "icinga/downtime.hpp" #include From 03a61347cd4a0ab9ccbbcfab2780318f05579b76 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Mon, 17 Dec 2018 13:47:50 +0100 Subject: [PATCH 122/219] Fix Service's name --- lib/redis/rediswriter-objects.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 898e3aee1..20557694e 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -318,6 +318,9 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr checksums->Set("host_id", GetObjectIdentifier(service->GetHost())); attributes->Set("display_name", service->GetDisplayName()); + // Overwrite name here, `object->name` is 'HostName!ServiceName' but we only want the name of the Service + attributes->Set("name", service->GetShortName()); + groups = service->GetGroups(); getGroup = &::GetObjectByName; } else { From d2dc0047a3ced0c60e79ee9f3da81c9d6ed6ccac Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Mon, 28 Jan 2019 15:52:09 +0100 Subject: [PATCH 123/219] Lock before queueing multiple queries --- lib/redis/redisconnection.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index 9f889ab2e..22386e6c9 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -187,6 +187,7 @@ void RedisConnection::ExecuteQuery(const std::vector& query, redisCallba void RedisConnection::ExecuteQueries(const std::vector >& queries, redisCallbackFn *fn, void *privdata) { + boost::mutex::scoped_lock lock = m_RedisConnectionWorkQueue.AcquireLock(); for (const auto& query : queries) { m_RedisConnectionWorkQueue.Enqueue(std::bind(&RedisConnection::SendMessageInternal, this, query, fn, privdata)); } From b95e39952e796052c899106e9eb2df3e0b369569 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Wed, 31 Oct 2018 15:39:27 +0100 Subject: [PATCH 124/219] Use transaction for config dump This wraps the config dump HMSETs into a transaction. --- lib/redis/rediswriter-objects.cpp | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 20557694e..e8248974f 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -108,6 +108,7 @@ void RedisWriter::UpdateAllConfigObjects() std::vector customVars = {"HMSET", m_PrefixConfigCustomVar + lcType}; std::vector checksums = {"HMSET", m_PrefixConfigCheckSum + lcType}; std::vector states = {"HMSET", m_PrefixStateObject + lcType }; + std::vector > transaction = {{"MULTI"}}; bool dumpState = (lcType == "host" || lcType == "service"); for (const ConfigObject::Ptr& object : type.first->GetObjects()) { @@ -124,31 +125,43 @@ void RedisWriter::UpdateAllConfigObjects() bulkCounter++; if (!bulkCounter % 100) { if (attributes.size() > 2) { - m_Rcon->ExecuteQuery(attributes); + transaction.push_back(attributes); attributes.erase(attributes.begin() + 2, attributes.end()); } if (customVars.size() > 2) { - m_Rcon->ExecuteQuery(customVars); + transaction.push_back(customVars); customVars.erase(customVars.begin() + 2, customVars.end()); } if (checksums.size() > 2) { - m_Rcon->ExecuteQuery(checksums); + transaction.push_back(checksums); checksums.erase(checksums.begin() + 2, checksums.end()); } if (states.size() > 2) { - m_Rcon->ExecuteQuery(states); + transaction.push_back(states); states.erase(states.begin() + 2, states.end()); } + + if (transaction.size() > 1) { + transaction.push_back({"EXEC"}); + m_Rcon->ExecuteQueries(transaction); + transaction.erase(transaction.begin() + 1, transaction.end()); + } } } + if (attributes.size() > 2) - m_Rcon->ExecuteQuery(attributes); + transaction.push_back(attributes); if (customVars.size() > 2) - m_Rcon->ExecuteQuery(customVars); + transaction.push_back(customVars); if (checksums.size() > 2) - m_Rcon->ExecuteQuery(checksums); + transaction.push_back(checksums); if (states.size() > 2) - m_Rcon->ExecuteQuery(states); + transaction.push_back(states); + + if (transaction.size() > 1) { + transaction.push_back({"EXEC"}); + m_Rcon->ExecuteQueries(transaction); + } m_Rcon->ExecuteQuery({"PUBLISH", "icinga:config:dump", lcType}); From 0ad008f465fd5ed7ed215369e3841c345bd996af Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Mon, 28 Jan 2019 16:08:31 +0100 Subject: [PATCH 125/219] Handle transaction vectors better --- lib/redis/rediswriter-objects.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index e8248974f..b865573be 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -107,7 +107,7 @@ void RedisWriter::UpdateAllConfigObjects() std::vector attributes = {"HMSET", m_PrefixConfigObject + lcType}; std::vector customVars = {"HMSET", m_PrefixConfigCustomVar + lcType}; std::vector checksums = {"HMSET", m_PrefixConfigCheckSum + lcType}; - std::vector states = {"HMSET", m_PrefixStateObject + lcType }; + std::vector states = {"HMSET", m_PrefixStateObject + lcType}; std::vector > transaction = {{"MULTI"}}; bool dumpState = (lcType == "host" || lcType == "service"); @@ -125,38 +125,38 @@ void RedisWriter::UpdateAllConfigObjects() bulkCounter++; if (!bulkCounter % 100) { if (attributes.size() > 2) { - transaction.push_back(attributes); - attributes.erase(attributes.begin() + 2, attributes.end()); + transaction.push_back(std::move(attributes)); + attributes = {"HMSET", m_PrefixConfigObject + lcType}; } if (customVars.size() > 2) { - transaction.push_back(customVars); - customVars.erase(customVars.begin() + 2, customVars.end()); + transaction.push_back(std::move(customVars)); + customVars = {"HMSET", m_PrefixConfigCustomVar + lcType}; } if (checksums.size() > 2) { - transaction.push_back(checksums); - checksums.erase(checksums.begin() + 2, checksums.end()); + transaction.push_back(std::move(checksums)); + checksums = {"HMSET", m_PrefixConfigCheckSum + lcType}; } if (states.size() > 2) { - transaction.push_back(states); - states.erase(states.begin() + 2, states.end()); + transaction.push_back(std::move(states)); + states = {"HMSET", m_PrefixStateObject + lcType}; } if (transaction.size() > 1) { transaction.push_back({"EXEC"}); m_Rcon->ExecuteQueries(transaction); - transaction.erase(transaction.begin() + 1, transaction.end()); + transaction = {{"MULTI"}}; } } } if (attributes.size() > 2) - transaction.push_back(attributes); + transaction.push_back(std::move(attributes)); if (customVars.size() > 2) - transaction.push_back(customVars); + transaction.push_back(std::move(customVars)); if (checksums.size() > 2) - transaction.push_back(checksums); + transaction.push_back(std::move(checksums)); if (states.size() > 2) - transaction.push_back(states); + transaction.push_back(std::move(states)); if (transaction.size() > 1) { transaction.push_back({"EXEC"}); From 7890c37357d3e7eece1c944a0867e095d63528fd Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Tue, 29 Jan 2019 15:44:49 +0100 Subject: [PATCH 126/219] Revert "Merge branch 'bugfix/execute-quieries' into 'feature/redis'" This reverts commit f6c1fc3b43b1895eb87c1657f3e3355bfe9ddaac, reversing changes made to ad1d448bcdead76a2bd45171a6efcd700b1c23d0. --- lib/redis/redisconnection.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index 22386e6c9..9f889ab2e 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -187,7 +187,6 @@ void RedisConnection::ExecuteQuery(const std::vector& query, redisCallba void RedisConnection::ExecuteQueries(const std::vector >& queries, redisCallbackFn *fn, void *privdata) { - boost::mutex::scoped_lock lock = m_RedisConnectionWorkQueue.AcquireLock(); for (const auto& query : queries) { m_RedisConnectionWorkQueue.Enqueue(std::bind(&RedisConnection::SendMessageInternal, this, query, fn, privdata)); } From 23099904c011b2880bd2356c340c0a1b0992aa76 Mon Sep 17 00:00:00 2001 From: Jean Flach Date: Mon, 28 Jan 2019 15:19:10 +0100 Subject: [PATCH 127/219] Publish when config_dump_in progress --- lib/redis/rediswriter.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index 076ad86ff..ed5a48b6a 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -121,6 +121,7 @@ void RedisWriter::TryToReconnect() /* Config dump */ m_ConfigDumpInProgress = true; + PublishStats(); UpdateAllConfigObjects(); @@ -224,6 +225,7 @@ void RedisWriter::PublishStats() return; Dictionary::Ptr status = GetStats(); + status->Set("config_dump_in_progress", m_ConfigDumpInProgress); String jsonStats = JsonEncode(status); m_Rcon->ExecuteQuery({ "PUBLISH", "icinga:stats", jsonStats }); From f601ba51e0357bd762320d8ab1432bcdafc40c38 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 13 May 2019 11:29:19 +0200 Subject: [PATCH 128/219] Revert "Eventqueue: Remove unused code" This reverts commit a7873da89dd835802fa957ebeefa7cd403d519a9. --- lib/remote/eventqueue.cpp | 19 +++++++++++++++++++ lib/remote/eventqueue.hpp | 2 ++ 2 files changed, 21 insertions(+) diff --git a/lib/remote/eventqueue.cpp b/lib/remote/eventqueue.cpp index 1125a4543..3c7504e6c 100644 --- a/lib/remote/eventqueue.cpp +++ b/lib/remote/eventqueue.cpp @@ -90,6 +90,25 @@ void EventQueue::SetFilter(std::unique_ptr filter) m_Filter.swap(filter); } +Dictionary::Ptr EventQueue::WaitForEvent(void *client, double timeout) +{ + boost::mutex::scoped_lock lock(m_Mutex); + + for (;;) { + auto it = m_Events.find(client); + ASSERT(it != m_Events.end()); + + if (!it->second.empty()) { + Dictionary::Ptr result = *it->second.begin(); + it->second.pop_front(); + return result; + } + + if (!m_CV.timed_wait(lock, boost::posix_time::milliseconds(long(timeout * 1000)))) + return nullptr; + } +} + std::vector EventQueue::GetQueuesForType(const String& type) { EventQueueRegistry::ItemMap queues = EventQueueRegistry::GetInstance()->GetItems(); diff --git a/lib/remote/eventqueue.hpp b/lib/remote/eventqueue.hpp index c8317cb74..33013836e 100644 --- a/lib/remote/eventqueue.hpp +++ b/lib/remote/eventqueue.hpp @@ -36,6 +36,8 @@ public: void SetTypes(const std::set& types); void SetFilter(std::unique_ptr filter); + Dictionary::Ptr WaitForEvent(void *client, double timeout = 5); + static std::vector GetQueuesForType(const String& type); static void UnregisterIfUnused(const String& name, const EventQueue::Ptr& queue); From c5b0884e4411b1c2644e2d2e2dc5856a5f1f9dee Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 13 May 2019 11:37:54 +0200 Subject: [PATCH 129/219] RedisWriter: Heartbeat should occur every second --- lib/redis/rediswriter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index ed5a48b6a..f4714cf78 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -78,7 +78,7 @@ void RedisWriter::Start(bool runtimeCreated) m_SubscriptionTimer->Start(); m_StatsTimer = new Timer(); - m_StatsTimer->SetInterval(10); + m_StatsTimer->SetInterval(1); m_StatsTimer->OnTimerExpired.connect(std::bind(&RedisWriter::PublishStatsTimerHandler, this)); m_StatsTimer->Start(); From a6ec60a9928cb1e492c67ee3a879809ea5bb61d8 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 14 May 2019 17:16:39 +0200 Subject: [PATCH 130/219] RedisWriter: Use IcingaApplication environment --- lib/redis/rediswriter-utility.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 7eb80fd45..1ef203450 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -77,11 +77,9 @@ String RedisWriter::FormatCommandLine(const Value& commandLine) return result; } -static Value l_DefaultEnv = "production"; - String RedisWriter::GetEnvironment() { - return ScriptGlobal::Get("Environment", &l_DefaultEnv); + return ConfigType::GetObjectsByType()[0]->GetEnvironment(); } String RedisWriter::GetObjectIdentifier(const ConfigObject::Ptr& object) From 50594ec1c83ae421f99cd5badd1bdb948e5cf361 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 28 Jun 2019 11:56:15 +0200 Subject: [PATCH 131/219] Replace std::bind() with lambdas refs #48 --- lib/redis/redisconnection.cpp | 10 +++++++--- lib/redis/rediswriter-objects.cpp | 29 +++++++++++++++++++---------- lib/redis/rediswriter.cpp | 21 +++++++++++---------- 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index 9f889ab2e..c55399def 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -43,7 +43,7 @@ void RedisConnection::Start() { RedisConnection::Connect(); - std::thread thread(std::bind(&RedisConnection::HandleRW, this)); + std::thread thread(&RedisConnection::HandleRW, this); thread.detach(); } @@ -181,14 +181,18 @@ void RedisConnection::DisconnectCallback(const redisAsyncContext *c, int status) void RedisConnection::ExecuteQuery(const std::vector& query, redisCallbackFn *fn, void *privdata) { - m_RedisConnectionWorkQueue.Enqueue(std::bind(&RedisConnection::SendMessageInternal, this, query, fn, privdata)); + m_RedisConnectionWorkQueue.Enqueue([this, query, fn, privdata]() { + SendMessageInternal(query, fn, privdata); + }); } void RedisConnection::ExecuteQueries(const std::vector >& queries, redisCallbackFn *fn, void *privdata) { for (const auto& query : queries) { - m_RedisConnectionWorkQueue.Enqueue(std::bind(&RedisConnection::SendMessageInternal, this, query, fn, privdata)); + m_RedisConnectionWorkQueue.Enqueue([this, query, fn, privdata]() { + SendMessageInternal(query, fn, privdata); + }); } } diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index b865573be..7ec008f32 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -53,20 +53,29 @@ INITIALIZE_ONCE(&RedisWriter::ConfigStaticInitialize); void RedisWriter::ConfigStaticInitialize() { /* triggered in ProcessCheckResult(), requires UpdateNextCheck() to be called before */ - Checkable::OnStateChange.connect(std::bind(&RedisWriter::StateChangeHandler, _1)); + Checkable::OnStateChange.connect([](const Checkable::Ptr& checkable, const CheckResult::Ptr&, StateType, const MessageOrigin::Ptr&) { + RedisWriter::StateChangeHandler(checkable); + }); + /* triggered when acknowledged host/service goes back to ok and when the acknowledgement gets deleted */ - Checkable::OnAcknowledgementCleared.connect(std::bind(&RedisWriter::StateChangeHandler, _1)); + Checkable::OnAcknowledgementCleared.connect([](const Checkable::Ptr& checkable, const MessageOrigin::Ptr&) { + RedisWriter::StateChangeHandler(checkable); + }); /* triggered on create, update and delete objects */ - ConfigObject::OnActiveChanged.connect(std::bind(&RedisWriter::VersionChangedHandler, _1)); - ConfigObject::OnVersionChanged.connect(std::bind(&RedisWriter::VersionChangedHandler, _1)); + ConfigObject::OnActiveChanged.connect([](const ConfigObject::Ptr& object, const Value&) { + RedisWriter::VersionChangedHandler(object); + }); + ConfigObject::OnVersionChanged.connect([](const ConfigObject::Ptr& object, const Value&) { + RedisWriter::VersionChangedHandler(object); + }); /* fixed downtime start */ - Downtime::OnDowntimeStarted.connect(std::bind(&RedisWriter::DowntimeChangedHandler, _1)); + Downtime::OnDowntimeStarted.connect(&RedisWriter::DowntimeChangedHandler); /* flexible downtime start */ - Downtime::OnDowntimeTriggered.connect(std::bind(&RedisWriter::DowntimeChangedHandler, _1)); + Downtime::OnDowntimeTriggered.connect(&RedisWriter::DowntimeChangedHandler); /* fixed/flexible downtime end */ - Downtime::OnDowntimeRemoved.connect(std::bind(&RedisWriter::DowntimeChangedHandler, _1)); + Downtime::OnDowntimeRemoved.connect(&RedisWriter::DowntimeChangedHandler); } void RedisWriter::UpdateAllConfigObjects() @@ -877,7 +886,7 @@ void RedisWriter::StateChangeHandler(const ConfigObject::Ptr &object) Type::Ptr type = object->GetReflectionType(); for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { - rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendStatusUpdate, rw, object)); + rw->m_WorkQueue.Enqueue([rw, object]() { rw->SendStatusUpdate(object); }); } } @@ -889,14 +898,14 @@ void RedisWriter::VersionChangedHandler(const ConfigObject::Ptr& object) // Create or update the object config for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { if (rw) - rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendConfigUpdate, rw, object, true)); + rw->m_WorkQueue.Enqueue([rw, object]() { rw->SendConfigUpdate(object, true); }); } } else if (!object->IsActive() && object->GetExtension("ConfigObjectDeleted")) { // same as in apilistener-configsync.cpp // Delete object config for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { if (rw) - rw->m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendConfigDelete, rw, object)); + rw->m_WorkQueue.Enqueue([rw, object]() { rw->SendConfigDelete(object); }); } } } diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index f4714cf78..373f6085f 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -26,6 +26,7 @@ #include "icinga/host.hpp" #include +#include using namespace icinga; @@ -64,27 +65,27 @@ void RedisWriter::Start(bool runtimeCreated) m_Rcon = new RedisConnection(GetHost(), GetPort(), GetPath(), GetPassword(), GetDbIndex()); m_Rcon->Start(); - m_WorkQueue.SetExceptionCallback(std::bind(&RedisWriter::ExceptionHandler, this, _1)); + m_WorkQueue.SetExceptionCallback([this](boost::exception_ptr exp) { ExceptionHandler(std::move(exp)); }); m_ReconnectTimer = new Timer(); m_ReconnectTimer->SetInterval(15); - m_ReconnectTimer->OnTimerExpired.connect(std::bind(&RedisWriter::ReconnectTimerHandler, this)); + m_ReconnectTimer->OnTimerExpired.connect([this](const Timer * const&) { ReconnectTimerHandler(); }); m_ReconnectTimer->Start(); m_ReconnectTimer->Reschedule(0); m_SubscriptionTimer = new Timer(); m_SubscriptionTimer->SetInterval(15); - m_SubscriptionTimer->OnTimerExpired.connect(std::bind(&RedisWriter::UpdateSubscriptionsTimerHandler, this)); + m_SubscriptionTimer->OnTimerExpired.connect([this](const Timer * const&) { UpdateSubscriptionsTimerHandler(); }); m_SubscriptionTimer->Start(); m_StatsTimer = new Timer(); m_StatsTimer->SetInterval(1); - m_StatsTimer->OnTimerExpired.connect(std::bind(&RedisWriter::PublishStatsTimerHandler, this)); + m_StatsTimer->OnTimerExpired.connect([this](const Timer * const&) { PublishStatsTimerHandler(); }); m_StatsTimer->Start(); m_WorkQueue.SetName("RedisWriter"); - boost::thread thread(std::bind(&RedisWriter::HandleEvents, this)); + boost::thread thread(&RedisWriter::HandleEvents, this); thread.detach(); } @@ -99,7 +100,7 @@ void RedisWriter::ExceptionHandler(boost::exception_ptr exp) void RedisWriter::ReconnectTimerHandler() { - m_WorkQueue.Enqueue(std::bind(&RedisWriter::TryToReconnect, this)); + m_WorkQueue.Enqueue([this]() { TryToReconnect(); }); } void RedisWriter::TryToReconnect() @@ -132,7 +133,7 @@ void RedisWriter::TryToReconnect() void RedisWriter::UpdateSubscriptionsTimerHandler() { - m_WorkQueue.Enqueue(std::bind(&RedisWriter::UpdateSubscriptions, this)); + m_WorkQueue.Enqueue([this]() { UpdateSubscriptions(); }); } void RedisWriter::UpdateSubscriptions() @@ -214,7 +215,7 @@ bool RedisWriter::GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi) void RedisWriter::PublishStatsTimerHandler(void) { - m_WorkQueue.Enqueue(std::bind(&RedisWriter::PublishStats, this)); + m_WorkQueue.Enqueue([this]() { PublishStats(); }); } void RedisWriter::PublishStats() @@ -260,7 +261,7 @@ void RedisWriter::HandleEvents() if (!event) continue; - m_WorkQueue.Enqueue(std::bind(&RedisWriter::SendEvent, this, event)); + m_WorkQueue.Enqueue([this, event]() { SendEvent(event); }); } queue->RemoveClient(this); @@ -318,7 +319,7 @@ void RedisWriter::SendEvent(const Dictionary::Ptr& event) checkable = Host::GetByName(event->Get("host")); } // Update State for icingaweb - m_WorkQueue.Enqueue(std::bind(&RedisWriter::UpdateState, this, checkable)); + m_WorkQueue.Enqueue([this, checkable]() { UpdateState(checkable); }); } if (type.Contains("Acknowledgement")) { From 1dcec6e77af1ec925c816e0f3a38e6c66dba1f82 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 4 Apr 2019 11:51:59 +0200 Subject: [PATCH 132/219] RedisConnection#SendMessageInternal(): lock only while actually using shared objects --- lib/redis/redisconnection.cpp | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index c55399def..7bf477980 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -200,12 +200,14 @@ void RedisConnection::SendMessageInternal(const std::vector& query, redi { AssertOnWorkQueue(); - boost::mutex::scoped_lock lock(m_CMutex); + { + boost::mutex::scoped_lock lock(m_CMutex); - if (!m_Context || !m_Connected) { - Log(LogCritical, "RedisWriter") - << "Not connected to Redis"; - return; + if (!m_Context || !m_Connected) { + Log(LogCritical, "RedisWriter") + << "Not connected to Redis"; + return; + } } const char **argv; @@ -224,7 +226,14 @@ void RedisConnection::SendMessageInternal(const std::vector& query, redi Log(LogDebug, "RedisWriter, Connection") << "Sending Command: " << debugstr; - int r = redisAsyncCommandArgv(m_Context, fn, privdata, query.size(), argv, argvlen); + + int r; + + { + boost::mutex::scoped_lock lock(m_CMutex); + + r = redisAsyncCommandArgv(m_Context, fn, privdata, query.size(), argv, argvlen); + } delete[] argv; delete[] argvlen; From 68c88b3edf07fcc7ea00f3b7701e31d50dc3018b Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 14 Jun 2019 11:52:34 +0200 Subject: [PATCH 133/219] Implement new Redis schema WIP WIP --- lib/redis/rediswriter-objects.cpp | 694 ++++++++++++++++++------------ lib/redis/rediswriter-utility.cpp | 21 +- lib/redis/rediswriter.cpp | 5 +- lib/redis/rediswriter.hpp | 9 +- 4 files changed, 426 insertions(+), 303 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 7ec008f32..a0d5a3626 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -107,23 +107,38 @@ void RedisWriter::UpdateAllConfigObjects() } } - upq.ParallelFor(types, [this](const TypePair& type) - { + const std::vector globalKeys = { + m_PrefixConfigObject + "customvar", + m_PrefixConfigObject + "action_url", + m_PrefixConfigObject + "notes_url", + m_PrefixConfigObject + "icon_image", + m_PrefixConfigObject + "commandargument", + m_PrefixConfigObject + "commandenvvar", + m_PrefixConfigObject + "timerange", + }; + DeleteKeys(globalKeys); + + upq.ParallelFor(types, [this, &globalKeys](const TypePair& type) { String lcType = type.second; - m_Rcon->ExecuteQuery( - {"DEL", m_PrefixConfigCheckSum + lcType, m_PrefixConfigObject + lcType, m_PrefixStateObject + lcType}); - size_t bulkCounter = 0; - std::vector attributes = {"HMSET", m_PrefixConfigObject + lcType}; - std::vector customVars = {"HMSET", m_PrefixConfigCustomVar + lcType}; - std::vector checksums = {"HMSET", m_PrefixConfigCheckSum + lcType}; - std::vector states = {"HMSET", m_PrefixStateObject + lcType}; - std::vector > transaction = {{"MULTI"}}; + + std::vector keys = GetTypeObjectKeys(lcType); + DeleteKeys(keys); + + keys.reserve(globalKeys.size()); + keys.insert(keys.end(), globalKeys.begin(), globalKeys.end()); + + std::map > statements = GenerateHmsetStatements(keys); + std::vector states = {"HMSET", m_PrefixStateObject + lcType}; + std::vector > transaction = {{"MULTI"}}; + bool dumpState = (lcType == "host" || lcType == "service"); + size_t bulkCounter = 0; for (const ConfigObject::Ptr& object : type.first->GetObjects()) { if (lcType != GetLowerCaseTypeNameDB(object)) continue; - CreateConfigUpdate(object, lcType, attributes, customVars, checksums, false); + + CreateConfigUpdate(object, lcType, statements, false); // Write out inital state for checkables if (dumpState) { @@ -133,23 +148,17 @@ void RedisWriter::UpdateAllConfigObjects() bulkCounter++; if (!bulkCounter % 100) { - if (attributes.size() > 2) { - transaction.push_back(std::move(attributes)); - attributes = {"HMSET", m_PrefixConfigObject + lcType}; - } - if (customVars.size() > 2) { - transaction.push_back(std::move(customVars)); - customVars = {"HMSET", m_PrefixConfigCustomVar + lcType}; - } - if (checksums.size() > 2) { - transaction.push_back(std::move(checksums)); - checksums = {"HMSET", m_PrefixConfigCheckSum + lcType}; - } + for (const auto& kv : statements) + if (kv.second.size() > 2) + transaction.push_back(kv.second); + if (states.size() > 2) { transaction.push_back(std::move(states)); states = {"HMSET", m_PrefixStateObject + lcType}; } + statements = GenerateHmsetStatements(keys); + if (transaction.size() > 1) { transaction.push_back({"EXEC"}); m_Rcon->ExecuteQueries(transaction); @@ -158,12 +167,10 @@ void RedisWriter::UpdateAllConfigObjects() } } - if (attributes.size() > 2) - transaction.push_back(std::move(attributes)); - if (customVars.size() > 2) - transaction.push_back(std::move(customVars)); - if (checksums.size() > 2) - transaction.push_back(std::move(checksums)); + for (const auto& kv : statements) + if (kv.second.size() > 2) + transaction.push_back(kv.second); + if (states.size() > 2) transaction.push_back(std::move(states)); @@ -181,9 +188,15 @@ void RedisWriter::UpdateAllConfigObjects() upq.Join(); if (upq.HasExceptions()) { - for (auto exc : upq.GetExceptions()) { - Log(LogCritical, "RedisWriter") - << "Exception during ConfigDump: " << exc; + for (boost::exception_ptr exc : upq.GetExceptions()) { + try { + if (exc) { + boost::rethrow_exception(exc); + } + } catch(const std::exception& e) { + Log(LogCritical, "RedisWriter") + << "Exception during ConfigDump: " << e.what(); + } } } @@ -191,10 +204,61 @@ void RedisWriter::UpdateAllConfigObjects() << "Initial config/status dump finished in " << Utility::GetTime() - startTime << " seconds."; } -void RedisWriter::UpdateState(const Checkable::Ptr& checkable) { - Dictionary::Ptr stateAttrs = SerializeState(checkable); +void RedisWriter::DeleteKeys(const std::vector& keys) { + std::vector query = {"DEL"}; + for (auto& key : keys) { + query.emplace_back(key); + } - m_Rcon->ExecuteQuery({"HSET", m_PrefixStateObject + GetLowerCaseTypeNameDB(checkable), GetObjectIdentifier(checkable), JsonEncode(stateAttrs)}); + m_Rcon->ExecuteQuery(query); +} + +std::map > RedisWriter::GenerateHmsetStatements(const std::vector& keys) +{ + std::map > statements; + for (auto& key : keys) { + std::vector statement = {"HMSET", key}; + statements.insert(std::pair >(key, statement)); + } + + return statements; +} + +std::vector RedisWriter::GetTypeObjectKeys(const String& type) +{ + std::vector keys = { + m_PrefixConfigObject + type, + m_PrefixConfigCheckSum + type, + m_PrefixConfigObject + type + ":customvar", + m_PrefixConfigCheckSum + type + ":customvar", + }; + + if (type == "host" || type == "service" || type == "user") { + keys.emplace_back(m_PrefixConfigObject + type + ":groupmember"); + keys.emplace_back(m_PrefixConfigCheckSum + type + ":groupmember"); + } else if (type == "timeperiod") { + keys.emplace_back(m_PrefixConfigObject + type + ":overwrite:include"); + keys.emplace_back(m_PrefixConfigCheckSum + type + ":overwrite:include"); + keys.emplace_back(m_PrefixConfigObject + type + ":overwrite:exclude"); + keys.emplace_back(m_PrefixConfigCheckSum + type + ":overwrite:exclude"); + keys.emplace_back(m_PrefixConfigObject + type + ":range"); + keys.emplace_back(m_PrefixConfigCheckSum + type + ":range"); + } else if (type == "zone") { + keys.emplace_back(m_PrefixConfigObject + type + ":parent"); + keys.emplace_back(m_PrefixConfigCheckSum + type + ":parent"); + } else if (type == "notification") { + keys.emplace_back(m_PrefixConfigObject + type + ":user"); + keys.emplace_back(m_PrefixConfigCheckSum + type + ":user"); + keys.emplace_back(m_PrefixConfigObject + type + ":usergroup"); + keys.emplace_back(m_PrefixConfigCheckSum + type + ":usergroup"); + } else if (type == "checkcommand" || type == "notificationcommand" || type == "eventcommand") { + keys.emplace_back(m_PrefixConfigObject + type + ":envvar"); + keys.emplace_back(m_PrefixConfigCheckSum + type + ":envvar"); + keys.emplace_back(m_PrefixConfigObject + type + ":argument"); + keys.emplace_back(m_PrefixConfigCheckSum + type + ":argument"); + } + + return keys; } template @@ -203,6 +267,270 @@ static ConfigObject::Ptr GetObjectByName(const String& name) return ConfigObject::GetObject(name); } +void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map >& statements) +{ + String objectKey = GetObjectIdentifier(object); + CustomVarObject::Ptr customVarObject = dynamic_pointer_cast(object); + String envId = CalculateCheckSumString(GetEnvironment()); + + if (customVarObject) { + auto vars(SerializeVars(customVarObject)); + if (vars) { + statements[m_PrefixConfigCheckSum + typeName + ":customvar"].emplace_back(objectKey); + statements[m_PrefixConfigCheckSum + typeName + ":customvar"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumVars(customVarObject)}}))); + + ObjectLock varsLock(vars); + Array::Ptr varsArray(new Array); + for (auto& kv : vars) { + statements[m_PrefixConfigObject + "customvar"].emplace_back(kv.first); + statements[m_PrefixConfigObject + "customvar"].emplace_back(JsonEncode(kv.second)); + varsArray->Add(kv.first); + } + + statements[m_PrefixConfigObject + typeName + ":customvar"].emplace_back(objectKey); + statements[m_PrefixConfigObject + typeName + ":customvar"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"customvars", varsArray}}))); + } + } + + Type::Ptr type = object->GetReflectionType(); + if (type == Host::TypeInstance || type == Service::TypeInstance) { + Checkable::Ptr checkable = static_pointer_cast(object); + + String actionUrl = checkable->GetActionUrl(); + String notesUrl = checkable->GetNotesUrl(); + String iconImage = checkable->GetIconImage(); + if (!actionUrl.IsEmpty()) { + statements[m_PrefixConfigObject + "action_url"].emplace_back(CalculateCheckSumArray(new Array({envId, actionUrl}))); + statements[m_PrefixConfigObject + "action_url"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"action_url", actionUrl}}))); + } + if (!notesUrl.IsEmpty()) { + statements[m_PrefixConfigObject + "notes_url"].emplace_back(CalculateCheckSumArray(new Array({envId, notesUrl}))); + statements[m_PrefixConfigObject + "notes_url"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"notes_url", notesUrl}}))); + } + if (!iconImage.IsEmpty()) { + statements[m_PrefixConfigObject + "icon_image"].emplace_back(CalculateCheckSumArray(new Array({envId, iconImage}))); + statements[m_PrefixConfigObject + "icon_image"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"icon_image", iconImage}}))); + } + + Host::Ptr host; + Service::Ptr service; + tie(host, service) = GetHostService(checkable); + + ConfigObject::Ptr (*getGroup)(const String& name); + Array::Ptr groups; + if (service) { + groups = service->GetGroups(); + getGroup = &::GetObjectByName; + } else { + groups = host->GetGroups(); + getGroup = &::GetObjectByName; + } + + if (groups) { + ObjectLock groupsLock(groups); + Array::Ptr groupIds(new Array); + for (auto& group : groups) { + groupIds->Add(GetObjectIdentifier((*getGroup)(group))); + } + + statements[m_PrefixConfigCheckSum + typeName + ":groupmember"].emplace_back(objectKey); + statements[m_PrefixConfigCheckSum + typeName + ":groupmember"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(groupIds)}}))); + statements[m_PrefixConfigObject + typeName + ":groupmember"].emplace_back(objectKey); + statements[m_PrefixConfigObject + typeName + ":groupmember"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"groups", groupIds}}))); + } + + return; + } + + if (type == TimePeriod::TypeInstance) { + TimePeriod::Ptr timeperiod = static_pointer_cast(object); + + Dictionary::Ptr ranges = timeperiod->GetRanges(); + if (ranges) { + ObjectLock rangesLock(ranges); + Array::Ptr rangeIds(new Array); + for (auto& kv : ranges) { + String id = CalculateCheckSumArray(new Array({envId, kv.first, kv.second})); + rangeIds->Add(id); + + statements[m_PrefixConfigObject + "timerange"].emplace_back(id); + statements[m_PrefixConfigObject + "timerange"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"range_key", kv.first}, {"range_value", kv.second}}))); + } + + statements[m_PrefixConfigCheckSum + typeName + ":range"].emplace_back(objectKey); + statements[m_PrefixConfigCheckSum + typeName + ":range"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(rangeIds)}}))); + statements[m_PrefixConfigObject + typeName + ":range"].emplace_back(objectKey); + statements[m_PrefixConfigObject + typeName + ":range"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"ranges", rangeIds}}))); + } + + Array::Ptr includes; + ConfigObject::Ptr (*getInclude)(const String& name); + includes = timeperiod->GetIncludes(); + getInclude = &::GetObjectByName; + + Array::Ptr includeChecksums = new Array(); + + ObjectLock includesLock(includes); + ObjectLock includeChecksumsLock(includeChecksums); + + for (auto include : includes) { + includeChecksums->Add(GetObjectIdentifier((*getInclude)(include.Get()))); + } + + statements[m_PrefixConfigCheckSum + typeName + ":overwrite:include"].emplace_back(objectKey); + statements[m_PrefixConfigCheckSum + typeName + ":overwrite:include"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(includes)}}))); + statements[m_PrefixConfigObject + typeName + ":overwrite:include"].emplace_back(objectKey); + statements[m_PrefixConfigObject + typeName + ":overwrite:include"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"includes", includeChecksums}}))); + + Array::Ptr excludes; + ConfigObject::Ptr (*getExclude)(const String& name); + + excludes = timeperiod->GetExcludes(); + getExclude = &::GetObjectByName; + + Array::Ptr excludeChecksums = new Array(); + + ObjectLock excludesLock(excludes); + ObjectLock excludeChecksumsLock(excludeChecksums); + + for (auto exclude : excludes) { + excludeChecksums->Add(GetObjectIdentifier((*getExclude)(exclude.Get()))); + } + + statements[m_PrefixConfigCheckSum + typeName + ":overwrite:exclude"].emplace_back(objectKey); + statements[m_PrefixConfigCheckSum + typeName + ":overwrite:exclude"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(excludes)}}))); + statements[m_PrefixConfigObject + typeName + ":overwrite:exclude"].emplace_back(objectKey); + statements[m_PrefixConfigObject + typeName + ":overwrite:exclude"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"excludes", excludeChecksums}}))); + + return; + } + + if (type == Zone::TypeInstance) { + Zone::Ptr zone = static_pointer_cast(object); + + Array::Ptr parents(new Array); + + for (auto& parent : zone->GetAllParentsRaw()) { + parents->Add(GetObjectIdentifier(parent)); + } + + statements[m_PrefixConfigCheckSum + typeName + ":parent"].emplace_back(objectKey); + statements[m_PrefixConfigCheckSum + typeName + ":parent"].emplace_back(JsonEncode(new Dictionary({{"checksum", HashValue(zone->GetAllParents())}}))); + statements[m_PrefixConfigObject + typeName + ":parent"].emplace_back(objectKey); + statements[m_PrefixConfigObject + typeName + ":parent"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"parents", parents}}))); + + return; + } + + if (type == User::TypeInstance) { + User::Ptr user = static_pointer_cast(object); + + Array::Ptr groups; + ConfigObject::Ptr (*getGroup)(const String& name); + + groups = user->GetGroups(); + getGroup = &::GetObjectByName; + + if (groups) { + ObjectLock groupsLock(groups); + Array::Ptr groupIds(new Array); + for (auto& group : groups) { + groupIds->Add(GetObjectIdentifier((*getGroup)(group))); + } + + statements[m_PrefixConfigCheckSum + typeName + ":groupmember"].emplace_back(objectKey); + statements[m_PrefixConfigCheckSum + typeName + ":groupmember"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(groupIds)}}))); + statements[m_PrefixConfigObject + typeName + ":groupmember"].emplace_back(objectKey); + statements[m_PrefixConfigObject + typeName + ":groupmember"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"groups", groupIds}}))); + } + + return; + } + + if (type == Notification::TypeInstance) { + Notification::Ptr notification = static_pointer_cast(object); + + std::set users = notification->GetUsers(); + Array::Ptr userIds = new Array(); + + auto usergroups(notification->GetUserGroups()); + Array::Ptr usergroupIds = new Array(); + + userIds->Reserve(users.size()); + + for (auto& user : users) { + userIds->Add(GetObjectIdentifier(user)); + } + + statements[m_PrefixConfigCheckSum + typeName + ":user"].emplace_back(objectKey); + statements[m_PrefixConfigCheckSum + typeName + ":user"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(userIds)}}))); + statements[m_PrefixConfigObject + typeName + ":user"].emplace_back(objectKey); + statements[m_PrefixConfigObject + typeName + ":user"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"users", userIds}}))); + + usergroupIds->Reserve(usergroups.size()); + + for (auto& usergroup : usergroups) { + usergroupIds->Add(GetObjectIdentifier(usergroup)); + } + + statements[m_PrefixConfigCheckSum + typeName + ":usergroup"].emplace_back(objectKey); + statements[m_PrefixConfigCheckSum + typeName + ":usergroup"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(usergroupIds)}}))); + statements[m_PrefixConfigObject + typeName + ":usergroup"].emplace_back(objectKey); + statements[m_PrefixConfigObject + typeName + ":usergroup"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"usergroups", usergroupIds}}))); + + return; + } + + if (type == CheckCommand::TypeInstance || type == NotificationCommand::TypeInstance || type == EventCommand::TypeInstance) { + Command::Ptr command = static_pointer_cast(object); + + Dictionary::Ptr arguments = command->GetArguments(); + if (arguments) { + ObjectLock argumentsLock(arguments); + Array::Ptr argumentIds(new Array); + for (auto& kv : arguments) { + String id = HashValue(kv.first + HashValue(kv.second)); + argumentIds->Add(id); + + statements[m_PrefixConfigObject + "commandargument"].emplace_back(id); + statements[m_PrefixConfigObject + "commandargument"].emplace_back(JsonEncode(kv.second)); + } + + statements[m_PrefixConfigCheckSum + typeName + ":argument"].emplace_back(objectKey); + statements[m_PrefixConfigCheckSum + typeName + ":argument"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(argumentIds)}}))); + statements[m_PrefixConfigObject + typeName + ":argument"].emplace_back(objectKey); + statements[m_PrefixConfigObject + typeName + ":argument"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"arguments", argumentIds}}))); + } + + Dictionary::Ptr envvars = command->GetArguments(); + if (envvars) { + ObjectLock envvarsLock(envvars); + Array::Ptr envvarIds(new Array); + for (auto& kv : envvars) { + String id = HashValue(kv.first + HashValue(kv.second)); + envvarIds->Add(id); + + statements[m_PrefixConfigObject + "commandenvvar"].emplace_back(id); + statements[m_PrefixConfigObject + "commandenvvar"].emplace_back(JsonEncode(kv.second)); + } + + statements[m_PrefixConfigCheckSum + typeName + ":envvar"].emplace_back(objectKey); + statements[m_PrefixConfigCheckSum + typeName + ":envvar"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(envvarIds)}}))); + statements[m_PrefixConfigObject + typeName + ":envvar"].emplace_back(objectKey); + statements[m_PrefixConfigObject + typeName + ":envvar"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"envvars", envvarIds}}))); + } + + return; + } +} + +void RedisWriter::UpdateState(const Checkable::Ptr& checkable) +{ + Dictionary::Ptr stateAttrs = SerializeState(checkable); + + m_Rcon->ExecuteQuery({"HSET", m_PrefixStateObject + GetLowerCaseTypeNameDB(checkable), GetObjectIdentifier(checkable), JsonEncode(stateAttrs)}); +} + // Used to update a single object, used for runtime updates void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate) { @@ -211,44 +539,45 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtime String typeName = GetLowerCaseTypeNameDB(object); - std::vector attribute = {"HSET", m_PrefixConfigObject + typeName}; - std::vector customVar = {"HSET", m_PrefixConfigCustomVar + typeName}; - std::vector checksum = {"HSET", m_PrefixConfigCheckSum + typeName}; - std::vector state = {"HSET", m_PrefixStateObject + typeName}; + std::vector keys = GetTypeObjectKeys(typeName); + std::map > statements = GenerateHmsetStatements(keys); + std::vector states = {"HMSET", m_PrefixStateObject + typeName}; - CreateConfigUpdate(object, typeName, attribute, customVar, checksum, runtimeUpdate); + CreateConfigUpdate(object, typeName, statements, runtimeUpdate); Checkable::Ptr checkable = dynamic_pointer_cast(object); if (checkable) { m_Rcon->ExecuteQuery({"HSET", m_PrefixStateObject + typeName, GetObjectIdentifier(checkable), JsonEncode(SerializeState(checkable))}); } - m_Rcon->ExecuteQuery(attribute); - m_Rcon->ExecuteQuery(checksum); - if (customVar.size() > 2) - m_Rcon->ExecuteQuery(customVar); + std::vector > transaction = {{"MULTI"}}; + for (const auto& kv : statements) + transaction.push_back(kv.second); + + if (transaction.size() > 1) { + transaction.push_back({"EXEC"}); + m_Rcon->ExecuteQueries(transaction); + } } // Takes object and collects IcingaDB relevant attributes and computes checksums. Returns whether the object is relevant // for IcingaDB. bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& attributes, Dictionary::Ptr& checksums) { - checksums->Set("name_checksum", CalculateCheckSumString(object->GetName())); - checksums->Set("environment_id", CalculateCheckSumString(GetEnvironment())); + attributes->Set("name_checksum", CalculateCheckSumString(object->GetName())); + attributes->Set("env_id", CalculateCheckSumString(GetEnvironment())); attributes->Set("name", object->GetName()); Zone::Ptr ObjectsZone = static_pointer_cast(object->GetZone()); if (ObjectsZone) { - checksums->Set("zone_id", GetObjectIdentifier(ObjectsZone)); + attributes->Set("zone_id", GetObjectIdentifier(ObjectsZone)); attributes->Set("zone", ObjectsZone->GetName()); } Type::Ptr type = object->GetReflectionType(); if (type == Endpoint::TypeInstance) { - checksums->Set("properties_checksum", HashValue(attributes)); - return true; } @@ -256,25 +585,12 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr Zone::Ptr zone = static_pointer_cast(object); attributes->Set("is_global", zone->GetGlobal()); - checksums->Set("properties_checksum", HashValue(attributes)); - Array::Ptr endpoints = new Array(); - endpoints->Resize(zone->GetEndpoints().size()); - - Array::SizeType i = 0; - for (auto& endpointObject : zone->GetEndpoints()) { - endpoints->Set(i++, endpointObject->GetName()); + Zone::Ptr parent = zone->GetParent(); + if (parent) { + attributes->Set("parent_id", GetObjectIdentifier(zone)); } - Array::Ptr parents(new Array); - - for (auto& parent : zone->GetAllParentsRaw()) { - parents->Add(GetObjectIdentifier(parent)); - } - - checksums->Set("parent_ids", parents); - checksums->Set("parents_checksum", HashValue(zone->GetAllParents())); - return true; } @@ -298,23 +614,23 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr attributes->Set("notes", checkable->GetNotes()); attributes->Set("icon_image_alt", checkable->GetIconImageAlt()); - checksums->Set("checkcommand_id", GetObjectIdentifier(checkable->GetCheckCommand())); + attributes->Set("checkcommand_id", GetObjectIdentifier(checkable->GetCheckCommand())); Endpoint::Ptr commandEndpoint = checkable->GetCommandEndpoint(); if (commandEndpoint) { - checksums->Set("command_endpoint_id", GetObjectIdentifier(commandEndpoint)); + attributes->Set("command_endpoint_id", GetObjectIdentifier(commandEndpoint)); attributes->Set("command_endpoint", commandEndpoint->GetName()); } TimePeriod::Ptr timePeriod = checkable->GetCheckPeriod(); if (timePeriod) { - checksums->Set("check_period_id", GetObjectIdentifier(timePeriod)); + attributes->Set("check_period_id", GetObjectIdentifier(timePeriod)); attributes->Set("check_period", timePeriod->GetName()); } EventCommand::Ptr eventCommand = checkable->GetEventCommand(); if (eventCommand) { - checksums->Set("eventcommand_id", GetObjectIdentifier(eventCommand)); + attributes->Set("eventcommand_id", GetObjectIdentifier(eventCommand)); attributes->Set("eventcommand", eventCommand->GetName()); } @@ -322,53 +638,29 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr String notesUrl = checkable->GetNotesUrl(); String iconImage = checkable->GetIconImage(); if (!actionUrl.IsEmpty()) - checksums->Set("action_url_id", CalculateCheckSumArray(new Array({GetEnvironment(), actionUrl}))); + attributes->Set("action_url_id", CalculateCheckSumArray(new Array({CalculateCheckSumString(GetEnvironment()), actionUrl}))); if (!notesUrl.IsEmpty()) - checksums->Set("notes_url_id", CalculateCheckSumArray(new Array({GetEnvironment(), notesUrl}))); + attributes->Set("notes_url_id", CalculateCheckSumArray(new Array({CalculateCheckSumString(GetEnvironment()), notesUrl}))); if (!iconImage.IsEmpty()) - checksums->Set("icon_image_id", CalculateCheckSumArray(new Array({GetEnvironment(), iconImage}))); + attributes->Set("icon_image_id", CalculateCheckSumArray(new Array({CalculateCheckSumString(GetEnvironment()), iconImage}))); Host::Ptr host; Service::Ptr service; tie(host, service) = GetHostService(checkable); - Array::Ptr groups; - ConfigObject::Ptr (*getGroup)(const String& name); - if (service) { - checksums->Set("host_id", GetObjectIdentifier(service->GetHost())); + attributes->Set("host_id", GetObjectIdentifier(service->GetHost())); attributes->Set("display_name", service->GetDisplayName()); // Overwrite name here, `object->name` is 'HostName!ServiceName' but we only want the name of the Service attributes->Set("name", service->GetShortName()); - - groups = service->GetGroups(); - getGroup = &::GetObjectByName; } else { attributes->Set("display_name", host->GetDisplayName()); attributes->Set("address", host->GetAddress()); attributes->Set("address6", host->GetAddress6()); - - groups = host->GetGroups(); - getGroup = &::GetObjectByName; } - checksums->Set("groups_checksum", CalculateCheckSumArray(groups)); - - Array::Ptr groupChecksums = new Array(); - - ObjectLock groupsLock(groups); - ObjectLock groupChecksumsLock(groupChecksums); - - for (auto group : groups) { - groupChecksums->Add(GetObjectIdentifier((*getGroup)(group.Get()))); - } - - checksums->Set("group_ids", groupChecksums); - - checksums->Set("properties_checksum", HashValue(attributes)); - return true; } @@ -382,29 +674,8 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr attributes->Set("states", user->GetStates()); attributes->Set("types", user->GetTypes()); - Array::Ptr groups; - ConfigObject::Ptr (*getGroup)(const String& name); - - groups = user->GetGroups(); - getGroup = &::GetObjectByName; - - checksums->Set("groups_checksum", CalculateCheckSumArray(groups)); - - Array::Ptr groupChecksums = new Array(); - - ObjectLock groupsLock(groups); - ObjectLock groupChecksumsLock(groupChecksums); - - for (auto group : groups) { - groupChecksums->Add(GetObjectIdentifier((*getGroup)(group.Get()))); - } - - checksums->Set("group_ids", groupChecksums); - if (user->GetPeriod()) - checksums->Set("period_id", GetObjectIdentifier(user->GetPeriod())); - - checksums->Set("properties_checksum", HashValue(attributes)); + attributes->Set("period_id", GetObjectIdentifier(user->GetPeriod())); return true; } @@ -414,52 +685,6 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr attributes->Set("display_name", timeperiod->GetDisplayName()); attributes->Set("prefer_includes", timeperiod->GetPreferIncludes()); - - checksums->Set("properties_checksum", HashValue(attributes)); - Dictionary::Ptr ranges = timeperiod->GetRanges(); - - attributes->Set("ranges", ranges); - checksums->Set("ranges_checksum", HashValue(ranges)); - - // Compute checksums for Includes (like groups) - Array::Ptr includes; - ConfigObject::Ptr (*getInclude)(const String& name); - includes = timeperiod->GetIncludes(); - getInclude = &::GetObjectByName; - - checksums->Set("includes_checksum", CalculateCheckSumArray(includes)); - - Array::Ptr includeChecksums = new Array(); - - ObjectLock includesLock(includes); - ObjectLock includeChecksumsLock(includeChecksums); - - for (auto include : includes) { - includeChecksums->Add(GetObjectIdentifier((*getInclude)(include.Get()))); - } - - checksums->Set("include_ids", includeChecksums); - - // Compute checksums for Excludes (like groups) - Array::Ptr excludes; - ConfigObject::Ptr (*getExclude)(const String& name); - - excludes = timeperiod->GetExcludes(); - getExclude = &::GetObjectByName; - - checksums->Set("excludes_checksum", CalculateCheckSumArray(excludes)); - - Array::Ptr excludeChecksums = new Array(); - - ObjectLock excludesLock(excludes); - ObjectLock excludeChecksumsLock(excludeChecksums); - - for (auto exclude : excludes) { - excludeChecksums->Add(GetObjectIdentifier((*getExclude)(exclude.Get()))); - } - - checksums->Set("exclude_ids", excludeChecksums); - return true; } @@ -468,53 +693,28 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr Host::Ptr host; Service::Ptr service; - std::set users = notification->GetUsers(); - Array::Ptr userChecksums = new Array(); - Array::Ptr userNames = new Array(); - auto usergroups(notification->GetUserGroups()); - Array::Ptr usergroupChecksums = new Array(); - Array::Ptr usergroupNames = new Array(); tie(host, service) = GetHostService(notification->GetCheckable()); - checksums->Set("properties_checksum", HashValue(attributes)); - checksums->Set("host_id", GetObjectIdentifier(host)); - checksums->Set("command_id", GetObjectIdentifier(notification->GetCommand())); + attributes->Set("host_id", GetObjectIdentifier(host)); + attributes->Set("command_id", GetObjectIdentifier(notification->GetCommand())); + + if (service) + attributes->Set("service_id", GetObjectIdentifier(service)); TimePeriod::Ptr timeperiod = notification->GetPeriod(); if (timeperiod) - checksums->Set("period_id", GetObjectIdentifier(timeperiod)); - - if (service) - checksums->Set("service_id", GetObjectIdentifier(service)); - - userChecksums->Reserve(users.size()); - userNames->Reserve(users.size()); - - for (auto& user : users) { - userChecksums->Add(GetObjectIdentifier(user)); - userNames->Add(user->GetName()); - } - - checksums->Set("user_ids", userChecksums); - checksums->Set("users_checksum", CalculateCheckSumArray(userNames)); - - usergroupChecksums->Reserve(usergroups.size()); - usergroupNames->Reserve(usergroups.size()); - - for (auto& usergroup : usergroups) { - usergroupChecksums->Add(GetObjectIdentifier(usergroup)); - usergroupNames->Add(usergroup->GetName()); - } - - checksums->Set("usergroup_ids", usergroupChecksums); - checksums->Set("usergroups_checksum", CalculateCheckSumArray(usergroupNames)); + attributes->Set("period_id", GetObjectIdentifier(timeperiod)); if (notification->GetTimes()) { attributes->Set("times_begin", notification->GetTimes()->Get("begin")); attributes->Set("times_end",notification->GetTimes()->Get("end")); } + attributes->Set("notification_interval", notification->GetInterval()); + attributes->Set("states", notification->GetStates()); + attributes->Set("types", notification->GetTypes()); + return true; } @@ -524,18 +724,17 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr attributes->Set("author", comment->GetAuthor()); attributes->Set("text", comment->GetText()); attributes->Set("entry_type", comment->GetEntryType()); + attributes->Set("entry_time", comment->GetEntryTime()); attributes->Set("is_persistent", comment->GetPersistent()); attributes->Set("expire_time", comment->GetExpireTime()); Host::Ptr host; Service::Ptr service; tie(host, service) = GetHostService(comment->GetCheckable()); - if (service) { - checksums->Set("service_id", GetObjectIdentifier(service)); - } else - checksums->Set("host_id", GetObjectIdentifier(host)); - - checksums->Set("properties_checksum", HashValue(attributes)); + if (service) + attributes->Set("service_id", GetObjectIdentifier(service)); + else + attributes->Set("host_id", GetObjectIdentifier(host)); return true; } @@ -559,11 +758,10 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr tie(host, service) = GetHostService(downtime->GetCheckable()); if (service) { - checksums->Set("service_id", GetObjectIdentifier(service)); + attributes->Set("service_id", GetObjectIdentifier(service)); } else - checksums->Set("host_id", GetObjectIdentifier(host)); + attributes->Set("host_id", GetObjectIdentifier(host)); - checksums->Set("properties_checksum", HashValue(attributes)); return true; } @@ -572,8 +770,6 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr attributes->Set("display_name", userGroup->GetDisplayName()); - checksums->Set("properties_checksum", HashValue(attributes)); - return true; } @@ -582,8 +778,6 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr attributes->Set("display_name", hostGroup->GetDisplayName()); - checksums->Set("properties_checksum", HashValue(attributes)); - return true; } @@ -592,58 +786,15 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr attributes->Set("display_name", serviceGroup->GetDisplayName()); - checksums->Set("properties_checksum", HashValue(attributes)); - return true; } if (type == CheckCommand::TypeInstance || type == NotificationCommand::TypeInstance || type == EventCommand::TypeInstance) { Command::Ptr command = static_pointer_cast(object); - if (dynamic_pointer_cast(object)) - attributes->Set("type", "CheckCommand"); - else if (dynamic_pointer_cast(object)) - attributes->Set("type", "EventCommand"); - else - attributes->Set("type", "NotificationCommand"); - attributes->Set("command", command->GetCommandLine()); attributes->Set("timeout", command->GetTimeout()); - checksums->Set("properties_checksum", HashValue(attributes)); - - Dictionary::Ptr arguments = command->GetArguments(); - Dictionary::Ptr argumentChecksums = new Dictionary; - - if (arguments) { - ObjectLock argumentsLock(arguments); - - for (auto& kv : arguments) { - argumentChecksums->Set(kv.first, HashValue(kv.second)); - } - - attributes->Set("arguments", arguments); - } - - checksums->Set("arguments_checksum", HashValue(arguments)); - checksums->Set("argument_ids", argumentChecksums); - - Dictionary::Ptr envvars = command->GetEnv(); - Dictionary::Ptr envvarChecksums = new Dictionary; - - if (envvars) { - ObjectLock argumentsLock(envvars); - - for (auto& kv : envvars) { - envvarChecksums->Set(kv.first, HashValue(kv.second)); - } - - attributes->Set("envvars", envvars); - } - - checksums->Set("envvars_checksum", HashValue(envvars)); - checksums->Set("envvar_ids", envvarChecksums); - return true; } @@ -656,8 +807,7 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr * icinga:config:object:downtime) need to be prepended. There is nothing to indicate success or failure. */ void -RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, const String typeName, std::vector& attributes, - std::vector& customVars, std::vector& checksums, +RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, const String typeName, std::map >& statements, bool runtimeUpdate) { /* TODO: This isn't essentially correct as we don't keep track of config objects ourselves. This would avoid duplicated config updates at startup. @@ -674,28 +824,15 @@ RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, const String ty if (!PrepareObject(object, attr, chksm)) return; + InsertObjectDependencies(object, typeName, statements); + String objectKey = GetObjectIdentifier(object); - attributes.emplace_back(objectKey); - attributes.emplace_back(JsonEncode(attr)); + statements[m_PrefixConfigObject + typeName].emplace_back(objectKey); + statements[m_PrefixConfigObject + typeName].emplace_back(JsonEncode(attr)); - CustomVarObject::Ptr customVarObject = dynamic_pointer_cast(object); - - if (customVarObject) { - chksm->Set("customvars_checksum", CalculateCheckSumVars(customVarObject)); - - auto vars(SerializeVars(customVarObject)); - - if (vars) { - auto varsJson(JsonEncode(vars)); - - customVars.emplace_back(objectKey); - customVars.emplace_back(varsJson); - } - } - - checksums.emplace_back(objectKey); - checksums.emplace_back(JsonEncode(chksm)); + statements[m_PrefixConfigCheckSum + typeName].emplace_back(objectKey);; + statements[m_PrefixConfigCheckSum + typeName].emplace_back(JsonEncode(new Dictionary({{"checksum", HashValue(attr)}}))); /* Send an update event to subscribers. */ if (runtimeUpdate) { @@ -717,6 +854,9 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) { + if (!m_Rcon || !m_Rcon->IsConnected()) + return; + Checkable::Ptr checkable = dynamic_pointer_cast(object); if (!checkable) return; diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 1ef203450..db136dcf9 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -277,32 +277,13 @@ Dictionary::Ptr RedisWriter::SerializeVars(const CustomVarObject::Ptr& object) ObjectLock olock(vars); for (auto& kv : vars) { - Dictionary::Ptr flatVars = new Dictionary(); - - { - auto it (scalarVars.find(kv.first)); - if (it != scalarVars.end()) { - for (auto& scalarVar : it->second) { - String strVal = Convert::ToString(scalarVar.second); - if (scalarVar.second.GetType() == ValueEmpty) - strVal = "NULL"; - - flatVars->Set(SHA1(PackObject(scalarVar.first)), (Dictionary::Ptr)new Dictionary({ - {"name", scalarVar.first}, - {"value", strVal} - })); - } - } - } - res->Set( SHA1(PackObject((Array::Ptr)new Array({env, kv.first, kv.second}))), (Dictionary::Ptr)new Dictionary({ - {"env_checksum", envChecksum}, + {"env_id", envChecksum}, {"name_checksum", SHA1(kv.first)}, {"name", kv.first}, {"value", kv.second}, - {"flat", flatVars} }) ); } diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index 373f6085f..ab03e804a 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -43,9 +43,8 @@ RedisWriter::RedisWriter() m_WorkQueue.SetName("RedisWriter"); - m_PrefixConfigObject = "icinga:config:object:"; - m_PrefixConfigCheckSum = "icinga:config:checksum:"; - m_PrefixConfigCustomVar = "icinga:config:customvar:"; + m_PrefixConfigObject = "icinga:config:"; + m_PrefixConfigCheckSum = "icinga:checksum:"; m_PrefixStateObject = "icinga:state:object:"; } diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index aa5cfe007..5c3123446 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -70,10 +70,14 @@ private: /* config & status dump */ void UpdateAllConfigObjects(); + void DeleteKeys(const std::vector& keys); + std::map > GenerateHmsetStatements(const std::vector& keys); + std::vector GetTypeObjectKeys(const String& type); + void InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map >& statements); void UpdateState(const Checkable::Ptr& checkable); void SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate); - void CreateConfigUpdate(const ConfigObject::Ptr& object, const String type, std::vector& attributes, - std::vector& customVars, std::vector& checksums, bool runtimeUpdate); + void CreateConfigUpdate(const ConfigObject::Ptr& object, const String type, std::map >& statements, + bool runtimeUpdate); void SendConfigDelete(const ConfigObject::Ptr& object); void SendStatusUpdate(const ConfigObject::Ptr& object); std::vector UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride); @@ -123,7 +127,6 @@ private: String m_PrefixConfigObject; String m_PrefixConfigCheckSum; - String m_PrefixConfigCustomVar; String m_PrefixStateObject; bool m_ConfigDumpInProgress; From 13295acb721f2acdc0b725cff64258815ac44426 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 19 Jun 2019 15:40:20 +0200 Subject: [PATCH 134/219] RedisWriter: Write objects of same type in parallel --- lib/redis/rediswriter-objects.cpp | 129 +++++++++++++++++++----------- lib/redis/rediswriter.hpp | 3 +- 2 files changed, 86 insertions(+), 46 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index a0d5a3626..9ff4c95a1 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -127,62 +127,80 @@ void RedisWriter::UpdateAllConfigObjects() keys.reserve(globalKeys.size()); keys.insert(keys.end(), globalKeys.begin(), globalKeys.end()); - std::map > statements = GenerateHmsetStatements(keys); - std::vector states = {"HMSET", m_PrefixStateObject + lcType}; - std::vector > transaction = {{"MULTI"}}; + std::vector objectChunks = ChunkObjects(type.first->GetObjects(), 500); - bool dumpState = (lcType == "host" || lcType == "service"); + WorkQueue upqObjectType(25000, Configuration::Concurrency); + upqObjectType.SetName("RedisWriter:ConfigDump:" + lcType); - size_t bulkCounter = 0; - for (const ConfigObject::Ptr& object : type.first->GetObjects()) { - if (lcType != GetLowerCaseTypeNameDB(object)) - continue; + upqObjectType.ParallelFor(objectChunks, [this, &type, &lcType, &keys](const Array::Ptr chunk) { + ObjectLock chunkLock(chunk); + std::map > statements = GenerateHmsetStatements(keys); + std::vector states = {"HMSET", m_PrefixStateObject + lcType}; + std::vector > transaction = {{"MULTI"}}; - CreateConfigUpdate(object, lcType, statements, false); + bool dumpState = (lcType == "host" || lcType == "service"); - // Write out inital state for checkables - if (dumpState) { - states.emplace_back(GetObjectIdentifier(object)); - states.emplace_back(JsonEncode(SerializeState(dynamic_pointer_cast(object)))); - } + size_t bulkCounter = 0; + for (const ConfigObject::Ptr& object : chunk) { + if (lcType != GetLowerCaseTypeNameDB(object)) + continue; - bulkCounter++; - if (!bulkCounter % 100) { - for (const auto& kv : statements) - if (kv.second.size() > 2) - transaction.push_back(kv.second); + CreateConfigUpdate(object, lcType, statements, false); - if (states.size() > 2) { - transaction.push_back(std::move(states)); - states = {"HMSET", m_PrefixStateObject + lcType}; + // Write out inital state for checkables + if (dumpState) { + states.emplace_back(GetObjectIdentifier(object)); + states.emplace_back(JsonEncode(SerializeState(dynamic_pointer_cast(object)))); } - statements = GenerateHmsetStatements(keys); + bulkCounter++; + if (!bulkCounter % 100) { + for (const auto& kv : statements) + if (kv.second.size() > 2) + transaction.push_back(kv.second); - if (transaction.size() > 1) { - transaction.push_back({"EXEC"}); - m_Rcon->ExecuteQueries(transaction); - transaction = {{"MULTI"}}; + if (states.size() > 2) { + transaction.push_back(std::move(states)); + states = {"HMSET", m_PrefixStateObject + lcType}; + } + + statements = GenerateHmsetStatements(keys); + + if (transaction.size() > 1) { + transaction.push_back({"EXEC"}); + m_Rcon->ExecuteQueries(transaction); + transaction = {{"MULTI"}}; + } + } + } + + for (const auto& kv : statements) + if (kv.second.size() > 2) + transaction.push_back(kv.second); + + if (states.size() > 2) + transaction.push_back(std::move(states)); + + if (transaction.size() > 1) { + transaction.push_back({"EXEC"}); + m_Rcon->ExecuteQueries(transaction); + } + + m_Rcon->ExecuteQuery({"PUBLISH", "icinga:config:dump", lcType}); + + Log(LogNotice, "RedisWriter") + << "Dumped " << bulkCounter << " objects of type " << type.second; + }); + + upqObjectType.Join(); + + if (upqObjectType.HasExceptions()) { + for (boost::exception_ptr exc : upqObjectType.GetExceptions()) { + if (exc) { + boost::rethrow_exception(exc); } } } - - for (const auto& kv : statements) - if (kv.second.size() > 2) - transaction.push_back(kv.second); - - if (states.size() > 2) - transaction.push_back(std::move(states)); - - if (transaction.size() > 1) { - transaction.push_back({"EXEC"}); - m_Rcon->ExecuteQueries(transaction); - } - - m_Rcon->ExecuteQuery({"PUBLISH", "icinga:config:dump", lcType}); - - Log(LogNotice, "RedisWriter") - << "Dumped " << bulkCounter << " objects of type " << type.second; }); upq.Join(); @@ -204,6 +222,27 @@ void RedisWriter::UpdateAllConfigObjects() << "Initial config/status dump finished in " << Utility::GetTime() - startTime << " seconds."; } +std::vector RedisWriter::ChunkObjects(std::vector > objects, size_t chunkSize) { + std::vector chunks; + Array::Ptr currentChunk(new Array); + size_t currentChunkSize = 0; + for (auto object : objects) { + currentChunk->Add(object); + currentChunkSize++; + if (currentChunkSize >= chunkSize) { + chunks.push_back(currentChunk); + currentChunk = new Array(); + currentChunkSize = 0; + } + } + + if (currentChunkSize > 0) { + chunks.push_back(currentChunk); + } + + return chunks; +} + void RedisWriter::DeleteKeys(const std::vector& keys) { std::vector query = {"DEL"}; for (auto& key : keys) { @@ -213,7 +252,7 @@ void RedisWriter::DeleteKeys(const std::vector& keys) { m_Rcon->ExecuteQuery(query); } -std::map > RedisWriter::GenerateHmsetStatements(const std::vector& keys) +std::map > RedisWriter::GenerateHmsetStatements(const std::vector keys) { std::map > statements; for (auto& key : keys) { diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 5c3123446..56ed9ce33 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -70,8 +70,9 @@ private: /* config & status dump */ void UpdateAllConfigObjects(); + std::vector ChunkObjects(std::vector > objects, size_t chunkSize); void DeleteKeys(const std::vector& keys); - std::map > GenerateHmsetStatements(const std::vector& keys); + std::map > GenerateHmsetStatements(const std::vector keys); std::vector GetTypeObjectKeys(const String& type); void InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map >& statements); void UpdateState(const Checkable::Ptr& checkable); From 7d0fc91c1bbc26c07ee7e4925057d45b858168a4 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 27 Jun 2019 15:00:59 +0200 Subject: [PATCH 135/219] RedisWriter#GetTypeObjectKeys(): move() the result --- lib/redis/rediswriter-objects.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 9ff4c95a1..2960ecdeb 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -45,6 +45,7 @@ #include "base/exception.hpp" #include #include +#include using namespace icinga; @@ -297,7 +298,7 @@ std::vector RedisWriter::GetTypeObjectKeys(const String& type) keys.emplace_back(m_PrefixConfigCheckSum + type + ":argument"); } - return keys; + return std::move(keys); } template From 0534141aaf1084539c3aecf3139438a219db0352 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 27 Jun 2019 15:03:14 +0200 Subject: [PATCH 136/219] Remove redundand vector#reserve() --- lib/redis/rediswriter-objects.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 2960ecdeb..a8ce587d8 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -125,7 +125,6 @@ void RedisWriter::UpdateAllConfigObjects() std::vector keys = GetTypeObjectKeys(lcType); DeleteKeys(keys); - keys.reserve(globalKeys.size()); keys.insert(keys.end(), globalKeys.begin(), globalKeys.end()); std::vector objectChunks = ChunkObjects(type.first->GetObjects(), 500); From 23448b0322604e9584e75e1420599cb8984a8182 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 27 Jun 2019 15:53:55 +0200 Subject: [PATCH 137/219] Make RedisWriter#ChunkObjects()'s result type more explicit --- lib/redis/rediswriter-objects.cpp | 35 +++++++++++++++---------------- lib/redis/rediswriter.hpp | 2 +- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index a8ce587d8..27d237940 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -43,6 +43,7 @@ #include "base/convert.hpp" #include "base/array.hpp" #include "base/exception.hpp" +#include #include #include #include @@ -127,13 +128,12 @@ void RedisWriter::UpdateAllConfigObjects() keys.insert(keys.end(), globalKeys.begin(), globalKeys.end()); - std::vector objectChunks = ChunkObjects(type.first->GetObjects(), 500); + auto objectChunks (ChunkObjects(type.first->GetObjects(), 500)); WorkQueue upqObjectType(25000, Configuration::Concurrency); upqObjectType.SetName("RedisWriter:ConfigDump:" + lcType); - upqObjectType.ParallelFor(objectChunks, [this, &type, &lcType, &keys](const Array::Ptr chunk) { - ObjectLock chunkLock(chunk); + upqObjectType.ParallelFor(objectChunks, [this, &type, &lcType, &keys](decltype(objectChunks)::const_reference chunk) { std::map > statements = GenerateHmsetStatements(keys); std::vector states = {"HMSET", m_PrefixStateObject + lcType}; std::vector > transaction = {{"MULTI"}}; @@ -222,25 +222,24 @@ void RedisWriter::UpdateAllConfigObjects() << "Initial config/status dump finished in " << Utility::GetTime() - startTime << " seconds."; } -std::vector RedisWriter::ChunkObjects(std::vector > objects, size_t chunkSize) { - std::vector chunks; - Array::Ptr currentChunk(new Array); - size_t currentChunkSize = 0; - for (auto object : objects) { - currentChunk->Add(object); - currentChunkSize++; - if (currentChunkSize >= chunkSize) { - chunks.push_back(currentChunk); - currentChunk = new Array(); - currentChunkSize = 0; - } +std::vector>> RedisWriter::ChunkObjects(std::vector> objects, size_t chunkSize) { + std::vector>> chunks; + auto offset (objects.begin()); + auto end (objects.end()); + + chunks.reserve((std::distance(offset, end) + chunkSize - 1) / chunkSize); + + while (std::distance(offset, end) >= chunkSize) { + auto until (offset + chunkSize); + chunks.emplace_back(offset, until); + offset = until; } - if (currentChunkSize > 0) { - chunks.push_back(currentChunk); + if (offset != end) { + chunks.emplace_back(offset, end); } - return chunks; + return std::move(chunks); } void RedisWriter::DeleteKeys(const std::vector& keys) { diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 56ed9ce33..26d7fb647 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -70,7 +70,7 @@ private: /* config & status dump */ void UpdateAllConfigObjects(); - std::vector ChunkObjects(std::vector > objects, size_t chunkSize); + std::vector>> ChunkObjects(std::vector> objects, size_t chunkSize); void DeleteKeys(const std::vector& keys); std::map > GenerateHmsetStatements(const std::vector keys); std::vector GetTypeObjectKeys(const String& type); From c24092a64e4c924d6b342fd5bbd360378cda75d7 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 27 Jun 2019 16:11:51 +0200 Subject: [PATCH 138/219] RedisWriter#GenerateHmsetStatements(): reduce memory allocations --- lib/redis/rediswriter-objects.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 27d237940..6a0bc5dc3 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -255,11 +255,10 @@ std::map > RedisWriter::GenerateHmsetStatements(cons { std::map > statements; for (auto& key : keys) { - std::vector statement = {"HMSET", key}; - statements.insert(std::pair >(key, statement)); + statements.emplace(key, std::vector({"HMSET", key})); } - return statements; + return std::move(statements); } std::vector RedisWriter::GetTypeObjectKeys(const String& type) From def3c7df325ceedb4178bd085953cc4bd69800d6 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 27 Jun 2019 16:27:19 +0200 Subject: [PATCH 139/219] RedisWriter#CreateConfigUpdate(): reduce memory allocations --- lib/redis/rediswriter-objects.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 6a0bc5dc3..fadda0776 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -864,12 +864,14 @@ RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, const String ty InsertObjectDependencies(object, typeName, statements); String objectKey = GetObjectIdentifier(object); + auto& attrs (statements.at(m_PrefixConfigObject + typeName)); + auto& chksms (statements.at(m_PrefixConfigCheckSum + typeName)); - statements[m_PrefixConfigObject + typeName].emplace_back(objectKey); - statements[m_PrefixConfigObject + typeName].emplace_back(JsonEncode(attr)); + attrs.emplace_back(objectKey); + attrs.emplace_back(JsonEncode(attr)); - statements[m_PrefixConfigCheckSum + typeName].emplace_back(objectKey);; - statements[m_PrefixConfigCheckSum + typeName].emplace_back(JsonEncode(new Dictionary({{"checksum", HashValue(attr)}}))); + chksms.emplace_back(objectKey); + chksms.emplace_back(JsonEncode(new Dictionary({{"checksum", HashValue(attr)}}))); /* Send an update event to subscribers. */ if (runtimeUpdate) { From 1d126b66e92bdffdc9250713baeb57e98701cb92 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 27 Jun 2019 17:28:00 +0200 Subject: [PATCH 140/219] RedisWriter#InsertObjectDependencies(): reduce memory allocations --- lib/redis/rediswriter-objects.cpp | 180 ++++++++++++++++++++---------- 1 file changed, 121 insertions(+), 59 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index fadda0776..628a58ce8 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -313,19 +313,26 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons if (customVarObject) { auto vars(SerializeVars(customVarObject)); if (vars) { - statements[m_PrefixConfigCheckSum + typeName + ":customvar"].emplace_back(objectKey); - statements[m_PrefixConfigCheckSum + typeName + ":customvar"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumVars(customVarObject)}}))); + auto& typeCvs (statements.at(m_PrefixConfigObject + typeName + ":customvar")); + auto& allCvs (statements.at(m_PrefixConfigObject + "customvar")); + auto& cvChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":customvar")); + + cvChksms.emplace_back(objectKey); + cvChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumVars(customVarObject)}}))); ObjectLock varsLock(vars); Array::Ptr varsArray(new Array); + + varsArray->Reserve(vars->GetLength()); + for (auto& kv : vars) { - statements[m_PrefixConfigObject + "customvar"].emplace_back(kv.first); - statements[m_PrefixConfigObject + "customvar"].emplace_back(JsonEncode(kv.second)); + allCvs.emplace_back(kv.first); + allCvs.emplace_back(JsonEncode(kv.second)); varsArray->Add(kv.first); } - statements[m_PrefixConfigObject + typeName + ":customvar"].emplace_back(objectKey); - statements[m_PrefixConfigObject + typeName + ":customvar"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"customvars", varsArray}}))); + typeCvs.emplace_back(objectKey); + typeCvs.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"customvars", varsArray}}))); } } @@ -337,16 +344,19 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons String notesUrl = checkable->GetNotesUrl(); String iconImage = checkable->GetIconImage(); if (!actionUrl.IsEmpty()) { - statements[m_PrefixConfigObject + "action_url"].emplace_back(CalculateCheckSumArray(new Array({envId, actionUrl}))); - statements[m_PrefixConfigObject + "action_url"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"action_url", actionUrl}}))); + auto& actionUrls (statements.at(m_PrefixConfigObject + "action_url")); + actionUrls.emplace_back(CalculateCheckSumArray(new Array({envId, actionUrl}))); + actionUrls.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"action_url", actionUrl}}))); } if (!notesUrl.IsEmpty()) { - statements[m_PrefixConfigObject + "notes_url"].emplace_back(CalculateCheckSumArray(new Array({envId, notesUrl}))); - statements[m_PrefixConfigObject + "notes_url"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"notes_url", notesUrl}}))); + auto& notesUrls (statements.at(m_PrefixConfigObject + "notes_url")); + notesUrls.emplace_back(CalculateCheckSumArray(new Array({envId, notesUrl}))); + notesUrls.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"notes_url", notesUrl}}))); } if (!iconImage.IsEmpty()) { - statements[m_PrefixConfigObject + "icon_image"].emplace_back(CalculateCheckSumArray(new Array({envId, iconImage}))); - statements[m_PrefixConfigObject + "icon_image"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"icon_image", iconImage}}))); + auto& iconImages (statements.at(m_PrefixConfigObject + "icon_image")); + iconImages.emplace_back(CalculateCheckSumArray(new Array({envId, iconImage}))); + iconImages.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"icon_image", iconImage}}))); } Host::Ptr host; @@ -366,14 +376,20 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons if (groups) { ObjectLock groupsLock(groups); Array::Ptr groupIds(new Array); + + groupIds->Reserve(groups->GetLength()); + for (auto& group : groups) { groupIds->Add(GetObjectIdentifier((*getGroup)(group))); } - statements[m_PrefixConfigCheckSum + typeName + ":groupmember"].emplace_back(objectKey); - statements[m_PrefixConfigCheckSum + typeName + ":groupmember"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(groupIds)}}))); - statements[m_PrefixConfigObject + typeName + ":groupmember"].emplace_back(objectKey); - statements[m_PrefixConfigObject + typeName + ":groupmember"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"groups", groupIds}}))); + auto& members (statements.at(m_PrefixConfigObject + typeName + ":groupmember")); + auto& memberChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":groupmember")); + + memberChksms.emplace_back(objectKey); + memberChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(groupIds)}}))); + members.emplace_back(objectKey); + members.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"groups", groupIds}}))); } return; @@ -386,18 +402,24 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons if (ranges) { ObjectLock rangesLock(ranges); Array::Ptr rangeIds(new Array); + auto& typeRanges (statements.at(m_PrefixConfigObject + typeName + ":range")); + auto& allRanges (statements.at(m_PrefixConfigObject + "timerange")); + auto& rangeChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":range")); + + rangeIds->Reserve(ranges->GetLength()); + for (auto& kv : ranges) { String id = CalculateCheckSumArray(new Array({envId, kv.first, kv.second})); rangeIds->Add(id); - statements[m_PrefixConfigObject + "timerange"].emplace_back(id); - statements[m_PrefixConfigObject + "timerange"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"range_key", kv.first}, {"range_value", kv.second}}))); + allRanges.emplace_back(id); + allRanges.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"range_key", kv.first}, {"range_value", kv.second}}))); } - statements[m_PrefixConfigCheckSum + typeName + ":range"].emplace_back(objectKey); - statements[m_PrefixConfigCheckSum + typeName + ":range"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(rangeIds)}}))); - statements[m_PrefixConfigObject + typeName + ":range"].emplace_back(objectKey); - statements[m_PrefixConfigObject + typeName + ":range"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"ranges", rangeIds}}))); + rangeChksms.emplace_back(objectKey); + rangeChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(rangeIds)}}))); + typeRanges.emplace_back(objectKey); + typeRanges.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"ranges", rangeIds}}))); } Array::Ptr includes; @@ -410,14 +432,19 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons ObjectLock includesLock(includes); ObjectLock includeChecksumsLock(includeChecksums); + includeChecksums->Reserve(includes->GetLength()); + for (auto include : includes) { includeChecksums->Add(GetObjectIdentifier((*getInclude)(include.Get()))); } - statements[m_PrefixConfigCheckSum + typeName + ":overwrite:include"].emplace_back(objectKey); - statements[m_PrefixConfigCheckSum + typeName + ":overwrite:include"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(includes)}}))); - statements[m_PrefixConfigObject + typeName + ":overwrite:include"].emplace_back(objectKey); - statements[m_PrefixConfigObject + typeName + ":overwrite:include"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"includes", includeChecksums}}))); + auto& includs (statements.at(m_PrefixConfigObject + typeName + ":overwrite:include")); + auto& includeChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":overwrite:include")); + + includeChksms.emplace_back(objectKey); + includeChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(includes)}}))); + includs.emplace_back(objectKey); + includs.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"includes", includeChecksums}}))); Array::Ptr excludes; ConfigObject::Ptr (*getExclude)(const String& name); @@ -430,14 +457,19 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons ObjectLock excludesLock(excludes); ObjectLock excludeChecksumsLock(excludeChecksums); + excludeChecksums->Reserve(excludes->GetLength()); + for (auto exclude : excludes) { excludeChecksums->Add(GetObjectIdentifier((*getExclude)(exclude.Get()))); } - statements[m_PrefixConfigCheckSum + typeName + ":overwrite:exclude"].emplace_back(objectKey); - statements[m_PrefixConfigCheckSum + typeName + ":overwrite:exclude"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(excludes)}}))); - statements[m_PrefixConfigObject + typeName + ":overwrite:exclude"].emplace_back(objectKey); - statements[m_PrefixConfigObject + typeName + ":overwrite:exclude"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"excludes", excludeChecksums}}))); + auto& excluds (statements.at(m_PrefixConfigObject + typeName + ":overwrite:exclude")); + auto& excludeChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":overwrite:exclude")); + + excludeChksms.emplace_back(objectKey); + excludeChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(excludes)}}))); + excluds.emplace_back(objectKey); + excluds.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"excludes", excludeChecksums}}))); return; } @@ -446,15 +478,21 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons Zone::Ptr zone = static_pointer_cast(object); Array::Ptr parents(new Array); + auto parentsRaw (zone->GetAllParentsRaw()); - for (auto& parent : zone->GetAllParentsRaw()) { + parents->Reserve(parentsRaw.size()); + + for (auto& parent : parentsRaw) { parents->Add(GetObjectIdentifier(parent)); } - statements[m_PrefixConfigCheckSum + typeName + ":parent"].emplace_back(objectKey); - statements[m_PrefixConfigCheckSum + typeName + ":parent"].emplace_back(JsonEncode(new Dictionary({{"checksum", HashValue(zone->GetAllParents())}}))); - statements[m_PrefixConfigObject + typeName + ":parent"].emplace_back(objectKey); - statements[m_PrefixConfigObject + typeName + ":parent"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"parents", parents}}))); + auto& parnts (statements.at(m_PrefixConfigObject + typeName + ":parent")); + auto& parentChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":parent")); + + parentChksms.emplace_back(objectKey); + parentChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", HashValue(zone->GetAllParents())}}))); + parnts.emplace_back(objectKey); + parnts.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"parents", parents}}))); return; } @@ -471,14 +509,20 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons if (groups) { ObjectLock groupsLock(groups); Array::Ptr groupIds(new Array); + + groupIds->Reserve(groups->GetLength()); + for (auto& group : groups) { groupIds->Add(GetObjectIdentifier((*getGroup)(group))); } - statements[m_PrefixConfigCheckSum + typeName + ":groupmember"].emplace_back(objectKey); - statements[m_PrefixConfigCheckSum + typeName + ":groupmember"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(groupIds)}}))); - statements[m_PrefixConfigObject + typeName + ":groupmember"].emplace_back(objectKey); - statements[m_PrefixConfigObject + typeName + ":groupmember"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"groups", groupIds}}))); + auto& members (statements.at(m_PrefixConfigObject + typeName + ":groupmember")); + auto& memberChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":groupmember")); + + memberChksms.emplace_back(objectKey); + memberChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(groupIds)}}))); + members.emplace_back(objectKey); + members.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"groups", groupIds}}))); } return; @@ -499,10 +543,13 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons userIds->Add(GetObjectIdentifier(user)); } - statements[m_PrefixConfigCheckSum + typeName + ":user"].emplace_back(objectKey); - statements[m_PrefixConfigCheckSum + typeName + ":user"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(userIds)}}))); - statements[m_PrefixConfigObject + typeName + ":user"].emplace_back(objectKey); - statements[m_PrefixConfigObject + typeName + ":user"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"users", userIds}}))); + auto& usrs (statements.at(m_PrefixConfigObject + typeName + ":user")); + auto& userChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":user")); + + userChksms.emplace_back(objectKey); + userChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(userIds)}}))); + usrs.emplace_back(objectKey); + usrs.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"users", userIds}}))); usergroupIds->Reserve(usergroups.size()); @@ -510,10 +557,13 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons usergroupIds->Add(GetObjectIdentifier(usergroup)); } - statements[m_PrefixConfigCheckSum + typeName + ":usergroup"].emplace_back(objectKey); - statements[m_PrefixConfigCheckSum + typeName + ":usergroup"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(usergroupIds)}}))); - statements[m_PrefixConfigObject + typeName + ":usergroup"].emplace_back(objectKey); - statements[m_PrefixConfigObject + typeName + ":usergroup"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"usergroups", usergroupIds}}))); + auto& groups (statements.at(m_PrefixConfigObject + typeName + ":usergroup")); + auto& groupChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":usergroup")); + + groupChksms.emplace_back(objectKey); + groupChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(usergroupIds)}}))); + groups.emplace_back(objectKey); + groups.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"usergroups", usergroupIds}}))); return; } @@ -525,36 +575,48 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons if (arguments) { ObjectLock argumentsLock(arguments); Array::Ptr argumentIds(new Array); + auto& typeArgs (statements.at(m_PrefixConfigObject + typeName + ":argument")); + auto& allArgs (statements.at(m_PrefixConfigObject + "commandargument")); + auto& argChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":argument")); + + argumentIds->Reserve(arguments->GetLength()); + for (auto& kv : arguments) { String id = HashValue(kv.first + HashValue(kv.second)); argumentIds->Add(id); - statements[m_PrefixConfigObject + "commandargument"].emplace_back(id); - statements[m_PrefixConfigObject + "commandargument"].emplace_back(JsonEncode(kv.second)); + allArgs.emplace_back(id); + allArgs.emplace_back(JsonEncode(kv.second)); } - statements[m_PrefixConfigCheckSum + typeName + ":argument"].emplace_back(objectKey); - statements[m_PrefixConfigCheckSum + typeName + ":argument"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(argumentIds)}}))); - statements[m_PrefixConfigObject + typeName + ":argument"].emplace_back(objectKey); - statements[m_PrefixConfigObject + typeName + ":argument"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"arguments", argumentIds}}))); + argChksms.emplace_back(objectKey); + argChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(argumentIds)}}))); + typeArgs.emplace_back(objectKey); + typeArgs.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"arguments", argumentIds}}))); } Dictionary::Ptr envvars = command->GetArguments(); if (envvars) { ObjectLock envvarsLock(envvars); Array::Ptr envvarIds(new Array); + auto& typeVars (statements.at(m_PrefixConfigObject + typeName + ":envvar")); + auto& allVars (statements.at(m_PrefixConfigObject + "commandenvvar")); + auto& varChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":envvar")); + + envvarIds->Reserve(envvars->GetLength()); + for (auto& kv : envvars) { String id = HashValue(kv.first + HashValue(kv.second)); envvarIds->Add(id); - statements[m_PrefixConfigObject + "commandenvvar"].emplace_back(id); - statements[m_PrefixConfigObject + "commandenvvar"].emplace_back(JsonEncode(kv.second)); + allVars.emplace_back(id); + allVars.emplace_back(JsonEncode(kv.second)); } - statements[m_PrefixConfigCheckSum + typeName + ":envvar"].emplace_back(objectKey); - statements[m_PrefixConfigCheckSum + typeName + ":envvar"].emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(envvarIds)}}))); - statements[m_PrefixConfigObject + typeName + ":envvar"].emplace_back(objectKey); - statements[m_PrefixConfigObject + typeName + ":envvar"].emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"envvars", envvarIds}}))); + varChksms.emplace_back(objectKey); + varChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(envvarIds)}}))); + typeVars.emplace_back(objectKey); + typeVars.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"envvars", envvarIds}}))); } return; From 07823c4b905164ab95852ef5ab2b15e2fba21fdb Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 28 Jun 2019 09:51:13 +0200 Subject: [PATCH 141/219] RedisWriter#CreateConfigUpdate(): don't require prepared commands --- lib/redis/rediswriter-objects.cpp | 113 +++++++++++++++--------------- lib/redis/rediswriter.hpp | 1 - 2 files changed, 55 insertions(+), 59 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 628a58ce8..8199198be 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -120,21 +120,19 @@ void RedisWriter::UpdateAllConfigObjects() }; DeleteKeys(globalKeys); - upq.ParallelFor(types, [this, &globalKeys](const TypePair& type) { + upq.ParallelFor(types, [this](const TypePair& type) { String lcType = type.second; std::vector keys = GetTypeObjectKeys(lcType); DeleteKeys(keys); - keys.insert(keys.end(), globalKeys.begin(), globalKeys.end()); - auto objectChunks (ChunkObjects(type.first->GetObjects(), 500)); WorkQueue upqObjectType(25000, Configuration::Concurrency); upqObjectType.SetName("RedisWriter:ConfigDump:" + lcType); - upqObjectType.ParallelFor(objectChunks, [this, &type, &lcType, &keys](decltype(objectChunks)::const_reference chunk) { - std::map > statements = GenerateHmsetStatements(keys); + upqObjectType.ParallelFor(objectChunks, [this, &type, &lcType](decltype(objectChunks)::const_reference chunk) { + std::map> statements; std::vector states = {"HMSET", m_PrefixStateObject + lcType}; std::vector > transaction = {{"MULTI"}}; @@ -155,16 +153,19 @@ void RedisWriter::UpdateAllConfigObjects() bulkCounter++; if (!bulkCounter % 100) { - for (const auto& kv : statements) - if (kv.second.size() > 2) - transaction.push_back(kv.second); + for (auto& kv : statements) { + if (!kv.second.empty()) { + kv.second.insert(kv.second.begin(), {"HMSET", kv.first}); + transaction.emplace_back(std::move(kv.second)); + } + } if (states.size() > 2) { transaction.push_back(std::move(states)); states = {"HMSET", m_PrefixStateObject + lcType}; } - statements = GenerateHmsetStatements(keys); + statements = decltype(statements)(); if (transaction.size() > 1) { transaction.push_back({"EXEC"}); @@ -174,9 +175,12 @@ void RedisWriter::UpdateAllConfigObjects() } } - for (const auto& kv : statements) - if (kv.second.size() > 2) - transaction.push_back(kv.second); + for (auto& kv : statements) { + if (!kv.second.empty()) { + kv.second.insert(kv.second.begin(), {"HMSET", kv.first}); + transaction.emplace_back(std::move(kv.second)); + } + } if (states.size() > 2) transaction.push_back(std::move(states)); @@ -251,16 +255,6 @@ void RedisWriter::DeleteKeys(const std::vector& keys) { m_Rcon->ExecuteQuery(query); } -std::map > RedisWriter::GenerateHmsetStatements(const std::vector keys) -{ - std::map > statements; - for (auto& key : keys) { - statements.emplace(key, std::vector({"HMSET", key})); - } - - return std::move(statements); -} - std::vector RedisWriter::GetTypeObjectKeys(const String& type) { std::vector keys = { @@ -313,9 +307,9 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons if (customVarObject) { auto vars(SerializeVars(customVarObject)); if (vars) { - auto& typeCvs (statements.at(m_PrefixConfigObject + typeName + ":customvar")); - auto& allCvs (statements.at(m_PrefixConfigObject + "customvar")); - auto& cvChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":customvar")); + auto& typeCvs (statements[m_PrefixConfigObject + typeName + ":customvar"]); + auto& allCvs (statements[m_PrefixConfigObject + "customvar"]); + auto& cvChksms (statements[m_PrefixConfigCheckSum + typeName + ":customvar"]); cvChksms.emplace_back(objectKey); cvChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumVars(customVarObject)}}))); @@ -344,17 +338,17 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons String notesUrl = checkable->GetNotesUrl(); String iconImage = checkable->GetIconImage(); if (!actionUrl.IsEmpty()) { - auto& actionUrls (statements.at(m_PrefixConfigObject + "action_url")); + auto& actionUrls (statements[m_PrefixConfigObject + "action_url"]); actionUrls.emplace_back(CalculateCheckSumArray(new Array({envId, actionUrl}))); actionUrls.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"action_url", actionUrl}}))); } if (!notesUrl.IsEmpty()) { - auto& notesUrls (statements.at(m_PrefixConfigObject + "notes_url")); + auto& notesUrls (statements[m_PrefixConfigObject + "notes_url"]); notesUrls.emplace_back(CalculateCheckSumArray(new Array({envId, notesUrl}))); notesUrls.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"notes_url", notesUrl}}))); } if (!iconImage.IsEmpty()) { - auto& iconImages (statements.at(m_PrefixConfigObject + "icon_image")); + auto& iconImages (statements[m_PrefixConfigObject + "icon_image"]); iconImages.emplace_back(CalculateCheckSumArray(new Array({envId, iconImage}))); iconImages.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"icon_image", iconImage}}))); } @@ -383,8 +377,8 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons groupIds->Add(GetObjectIdentifier((*getGroup)(group))); } - auto& members (statements.at(m_PrefixConfigObject + typeName + ":groupmember")); - auto& memberChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":groupmember")); + auto& members (statements[m_PrefixConfigObject + typeName + ":groupmember"]); + auto& memberChksms (statements[m_PrefixConfigCheckSum + typeName + ":groupmember"]); memberChksms.emplace_back(objectKey); memberChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(groupIds)}}))); @@ -402,9 +396,9 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons if (ranges) { ObjectLock rangesLock(ranges); Array::Ptr rangeIds(new Array); - auto& typeRanges (statements.at(m_PrefixConfigObject + typeName + ":range")); - auto& allRanges (statements.at(m_PrefixConfigObject + "timerange")); - auto& rangeChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":range")); + auto& typeRanges (statements[m_PrefixConfigObject + typeName + ":range"]); + auto& allRanges (statements[m_PrefixConfigObject + "timerange"]); + auto& rangeChksms (statements[m_PrefixConfigCheckSum + typeName + ":range"]); rangeIds->Reserve(ranges->GetLength()); @@ -438,8 +432,8 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons includeChecksums->Add(GetObjectIdentifier((*getInclude)(include.Get()))); } - auto& includs (statements.at(m_PrefixConfigObject + typeName + ":overwrite:include")); - auto& includeChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":overwrite:include")); + auto& includs (statements[m_PrefixConfigObject + typeName + ":overwrite:include"]); + auto& includeChksms (statements[m_PrefixConfigCheckSum + typeName + ":overwrite:include"]); includeChksms.emplace_back(objectKey); includeChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(includes)}}))); @@ -463,8 +457,8 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons excludeChecksums->Add(GetObjectIdentifier((*getExclude)(exclude.Get()))); } - auto& excluds (statements.at(m_PrefixConfigObject + typeName + ":overwrite:exclude")); - auto& excludeChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":overwrite:exclude")); + auto& excluds (statements[m_PrefixConfigObject + typeName + ":overwrite:exclude"]); + auto& excludeChksms (statements[m_PrefixConfigCheckSum + typeName + ":overwrite:exclude"]); excludeChksms.emplace_back(objectKey); excludeChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(excludes)}}))); @@ -486,8 +480,8 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons parents->Add(GetObjectIdentifier(parent)); } - auto& parnts (statements.at(m_PrefixConfigObject + typeName + ":parent")); - auto& parentChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":parent")); + auto& parnts (statements[m_PrefixConfigObject + typeName + ":parent"]); + auto& parentChksms (statements[m_PrefixConfigCheckSum + typeName + ":parent"]); parentChksms.emplace_back(objectKey); parentChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", HashValue(zone->GetAllParents())}}))); @@ -516,8 +510,8 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons groupIds->Add(GetObjectIdentifier((*getGroup)(group))); } - auto& members (statements.at(m_PrefixConfigObject + typeName + ":groupmember")); - auto& memberChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":groupmember")); + auto& members (statements[m_PrefixConfigObject + typeName + ":groupmember"]); + auto& memberChksms (statements[m_PrefixConfigCheckSum + typeName + ":groupmember"]); memberChksms.emplace_back(objectKey); memberChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(groupIds)}}))); @@ -543,8 +537,8 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons userIds->Add(GetObjectIdentifier(user)); } - auto& usrs (statements.at(m_PrefixConfigObject + typeName + ":user")); - auto& userChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":user")); + auto& usrs (statements[m_PrefixConfigObject + typeName + ":user"]); + auto& userChksms (statements[m_PrefixConfigCheckSum + typeName + ":user"]); userChksms.emplace_back(objectKey); userChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(userIds)}}))); @@ -557,8 +551,8 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons usergroupIds->Add(GetObjectIdentifier(usergroup)); } - auto& groups (statements.at(m_PrefixConfigObject + typeName + ":usergroup")); - auto& groupChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":usergroup")); + auto& groups (statements[m_PrefixConfigObject + typeName + ":usergroup"]); + auto& groupChksms (statements[m_PrefixConfigCheckSum + typeName + ":usergroup"]); groupChksms.emplace_back(objectKey); groupChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(usergroupIds)}}))); @@ -575,9 +569,9 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons if (arguments) { ObjectLock argumentsLock(arguments); Array::Ptr argumentIds(new Array); - auto& typeArgs (statements.at(m_PrefixConfigObject + typeName + ":argument")); - auto& allArgs (statements.at(m_PrefixConfigObject + "commandargument")); - auto& argChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":argument")); + auto& typeArgs (statements[m_PrefixConfigObject + typeName + ":argument"]); + auto& allArgs (statements[m_PrefixConfigObject + "commandargument"]); + auto& argChksms (statements[m_PrefixConfigCheckSum + typeName + ":argument"]); argumentIds->Reserve(arguments->GetLength()); @@ -599,9 +593,9 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons if (envvars) { ObjectLock envvarsLock(envvars); Array::Ptr envvarIds(new Array); - auto& typeVars (statements.at(m_PrefixConfigObject + typeName + ":envvar")); - auto& allVars (statements.at(m_PrefixConfigObject + "commandenvvar")); - auto& varChksms (statements.at(m_PrefixConfigCheckSum + typeName + ":envvar")); + auto& typeVars (statements[m_PrefixConfigObject + typeName + ":envvar"]); + auto& allVars (statements[m_PrefixConfigObject + "commandenvvar"]); + auto& varChksms (statements[m_PrefixConfigCheckSum + typeName + ":envvar"]); envvarIds->Reserve(envvars->GetLength()); @@ -638,9 +632,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtime String typeName = GetLowerCaseTypeNameDB(object); - std::vector keys = GetTypeObjectKeys(typeName); - - std::map > statements = GenerateHmsetStatements(keys); + std::map> statements; std::vector states = {"HMSET", m_PrefixStateObject + typeName}; CreateConfigUpdate(object, typeName, statements, runtimeUpdate); @@ -651,8 +643,13 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtime } std::vector > transaction = {{"MULTI"}}; - for (const auto& kv : statements) - transaction.push_back(kv.second); + + for (auto& kv : statements) { + if (!kv.second.empty()) { + kv.second.insert(kv.second.begin(), {"HMSET", kv.first}); + transaction.emplace_back(std::move(kv.second)); + } + } if (transaction.size() > 1) { transaction.push_back({"EXEC"}); @@ -926,8 +923,8 @@ RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, const String ty InsertObjectDependencies(object, typeName, statements); String objectKey = GetObjectIdentifier(object); - auto& attrs (statements.at(m_PrefixConfigObject + typeName)); - auto& chksms (statements.at(m_PrefixConfigCheckSum + typeName)); + auto& attrs (statements[m_PrefixConfigObject + typeName]); + auto& chksms (statements[m_PrefixConfigCheckSum + typeName]); attrs.emplace_back(objectKey); attrs.emplace_back(JsonEncode(attr)); diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 26d7fb647..16a9a83ca 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -72,7 +72,6 @@ private: void UpdateAllConfigObjects(); std::vector>> ChunkObjects(std::vector> objects, size_t chunkSize); void DeleteKeys(const std::vector& keys); - std::map > GenerateHmsetStatements(const std::vector keys); std::vector GetTypeObjectKeys(const String& type); void InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map >& statements); void UpdateState(const Checkable::Ptr& checkable); From 6fd6f74b0f3be2515f96800655ab476acf35f521 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 28 Jun 2019 10:02:59 +0200 Subject: [PATCH 142/219] Fix missing ( ) --- lib/redis/rediswriter-objects.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 8199198be..d7b9ddeeb 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -152,7 +152,7 @@ void RedisWriter::UpdateAllConfigObjects() } bulkCounter++; - if (!bulkCounter % 100) { + if (!(bulkCounter % 100)) { for (auto& kv : statements) { if (!kv.second.empty()) { kv.second.insert(kv.second.begin(), {"HMSET", kv.first}); From 0bfdaccc7af31e91773d68a4cc1df581f42fff72 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 17 Jul 2019 14:30:50 +0200 Subject: [PATCH 143/219] Split up redis arrays into single entries --- lib/redis/rediswriter-objects.cpp | 118 +++++++++++++++++------------- 1 file changed, 68 insertions(+), 50 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index d7b9ddeeb..b05ae3bd0 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -322,11 +322,10 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons for (auto& kv : vars) { allCvs.emplace_back(kv.first); allCvs.emplace_back(JsonEncode(kv.second)); - varsArray->Add(kv.first); + String id = CalculateCheckSumArray(new Array({envId, kv.first, objectKey})); + typeCvs.emplace_back(id); + typeCvs.emplace_back(JsonEncode(new Dictionary({{"object_id", objectKey}, {"env_id", envId}, {"customvar_id", kv.first}}))); } - - typeCvs.emplace_back(objectKey); - typeCvs.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"customvars", varsArray}}))); } } @@ -373,17 +372,20 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons groupIds->Reserve(groups->GetLength()); - for (auto& group : groups) { - groupIds->Add(GetObjectIdentifier((*getGroup)(group))); - } - auto& members (statements[m_PrefixConfigObject + typeName + ":groupmember"]); auto& memberChksms (statements[m_PrefixConfigCheckSum + typeName + ":groupmember"]); + for (auto& group : groups) { + String groupId = GetObjectIdentifier((*getGroup)(group)); + String id = CalculateCheckSumArray(new Array({envId, groupId, objectKey})); + members.emplace_back(id); + members.emplace_back(JsonEncode(new Dictionary({{"object_id", objectKey}, {"env_id", envId}, {"group_id", groupId}}))); + + groupIds->Add(groupId); + } + memberChksms.emplace_back(objectKey); memberChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(groupIds)}}))); - members.emplace_back(objectKey); - members.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"groups", groupIds}}))); } return; @@ -403,17 +405,19 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons rangeIds->Reserve(ranges->GetLength()); for (auto& kv : ranges) { - String id = CalculateCheckSumArray(new Array({envId, kv.first, kv.second})); - rangeIds->Add(id); + String rangeId = CalculateCheckSumArray(new Array({envId, kv.first, kv.second})); + rangeIds->Add(rangeId); - allRanges.emplace_back(id); + String id = CalculateCheckSumArray(new Array({envId, rangeId, objectKey})); + typeRanges.emplace_back(id); + typeRanges.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"timeperiod_id", objectKey}, {"range_id", rangeId}}))); + + allRanges.emplace_back(rangeId); allRanges.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"range_key", kv.first}, {"range_value", kv.second}}))); } rangeChksms.emplace_back(objectKey); rangeChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(rangeIds)}}))); - typeRanges.emplace_back(objectKey); - typeRanges.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"ranges", rangeIds}}))); } Array::Ptr includes; @@ -428,17 +432,20 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons includeChecksums->Reserve(includes->GetLength()); - for (auto include : includes) { - includeChecksums->Add(GetObjectIdentifier((*getInclude)(include.Get()))); - } auto& includs (statements[m_PrefixConfigObject + typeName + ":overwrite:include"]); auto& includeChksms (statements[m_PrefixConfigCheckSum + typeName + ":overwrite:include"]); + for (auto include : includes) { + String includeId = GetObjectIdentifier((*getInclude)(include.Get())); + includeChecksums->Add(includeId); + + String id = CalculateCheckSumArray(new Array({envId, includeId, objectKey})); + includs.emplace_back(id); + includs.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"timeperiod_id", objectKey}, {"include_id", includeId}}))); + } includeChksms.emplace_back(objectKey); includeChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(includes)}}))); - includs.emplace_back(objectKey); - includs.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"includes", includeChecksums}}))); Array::Ptr excludes; ConfigObject::Ptr (*getExclude)(const String& name); @@ -453,17 +460,20 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons excludeChecksums->Reserve(excludes->GetLength()); - for (auto exclude : excludes) { - excludeChecksums->Add(GetObjectIdentifier((*getExclude)(exclude.Get()))); - } - auto& excluds (statements[m_PrefixConfigObject + typeName + ":overwrite:exclude"]); auto& excludeChksms (statements[m_PrefixConfigCheckSum + typeName + ":overwrite:exclude"]); + for (auto exclude : excludes) { + String excludeId = GetObjectIdentifier((*getExclude)(exclude.Get())); + excludeChecksums->Add(excludeId); + + String id = CalculateCheckSumArray(new Array({envId, excludeId, objectKey})); + excluds.emplace_back(id); + excluds.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"timeperiod_id", objectKey}, {"exclude_id", excludeId}}))); + } + excludeChksms.emplace_back(objectKey); excludeChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(excludes)}}))); - excluds.emplace_back(objectKey); - excluds.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"excludes", excludeChecksums}}))); return; } @@ -476,17 +486,19 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons parents->Reserve(parentsRaw.size()); - for (auto& parent : parentsRaw) { - parents->Add(GetObjectIdentifier(parent)); - } - auto& parnts (statements[m_PrefixConfigObject + typeName + ":parent"]); auto& parentChksms (statements[m_PrefixConfigCheckSum + typeName + ":parent"]); + for (auto& parent : parentsRaw) { + String parentId = GetObjectIdentifier(parent); + String id = CalculateCheckSumArray(new Array({envId, parentId, objectKey})); + parnts.emplace_back(id); + parnts.emplace_back(JsonEncode(new Dictionary({{"zone_id", objectKey}, {"env_id", envId}, {"parent_id", parentId}}))); + parents->Add(GetObjectIdentifier(parent)); + } + parentChksms.emplace_back(objectKey); parentChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", HashValue(zone->GetAllParents())}}))); - parnts.emplace_back(objectKey); - parnts.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"parents", parents}}))); return; } @@ -506,17 +518,19 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons groupIds->Reserve(groups->GetLength()); - for (auto& group : groups) { - groupIds->Add(GetObjectIdentifier((*getGroup)(group))); - } - auto& members (statements[m_PrefixConfigObject + typeName + ":groupmember"]); auto& memberChksms (statements[m_PrefixConfigCheckSum + typeName + ":groupmember"]); + for (auto& group : groups) { + String groupId = GetObjectIdentifier((*getGroup)(group)); + String id = CalculateCheckSumArray(new Array({envId, groupId, objectKey})); + members.emplace_back(id); + members.emplace_back(JsonEncode(new Dictionary({{"user_id", objectKey}, {"env_id", envId}, {"group_id", groupId}}))); + groupIds->Add(groupId); + } + memberChksms.emplace_back(objectKey); memberChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(groupIds)}}))); - members.emplace_back(objectKey); - members.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"groups", groupIds}}))); } return; @@ -533,31 +547,35 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons userIds->Reserve(users.size()); - for (auto& user : users) { - userIds->Add(GetObjectIdentifier(user)); - } - auto& usrs (statements[m_PrefixConfigObject + typeName + ":user"]); auto& userChksms (statements[m_PrefixConfigCheckSum + typeName + ":user"]); + for (auto& user : users) { + String userId = GetObjectIdentifier(user); + String id = CalculateCheckSumArray(new Array({envId, userId, objectKey})); + usrs.emplace_back(id); + usrs.emplace_back(JsonEncode(new Dictionary({{"notification_id", objectKey}, {"env_id", envId}, {"user_id", userId}}))); + userIds->Add(userId); + } + userChksms.emplace_back(objectKey); userChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(userIds)}}))); - usrs.emplace_back(objectKey); - usrs.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"users", userIds}}))); usergroupIds->Reserve(usergroups.size()); - for (auto& usergroup : usergroups) { - usergroupIds->Add(GetObjectIdentifier(usergroup)); - } - auto& groups (statements[m_PrefixConfigObject + typeName + ":usergroup"]); auto& groupChksms (statements[m_PrefixConfigCheckSum + typeName + ":usergroup"]); + for (auto& usergroup : usergroups) { + String usergroupId = GetObjectIdentifier(usergroup); + String id = CalculateCheckSumArray(new Array({envId, usergroupId, objectKey})); + groups.emplace_back(id); + groups.emplace_back(JsonEncode(new Dictionary({{"notification_id", objectKey}, {"env_id", envId}, {"usergroup_id", usergroupId}}))); + usergroupIds->Add(usergroupId); + } + groupChksms.emplace_back(objectKey); groupChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(usergroupIds)}}))); - groups.emplace_back(objectKey); - groups.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"usergroups", usergroupIds}}))); return; } From fc718d99a725029f5105eb20fc9f62f6cc31a58e Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 17 Jul 2019 15:08:53 +0200 Subject: [PATCH 144/219] Rename :overwrite: keys to :override: --- lib/redis/rediswriter-objects.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index b05ae3bd0..12000791a 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -268,10 +268,10 @@ std::vector RedisWriter::GetTypeObjectKeys(const String& type) keys.emplace_back(m_PrefixConfigObject + type + ":groupmember"); keys.emplace_back(m_PrefixConfigCheckSum + type + ":groupmember"); } else if (type == "timeperiod") { - keys.emplace_back(m_PrefixConfigObject + type + ":overwrite:include"); - keys.emplace_back(m_PrefixConfigCheckSum + type + ":overwrite:include"); - keys.emplace_back(m_PrefixConfigObject + type + ":overwrite:exclude"); - keys.emplace_back(m_PrefixConfigCheckSum + type + ":overwrite:exclude"); + keys.emplace_back(m_PrefixConfigObject + type + ":override:include"); + keys.emplace_back(m_PrefixConfigCheckSum + type + ":override:include"); + keys.emplace_back(m_PrefixConfigObject + type + ":override:exclude"); + keys.emplace_back(m_PrefixConfigCheckSum + type + ":override:exclude"); keys.emplace_back(m_PrefixConfigObject + type + ":range"); keys.emplace_back(m_PrefixConfigCheckSum + type + ":range"); } else if (type == "zone") { @@ -433,8 +433,8 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons includeChecksums->Reserve(includes->GetLength()); - auto& includs (statements[m_PrefixConfigObject + typeName + ":overwrite:include"]); - auto& includeChksms (statements[m_PrefixConfigCheckSum + typeName + ":overwrite:include"]); + auto& includs (statements[m_PrefixConfigObject + typeName + ":override:include"]); + auto& includeChksms (statements[m_PrefixConfigCheckSum + typeName + ":override:include"]); for (auto include : includes) { String includeId = GetObjectIdentifier((*getInclude)(include.Get())); includeChecksums->Add(includeId); @@ -460,8 +460,8 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons excludeChecksums->Reserve(excludes->GetLength()); - auto& excluds (statements[m_PrefixConfigObject + typeName + ":overwrite:exclude"]); - auto& excludeChksms (statements[m_PrefixConfigCheckSum + typeName + ":overwrite:exclude"]); + auto& excluds (statements[m_PrefixConfigObject + typeName + ":override:exclude"]); + auto& excludeChksms (statements[m_PrefixConfigCheckSum + typeName + ":override:exclude"]); for (auto exclude : excludes) { String excludeId = GetObjectIdentifier((*getExclude)(exclude.Get())); From 74b683970407976001ec48782c1e9fe84051de31 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Thu, 18 Jul 2019 15:02:47 +0200 Subject: [PATCH 145/219] Add JsonEncode to customvar value --- lib/redis/rediswriter-utility.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index db136dcf9..643dab0fe 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -283,7 +283,7 @@ Dictionary::Ptr RedisWriter::SerializeVars(const CustomVarObject::Ptr& object) {"env_id", envChecksum}, {"name_checksum", SHA1(kv.first)}, {"name", kv.first}, - {"value", kv.second}, + {"value", JsonEncode(kv.second)}, }) ); } From e41bcbc81c6018e3fd35ae5a7c6dff2e76f5293d Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 24 Jul 2019 10:55:47 +0200 Subject: [PATCH 146/219] Do not store timeperiod range in separate key (timerange) --- lib/redis/rediswriter-objects.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 12000791a..fb26f1c9e 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -116,7 +116,6 @@ void RedisWriter::UpdateAllConfigObjects() m_PrefixConfigObject + "icon_image", m_PrefixConfigObject + "commandargument", m_PrefixConfigObject + "commandenvvar", - m_PrefixConfigObject + "timerange", }; DeleteKeys(globalKeys); @@ -399,7 +398,6 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons ObjectLock rangesLock(ranges); Array::Ptr rangeIds(new Array); auto& typeRanges (statements[m_PrefixConfigObject + typeName + ":range"]); - auto& allRanges (statements[m_PrefixConfigObject + "timerange"]); auto& rangeChksms (statements[m_PrefixConfigCheckSum + typeName + ":range"]); rangeIds->Reserve(ranges->GetLength()); @@ -410,10 +408,7 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons String id = CalculateCheckSumArray(new Array({envId, rangeId, objectKey})); typeRanges.emplace_back(id); - typeRanges.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"timeperiod_id", objectKey}, {"range_id", rangeId}}))); - - allRanges.emplace_back(rangeId); - allRanges.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"range_key", kv.first}, {"range_value", kv.second}}))); + typeRanges.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"timeperiod_id", objectKey}, {"range_key", kv.first}, {"range_value", kv.second}}))); } rangeChksms.emplace_back(objectKey); From 9308acfcd50bf2b0d6e10e3cfc0c6d8ba86c5af1 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 24 Jul 2019 15:47:40 +0200 Subject: [PATCH 147/219] RedisWriter: Json encode command field command --- lib/redis/rediswriter-objects.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index fb26f1c9e..780279821 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -901,7 +901,7 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr if (type == CheckCommand::TypeInstance || type == NotificationCommand::TypeInstance || type == EventCommand::TypeInstance) { Command::Ptr command = static_pointer_cast(object); - attributes->Set("command", command->GetCommandLine()); + attributes->Set("command", JsonEncode(command->GetCommandLine())); attributes->Set("timeout", command->GetTimeout()); return true; From 8421a98a2c151771075040e3af170ca196a8f9a8 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Fri, 26 Jul 2019 16:28:45 +0200 Subject: [PATCH 148/219] Fix command arguments and envvars --- lib/redis/rediswriter-objects.cpp | 65 ++++++++++++++++++------------- 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 780279821..343ad3130 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -114,8 +114,6 @@ void RedisWriter::UpdateAllConfigObjects() m_PrefixConfigObject + "action_url", m_PrefixConfigObject + "notes_url", m_PrefixConfigObject + "icon_image", - m_PrefixConfigObject + "commandargument", - m_PrefixConfigObject + "commandenvvar", }; DeleteKeys(globalKeys); @@ -581,49 +579,64 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons Dictionary::Ptr arguments = command->GetArguments(); if (arguments) { ObjectLock argumentsLock(arguments); - Array::Ptr argumentIds(new Array); auto& typeArgs (statements[m_PrefixConfigObject + typeName + ":argument"]); - auto& allArgs (statements[m_PrefixConfigObject + "commandargument"]); auto& argChksms (statements[m_PrefixConfigCheckSum + typeName + ":argument"]); - argumentIds->Reserve(arguments->GetLength()); - for (auto& kv : arguments) { - String id = HashValue(kv.first + HashValue(kv.second)); - argumentIds->Add(id); + Dictionary::Ptr values; + if (kv.second.IsObjectType()) { + values = kv.second; + } else if (kv.second.IsObjectType()) { + values = new Dictionary({{"value", JsonEncode(kv.second)}}); + } else { + values = new Dictionary({{"value", kv.second}}); + } - allArgs.emplace_back(id); - allArgs.emplace_back(JsonEncode(kv.second)); + values->Set("value", JsonEncode(values->Get("value"))); + values->Set("command_id", objectKey); + values->Set("argument_key", kv.first); + values->Set("env_id", envId); + + String id = HashValue(objectKey + kv.first + envId); + + typeArgs.emplace_back(id); + typeArgs.emplace_back(JsonEncode(values)); + argChksms.emplace_back(id); + argChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", HashValue(kv.second)}}))); } - - argChksms.emplace_back(objectKey); - argChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(argumentIds)}}))); - typeArgs.emplace_back(objectKey); - typeArgs.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"arguments", argumentIds}}))); } - Dictionary::Ptr envvars = command->GetArguments(); + Dictionary::Ptr envvars = command->GetEnv(); if (envvars) { ObjectLock envvarsLock(envvars); Array::Ptr envvarIds(new Array); auto& typeVars (statements[m_PrefixConfigObject + typeName + ":envvar"]); - auto& allVars (statements[m_PrefixConfigObject + "commandenvvar"]); auto& varChksms (statements[m_PrefixConfigCheckSum + typeName + ":envvar"]); envvarIds->Reserve(envvars->GetLength()); for (auto& kv : envvars) { - String id = HashValue(kv.first + HashValue(kv.second)); - envvarIds->Add(id); + Dictionary::Ptr values; + if (kv.second.IsObjectType()) { + values = kv.second; + } else if (kv.second.IsObjectType()) { + values = new Dictionary({{"value", JsonEncode(kv.second)}}); + } else { + values = new Dictionary({{"value", kv.second}}); + } - allVars.emplace_back(id); - allVars.emplace_back(JsonEncode(kv.second)); + values->Set("value", JsonEncode(values->Get("value"))); + values->Set("command_id", objectKey); + values->Set("envvar_key", kv.first); + values->Set("env_id", envId); + + String id = HashValue(objectKey + kv.first + envId); + + typeVars.emplace_back(id); + typeVars.emplace_back(JsonEncode(values)); + varChksms.emplace_back(id); + varChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", HashValue(kv.second)}}))); } - - varChksms.emplace_back(objectKey); - varChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumArray(envvarIds)}}))); - typeVars.emplace_back(objectKey); - typeVars.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"envvars", envvarIds}}))); } return; From 11d541519332eb7007416f250626fe2184f22e68 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 29 Jul 2019 14:43:09 +0200 Subject: [PATCH 149/219] RedisWriter: Add depth attribute to zone --- lib/redis/rediswriter-objects.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 343ad3130..2b17216de 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -713,6 +713,9 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr attributes->Set("parent_id", GetObjectIdentifier(zone)); } + auto parentsRaw (zone->GetAllParentsRaw()); + attributes->Set("depth", parentsRaw.size()); + return true; } From 74eccf60f2688bb79fb471e1d8ac7f5d7387ca1c Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 30 Oct 2018 12:51:55 +0100 Subject: [PATCH 150/219] Fix memory leak --- lib/redis/rediswriter.cpp | 18 +++++++++++++----- lib/redis/rediswriter.hpp | 3 ++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index ab03e804a..aadd87fa2 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -25,6 +25,7 @@ #include "icinga/checkable.hpp" #include "icinga/host.hpp" +#include #include #include @@ -189,7 +190,7 @@ void RedisWriter::UpdateSubscriptions() bool RedisWriter::GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi) { try { - redisReply *redisReply = RedisGet({ "SMEMBERS", key }); + auto redisReply = RedisGet({ "SMEMBERS", key }); VERIFY(redisReply->type == REDIS_REPLY_ARRAY); if (redisReply->elements == 0) @@ -280,11 +281,11 @@ void RedisWriter::HandleEvent(const Dictionary::Ptr& event) String body = JsonEncode(event); - redisReply *maxExists = RedisGet({ "EXISTS", "icinga:subscription:" + name + ":limit" }); + auto maxExists = RedisGet({ "EXISTS", "icinga:subscription:" + name + ":limit" }); long maxEvents = MAX_EVENTS_DEFAULT; if (maxExists->integer) { - redisReply *redisReply = RedisGet({ "GET", "icinga:subscription:" + name + ":limit"}); + auto redisReply = RedisGet({ "GET", "icinga:subscription:" + name + ":limit"}); VERIFY(redisReply->type == REDIS_REPLY_STRING); Log(LogInformation, "RedisWriter") @@ -399,8 +400,15 @@ void RedisWriter::RedisQueryCallback(redisAsyncContext *c, void *r, void *p) { wait->cv.notify_all(); } +struct RedisReplyDeleter +{ + inline void operator() (redisReply *reply) + { + freeReplyObject(reply); + } +}; -redisReply* RedisWriter::RedisGet(const std::vector& query) { +std::shared_ptr RedisWriter::RedisGet(const std::vector& query) { auto *wait = new synchronousWait; wait->ready = false; @@ -413,5 +421,5 @@ redisReply* RedisWriter::RedisGet(const std::vector& query) { wait->ready = true; } - return wait->reply; + return std::shared_ptr(wait->reply, RedisReplyDeleter()); } \ No newline at end of file diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 16a9a83ca..5fbf3514f 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -29,6 +29,7 @@ #include "icinga/checkable.hpp" #include "icinga/service.hpp" #include "icinga/downtime.hpp" +#include #include namespace icinga @@ -114,7 +115,7 @@ private: void ExceptionHandler(boost::exception_ptr exp); //Used to get a reply from the asyncronous connection - redisReply* RedisGet(const std::vector& query); + std::shared_ptr RedisGet(const std::vector& query); static void RedisQueryCallback(redisAsyncContext *c, void *r, void *p); static redisReply* dupReplyObject(redisReply* reply); From bb333b535b6e1141a916dccb9b34cbbdd1c15f99 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 25 Jul 2019 17:41:07 +0200 Subject: [PATCH 151/219] RedisConnection#ExecuteQueries(): fire all queries at once refs #49 --- lib/redis/redisconnection.cpp | 15 ++++++++++----- lib/redis/redisconnection.hpp | 1 + 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index 7bf477980..0849bce1b 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -189,11 +189,9 @@ void RedisConnection::ExecuteQuery(const std::vector& query, redisCallba void RedisConnection::ExecuteQueries(const std::vector >& queries, redisCallbackFn *fn, void *privdata) { - for (const auto& query : queries) { - m_RedisConnectionWorkQueue.Enqueue([this, query, fn, privdata]() { - SendMessageInternal(query, fn, privdata); - }); - } + m_RedisConnectionWorkQueue.Enqueue([this, queries, fn, privdata]() { + SendMessagesInternal(queries, fn, privdata); + }); } void RedisConnection::SendMessageInternal(const std::vector& query, redisCallbackFn *fn, void *privdata) @@ -248,3 +246,10 @@ void RedisConnection::SendMessageInternal(const std::vector& query, redi ); } } + +void RedisConnection::SendMessagesInternal(const std::vector>& queries, redisCallbackFn *fn, void *privdata) +{ + for (const auto& query : queries) { + SendMessageInternal(query, fn, privdata); + } +} diff --git a/lib/redis/redisconnection.hpp b/lib/redis/redisconnection.hpp index 3dee85e93..4f1af40f7 100644 --- a/lib/redis/redisconnection.hpp +++ b/lib/redis/redisconnection.hpp @@ -69,6 +69,7 @@ namespace icinga static void StaticInitialize(); void SendMessageInternal(const std::vector& query, redisCallbackFn *fn, void *privdata); + void SendMessagesInternal(const std::vector>& queries, redisCallbackFn *fn, void *privdata); void AssertOnWorkQueue(); From fb98d3edef79b7171427aeabb2e1e377ceea55c0 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 25 Jul 2019 18:00:08 +0200 Subject: [PATCH 152/219] RedisConnection#ExecuteQuer{y,ies}(): std::move() queries refs #49 --- lib/redis/redisconnection.cpp | 19 ++++++++++++------- lib/redis/redisconnection.hpp | 4 ++-- lib/redis/rediswriter-objects.cpp | 10 +++++----- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index 0849bce1b..32b4272ea 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -25,7 +25,8 @@ #include "base/utility.hpp" #include "redis/rediswriter.hpp" #include "hiredis/hiredis.h" - +#include +#include using namespace icinga; @@ -179,18 +180,22 @@ void RedisConnection::DisconnectCallback(const redisAsyncContext *c, int status) rc->m_Connected = false; } -void RedisConnection::ExecuteQuery(const std::vector& query, redisCallbackFn *fn, void *privdata) +void RedisConnection::ExecuteQuery(std::vector query, redisCallbackFn *fn, void *privdata) { - m_RedisConnectionWorkQueue.Enqueue([this, query, fn, privdata]() { - SendMessageInternal(query, fn, privdata); + auto queryPtr (std::make_shared(std::move(query))); + + m_RedisConnectionWorkQueue.Enqueue([this, queryPtr, fn, privdata]() { + SendMessageInternal(*queryPtr, fn, privdata); }); } void -RedisConnection::ExecuteQueries(const std::vector >& queries, redisCallbackFn *fn, void *privdata) +RedisConnection::ExecuteQueries(std::vector> queries, redisCallbackFn *fn, void *privdata) { - m_RedisConnectionWorkQueue.Enqueue([this, queries, fn, privdata]() { - SendMessagesInternal(queries, fn, privdata); + auto queriesPtr (std::make_shared(std::move(queries))); + + m_RedisConnectionWorkQueue.Enqueue([this, queriesPtr, fn, privdata]() { + SendMessagesInternal(*queriesPtr, fn, privdata); }); } diff --git a/lib/redis/redisconnection.hpp b/lib/redis/redisconnection.hpp index 4f1af40f7..7b1e21404 100644 --- a/lib/redis/redisconnection.hpp +++ b/lib/redis/redisconnection.hpp @@ -60,9 +60,9 @@ namespace icinga bool IsConnected(); - void ExecuteQuery(const std::vector& query, redisCallbackFn *fn = NULL, void *privdata = NULL); + void ExecuteQuery(std::vector query, redisCallbackFn *fn = NULL, void *privdata = NULL); - void ExecuteQueries(const std::vector >& queries, redisCallbackFn *fn = NULL, + void ExecuteQueries(std::vector> queries, redisCallbackFn *fn = NULL, void *privdata = NULL); private: diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 2b17216de..143bf671b 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -166,7 +166,7 @@ void RedisWriter::UpdateAllConfigObjects() if (transaction.size() > 1) { transaction.push_back({"EXEC"}); - m_Rcon->ExecuteQueries(transaction); + m_Rcon->ExecuteQueries(std::move(transaction)); transaction = {{"MULTI"}}; } } @@ -184,7 +184,7 @@ void RedisWriter::UpdateAllConfigObjects() if (transaction.size() > 1) { transaction.push_back({"EXEC"}); - m_Rcon->ExecuteQueries(transaction); + m_Rcon->ExecuteQueries(std::move(transaction)); } m_Rcon->ExecuteQuery({"PUBLISH", "icinga:config:dump", lcType}); @@ -249,7 +249,7 @@ void RedisWriter::DeleteKeys(const std::vector& keys) { query.emplace_back(key); } - m_Rcon->ExecuteQuery(query); + m_Rcon->ExecuteQuery(std::move(query)); } std::vector RedisWriter::GetTypeObjectKeys(const String& type) @@ -679,7 +679,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtime if (transaction.size() > 1) { transaction.push_back({"EXEC"}); - m_Rcon->ExecuteQueries(transaction); + m_Rcon->ExecuteQueries(std::move(transaction)); } } @@ -1008,7 +1008,7 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) streamadd.emplace_back(kv.second); } - m_Rcon->ExecuteQuery(streamadd); + m_Rcon->ExecuteQuery(std::move(streamadd)); } Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) From 53d04cc4e8051e44e928ad60f7f779a4d391d6e2 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 25 Jul 2019 18:03:42 +0200 Subject: [PATCH 153/219] RedisWriter#RedisGet(): std::move() query refs #49 --- lib/redis/rediswriter.cpp | 6 +++--- lib/redis/rediswriter.hpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index aadd87fa2..1ec7a9823 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -408,11 +408,11 @@ struct RedisReplyDeleter } }; -std::shared_ptr RedisWriter::RedisGet(const std::vector& query) { +std::shared_ptr RedisWriter::RedisGet(std::vector query) { auto *wait = new synchronousWait; wait->ready = false; - m_Rcon->ExecuteQuery(query, RedisQueryCallback, wait); + m_Rcon->ExecuteQuery(std::move(query), RedisQueryCallback, wait); boost::mutex::scoped_lock lock(wait->mtx); while (!wait->ready) { @@ -422,4 +422,4 @@ std::shared_ptr RedisWriter::RedisGet(const std::vector& que } return std::shared_ptr(wait->reply, RedisReplyDeleter()); -} \ No newline at end of file +} diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 5fbf3514f..17fbc10a1 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -115,7 +115,7 @@ private: void ExceptionHandler(boost::exception_ptr exp); //Used to get a reply from the asyncronous connection - std::shared_ptr RedisGet(const std::vector& query); + std::shared_ptr RedisGet(std::vector query); static void RedisQueryCallback(redisAsyncContext *c, void *r, void *p); static redisReply* dupReplyObject(redisReply* reply); From d5d3f3e60cff650f8b165393527691e60a340d50 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 7 Aug 2019 15:50:16 +0200 Subject: [PATCH 154/219] Remove unused function CollectScalarVars() --- lib/redis/rediswriter-utility.cpp | 96 +------------------------------ 1 file changed, 1 insertion(+), 95 deletions(-) diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 643dab0fe..2ca9cd739 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -130,87 +130,6 @@ String RedisWriter::CalculateCheckSumVars(const CustomVarObject::Ptr& object) return HashValue(vars); } -/** - * Collect the leaves of haystack in needles - * - * haystack = { - * "disks": { - * "disk": {}, - * "disk /": { - * "disk_partitions": "/" - * } - * } - * } - * - * path = [] - * - * needles = { - * "disks": [ - * [ - * ["disks", "disk /", "disk_partitions"], - * "/" - * ] - * ] - * } - * - * @param haystack A config object's custom vars as returned by {@link CustomVarObject#GetVars()} - * @param path Used for buffering only, shall be empty - * @param needles The result, a mapping from the top-level custom var key to a list of leaves with full path and value - */ -static void CollectScalarVars(Value haystack, std::vector& path, std::map>>& needles) -{ - switch (haystack.GetType()) { - case ValueObject: - { - const Object::Ptr& obj = haystack.Get(); - - Dictionary::Ptr dict = dynamic_pointer_cast(obj); - - if (dict) { - ObjectLock olock(dict); - - for (auto& kv : dict) { - path.emplace_back(kv.first); - CollectScalarVars(kv.second, path, needles); - path.pop_back(); - } - - break; - } - - Array::Ptr arr = dynamic_pointer_cast(obj); - - if (arr) { - double index = 0.0; - - ObjectLock xlock(arr); - - for (auto& v : arr) { - path.emplace_back(index); - CollectScalarVars(v, path, needles); - path.pop_back(); - - index += 1.0; - } - - break; - } - } - - haystack = Empty; - - case ValueString: - case ValueNumber: - case ValueBoolean: - case ValueEmpty: - needles[path[0].Get()].emplace_back(Array::FromVector(path), haystack); - break; - - default: - VERIFY(!"Invalid variant type."); - } -} - /** * Prepare object's custom vars for being written to Redis * @@ -234,7 +153,7 @@ static void CollectScalarVars(Value haystack, std::vector& path, std::map * } * } * ])): { - * "env_checksum": SHA1(Environment), + * "envId": SHA1(Environment), * "name_checksum": SHA1("disks"), * "name": "disks", * "value": { @@ -242,12 +161,6 @@ static void CollectScalarVars(Value haystack, std::vector& path, std::map * "disk /": { * "disk_partitions": "/" * } - * }, - * "flat": { - * SHA1(PackObject(["disks", "disk /", "disk_partitions"])): { - * "name": ["disks", "disk /", "disk_partitions"], - * "value": "/" - * } * } * } * } @@ -263,13 +176,6 @@ Dictionary::Ptr RedisWriter::SerializeVars(const CustomVarObject::Ptr& object) if (!vars) return nullptr; - std::map>> scalarVars; - - { - std::vector pathBuf; - CollectScalarVars(vars, pathBuf, scalarVars); - } - Dictionary::Ptr res = new Dictionary(); auto env (GetEnvironment()); auto envChecksum (SHA1(env)); From e60ca9bca91ca28046c7730ede04128e28b06164 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 8 Aug 2019 18:03:17 +0200 Subject: [PATCH 155/219] RedisWriter#InsertObjectDependencies(): don't change config objects' attributes in-place refs #52 --- lib/redis/rediswriter-objects.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 143bf671b..9fba8a067 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -586,6 +586,7 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons Dictionary::Ptr values; if (kv.second.IsObjectType()) { values = kv.second; + values = values->ShallowClone(); } else if (kv.second.IsObjectType()) { values = new Dictionary({{"value", JsonEncode(kv.second)}}); } else { @@ -619,6 +620,7 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons Dictionary::Ptr values; if (kv.second.IsObjectType()) { values = kv.second; + values = values->ShallowClone(); } else if (kv.second.IsObjectType()) { values = new Dictionary({{"value", JsonEncode(kv.second)}}); } else { From 752c5998df39704dcd7de740e661d9ec7c516888 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 31 Jul 2019 11:27:17 +0200 Subject: [PATCH 156/219] Prefer vector#emplace_back over #push_back --- lib/redis/rediswriter-objects.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 9fba8a067..2c930e380 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -158,7 +158,7 @@ void RedisWriter::UpdateAllConfigObjects() } if (states.size() > 2) { - transaction.push_back(std::move(states)); + transaction.emplace_back(std::move(states)); states = {"HMSET", m_PrefixStateObject + lcType}; } @@ -180,7 +180,7 @@ void RedisWriter::UpdateAllConfigObjects() } if (states.size() > 2) - transaction.push_back(std::move(states)); + transaction.emplace_back(std::move(states)); if (transaction.size() > 1) { transaction.push_back({"EXEC"}); From a5971df039b87541458e5f99954a36b79f3e1f9e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 5 Aug 2019 13:30:09 +0200 Subject: [PATCH 157/219] RedisConnection: I/O the Redis protocol by itself (PoC) --- lib/redis/redisconnection.cpp | 433 ++++++++++++++++-------------- lib/redis/redisconnection.hpp | 317 +++++++++++++++++++--- lib/redis/rediswriter-objects.cpp | 22 +- lib/redis/rediswriter.cpp | 85 ++---- lib/redis/rediswriter.hpp | 2 - 5 files changed, 549 insertions(+), 310 deletions(-) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index 32b4272ea..080b64c4c 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -17,244 +17,283 @@ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * ******************************************************************************/ -#include "base/object.hpp" #include "redis/redisconnection.hpp" -#include "base/workqueue.hpp" -#include "base/logger.hpp" +#include "base/array.hpp" #include "base/convert.hpp" -#include "base/utility.hpp" -#include "redis/rediswriter.hpp" -#include "hiredis/hiredis.h" +#include "base/defer.hpp" +#include "base/io-engine.hpp" +#include "base/logger.hpp" +#include "base/objectlock.hpp" +#include "base/string.hpp" +#include "base/tcpsocket.hpp" +#include +#include +#include +#include +#include +#include #include #include using namespace icinga; +namespace asio = boost::asio; RedisConnection::RedisConnection(const String host, const int port, const String path, const String password, const int db) : - m_Host(host), m_Port(port), m_Path(path), m_Password(password), m_DbIndex(db), m_Context(NULL), m_Connected(false) + RedisConnection(IoEngine::Get().GetIoService(), host, port, path, password, db) { - m_RedisConnectionWorkQueue.SetName("RedisConnection"); } -void RedisConnection::StaticInitialize() +RedisConnection::RedisConnection(boost::asio::io_context& io, String host, int port, String path, String password, int db) + : m_Host(std::move(host)), m_Port(port), m_Path(std::move(path)), m_Password(std::move(password)), m_DbIndex(db), + m_Connecting(false), m_Connected(false), m_Strand(io), m_QueuedWrites(io), m_QueuedReads(io) { } void RedisConnection::Start() { - RedisConnection::Connect(); + if (!m_Connecting.exchange(true)) { + Ptr keepAlive (this); - std::thread thread(&RedisConnection::HandleRW, this); - thread.detach(); -} - -void RedisConnection::AssertOnWorkQueue() -{ - ASSERT(m_RedisConnectionWorkQueue.IsWorkerThread()); -} - -void RedisConnection::HandleRW() -{ - Utility::SetThreadName("RedisConnection Handler"); - - for (;;) { - try { - { - boost::mutex::scoped_lock lock(m_CMutex); - if (!m_Connected) - return; - redisAsyncHandleWrite(m_Context); - redisAsyncHandleRead(m_Context); - } - Utility::Sleep(0.1); - } catch (const std::exception&) { - Log(LogCritical, "RedisWriter", "Internal Redis Error"); - } + asio::spawn(m_Strand, [this, keepAlive](asio::yield_context yc) { Connect(yc); }); } } - -void RedisConnection::RedisInitialCallback(redisAsyncContext *c, void *r, void *p) -{ - auto state = (ConnectionState *) p; - if (state->state != Starting && !r) { - Log(LogCritical, "RedisConnection") - << "No answer from Redis during initial connection, is the Redis server running?"; - return; - } else if (r != nullptr) { - redisReply *rep = (redisReply *) r; - if (rep->type == REDIS_REPLY_ERROR) { - Log(LogCritical, "RedisConnection") - << "Failed to connect to Redis: " << rep->str; - state->conn->m_Connected = false; - return; - } - } - - if (state->state == Starting) { - state->state = Auth; - if (!state->conn->m_Password.IsEmpty()) { - boost::mutex::scoped_lock lock(state->conn->m_CMutex); - redisAsyncCommand(c, &RedisInitialCallback, p, "AUTH %s", state->conn->m_Password.CStr()); - return; - } - } - if (state->state == Auth) - { - state->state = DBSelect; - if (state->conn->m_DbIndex != 0) { - boost::mutex::scoped_lock lock(state->conn->m_CMutex); - redisAsyncCommand(c, &RedisInitialCallback, p, "SELECT %d", state->conn->m_DbIndex); - return; - } - } - if (state->state == DBSelect) - state->conn->m_Connected = true; -} bool RedisConnection::IsConnected() { - return m_Connected; + return m_Connected.load(); } - -void RedisConnection::Connect() +void RedisConnection::FireAndForgetQuery(RedisConnection::Query query) { - if (m_Connected) - return; + auto item (std::make_shared(std::move(query))); - Log(LogInformation, "RedisWriter", "Trying to connect to redis server Async"); - { - boost::mutex::scoped_lock lock(m_CMutex); - - if (m_Path.IsEmpty()) - m_Context = redisAsyncConnect(m_Host.CStr(), m_Port); - else - m_Context = redisAsyncConnectUnix(m_Path.CStr()); - - m_Context->data = (void*) this; - - redisAsyncSetConnectCallback(m_Context, &ConnectCallback); - redisAsyncSetDisconnectCallback(m_Context, &DisconnectCallback); - } - - m_State = ConnectionState{Starting, this}; - RedisInitialCallback(m_Context, nullptr, (void*)&m_State); + asio::post(m_Strand, [this, item]() { + m_Queues.FireAndForgetQuery.emplace(std::move(*item)); + m_QueuedWrites.Set(); + }); } -void RedisConnection::Disconnect() +void RedisConnection::FireAndForgetQueries(RedisConnection::Queries queries) { - redisAsyncDisconnect(m_Context); + auto item (std::make_shared(std::move(queries))); + + asio::post(m_Strand, [this, item]() { + m_Queues.FireAndForgetQueries.emplace(std::move(*item)); + m_QueuedWrites.Set(); + }); } -void RedisConnection::ConnectCallback(const redisAsyncContext *c, int status) +RedisConnection::Reply RedisConnection::GetResultOfQuery(RedisConnection::Query query) { - auto *rc = (RedisConnection* ) const_cast(c)->data; - if (status != REDIS_OK) { - if (c->err != 0) { - Log(LogCritical, "RedisConnection") - << "Redis connection failure: " << c->errstr; + std::promise promise; + auto future (promise.get_future()); + auto item (std::make_shared(std::move(query), std::move(promise))); + + asio::post(m_Strand, [this, item]() { + m_Queues.GetResultOfQuery.emplace(std::move(*item)); + m_QueuedWrites.Set(); + }); + + item = nullptr; + future.wait(); + return future.get(); +} + +RedisConnection::Replies RedisConnection::GetResultsOfQueries(RedisConnection::Queries queries) +{ + std::promise promise; + auto future (promise.get_future()); + auto item (std::make_shared(std::move(queries), std::move(promise))); + + asio::post(m_Strand, [this, item]() { + m_Queues.GetResultsOfQueries.emplace(std::move(*item)); + m_QueuedWrites.Set(); + }); + + item = nullptr; + future.wait(); + return future.get(); +} + +void RedisConnection::Connect(asio::yield_context& yc) +{ + Defer notConnecting ([this]() { + if (!m_Connected.load()) { + m_Connecting.store(false); + } + }); + + Log(LogInformation, "RedisWriter", "Trying to connect to Redis server (async)"); + + try { + if (m_Path.IsEmpty()) { + m_TcpConn = decltype(m_TcpConn)(new TcpConn(m_Strand.context())); + icinga::Connect(m_TcpConn->next_layer(), m_Host, Convert::ToString(m_Port), yc); } else { - Log(LogCritical, "RedisConnection") - << "Redis connection failure"; + m_UnixConn = decltype(m_UnixConn)(new UnixConn(m_Strand.context())); + m_UnixConn->next_layer().async_connect(Unix::endpoint(m_Path.CStr()), yc); } - rc->m_Connected = false; - } else { - Log(LogInformation, "RedisConnection") - << "Redis Connection: O N L I N E"; - } -} -// It's unfortunate we can not pass any user data here. All we get to do is log a message and hope for the best -void RedisConnection::DisconnectCallback(const redisAsyncContext *c, int status) -{ - auto *rc = (RedisConnection* ) const_cast(c)->data; - boost::mutex::scoped_lock lock(rc->m_CMutex); - if (status == REDIS_OK) - Log(LogInformation, "RedisConnection") << "Redis disconnected by us"; - else { - if (c->err != 0) - Log(LogCritical, "RedisConnection") << "Redis disconnected by server. Reason: " << c->errstr; - else - Log(LogCritical, "RedisConnection") << "Redis disconnected by server"; - } + { + Ptr keepAlive (this); - rc->m_Connected = false; -} - -void RedisConnection::ExecuteQuery(std::vector query, redisCallbackFn *fn, void *privdata) -{ - auto queryPtr (std::make_shared(std::move(query))); - - m_RedisConnectionWorkQueue.Enqueue([this, queryPtr, fn, privdata]() { - SendMessageInternal(*queryPtr, fn, privdata); - }); -} - -void -RedisConnection::ExecuteQueries(std::vector> queries, redisCallbackFn *fn, void *privdata) -{ - auto queriesPtr (std::make_shared(std::move(queries))); - - m_RedisConnectionWorkQueue.Enqueue([this, queriesPtr, fn, privdata]() { - SendMessagesInternal(*queriesPtr, fn, privdata); - }); -} - -void RedisConnection::SendMessageInternal(const std::vector& query, redisCallbackFn *fn, void *privdata) -{ - AssertOnWorkQueue(); - - { - boost::mutex::scoped_lock lock(m_CMutex); - - if (!m_Context || !m_Connected) { - Log(LogCritical, "RedisWriter") - << "Not connected to Redis"; - return; + asio::spawn(m_Strand, [this, keepAlive](asio::yield_context yc) { ReadLoop(yc); }); + asio::spawn(m_Strand, [this, keepAlive](asio::yield_context yc) { WriteLoop(yc); }); } - } - const char **argv; - size_t *argvlen; - - argv = new const char *[query.size()]; - argvlen = new size_t[query.size()]; - String debugstr; - - for (std::vector::size_type i = 0; i < query.size(); i++) { - argv[i] = query[i].CStr(); - argvlen[i] = query[i].GetLength(); - debugstr += argv[i]; - debugstr += " "; - } - - Log(LogDebug, "RedisWriter, Connection") - << "Sending Command: " << debugstr; - - int r; - - { - boost::mutex::scoped_lock lock(m_CMutex); - - r = redisAsyncCommandArgv(m_Context, fn, privdata, query.size(), argv, argvlen); - } - - delete[] argv; - delete[] argvlen; - - if (r == REDIS_REPLY_ERROR) { + m_Connected.store(true); + } catch (const boost::coroutines::detail::forced_unwind&) { + throw; + } catch (const std::exception& ex) { Log(LogCritical, "RedisWriter") - << "Redis Async query failed"; - - BOOST_THROW_EXCEPTION( - redis_error() - << errinfo_redis_query(Utility::Join(Array::FromVector(query), ' ', false)) - ); + << "Cannot connect to " << m_Host << ":" << m_Port << ": " << ex.what(); } } -void RedisConnection::SendMessagesInternal(const std::vector>& queries, redisCallbackFn *fn, void *privdata) +void RedisConnection::ReadLoop(asio::yield_context& yc) { - for (const auto& query : queries) { - SendMessageInternal(query, fn, privdata); + for (;;) { + m_QueuedReads.Wait(yc); + + do { + auto item (std::move(m_Queues.FutureResponseActions.front())); + m_Queues.FutureResponseActions.pop(); + + switch (item.Action) { + case ResponseAction::Ignore: + for (auto i (item.Amount); i; --i) { + ReadOne(yc); + } + break; + case ResponseAction::Deliver: + for (auto i (item.Amount); i; --i) { + auto promise (std::move(m_Queues.ReplyPromises.front())); + m_Queues.ReplyPromises.pop(); + + promise.set_value(ReadOne(yc)); + } + break; + case ResponseAction::DeliverBulk: + { + auto promise (std::move(m_Queues.RepliesPromises.front())); + m_Queues.RepliesPromises.pop(); + + Replies replies; + replies.reserve(item.Amount); + + for (auto i (item.Amount); i; --i) { + replies.emplace_back(ReadOne(yc)); + } + + promise.set_value(std::move(replies)); + } + } + } while (!m_Queues.FutureResponseActions.empty()); + + m_QueuedReads.Clear(); + } +} + +void RedisConnection::WriteLoop(asio::yield_context& yc) +{ + for (;;) { + m_QueuedWrites.Wait(yc); + + bool writtenAll = true; + + do { + writtenAll = true; + + if (!m_Queues.FireAndForgetQuery.empty()) { + auto item (std::move(m_Queues.FireAndForgetQuery.front())); + m_Queues.FireAndForgetQuery.pop(); + + if (m_Queues.FutureResponseActions.empty() || m_Queues.FutureResponseActions.back().Action != ResponseAction::Ignore) { + m_Queues.FutureResponseActions.emplace(FutureResponseAction{1, ResponseAction::Ignore}); + } else { + ++m_Queues.FutureResponseActions.back().Amount; + } + + m_QueuedReads.Set(); + writtenAll = false; + + WriteOne(item, yc); + } + + if (!m_Queues.FireAndForgetQueries.empty()) { + auto item (std::move(m_Queues.FireAndForgetQueries.front())); + m_Queues.FireAndForgetQueries.pop(); + + if (m_Queues.FutureResponseActions.empty() || m_Queues.FutureResponseActions.back().Action != ResponseAction::Ignore) { + m_Queues.FutureResponseActions.emplace(FutureResponseAction{item.size(), ResponseAction::Ignore}); + } else { + m_Queues.FutureResponseActions.back().Amount += item.size(); + } + + m_QueuedReads.Set(); + writtenAll = false; + + for (auto& query : item) { + WriteOne(query, yc); + } + } + + if (!m_Queues.GetResultOfQuery.empty()) { + auto item (std::move(m_Queues.GetResultOfQuery.front())); + m_Queues.GetResultOfQuery.pop(); + m_Queues.ReplyPromises.emplace(std::move(item.second)); + + if (m_Queues.FutureResponseActions.empty() || m_Queues.FutureResponseActions.back().Action != ResponseAction::Deliver) { + m_Queues.FutureResponseActions.emplace(FutureResponseAction{1, ResponseAction::Deliver}); + } else { + ++m_Queues.FutureResponseActions.back().Amount; + } + + m_QueuedReads.Set(); + writtenAll = false; + + WriteOne(item.first, yc); + } + + if (!m_Queues.GetResultsOfQueries.empty()) { + auto item (std::move(m_Queues.GetResultsOfQueries.front())); + m_Queues.GetResultsOfQueries.pop(); + m_Queues.RepliesPromises.emplace(std::move(item.second)); + m_Queues.FutureResponseActions.emplace(FutureResponseAction{item.first.size(), ResponseAction::DeliverBulk}); + + m_QueuedReads.Set(); + writtenAll = false; + + for (auto& query : item.first) { + WriteOne(query, yc); + } + } + } while (!writtenAll); + + m_QueuedWrites.Clear(); + + if (m_Path.IsEmpty()) { + m_TcpConn->async_flush(yc); + } else { + m_UnixConn->async_flush(yc); + } + + } +} + +RedisConnection::Reply RedisConnection::ReadOne(boost::asio::yield_context& yc) +{ + if (m_Path.IsEmpty()) { + return ReadRESP(*m_TcpConn, yc); + } else { + return ReadRESP(*m_UnixConn, yc); + } +} + +void RedisConnection::WriteOne(RedisConnection::Query& query, asio::yield_context& yc) +{ + if (m_Path.IsEmpty()) { + WriteRESP(*m_TcpConn, query, yc); + } else { + WriteRESP(*m_UnixConn, query, yc); } } diff --git a/lib/redis/redisconnection.hpp b/lib/redis/redisconnection.hpp index 7b1e21404..b6b5c005d 100644 --- a/lib/redis/redisconnection.hpp +++ b/lib/redis/redisconnection.hpp @@ -20,9 +20,35 @@ #ifndef REDISCONNECTION_H #define REDISCONNECTION_H -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "base/array.hpp" +#include "base/atomic.hpp" +#include "base/io-engine.hpp" #include "base/object.hpp" -#include "base/workqueue.hpp" +#include "base/string.hpp" +#include "base/value.hpp" namespace icinga { @@ -31,77 +57,288 @@ namespace icinga * * @ingroup redis */ - - enum conn_state{ - Starting, - Auth, - DBSelect, - Done, - }; - - class RedisConnection; - struct ConnectionState { - conn_state state; - RedisConnection *conn; - }; - class RedisConnection final : public Object { public: DECLARE_PTR_TYPEDEFS(RedisConnection); + typedef std::vector Query; + typedef std::vector Queries; + typedef Value Reply; + typedef std::vector Replies; + RedisConnection(const String host, const int port, const String path, const String password = "", const int db = 0); void Start(); - void Connect(); - - void Disconnect(); - bool IsConnected(); - void ExecuteQuery(std::vector query, redisCallbackFn *fn = NULL, void *privdata = NULL); + void FireAndForgetQuery(Query query); + void FireAndForgetQueries(Queries queries); - void ExecuteQueries(std::vector> queries, redisCallbackFn *fn = NULL, - void *privdata = NULL); + Reply GetResultOfQuery(Query query); + Replies GetResultsOfQueries(Queries queries); private: - static void StaticInitialize(); + enum class ResponseAction : unsigned char + { + Ignore, Deliver, DeliverBulk + }; - void SendMessageInternal(const std::vector& query, redisCallbackFn *fn, void *privdata); - void SendMessagesInternal(const std::vector>& queries, redisCallbackFn *fn, void *privdata); + struct FutureResponseAction + { + size_t Amount; + ResponseAction Action; + }; - void AssertOnWorkQueue(); + typedef boost::asio::ip::tcp Tcp; + typedef boost::asio::local::stream_protocol Unix; - void HandleRW(); + typedef boost::asio::buffered_stream TcpConn; + typedef boost::asio::buffered_stream UnixConn; - static void DisconnectCallback(const redisAsyncContext *c, int status); - static void ConnectCallback(const redisAsyncContext *c, int status); + template + static Value ReadRESP(AsyncReadStream& stream, boost::asio::yield_context& yc); - static void RedisInitialCallback(redisAsyncContext *c, void *r, void *p); + template + static std::vector ReadLine(AsyncReadStream& stream, boost::asio::yield_context& yc, size_t hint = 0); + template + static void WriteRESP(AsyncWriteStream& stream, const Query& query, boost::asio::yield_context& yc); - WorkQueue m_RedisConnectionWorkQueue{100000}; - Timer::Ptr m_EventLoop; + template + static void WriteInt(AsyncWriteStream& stream, intmax_t i, boost::asio::yield_context& yc); - redisAsyncContext *m_Context; + RedisConnection(boost::asio::io_context& io, String host, int port, String path, String password, int db); + + void Connect(boost::asio::yield_context& yc); + void ReadLoop(boost::asio::yield_context& yc); + void WriteLoop(boost::asio::yield_context& yc); + Reply ReadOne(boost::asio::yield_context& yc); + void WriteOne(Query& query, boost::asio::yield_context& yc); String m_Path; String m_Host; int m_Port; String m_Password; int m_DbIndex; - bool m_Connected; - boost::mutex m_CMutex; - ConnectionState m_State; + boost::asio::io_context::strand m_Strand; + std::unique_ptr m_TcpConn; + std::unique_ptr m_UnixConn; + Atomic m_Connecting, m_Connected; + struct { + std::queue FireAndForgetQuery; + std::queue FireAndForgetQueries; + std::queue>> GetResultOfQuery; + std::queue>> GetResultsOfQueries; + std::queue> ReplyPromises; + std::queue> RepliesPromises; + std::queue FutureResponseActions; + } m_Queues; + + AsioConditionVariable m_QueuedWrites, m_QueuedReads; }; - struct redis_error : virtual std::exception, virtual boost::exception { }; +class RedisError final : public Object +{ +public: + DECLARE_PTR_TYPEDEFS(RedisError); - struct errinfo_redis_query_; - typedef boost::error_info errinfo_redis_query; + inline RedisError(String message) : m_Message(std::move(message)) + { + } + + inline const String& GetMessage() + { + return m_Message; + } + +private: + String m_Message; +}; + +class RedisProtocolError : public std::runtime_error +{ +protected: + inline RedisProtocolError() : runtime_error("") + { + } +}; + +class BadRedisType : public RedisProtocolError +{ +public: + inline BadRedisType(char type) : m_What{type, 0} + { + } + + virtual const char * what() const noexcept override + { + return m_What; + } + +private: + char m_What[2]; +}; + +class BadRedisInt : public RedisProtocolError +{ +public: + inline BadRedisInt(std::vector intStr) : m_What(std::move(intStr)) + { + m_What.emplace_back(0); + } + + virtual const char * what() const noexcept override + { + return m_What.data(); + } + +private: + std::vector m_What; +}; + +template +Value RedisConnection::ReadRESP(AsyncReadStream& stream, boost::asio::yield_context& yc) +{ + namespace asio = boost::asio; + + char type = 0; + asio::async_read(stream, asio::mutable_buffer(&type, 1), yc); + + switch (type) { + case '+': + { + auto buf (ReadLine(stream, yc)); + return String(buf.begin(), buf.end()); + } + case '-': + { + auto buf (ReadLine(stream, yc)); + return new RedisError(String(buf.begin(), buf.end())); + } + case ':': + { + auto buf (ReadLine(stream, yc, 21)); + intmax_t i = 0; + + try { + i = boost::lexical_cast(boost::string_view(buf.data(), buf.size())); + } catch (...) { + throw BadRedisInt(std::move(buf)); + } + + return (double)i; + } + case '$': + { + auto buf (ReadLine(stream, yc, 21)); + intmax_t i = 0; + + try { + i = boost::lexical_cast(boost::string_view(buf.data(), buf.size())); + } catch (...) { + throw BadRedisInt(std::move(buf)); + } + + if (i < 0) { + return Value(); + } + + buf.clear(); + buf.insert(buf.end(), i, 0); + asio::async_read(stream, asio::mutable_buffer(buf.data(), buf.size()), yc); + + { + char crlf[2]; + asio::async_read(stream, asio::mutable_buffer(crlf, 2), yc); + } + + return String(buf.begin(), buf.end()); + } + case '*': + { + auto buf (ReadLine(stream, yc, 21)); + intmax_t i = 0; + + try { + i = boost::lexical_cast(boost::string_view(buf.data(), buf.size())); + } catch (...) { + throw BadRedisInt(std::move(buf)); + } + + Array::Ptr arr = new Array(); + + if (i < 0) { + i = 0; + } + + arr->Reserve(i); + + for (; i; --i) { + arr->Add(ReadRESP(stream, yc)); + } + + return arr; + } + default: + throw BadRedisType(type); + } +} + +template +std::vector RedisConnection::ReadLine(AsyncReadStream& stream, boost::asio::yield_context& yc, size_t hint) +{ + namespace asio = boost::asio; + + std::vector line; + line.reserve(hint); + + char next = 0; + asio::mutable_buffer buf (&next, 1); + + for (;;) { + asio::async_read(stream, buf, yc); + + if (next == '\r') { + asio::async_read(stream, buf, yc); + return std::move(line); + } + + line.emplace_back(next); + } +} + +template +void RedisConnection::WriteRESP(AsyncWriteStream& stream, const Query& query, boost::asio::yield_context& yc) +{ + namespace asio = boost::asio; + + asio::async_write(stream, asio::const_buffer("*", 1), yc); + WriteInt(stream, query.size(), yc); + asio::async_write(stream, asio::const_buffer("\r\n", 2), yc); + + for (auto& arg : query) { + asio::async_write(stream, asio::const_buffer("$", 1), yc); + WriteInt(stream, arg.GetLength(), yc); + asio::async_write(stream, asio::const_buffer("\r\n", 2), yc); + asio::async_write(stream, asio::const_buffer(arg.CStr(), arg.GetLength()), yc); + asio::async_write(stream, asio::const_buffer("\r\n", 2), yc); + } +} + +template +void RedisConnection::WriteInt(AsyncWriteStream& stream, intmax_t i, boost::asio::yield_context& yc) +{ + namespace asio = boost::asio; + + char buf[21] = {}; + sprintf(buf, "%jd", i); + + asio::async_write(stream, asio::const_buffer(buf, strlen(buf)), yc); +} } diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 2c930e380..e02d33bc6 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -166,7 +166,7 @@ void RedisWriter::UpdateAllConfigObjects() if (transaction.size() > 1) { transaction.push_back({"EXEC"}); - m_Rcon->ExecuteQueries(std::move(transaction)); + m_Rcon->FireAndForgetQueries(std::move(transaction)); transaction = {{"MULTI"}}; } } @@ -184,10 +184,10 @@ void RedisWriter::UpdateAllConfigObjects() if (transaction.size() > 1) { transaction.push_back({"EXEC"}); - m_Rcon->ExecuteQueries(std::move(transaction)); + m_Rcon->FireAndForgetQueries(std::move(transaction)); } - m_Rcon->ExecuteQuery({"PUBLISH", "icinga:config:dump", lcType}); + m_Rcon->FireAndForgetQuery({"PUBLISH", "icinga:config:dump", lcType}); Log(LogNotice, "RedisWriter") << "Dumped " << bulkCounter << " objects of type " << type.second; @@ -249,7 +249,7 @@ void RedisWriter::DeleteKeys(const std::vector& keys) { query.emplace_back(key); } - m_Rcon->ExecuteQuery(std::move(query)); + m_Rcon->FireAndForgetQuery(std::move(query)); } std::vector RedisWriter::GetTypeObjectKeys(const String& type) @@ -649,7 +649,7 @@ void RedisWriter::UpdateState(const Checkable::Ptr& checkable) { Dictionary::Ptr stateAttrs = SerializeState(checkable); - m_Rcon->ExecuteQuery({"HSET", m_PrefixStateObject + GetLowerCaseTypeNameDB(checkable), GetObjectIdentifier(checkable), JsonEncode(stateAttrs)}); + m_Rcon->FireAndForgetQuery({"HSET", m_PrefixStateObject + GetLowerCaseTypeNameDB(checkable), GetObjectIdentifier(checkable), JsonEncode(stateAttrs)}); } // Used to update a single object, used for runtime updates @@ -666,7 +666,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtime CreateConfigUpdate(object, typeName, statements, runtimeUpdate); Checkable::Ptr checkable = dynamic_pointer_cast(object); if (checkable) { - m_Rcon->ExecuteQuery({"HSET", m_PrefixStateObject + typeName, + m_Rcon->FireAndForgetQuery({"HSET", m_PrefixStateObject + typeName, GetObjectIdentifier(checkable), JsonEncode(SerializeState(checkable))}); } @@ -681,7 +681,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtime if (transaction.size() > 1) { transaction.push_back({"EXEC"}); - m_Rcon->ExecuteQueries(std::move(transaction)); + m_Rcon->FireAndForgetQueries(std::move(transaction)); } } @@ -965,7 +965,7 @@ RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, const String ty /* Send an update event to subscribers. */ if (runtimeUpdate) { - m_Rcon->ExecuteQuery({"PUBLISH", "icinga:config:update", typeName + ":" + objectKey}); + m_Rcon->FireAndForgetQuery({"PUBLISH", "icinga:config:update", typeName + ":" + objectKey}); } } @@ -974,7 +974,7 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) String typeName = object->GetReflectionType()->GetName().ToLower(); String objectKey = GetObjectIdentifier(object); - m_Rcon->ExecuteQueries({ + m_Rcon->FireAndForgetQueries({ {"HDEL", m_PrefixConfigObject + typeName, objectKey}, {"DEL", m_PrefixStateObject + typeName + ":" + objectKey}, {"PUBLISH", "icinga:config:delete", typeName + ":" + objectKey} @@ -1010,7 +1010,7 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) streamadd.emplace_back(kv.second); } - m_Rcon->ExecuteQuery(std::move(streamadd)); + m_Rcon->FireAndForgetQuery(std::move(streamadd)); } Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) @@ -1147,7 +1147,7 @@ RedisWriter::UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, typeName = typeNameOverride.ToLower(); return {GetObjectIdentifier(object), JsonEncode(attrs)}; - //m_Rcon->ExecuteQuery({"HSET", keyPrefix + typeName, GetObjectIdentifier(object), JsonEncode(attrs)}); + //m_Rcon->FireAndForgetQuery({"HSET", keyPrefix + typeName, GetObjectIdentifier(object), JsonEncode(attrs)}); } void RedisWriter::StateChangeHandler(const ConfigObject::Ptr &object) diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index 1ec7a9823..1b9043176 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -150,29 +150,23 @@ void RedisWriter::UpdateSubscriptions() if (!m_Rcon || !m_Rcon->IsConnected()) return; - long long cursor = 0; - + String cursor = "0"; String keyPrefix = "icinga:subscription:"; do { - auto reply = RedisGet({ "SCAN", Convert::ToString(cursor), "MATCH", keyPrefix + "*", "COUNT", "1000" }); + Array::Ptr reply = m_Rcon->GetResultOfQuery({ "SCAN", cursor, "MATCH", keyPrefix + "*", "COUNT", "1000" }); + VERIFY(reply->GetLength() % 2u == 0u); - VERIFY(reply->type == REDIS_REPLY_ARRAY); - VERIFY(reply->elements % 2 == 0); + cursor = reply->Get(0); - redisReply *cursorReply = reply->element[0]; - cursor = Convert::ToLong(cursorReply->str); + Array::Ptr keys = reply->Get(1); + ObjectLock oLock (keys); - redisReply *keysReply = reply->element[1]; - - for (size_t i = 0; i < keysReply->elements; i++) { - if (boost::algorithm::ends_with(keysReply->element[i]->str, ":limit")) + for (String key : keys) { + if (boost::algorithm::ends_with(key, ":limit")) continue; - redisReply *keyReply = keysReply->element[i]; - VERIFY(keyReply->type == REDIS_REPLY_STRING); RedisSubscriptionInfo rsi; - String key = keysReply->element[i]->str; if (!RedisWriter::GetSubscriptionTypes(key, rsi)) { Log(LogInformation, "RedisWriter") @@ -181,7 +175,7 @@ void RedisWriter::UpdateSubscriptions() m_Subscriptions[key.SubStr(keyPrefix.GetLength())] = rsi; } } - } while (cursor != 0); + } while (cursor != "0"); Log(LogInformation, "RedisWriter") << "Current Redis event subscriptions: " << m_Subscriptions.size(); @@ -190,14 +184,17 @@ void RedisWriter::UpdateSubscriptions() bool RedisWriter::GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi) { try { - auto redisReply = RedisGet({ "SMEMBERS", key }); - VERIFY(redisReply->type == REDIS_REPLY_ARRAY); + Array::Ptr redisReply = m_Rcon->GetResultOfQuery({ "SMEMBERS", key }); - if (redisReply->elements == 0) + if (redisReply->GetLength() == 0) return false; - for (size_t j = 0; j < redisReply->elements; j++) { - rsi.EventTypes.insert(redisReply->element[j]->str); + { + ObjectLock oLock (redisReply); + + for (String member : redisReply) { + rsi.EventTypes.insert(member); + } } Log(LogInformation, "RedisWriter") @@ -229,7 +226,7 @@ void RedisWriter::PublishStats() status->Set("config_dump_in_progress", m_ConfigDumpInProgress); String jsonStats = JsonEncode(status); - m_Rcon->ExecuteQuery({ "PUBLISH", "icinga:stats", jsonStats }); + m_Rcon->FireAndForgetQuery({ "PUBLISH", "icinga:stats", jsonStats }); } void RedisWriter::HandleEvents() @@ -281,20 +278,19 @@ void RedisWriter::HandleEvent(const Dictionary::Ptr& event) String body = JsonEncode(event); - auto maxExists = RedisGet({ "EXISTS", "icinga:subscription:" + name + ":limit" }); + double maxExists = m_Rcon->GetResultOfQuery({ "EXISTS", "icinga:subscription:" + name + ":limit" }); long maxEvents = MAX_EVENTS_DEFAULT; - if (maxExists->integer) { - auto redisReply = RedisGet({ "GET", "icinga:subscription:" + name + ":limit"}); - VERIFY(redisReply->type == REDIS_REPLY_STRING); + if (maxExists != 0) { + String redisReply = m_Rcon->GetResultOfQuery({ "GET", "icinga:subscription:" + name + ":limit"}); Log(LogInformation, "RedisWriter") - << "Got limit " << redisReply->str << " for " << name; + << "Got limit " << redisReply << " for " << name; - maxEvents = Convert::ToLong(redisReply->str); + maxEvents = Convert::ToLong(redisReply); } - m_Rcon->ExecuteQueries({ + m_Rcon->FireAndForgetQueries({ { "MULTI" }, { "LPUSH", "icinga:event:" + name, body }, { "LTRIM", "icinga:event:" + name, "0", String(maxEvents - 1)}, @@ -354,7 +350,7 @@ void RedisWriter::SendEvent(const Dictionary::Ptr& event) // Log(LogInformation, "RedisWriter") // << "Sending event \"" << body << "\""; - m_Rcon->ExecuteQueries({ + m_Rcon->FireAndForgetQueries({ { "PUBLISH", "icinga:event:all", body }, { "PUBLISH", "icinga:event:" + event->Get("type"), body }}); } @@ -385,21 +381,6 @@ struct synchronousWait { redisReply* reply; }; -void RedisWriter::RedisQueryCallback(redisAsyncContext *c, void *r, void *p) { - auto wait = (struct synchronousWait*) p; - auto rp = reinterpret_cast(r); - - - if (r == NULL) - wait->reply = nullptr; - else - wait->reply = RedisWriter::dupReplyObject(rp); - - boost::mutex::scoped_lock lock(wait->mtx); - wait->ready = true; - wait->cv.notify_all(); -} - struct RedisReplyDeleter { inline void operator() (redisReply *reply) @@ -407,19 +388,3 @@ struct RedisReplyDeleter freeReplyObject(reply); } }; - -std::shared_ptr RedisWriter::RedisGet(std::vector query) { - auto *wait = new synchronousWait; - wait->ready = false; - - m_Rcon->ExecuteQuery(std::move(query), RedisQueryCallback, wait); - - boost::mutex::scoped_lock lock(wait->mtx); - while (!wait->ready) { - wait->cv.wait(lock); - if (!wait->ready) - wait->ready = true; - } - - return std::shared_ptr(wait->reply, RedisReplyDeleter()); -} diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 17fbc10a1..b62a82cb4 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -115,8 +115,6 @@ private: void ExceptionHandler(boost::exception_ptr exp); //Used to get a reply from the asyncronous connection - std::shared_ptr RedisGet(std::vector query); - static void RedisQueryCallback(redisAsyncContext *c, void *r, void *p); static redisReply* dupReplyObject(redisReply* reply); From 132b2dcb77755edf145169856eb9d3a27e616f98 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 5 Aug 2019 13:53:05 +0200 Subject: [PATCH 158/219] Get rid of hiredis --- CMakeLists.txt | 4 - lib/redis/CMakeLists.txt | 2 - lib/redis/rediswriter-utility.cpp | 31 - lib/redis/rediswriter.cpp | 21 - lib/redis/rediswriter.hpp | 5 - third-party/CMakeLists.txt | 4 - third-party/hiredis/.gitignore | 7 - third-party/hiredis/.travis.yml | 24 - third-party/hiredis/CHANGELOG.md | 110 -- third-party/hiredis/CMakeLists.txt | 28 - third-party/hiredis/COPYING | 29 - third-party/hiredis/Makefile | 217 ---- third-party/hiredis/README.md | 392 ------ third-party/hiredis/adapters/ae.h | 127 -- third-party/hiredis/adapters/glib.h | 153 --- third-party/hiredis/adapters/ivykis.h | 81 -- third-party/hiredis/adapters/libev.h | 147 --- third-party/hiredis/adapters/libevent.h | 108 -- third-party/hiredis/adapters/libuv.h | 121 -- third-party/hiredis/adapters/macosx.h | 114 -- third-party/hiredis/adapters/qt.h | 135 -- third-party/hiredis/async.c | 687 ----------- third-party/hiredis/async.h | 129 -- third-party/hiredis/dict.c | 338 ----- third-party/hiredis/dict.h | 126 -- third-party/hiredis/examples/example-ae.c | 62 - third-party/hiredis/examples/example-glib.c | 73 -- third-party/hiredis/examples/example-ivykis.c | 58 - third-party/hiredis/examples/example-libev.c | 52 - .../hiredis/examples/example-libevent.c | 53 - third-party/hiredis/examples/example-libuv.c | 53 - third-party/hiredis/examples/example-macosx.c | 66 - third-party/hiredis/examples/example-qt.cpp | 46 - third-party/hiredis/examples/example-qt.h | 32 - third-party/hiredis/examples/example.c | 78 -- third-party/hiredis/fmacros.h | 21 - third-party/hiredis/hiredis.c | 1021 --------------- third-party/hiredis/hiredis.h | 223 ---- third-party/hiredis/net.c | 458 ------- third-party/hiredis/net.h | 53 - third-party/hiredis/read.c | 525 -------- third-party/hiredis/read.h | 116 -- third-party/hiredis/sds.c | 1095 ----------------- third-party/hiredis/sds.h | 105 -- third-party/hiredis/test.c | 807 ------------ third-party/hiredis/win32.h | 42 - 46 files changed, 8179 deletions(-) delete mode 100644 third-party/hiredis/.gitignore delete mode 100644 third-party/hiredis/.travis.yml delete mode 100644 third-party/hiredis/CHANGELOG.md delete mode 100644 third-party/hiredis/CMakeLists.txt delete mode 100644 third-party/hiredis/COPYING delete mode 100644 third-party/hiredis/Makefile delete mode 100644 third-party/hiredis/README.md delete mode 100644 third-party/hiredis/adapters/ae.h delete mode 100644 third-party/hiredis/adapters/glib.h delete mode 100644 third-party/hiredis/adapters/ivykis.h delete mode 100644 third-party/hiredis/adapters/libev.h delete mode 100644 third-party/hiredis/adapters/libevent.h delete mode 100644 third-party/hiredis/adapters/libuv.h delete mode 100644 third-party/hiredis/adapters/macosx.h delete mode 100644 third-party/hiredis/adapters/qt.h delete mode 100644 third-party/hiredis/async.c delete mode 100644 third-party/hiredis/async.h delete mode 100644 third-party/hiredis/dict.c delete mode 100644 third-party/hiredis/dict.h delete mode 100644 third-party/hiredis/examples/example-ae.c delete mode 100644 third-party/hiredis/examples/example-glib.c delete mode 100644 third-party/hiredis/examples/example-ivykis.c delete mode 100644 third-party/hiredis/examples/example-libev.c delete mode 100644 third-party/hiredis/examples/example-libevent.c delete mode 100644 third-party/hiredis/examples/example-libuv.c delete mode 100644 third-party/hiredis/examples/example-macosx.c delete mode 100644 third-party/hiredis/examples/example-qt.cpp delete mode 100644 third-party/hiredis/examples/example-qt.h delete mode 100644 third-party/hiredis/examples/example.c delete mode 100644 third-party/hiredis/fmacros.h delete mode 100644 third-party/hiredis/hiredis.c delete mode 100644 third-party/hiredis/hiredis.h delete mode 100644 third-party/hiredis/net.c delete mode 100644 third-party/hiredis/net.h delete mode 100644 third-party/hiredis/read.c delete mode 100644 third-party/hiredis/read.h delete mode 100644 third-party/hiredis/sds.c delete mode 100644 third-party/hiredis/sds.h delete mode 100644 third-party/hiredis/test.c delete mode 100644 third-party/hiredis/win32.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e1b016d1..59e0533d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -204,10 +204,6 @@ if(HAVE_SYSTEMD) list(APPEND base_DEPS systemd) endif() -if(ICINGA2_WITH_REDIS) - list(APPEND base_OBJS $) -endif() - if(EDITLINE_FOUND) list(APPEND base_DEPS ${EDITLINE_LIBRARIES}) include_directories(${EDITLINE_INCLUDE_DIR}) diff --git a/lib/redis/CMakeLists.txt b/lib/redis/CMakeLists.txt index 5e953ad7e..bb00ca343 100644 --- a/lib/redis/CMakeLists.txt +++ b/lib/redis/CMakeLists.txt @@ -31,8 +31,6 @@ include_directories(${icinga2_SOURCE_DIR}/third-party) add_dependencies(redis base config icinga remote) -link_directories(${icinga2_BINARY_DIR}/third-party/hiredis) - set_target_properties ( redis PROPERTIES FOLDER Components diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 2ca9cd739..cb65f1d2e 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -277,34 +277,3 @@ String RedisWriter::GetLowerCaseTypeNameDB(const ConfigObject::Ptr& obj) return typeName; } - -//Used to duplicate a redisReply, needed as redisReplies are freed when the async callback finishes -redisReply* RedisWriter::dupReplyObject(redisReply* reply) -{ - redisReply* r = (redisReply*)calloc(1, sizeof(*r)); - memcpy(r, reply, sizeof(*r)); - if(REDIS_REPLY_ERROR==reply->type || REDIS_REPLY_STRING==reply->type || REDIS_REPLY_STATUS==reply->type) //copy str - { - r->str = (char*)malloc(reply->len+1); - memcpy(r->str, reply->str, reply->len); - r->str[reply->len] = '\0'; - } - else if(REDIS_REPLY_ARRAY==reply->type) //copy array - { - r->element = (redisReply**)calloc(reply->elements, sizeof(redisReply*)); - memset(r->element, 0, r->elements*sizeof(redisReply*)); - for(uint32_t i=0; ielements; ++i) - { - if(NULL!=reply->element[i]) - { - if( NULL == (r->element[i] = dupReplyObject(reply->element[i])) ) - { - //clone child failed, free current reply, and return NULL - freeReplyObject(r); - return NULL; - } - } - } - } - return r; -} diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index 1b9043176..7670537e1 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -367,24 +367,3 @@ void RedisWriter::AssertOnWorkQueue() { ASSERT(m_WorkQueue.IsWorkerThread()); } - - -/* - * This whole spiel is required as we mostly use a "fire and forget" approach with the Redis Connection. To wait for a - * reply from Redis we have to wait for the callback to finish, this is done with the help of this struct. ready, cv - * and mtx are used for making sure we have the redisReply when we return. - */ -struct synchronousWait { - bool ready; - boost::condition_variable cv; - boost::mutex mtx; - redisReply* reply; -}; - -struct RedisReplyDeleter -{ - inline void operator() (redisReply *reply) - { - freeReplyObject(reply); - } -}; diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index b62a82cb4..240af965d 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -30,7 +30,6 @@ #include "icinga/service.hpp" #include "icinga/downtime.hpp" #include -#include namespace icinga { @@ -114,10 +113,6 @@ private: void ExceptionHandler(boost::exception_ptr exp); - //Used to get a reply from the asyncronous connection - static redisReply* dupReplyObject(redisReply* reply); - - Timer::Ptr m_StatsTimer; Timer::Ptr m_ReconnectTimer; Timer::Ptr m_SubscriptionTimer; diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index fa5f5bfce..fea750f1d 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -6,8 +6,4 @@ if(UNIX OR CYGWIN) add_subdirectory(execvpe) endif() -if(ICINGA2_WITH_REDIS) - add_subdirectory(hiredis) -endif() - add_subdirectory(socketpair) diff --git a/third-party/hiredis/.gitignore b/third-party/hiredis/.gitignore deleted file mode 100644 index c44b5c537..000000000 --- a/third-party/hiredis/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -/hiredis-test -/examples/hiredis-example* -/*.o -/*.so -/*.dylib -/*.a -/*.pc diff --git a/third-party/hiredis/.travis.yml b/third-party/hiredis/.travis.yml deleted file mode 100644 index 1e1ce3006..000000000 --- a/third-party/hiredis/.travis.yml +++ /dev/null @@ -1,24 +0,0 @@ -language: c -sudo: false -compiler: - - gcc - - clang - -addons: - apt: - packages: - - libc6-dbg - - libc6-dev - - libc6:i386 - - libc6-dev-i386 - - libc6-dbg:i386 - - gcc-multilib - - valgrind - -env: - - CFLAGS="-Werror" - - PRE="valgrind --track-origins=yes --leak-check=full" - - TARGET="32bit" TARGET_VARS="32bit-vars" CFLAGS="-Werror" - - TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" - -script: make $TARGET CFLAGS="$CFLAGS" && make check PRE="$PRE" && make $TARGET_VARS hiredis-example diff --git a/third-party/hiredis/CHANGELOG.md b/third-party/hiredis/CHANGELOG.md deleted file mode 100644 index a5015c5da..000000000 --- a/third-party/hiredis/CHANGELOG.md +++ /dev/null @@ -1,110 +0,0 @@ -### 0.13.3 (2015-09-16) - -* Revert "Clear `REDIS_CONNECTED` flag when connection is closed". -* Make tests pass on FreeBSD (Thanks, Giacomo Olgeni) - - -If the `REDIS_CONNECTED` flag is cleared, -the async onDisconnect callback function will never be called. -This causes problems as the disconnect is never reported back to the user. - -### 0.13.2 (2015-08-25) - -* Prevent crash on pending replies in async code (Thanks, @switch-st) -* Clear `REDIS_CONNECTED` flag when connection is closed (Thanks, Jerry Jacobs) -* Add MacOS X addapter (Thanks, @dizzus) -* Add Qt adapter (Thanks, Pietro Cerutti) -* Add Ivykis adapter (Thanks, Gergely Nagy) - -All adapters are provided as is and are only tested where possible. - -### 0.13.1 (2015-05-03) - -This is a bug fix release. -The new `reconnect` method introduced new struct members, which clashed with pre-defined names in pre-C99 code. -Another commit forced C99 compilation just to make it work, but of course this is not desirable for outside projects. -Other non-C99 code can now use hiredis as usual again. -Sorry for the inconvenience. - -* Fix memory leak in async reply handling (Salvatore Sanfilippo) -* Rename struct member to avoid name clash with pre-c99 code (Alex Balashov, ncopa) - -### 0.13.0 (2015-04-16) - -This release adds a minimal Windows compatibility layer. -The parser, standalone since v0.12.0, can now be compiled on Windows -(and thus used in other client libraries as well) - -* Windows compatibility layer for parser code (tzickel) -* Properly escape data printed to PKGCONF file (Dan Skorupski) -* Fix tests when assert() undefined (Keith Bennett, Matt Stancliff) -* Implement a reconnect method for the client context, this changes the structure of `redisContext` (Aaron Bedra) - -### 0.12.1 (2015-01-26) - -* Fix `make install`: DESTDIR support, install all required files, install PKGCONF in proper location -* Fix `make test` as 32 bit build on 64 bit platform - -### 0.12.0 (2015-01-22) - -* Add optional KeepAlive support - -* Try again on EINTR errors - -* Add libuv adapter - -* Add IPv6 support - -* Remove possiblity of multiple close on same fd - -* Add ability to bind source address on connect - -* Add redisConnectFd() and redisFreeKeepFd() - -* Fix getaddrinfo() memory leak - -* Free string if it is unused (fixes memory leak) - -* Improve redisAppendCommandArgv performance 2.5x - -* Add support for SO_REUSEADDR - -* Fix redisvFormatCommand format parsing - -* Add GLib 2.0 adapter - -* Refactor reading code into read.c - -* Fix errno error buffers to not clobber errors - -* Generate pkgconf during build - -* Silence _BSD_SOURCE warnings - -* Improve digit counting for multibulk creation - - -### 0.11.0 - -* Increase the maximum multi-bulk reply depth to 7. - -* Increase the read buffer size from 2k to 16k. - -* Use poll(2) instead of select(2) to support large fds (>= 1024). - -### 0.10.1 - -* Makefile overhaul. Important to check out if you override one or more - variables using environment variables or via arguments to the "make" tool. - -* Issue #45: Fix potential memory leak for a multi bulk reply with 0 elements - being created by the default reply object functions. - -* Issue #43: Don't crash in an asynchronous context when Redis returns an error - reply after the connection has been made (this happens when the maximum - number of connections is reached). - -### 0.10.0 - -* See commit log. - diff --git a/third-party/hiredis/CMakeLists.txt b/third-party/hiredis/CMakeLists.txt deleted file mode 100644 index e1a00a443..000000000 --- a/third-party/hiredis/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -# Icinga 2 -# Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation -# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. - -add_library(hiredis OBJECT net.c net.h hiredis.c hiredis.h sds.c sds.h async.c async.h read.c read.h) - -if(HAVE_VISIBILITY_HIDDEN) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=default") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=default") -endif() - -set_target_properties ( - hiredis PROPERTIES - FOLDER Lib -) diff --git a/third-party/hiredis/COPYING b/third-party/hiredis/COPYING deleted file mode 100644 index a5fc97395..000000000 --- a/third-party/hiredis/COPYING +++ /dev/null @@ -1,29 +0,0 @@ -Copyright (c) 2009-2011, Salvatore Sanfilippo -Copyright (c) 2010-2011, Pieter Noordhuis - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of Redis nor the names of its contributors may be used - to endorse or promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/third-party/hiredis/Makefile b/third-party/hiredis/Makefile deleted file mode 100644 index cff2a84ce..000000000 --- a/third-party/hiredis/Makefile +++ /dev/null @@ -1,217 +0,0 @@ -# Hiredis Makefile -# Copyright (C) 2010-2011 Salvatore Sanfilippo -# Copyright (C) 2010-2011 Pieter Noordhuis -# This file is released under the BSD license, see the COPYING file - -OBJ=net.o hiredis.o sds.o async.o read.o -EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib -TESTS=hiredis-test -LIBNAME=libhiredis -PKGCONFNAME=hiredis.pc - -HIREDIS_MAJOR=$(shell grep HIREDIS_MAJOR hiredis.h | awk '{print $$3}') -HIREDIS_MINOR=$(shell grep HIREDIS_MINOR hiredis.h | awk '{print $$3}') -HIREDIS_PATCH=$(shell grep HIREDIS_PATCH hiredis.h | awk '{print $$3}') -HIREDIS_SONAME=$(shell grep HIREDIS_SONAME hiredis.h | awk '{print $$3}') - -# Installation related variables and target -PREFIX?=/usr/local -INCLUDE_PATH?=include/hiredis -LIBRARY_PATH?=lib -PKGCONF_PATH?=pkgconfig -INSTALL_INCLUDE_PATH= $(DESTDIR)$(PREFIX)/$(INCLUDE_PATH) -INSTALL_LIBRARY_PATH= $(DESTDIR)$(PREFIX)/$(LIBRARY_PATH) -INSTALL_PKGCONF_PATH= $(INSTALL_LIBRARY_PATH)/$(PKGCONF_PATH) - -# redis-server configuration used for testing -REDIS_PORT=56379 -REDIS_SERVER=redis-server -define REDIS_TEST_CONFIG - daemonize yes - pidfile /tmp/hiredis-test-redis.pid - port $(REDIS_PORT) - bind 127.0.0.1 - unixsocket /tmp/hiredis-test-redis.sock -endef -export REDIS_TEST_CONFIG - -# Fallback to gcc when $CC is not in $PATH. -CC:=$(shell sh -c 'type $(CC) >/dev/null 2>/dev/null && echo $(CC) || echo gcc') -CXX:=$(shell sh -c 'type $(CXX) >/dev/null 2>/dev/null && echo $(CXX) || echo g++') -OPTIMIZATION?=-O3 -WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings -DEBUG?= -g -ggdb -REAL_CFLAGS=$(OPTIMIZATION) -fPIC $(CFLAGS) $(WARNINGS) $(DEBUG) $(ARCH) -REAL_LDFLAGS=$(LDFLAGS) $(ARCH) - -DYLIBSUFFIX=so -STLIBSUFFIX=a -DYLIB_MINOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME) -DYLIB_MAJOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR) -DYLIBNAME=$(LIBNAME).$(DYLIBSUFFIX) -DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) -STLIBNAME=$(LIBNAME).$(STLIBSUFFIX) -STLIB_MAKE_CMD=ar rcs $(STLIBNAME) - -# Platform-specific overrides -uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') -ifeq ($(uname_S),SunOS) - REAL_LDFLAGS+= -ldl -lnsl -lsocket - DYLIB_MAKE_CMD=$(CC) -G -o $(DYLIBNAME) -h $(DYLIB_MINOR_NAME) $(LDFLAGS) - INSTALL= cp -r -endif -ifeq ($(uname_S),Darwin) - DYLIBSUFFIX=dylib - DYLIB_MINOR_NAME=$(LIBNAME).$(HIREDIS_SONAME).$(DYLIBSUFFIX) - DYLIB_MAKE_CMD=$(CC) -shared -Wl,-install_name,$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) -endif - -all: $(DYLIBNAME) $(STLIBNAME) hiredis-test $(PKGCONFNAME) - -# Deps (use make dep to generate this) -async.o: async.c fmacros.h async.h hiredis.h read.h sds.h net.h dict.c dict.h -dict.o: dict.c fmacros.h dict.h -hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h -net.o: net.c fmacros.h net.h hiredis.h read.h sds.h -read.o: read.c fmacros.h read.h sds.h -sds.o: sds.c sds.h -test.o: test.c fmacros.h hiredis.h read.h sds.h - -$(DYLIBNAME): $(OBJ) - $(DYLIB_MAKE_CMD) $(OBJ) - -$(STLIBNAME): $(OBJ) - $(STLIB_MAKE_CMD) $(OBJ) - -dynamic: $(DYLIBNAME) -static: $(STLIBNAME) - -# Binaries: -hiredis-example-libevent: examples/example-libevent.c adapters/libevent.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -levent $(STLIBNAME) - -hiredis-example-libev: examples/example-libev.c adapters/libev.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -lev $(STLIBNAME) - -hiredis-example-glib: examples/example-glib.c adapters/glib.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) $(shell pkg-config --cflags --libs glib-2.0) -I. $< $(STLIBNAME) - -hiredis-example-ivykis: examples/example-ivykis.c adapters/ivykis.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -livykis $(STLIBNAME) - -hiredis-example-macosx: examples/example-macosx.c adapters/macosx.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) - -ifndef AE_DIR -hiredis-example-ae: - @echo "Please specify AE_DIR (e.g. /src)" - @false -else -hiredis-example-ae: examples/example-ae.c adapters/ae.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(AE_DIR) $< $(AE_DIR)/ae.o $(AE_DIR)/zmalloc.o $(AE_DIR)/../deps/jemalloc/lib/libjemalloc.a -pthread $(STLIBNAME) -endif - -ifndef LIBUV_DIR -hiredis-example-libuv: - @echo "Please specify LIBUV_DIR (e.g. ../libuv/)" - @false -else -hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) -endif - -ifeq ($(and $(QT_MOC),$(QT_INCLUDE_DIR),$(QT_LIBRARY_DIR)),) -hiredis-example-qt: - @echo "Please specify QT_MOC, QT_INCLUDE_DIR AND QT_LIBRARY_DIR" - @false -else -hiredis-example-qt: examples/example-qt.cpp adapters/qt.h $(STLIBNAME) - $(QT_MOC) adapters/qt.h -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore | \ - $(CXX) -x c++ -o qt-adapter-moc.o -c - $(REAL_CFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore - $(QT_MOC) examples/example-qt.h -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore | \ - $(CXX) -x c++ -o qt-example-moc.o -c - $(REAL_CFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore - $(CXX) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore -L$(QT_LIBRARY_DIR) qt-adapter-moc.o qt-example-moc.o $< -pthread $(STLIBNAME) -lQtCore -endif - -hiredis-example: examples/example.c $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< $(STLIBNAME) - -examples: $(EXAMPLES) - -hiredis-test: test.o $(STLIBNAME) - -hiredis-%: %.o $(STLIBNAME) - $(CC) $(REAL_CFLAGS) -o $@ $(REAL_LDFLAGS) $< $(STLIBNAME) - -test: hiredis-test - ./hiredis-test - -check: hiredis-test - @echo "$$REDIS_TEST_CONFIG" | $(REDIS_SERVER) - - $(PRE) ./hiredis-test -h 127.0.0.1 -p $(REDIS_PORT) -s /tmp/hiredis-test-redis.sock || \ - ( kill `cat /tmp/hiredis-test-redis.pid` && false ) - kill `cat /tmp/hiredis-test-redis.pid` - -.c.o: - $(CC) -std=c99 -pedantic -c $(REAL_CFLAGS) $< - -clean: - rm -rf $(DYLIBNAME) $(STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov - -dep: - $(CC) -MM *.c - -ifeq ($(uname_S),SunOS) - INSTALL?= cp -r -endif - -INSTALL?= cp -a - -$(PKGCONFNAME): hiredis.h - @echo "Generating $@ for pkgconfig..." - @echo prefix=$(PREFIX) > $@ - @echo exec_prefix=\$${prefix} >> $@ - @echo libdir=$(PREFIX)/$(LIBRARY_PATH) >> $@ - @echo includedir=$(PREFIX)/$(INCLUDE_PATH) >> $@ - @echo >> $@ - @echo Name: hiredis >> $@ - @echo Description: Minimalistic C client library for Redis. >> $@ - @echo Version: $(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(HIREDIS_PATCH) >> $@ - @echo Libs: -L\$${libdir} -lhiredis >> $@ - @echo Cflags: -I\$${includedir} -D_FILE_OFFSET_BITS=64 >> $@ - -install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) - mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_LIBRARY_PATH) - $(INSTALL) hiredis.h async.h read.h sds.h adapters $(INSTALL_INCLUDE_PATH) - $(INSTALL) $(DYLIBNAME) $(INSTALL_LIBRARY_PATH)/$(DYLIB_MINOR_NAME) - cd $(INSTALL_LIBRARY_PATH) && ln -sf $(DYLIB_MINOR_NAME) $(DYLIBNAME) - $(INSTALL) $(STLIBNAME) $(INSTALL_LIBRARY_PATH) - mkdir -p $(INSTALL_PKGCONF_PATH) - $(INSTALL) $(PKGCONFNAME) $(INSTALL_PKGCONF_PATH) - -32bit: - @echo "" - @echo "WARNING: if this fails under Linux you probably need to install libc6-dev-i386" - @echo "" - $(MAKE) CFLAGS="-m32" LDFLAGS="-m32" - -32bit-vars: - $(eval CFLAGS=-m32) - $(eval LDFLAGS=-m32) - -gprof: - $(MAKE) CFLAGS="-pg" LDFLAGS="-pg" - -gcov: - $(MAKE) CFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS="-fprofile-arcs" - -coverage: gcov - make check - mkdir -p tmp/lcov - lcov -d . -c -o tmp/lcov/hiredis.info - genhtml --legend -o tmp/lcov/report tmp/lcov/hiredis.info - -noopt: - $(MAKE) OPTIMIZATION="" - -.PHONY: all test check clean dep install 32bit 32bit-vars gprof gcov noopt diff --git a/third-party/hiredis/README.md b/third-party/hiredis/README.md deleted file mode 100644 index 4f1a58d2a..000000000 --- a/third-party/hiredis/README.md +++ /dev/null @@ -1,392 +0,0 @@ -[![Build Status](https://travis-ci.org/redis/hiredis.png)](https://travis-ci.org/redis/hiredis) - -# HIREDIS - -Hiredis is a minimalistic C client library for the [Redis](http://redis.io/) database. - -It is minimalistic because it just adds minimal support for the protocol, but -at the same time it uses a high level printf-alike API in order to make it -much higher level than otherwise suggested by its minimal code base and the -lack of explicit bindings for every Redis command. - -Apart from supporting sending commands and receiving replies, it comes with -a reply parser that is decoupled from the I/O layer. It -is a stream parser designed for easy reusability, which can for instance be used -in higher level language bindings for efficient reply parsing. - -Hiredis only supports the binary-safe Redis protocol, so you can use it with any -Redis version >= 1.2.0. - -The library comes with multiple APIs. There is the -*synchronous API*, the *asynchronous API* and the *reply parsing API*. - -## UPGRADING - -Version 0.9.0 is a major overhaul of hiredis in every aspect. However, upgrading existing -code using hiredis should not be a big pain. The key thing to keep in mind when -upgrading is that hiredis >= 0.9.0 uses a `redisContext*` to keep state, in contrast to -the stateless 0.0.1 that only has a file descriptor to work with. - -## Synchronous API - -To consume the synchronous API, there are only a few function calls that need to be introduced: - -```c -redisContext *redisConnect(const char *ip, int port); -void *redisCommand(redisContext *c, const char *format, ...); -void freeReplyObject(void *reply); -``` - -### Connecting - -The function `redisConnect` is used to create a so-called `redisContext`. The -context is where Hiredis holds state for a connection. The `redisContext` -struct has an integer `err` field that is non-zero when the connection is in -an error state. The field `errstr` will contain a string with a description of -the error. More information on errors can be found in the **Errors** section. -After trying to connect to Redis using `redisConnect` you should -check the `err` field to see if establishing the connection was successful: -```c -redisContext *c = redisConnect("127.0.0.1", 6379); -if (c != NULL && c->err) { - printf("Error: %s\n", c->errstr); - // handle error -} -``` - -### Sending commands - -There are several ways to issue commands to Redis. The first that will be introduced is -`redisCommand`. This function takes a format similar to printf. In the simplest form, -it is used like this: -```c -reply = redisCommand(context, "SET foo bar"); -``` - -The specifier `%s` interpolates a string in the command, and uses `strlen` to -determine the length of the string: -```c -reply = redisCommand(context, "SET foo %s", value); -``` -When you need to pass binary safe strings in a command, the `%b` specifier can be -used. Together with a pointer to the string, it requires a `size_t` length argument -of the string: -```c -reply = redisCommand(context, "SET foo %b", value, (size_t) valuelen); -``` -Internally, Hiredis splits the command in different arguments and will -convert it to the protocol used to communicate with Redis. -One or more spaces separates arguments, so you can use the specifiers -anywhere in an argument: -```c -reply = redisCommand(context, "SET key:%s %s", myid, value); -``` - -### Using replies - -The return value of `redisCommand` holds a reply when the command was -successfully executed. When an error occurs, the return value is `NULL` and -the `err` field in the context will be set (see section on **Errors**). -Once an error is returned the context cannot be reused and you should set up -a new connection. - -The standard replies that `redisCommand` are of the type `redisReply`. The -`type` field in the `redisReply` should be used to test what kind of reply -was received: - -* **`REDIS_REPLY_STATUS`**: - * The command replied with a status reply. The status string can be accessed using `reply->str`. - The length of this string can be accessed using `reply->len`. - -* **`REDIS_REPLY_ERROR`**: - * The command replied with an error. The error string can be accessed identical to `REDIS_REPLY_STATUS`. - -* **`REDIS_REPLY_INTEGER`**: - * The command replied with an integer. The integer value can be accessed using the - `reply->integer` field of type `long long`. - -* **`REDIS_REPLY_NIL`**: - * The command replied with a **nil** object. There is no data to access. - -* **`REDIS_REPLY_STRING`**: - * A bulk (string) reply. The value of the reply can be accessed using `reply->str`. - The length of this string can be accessed using `reply->len`. - -* **`REDIS_REPLY_ARRAY`**: - * A multi bulk reply. The number of elements in the multi bulk reply is stored in - `reply->elements`. Every element in the multi bulk reply is a `redisReply` object as well - and can be accessed via `reply->element[..index..]`. - Redis may reply with nested arrays but this is fully supported. - -Replies should be freed using the `freeReplyObject()` function. -Note that this function will take care of freeing sub-reply objects -contained in arrays and nested arrays, so there is no need for the user to -free the sub replies (it is actually harmful and will corrupt the memory). - -**Important:** the current version of hiredis (0.10.0) frees replies when the -asynchronous API is used. This means you should not call `freeReplyObject` when -you use this API. The reply is cleaned up by hiredis _after_ the callback -returns. This behavior will probably change in future releases, so make sure to -keep an eye on the changelog when upgrading (see issue #39). - -### Cleaning up - -To disconnect and free the context the following function can be used: -```c -void redisFree(redisContext *c); -``` -This function immediately closes the socket and then frees the allocations done in -creating the context. - -### Sending commands (cont'd) - -Together with `redisCommand`, the function `redisCommandArgv` can be used to issue commands. -It has the following prototype: -```c -void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen); -``` -It takes the number of arguments `argc`, an array of strings `argv` and the lengths of the -arguments `argvlen`. For convenience, `argvlen` may be set to `NULL` and the function will -use `strlen(3)` on every argument to determine its length. Obviously, when any of the arguments -need to be binary safe, the entire array of lengths `argvlen` should be provided. - -The return value has the same semantic as `redisCommand`. - -### Pipelining - -To explain how Hiredis supports pipelining in a blocking connection, there needs to be -understanding of the internal execution flow. - -When any of the functions in the `redisCommand` family is called, Hiredis first formats the -command according to the Redis protocol. The formatted command is then put in the output buffer -of the context. This output buffer is dynamic, so it can hold any number of commands. -After the command is put in the output buffer, `redisGetReply` is called. This function has the -following two execution paths: - -1. The input buffer is non-empty: - * Try to parse a single reply from the input buffer and return it - * If no reply could be parsed, continue at *2* -2. The input buffer is empty: - * Write the **entire** output buffer to the socket - * Read from the socket until a single reply could be parsed - -The function `redisGetReply` is exported as part of the Hiredis API and can be used when a reply -is expected on the socket. To pipeline commands, the only things that needs to be done is -filling up the output buffer. For this cause, two commands can be used that are identical -to the `redisCommand` family, apart from not returning a reply: -```c -void redisAppendCommand(redisContext *c, const char *format, ...); -void redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen); -``` -After calling either function one or more times, `redisGetReply` can be used to receive the -subsequent replies. The return value for this function is either `REDIS_OK` or `REDIS_ERR`, where -the latter means an error occurred while reading a reply. Just as with the other commands, -the `err` field in the context can be used to find out what the cause of this error is. - -The following examples shows a simple pipeline (resulting in only a single call to `write(2)` and -a single call to `read(2)`): -```c -redisReply *reply; -redisAppendCommand(context,"SET foo bar"); -redisAppendCommand(context,"GET foo"); -redisGetReply(context,&reply); // reply for SET -freeReplyObject(reply); -redisGetReply(context,&reply); // reply for GET -freeReplyObject(reply); -``` -This API can also be used to implement a blocking subscriber: -```c -reply = redisCommand(context,"SUBSCRIBE foo"); -freeReplyObject(reply); -while(redisGetReply(context,&reply) == REDIS_OK) { - // consume message - freeReplyObject(reply); -} -``` -### Errors - -When a function call is not successful, depending on the function either `NULL` or `REDIS_ERR` is -returned. The `err` field inside the context will be non-zero and set to one of the -following constants: - -* **`REDIS_ERR_IO`**: - There was an I/O error while creating the connection, trying to write - to the socket or read from the socket. If you included `errno.h` in your - application, you can use the global `errno` variable to find out what is - wrong. - -* **`REDIS_ERR_EOF`**: - The server closed the connection which resulted in an empty read. - -* **`REDIS_ERR_PROTOCOL`**: - There was an error while parsing the protocol. - -* **`REDIS_ERR_OTHER`**: - Any other error. Currently, it is only used when a specified hostname to connect - to cannot be resolved. - -In every case, the `errstr` field in the context will be set to hold a string representation -of the error. - -## Asynchronous API - -Hiredis comes with an asynchronous API that works easily with any event library. -Examples are bundled that show using Hiredis with [libev](http://software.schmorp.de/pkg/libev.html) -and [libevent](http://monkey.org/~provos/libevent/). - -### Connecting - -The function `redisAsyncConnect` can be used to establish a non-blocking connection to -Redis. It returns a pointer to the newly created `redisAsyncContext` struct. The `err` field -should be checked after creation to see if there were errors creating the connection. -Because the connection that will be created is non-blocking, the kernel is not able to -instantly return if the specified host and port is able to accept a connection. -```c -redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); -if (c->err) { - printf("Error: %s\n", c->errstr); - // handle error -} -``` - -The asynchronous context can hold a disconnect callback function that is called when the -connection is disconnected (either because of an error or per user request). This function should -have the following prototype: -```c -void(const redisAsyncContext *c, int status); -``` -On a disconnect, the `status` argument is set to `REDIS_OK` when disconnection was initiated by the -user, or `REDIS_ERR` when the disconnection was caused by an error. When it is `REDIS_ERR`, the `err` -field in the context can be accessed to find out the cause of the error. - -The context object is always freed after the disconnect callback fired. When a reconnect is needed, -the disconnect callback is a good point to do so. - -Setting the disconnect callback can only be done once per context. For subsequent calls it will -return `REDIS_ERR`. The function to set the disconnect callback has the following prototype: -```c -int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); -``` -### Sending commands and their callbacks - -In an asynchronous context, commands are automatically pipelined due to the nature of an event loop. -Therefore, unlike the synchronous API, there is only a single way to send commands. -Because commands are sent to Redis asynchronously, issuing a command requires a callback function -that is called when the reply is received. Reply callbacks should have the following prototype: -```c -void(redisAsyncContext *c, void *reply, void *privdata); -``` -The `privdata` argument can be used to curry arbitrary data to the callback from the point where -the command is initially queued for execution. - -The functions that can be used to issue commands in an asynchronous context are: -```c -int redisAsyncCommand( - redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, - const char *format, ...); -int redisAsyncCommandArgv( - redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, - int argc, const char **argv, const size_t *argvlen); -``` -Both functions work like their blocking counterparts. The return value is `REDIS_OK` when the command -was successfully added to the output buffer and `REDIS_ERR` otherwise. Example: when the connection -is being disconnected per user-request, no new commands may be added to the output buffer and `REDIS_ERR` is -returned on calls to the `redisAsyncCommand` family. - -If the reply for a command with a `NULL` callback is read, it is immediately freed. When the callback -for a command is non-`NULL`, the memory is freed immediately following the callback: the reply is only -valid for the duration of the callback. - -All pending callbacks are called with a `NULL` reply when the context encountered an error. - -### Disconnecting - -An asynchronous connection can be terminated using: -```c -void redisAsyncDisconnect(redisAsyncContext *ac); -``` -When this function is called, the connection is **not** immediately terminated. Instead, new -commands are no longer accepted and the connection is only terminated when all pending commands -have been written to the socket, their respective replies have been read and their respective -callbacks have been executed. After this, the disconnection callback is executed with the -`REDIS_OK` status and the context object is freed. - -### Hooking it up to event library *X* - -There are a few hooks that need to be set on the context object after it is created. -See the `adapters/` directory for bindings to *libev* and *libevent*. - -## Reply parsing API - -Hiredis comes with a reply parsing API that makes it easy for writing higher -level language bindings. - -The reply parsing API consists of the following functions: -```c -redisReader *redisReaderCreate(void); -void redisReaderFree(redisReader *reader); -int redisReaderFeed(redisReader *reader, const char *buf, size_t len); -int redisReaderGetReply(redisReader *reader, void **reply); -``` -The same set of functions are used internally by hiredis when creating a -normal Redis context, the above API just exposes it to the user for a direct -usage. - -### Usage - -The function `redisReaderCreate` creates a `redisReader` structure that holds a -buffer with unparsed data and state for the protocol parser. - -Incoming data -- most likely from a socket -- can be placed in the internal -buffer of the `redisReader` using `redisReaderFeed`. This function will make a -copy of the buffer pointed to by `buf` for `len` bytes. This data is parsed -when `redisReaderGetReply` is called. This function returns an integer status -and a reply object (as described above) via `void **reply`. The returned status -can be either `REDIS_OK` or `REDIS_ERR`, where the latter means something went -wrong (either a protocol error, or an out of memory error). - -The parser limits the level of nesting for multi bulk payloads to 7. If the -multi bulk nesting level is higher than this, the parser returns an error. - -### Customizing replies - -The function `redisReaderGetReply` creates `redisReply` and makes the function -argument `reply` point to the created `redisReply` variable. For instance, if -the response of type `REDIS_REPLY_STATUS` then the `str` field of `redisReply` -will hold the status as a vanilla C string. However, the functions that are -responsible for creating instances of the `redisReply` can be customized by -setting the `fn` field on the `redisReader` struct. This should be done -immediately after creating the `redisReader`. - -For example, [hiredis-rb](https://github.com/pietern/hiredis-rb/blob/master/ext/hiredis_ext/reader.c) -uses customized reply object functions to create Ruby objects. - -### Reader max buffer - -Both when using the Reader API directly or when using it indirectly via a -normal Redis context, the redisReader structure uses a buffer in order to -accumulate data from the server. -Usually this buffer is destroyed when it is empty and is larger than 16 -KiB in order to avoid wasting memory in unused buffers - -However when working with very big payloads destroying the buffer may slow -down performances considerably, so it is possible to modify the max size of -an idle buffer changing the value of the `maxbuf` field of the reader structure -to the desired value. The special value of 0 means that there is no maximum -value for an idle buffer, so the buffer will never get freed. - -For instance if you have a normal Redis context you can set the maximum idle -buffer to zero (unlimited) just with: -```c -context->reader->maxbuf = 0; -``` -This should be done only in order to maximize performances when working with -large payloads. The context should be set back to `REDIS_READER_MAX_BUF` again -as soon as possible in order to prevent allocation of useless memory. - -## AUTHORS - -Hiredis was written by Salvatore Sanfilippo (antirez at gmail) and -Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. -Hiredis is currently maintained by Matt Stancliff (matt at genges dot com) and -Jan-Erik Rediger (janerik at fnordig dot com) diff --git a/third-party/hiredis/adapters/ae.h b/third-party/hiredis/adapters/ae.h deleted file mode 100644 index 5c551c2ed..000000000 --- a/third-party/hiredis/adapters/ae.h +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2010-2011, Pieter Noordhuis - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __HIREDIS_AE_H__ -#define __HIREDIS_AE_H__ -#include -#include -#include "../hiredis.h" -#include "../async.h" - -typedef struct redisAeEvents { - redisAsyncContext *context; - aeEventLoop *loop; - int fd; - int reading, writing; -} redisAeEvents; - -static void redisAeReadEvent(aeEventLoop *el, int fd, void *privdata, int mask) { - ((void)el); ((void)fd); ((void)mask); - - redisAeEvents *e = (redisAeEvents*)privdata; - redisAsyncHandleRead(e->context); -} - -static void redisAeWriteEvent(aeEventLoop *el, int fd, void *privdata, int mask) { - ((void)el); ((void)fd); ((void)mask); - - redisAeEvents *e = (redisAeEvents*)privdata; - redisAsyncHandleWrite(e->context); -} - -static void redisAeAddRead(void *privdata) { - redisAeEvents *e = (redisAeEvents*)privdata; - aeEventLoop *loop = e->loop; - if (!e->reading) { - e->reading = 1; - aeCreateFileEvent(loop,e->fd,AE_READABLE,redisAeReadEvent,e); - } -} - -static void redisAeDelRead(void *privdata) { - redisAeEvents *e = (redisAeEvents*)privdata; - aeEventLoop *loop = e->loop; - if (e->reading) { - e->reading = 0; - aeDeleteFileEvent(loop,e->fd,AE_READABLE); - } -} - -static void redisAeAddWrite(void *privdata) { - redisAeEvents *e = (redisAeEvents*)privdata; - aeEventLoop *loop = e->loop; - if (!e->writing) { - e->writing = 1; - aeCreateFileEvent(loop,e->fd,AE_WRITABLE,redisAeWriteEvent,e); - } -} - -static void redisAeDelWrite(void *privdata) { - redisAeEvents *e = (redisAeEvents*)privdata; - aeEventLoop *loop = e->loop; - if (e->writing) { - e->writing = 0; - aeDeleteFileEvent(loop,e->fd,AE_WRITABLE); - } -} - -static void redisAeCleanup(void *privdata) { - redisAeEvents *e = (redisAeEvents*)privdata; - redisAeDelRead(privdata); - redisAeDelWrite(privdata); - free(e); -} - -static int redisAeAttach(aeEventLoop *loop, redisAsyncContext *ac) { - redisContext *c = &(ac->c); - redisAeEvents *e; - - /* Nothing should be attached when something is already attached */ - if (ac->ev.data != NULL) - return REDIS_ERR; - - /* Create container for context and r/w events */ - e = (redisAeEvents*)malloc(sizeof(*e)); - e->context = ac; - e->loop = loop; - e->fd = c->fd; - e->reading = e->writing = 0; - - /* Register functions to start/stop listening for events */ - ac->ev.addRead = redisAeAddRead; - ac->ev.delRead = redisAeDelRead; - ac->ev.addWrite = redisAeAddWrite; - ac->ev.delWrite = redisAeDelWrite; - ac->ev.cleanup = redisAeCleanup; - ac->ev.data = e; - - return REDIS_OK; -} -#endif diff --git a/third-party/hiredis/adapters/glib.h b/third-party/hiredis/adapters/glib.h deleted file mode 100644 index e0a6411d3..000000000 --- a/third-party/hiredis/adapters/glib.h +++ /dev/null @@ -1,153 +0,0 @@ -#ifndef __HIREDIS_GLIB_H__ -#define __HIREDIS_GLIB_H__ - -#include - -#include "../hiredis.h" -#include "../async.h" - -typedef struct -{ - GSource source; - redisAsyncContext *ac; - GPollFD poll_fd; -} RedisSource; - -static void -redis_source_add_read (gpointer data) -{ - RedisSource *source = (RedisSource *)data; - g_return_if_fail(source); - source->poll_fd.events |= G_IO_IN; - g_main_context_wakeup(g_source_get_context((GSource *)data)); -} - -static void -redis_source_del_read (gpointer data) -{ - RedisSource *source = (RedisSource *)data; - g_return_if_fail(source); - source->poll_fd.events &= ~G_IO_IN; - g_main_context_wakeup(g_source_get_context((GSource *)data)); -} - -static void -redis_source_add_write (gpointer data) -{ - RedisSource *source = (RedisSource *)data; - g_return_if_fail(source); - source->poll_fd.events |= G_IO_OUT; - g_main_context_wakeup(g_source_get_context((GSource *)data)); -} - -static void -redis_source_del_write (gpointer data) -{ - RedisSource *source = (RedisSource *)data; - g_return_if_fail(source); - source->poll_fd.events &= ~G_IO_OUT; - g_main_context_wakeup(g_source_get_context((GSource *)data)); -} - -static void -redis_source_cleanup (gpointer data) -{ - RedisSource *source = (RedisSource *)data; - - g_return_if_fail(source); - - redis_source_del_read(source); - redis_source_del_write(source); - /* - * It is not our responsibility to remove ourself from the - * current main loop. However, we will remove the GPollFD. - */ - if (source->poll_fd.fd >= 0) { - g_source_remove_poll((GSource *)data, &source->poll_fd); - source->poll_fd.fd = -1; - } -} - -static gboolean -redis_source_prepare (GSource *source, - gint *timeout_) -{ - RedisSource *redis = (RedisSource *)source; - *timeout_ = -1; - return !!(redis->poll_fd.events & redis->poll_fd.revents); -} - -static gboolean -redis_source_check (GSource *source) -{ - RedisSource *redis = (RedisSource *)source; - return !!(redis->poll_fd.events & redis->poll_fd.revents); -} - -static gboolean -redis_source_dispatch (GSource *source, - GSourceFunc callback, - gpointer user_data) -{ - RedisSource *redis = (RedisSource *)source; - - if ((redis->poll_fd.revents & G_IO_OUT)) { - redisAsyncHandleWrite(redis->ac); - redis->poll_fd.revents &= ~G_IO_OUT; - } - - if ((redis->poll_fd.revents & G_IO_IN)) { - redisAsyncHandleRead(redis->ac); - redis->poll_fd.revents &= ~G_IO_IN; - } - - if (callback) { - return callback(user_data); - } - - return TRUE; -} - -static void -redis_source_finalize (GSource *source) -{ - RedisSource *redis = (RedisSource *)source; - - if (redis->poll_fd.fd >= 0) { - g_source_remove_poll(source, &redis->poll_fd); - redis->poll_fd.fd = -1; - } -} - -static GSource * -redis_source_new (redisAsyncContext *ac) -{ - static GSourceFuncs source_funcs = { - .prepare = redis_source_prepare, - .check = redis_source_check, - .dispatch = redis_source_dispatch, - .finalize = redis_source_finalize, - }; - redisContext *c = &ac->c; - RedisSource *source; - - g_return_val_if_fail(ac != NULL, NULL); - - source = (RedisSource *)g_source_new(&source_funcs, sizeof *source); - source->ac = ac; - source->poll_fd.fd = c->fd; - source->poll_fd.events = 0; - source->poll_fd.revents = 0; - g_source_add_poll((GSource *)source, &source->poll_fd); - - ac->ev.addRead = redis_source_add_read; - ac->ev.delRead = redis_source_del_read; - ac->ev.addWrite = redis_source_add_write; - ac->ev.delWrite = redis_source_del_write; - ac->ev.cleanup = redis_source_cleanup; - ac->ev.data = source; - - return (GSource *)source; -} - -#endif /* __HIREDIS_GLIB_H__ */ diff --git a/third-party/hiredis/adapters/ivykis.h b/third-party/hiredis/adapters/ivykis.h deleted file mode 100644 index 6a12a868a..000000000 --- a/third-party/hiredis/adapters/ivykis.h +++ /dev/null @@ -1,81 +0,0 @@ -#ifndef __HIREDIS_IVYKIS_H__ -#define __HIREDIS_IVYKIS_H__ -#include -#include "../hiredis.h" -#include "../async.h" - -typedef struct redisIvykisEvents { - redisAsyncContext *context; - struct iv_fd fd; -} redisIvykisEvents; - -static void redisIvykisReadEvent(void *arg) { - redisAsyncContext *context = (redisAsyncContext *)arg; - redisAsyncHandleRead(context); -} - -static void redisIvykisWriteEvent(void *arg) { - redisAsyncContext *context = (redisAsyncContext *)arg; - redisAsyncHandleWrite(context); -} - -static void redisIvykisAddRead(void *privdata) { - redisIvykisEvents *e = (redisIvykisEvents*)privdata; - iv_fd_set_handler_in(&e->fd, redisIvykisReadEvent); -} - -static void redisIvykisDelRead(void *privdata) { - redisIvykisEvents *e = (redisIvykisEvents*)privdata; - iv_fd_set_handler_in(&e->fd, NULL); -} - -static void redisIvykisAddWrite(void *privdata) { - redisIvykisEvents *e = (redisIvykisEvents*)privdata; - iv_fd_set_handler_out(&e->fd, redisIvykisWriteEvent); -} - -static void redisIvykisDelWrite(void *privdata) { - redisIvykisEvents *e = (redisIvykisEvents*)privdata; - iv_fd_set_handler_out(&e->fd, NULL); -} - -static void redisIvykisCleanup(void *privdata) { - redisIvykisEvents *e = (redisIvykisEvents*)privdata; - - iv_fd_unregister(&e->fd); - free(e); -} - -static int redisIvykisAttach(redisAsyncContext *ac) { - redisContext *c = &(ac->c); - redisIvykisEvents *e; - - /* Nothing should be attached when something is already attached */ - if (ac->ev.data != NULL) - return REDIS_ERR; - - /* Create container for context and r/w events */ - e = (redisIvykisEvents*)malloc(sizeof(*e)); - e->context = ac; - - /* Register functions to start/stop listening for events */ - ac->ev.addRead = redisIvykisAddRead; - ac->ev.delRead = redisIvykisDelRead; - ac->ev.addWrite = redisIvykisAddWrite; - ac->ev.delWrite = redisIvykisDelWrite; - ac->ev.cleanup = redisIvykisCleanup; - ac->ev.data = e; - - /* Initialize and install read/write events */ - IV_FD_INIT(&e->fd); - e->fd.fd = c->fd; - e->fd.handler_in = redisIvykisReadEvent; - e->fd.handler_out = redisIvykisWriteEvent; - e->fd.handler_err = NULL; - e->fd.cookie = e->context; - - iv_fd_register(&e->fd); - - return REDIS_OK; -} -#endif diff --git a/third-party/hiredis/adapters/libev.h b/third-party/hiredis/adapters/libev.h deleted file mode 100644 index 2bf8d521f..000000000 --- a/third-party/hiredis/adapters/libev.h +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (c) 2010-2011, Pieter Noordhuis - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __HIREDIS_LIBEV_H__ -#define __HIREDIS_LIBEV_H__ -#include -#include -#include -#include "../hiredis.h" -#include "../async.h" - -typedef struct redisLibevEvents { - redisAsyncContext *context; - struct ev_loop *loop; - int reading, writing; - ev_io rev, wev; -} redisLibevEvents; - -static void redisLibevReadEvent(EV_P_ ev_io *watcher, int revents) { -#if EV_MULTIPLICITY - ((void)loop); -#endif - ((void)revents); - - redisLibevEvents *e = (redisLibevEvents*)watcher->data; - redisAsyncHandleRead(e->context); -} - -static void redisLibevWriteEvent(EV_P_ ev_io *watcher, int revents) { -#if EV_MULTIPLICITY - ((void)loop); -#endif - ((void)revents); - - redisLibevEvents *e = (redisLibevEvents*)watcher->data; - redisAsyncHandleWrite(e->context); -} - -static void redisLibevAddRead(void *privdata) { - redisLibevEvents *e = (redisLibevEvents*)privdata; - struct ev_loop *loop = e->loop; - ((void)loop); - if (!e->reading) { - e->reading = 1; - ev_io_start(EV_A_ &e->rev); - } -} - -static void redisLibevDelRead(void *privdata) { - redisLibevEvents *e = (redisLibevEvents*)privdata; - struct ev_loop *loop = e->loop; - ((void)loop); - if (e->reading) { - e->reading = 0; - ev_io_stop(EV_A_ &e->rev); - } -} - -static void redisLibevAddWrite(void *privdata) { - redisLibevEvents *e = (redisLibevEvents*)privdata; - struct ev_loop *loop = e->loop; - ((void)loop); - if (!e->writing) { - e->writing = 1; - ev_io_start(EV_A_ &e->wev); - } -} - -static void redisLibevDelWrite(void *privdata) { - redisLibevEvents *e = (redisLibevEvents*)privdata; - struct ev_loop *loop = e->loop; - ((void)loop); - if (e->writing) { - e->writing = 0; - ev_io_stop(EV_A_ &e->wev); - } -} - -static void redisLibevCleanup(void *privdata) { - redisLibevEvents *e = (redisLibevEvents*)privdata; - redisLibevDelRead(privdata); - redisLibevDelWrite(privdata); - free(e); -} - -static int redisLibevAttach(EV_P_ redisAsyncContext *ac) { - redisContext *c = &(ac->c); - redisLibevEvents *e; - - /* Nothing should be attached when something is already attached */ - if (ac->ev.data != NULL) - return REDIS_ERR; - - /* Create container for context and r/w events */ - e = (redisLibevEvents*)malloc(sizeof(*e)); - e->context = ac; -#if EV_MULTIPLICITY - e->loop = loop; -#else - e->loop = NULL; -#endif - e->reading = e->writing = 0; - e->rev.data = e; - e->wev.data = e; - - /* Register functions to start/stop listening for events */ - ac->ev.addRead = redisLibevAddRead; - ac->ev.delRead = redisLibevDelRead; - ac->ev.addWrite = redisLibevAddWrite; - ac->ev.delWrite = redisLibevDelWrite; - ac->ev.cleanup = redisLibevCleanup; - ac->ev.data = e; - - /* Initialize read/write events */ - ev_io_init(&e->rev,redisLibevReadEvent,c->fd,EV_READ); - ev_io_init(&e->wev,redisLibevWriteEvent,c->fd,EV_WRITE); - return REDIS_OK; -} - -#endif diff --git a/third-party/hiredis/adapters/libevent.h b/third-party/hiredis/adapters/libevent.h deleted file mode 100644 index 1c2b271bb..000000000 --- a/third-party/hiredis/adapters/libevent.h +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2010-2011, Pieter Noordhuis - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __HIREDIS_LIBEVENT_H__ -#define __HIREDIS_LIBEVENT_H__ -#include -#include "../hiredis.h" -#include "../async.h" - -typedef struct redisLibeventEvents { - redisAsyncContext *context; - struct event rev, wev; -} redisLibeventEvents; - -static void redisLibeventReadEvent(int fd, short event, void *arg) { - ((void)fd); ((void)event); - redisLibeventEvents *e = (redisLibeventEvents*)arg; - redisAsyncHandleRead(e->context); -} - -static void redisLibeventWriteEvent(int fd, short event, void *arg) { - ((void)fd); ((void)event); - redisLibeventEvents *e = (redisLibeventEvents*)arg; - redisAsyncHandleWrite(e->context); -} - -static void redisLibeventAddRead(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_add(&e->rev,NULL); -} - -static void redisLibeventDelRead(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(&e->rev); -} - -static void redisLibeventAddWrite(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_add(&e->wev,NULL); -} - -static void redisLibeventDelWrite(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(&e->wev); -} - -static void redisLibeventCleanup(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(&e->rev); - event_del(&e->wev); - free(e); -} - -static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { - redisContext *c = &(ac->c); - redisLibeventEvents *e; - - /* Nothing should be attached when something is already attached */ - if (ac->ev.data != NULL) - return REDIS_ERR; - - /* Create container for context and r/w events */ - e = (redisLibeventEvents*)malloc(sizeof(*e)); - e->context = ac; - - /* Register functions to start/stop listening for events */ - ac->ev.addRead = redisLibeventAddRead; - ac->ev.delRead = redisLibeventDelRead; - ac->ev.addWrite = redisLibeventAddWrite; - ac->ev.delWrite = redisLibeventDelWrite; - ac->ev.cleanup = redisLibeventCleanup; - ac->ev.data = e; - - /* Initialize and install read/write events */ - event_set(&e->rev,c->fd,EV_READ,redisLibeventReadEvent,e); - event_set(&e->wev,c->fd,EV_WRITE,redisLibeventWriteEvent,e); - event_base_set(base,&e->rev); - event_base_set(base,&e->wev); - return REDIS_OK; -} -#endif diff --git a/third-party/hiredis/adapters/libuv.h b/third-party/hiredis/adapters/libuv.h deleted file mode 100644 index 3cdf3d394..000000000 --- a/third-party/hiredis/adapters/libuv.h +++ /dev/null @@ -1,121 +0,0 @@ -#ifndef __HIREDIS_LIBUV_H__ -#define __HIREDIS_LIBUV_H__ -#include -#include -#include "../hiredis.h" -#include "../async.h" -#include - -typedef struct redisLibuvEvents { - redisAsyncContext* context; - uv_poll_t handle; - int events; -} redisLibuvEvents; - - -static void redisLibuvPoll(uv_poll_t* handle, int status, int events) { - redisLibuvEvents* p = (redisLibuvEvents*)handle->data; - - if (status != 0) { - return; - } - - if (events & UV_READABLE) { - redisAsyncHandleRead(p->context); - } - if (events & UV_WRITABLE) { - redisAsyncHandleWrite(p->context); - } -} - - -static void redisLibuvAddRead(void *privdata) { - redisLibuvEvents* p = (redisLibuvEvents*)privdata; - - p->events |= UV_READABLE; - - uv_poll_start(&p->handle, p->events, redisLibuvPoll); -} - - -static void redisLibuvDelRead(void *privdata) { - redisLibuvEvents* p = (redisLibuvEvents*)privdata; - - p->events &= ~UV_READABLE; - - if (p->events) { - uv_poll_start(&p->handle, p->events, redisLibuvPoll); - } else { - uv_poll_stop(&p->handle); - } -} - - -static void redisLibuvAddWrite(void *privdata) { - redisLibuvEvents* p = (redisLibuvEvents*)privdata; - - p->events |= UV_WRITABLE; - - uv_poll_start(&p->handle, p->events, redisLibuvPoll); -} - - -static void redisLibuvDelWrite(void *privdata) { - redisLibuvEvents* p = (redisLibuvEvents*)privdata; - - p->events &= ~UV_WRITABLE; - - if (p->events) { - uv_poll_start(&p->handle, p->events, redisLibuvPoll); - } else { - uv_poll_stop(&p->handle); - } -} - - -static void on_close(uv_handle_t* handle) { - redisLibuvEvents* p = (redisLibuvEvents*)handle->data; - - free(p); -} - - -static void redisLibuvCleanup(void *privdata) { - redisLibuvEvents* p = (redisLibuvEvents*)privdata; - - uv_close((uv_handle_t*)&p->handle, on_close); -} - - -static int redisLibuvAttach(redisAsyncContext* ac, uv_loop_t* loop) { - redisContext *c = &(ac->c); - - if (ac->ev.data != NULL) { - return REDIS_ERR; - } - - ac->ev.addRead = redisLibuvAddRead; - ac->ev.delRead = redisLibuvDelRead; - ac->ev.addWrite = redisLibuvAddWrite; - ac->ev.delWrite = redisLibuvDelWrite; - ac->ev.cleanup = redisLibuvCleanup; - - redisLibuvEvents* p = (redisLibuvEvents*)malloc(sizeof(*p)); - - if (!p) { - return REDIS_ERR; - } - - memset(p, 0, sizeof(*p)); - - if (uv_poll_init(loop, &p->handle, c->fd) != 0) { - return REDIS_ERR; - } - - ac->ev.data = p; - p->handle.data = p; - p->context = ac; - - return REDIS_OK; -} -#endif diff --git a/third-party/hiredis/adapters/macosx.h b/third-party/hiredis/adapters/macosx.h deleted file mode 100644 index 72121f606..000000000 --- a/third-party/hiredis/adapters/macosx.h +++ /dev/null @@ -1,114 +0,0 @@ -// -// Created by Дмитрий Бахвалов on 13.07.15. -// Copyright (c) 2015 Dmitry Bakhvalov. All rights reserved. -// - -#ifndef __HIREDIS_MACOSX_H__ -#define __HIREDIS_MACOSX_H__ - -#include - -#include "../hiredis.h" -#include "../async.h" - -typedef struct { - redisAsyncContext *context; - CFSocketRef socketRef; - CFRunLoopSourceRef sourceRef; -} RedisRunLoop; - -static int freeRedisRunLoop(RedisRunLoop* redisRunLoop) { - if( redisRunLoop != NULL ) { - if( redisRunLoop->sourceRef != NULL ) { - CFRunLoopSourceInvalidate(redisRunLoop->sourceRef); - CFRelease(redisRunLoop->sourceRef); - } - if( redisRunLoop->socketRef != NULL ) { - CFSocketInvalidate(redisRunLoop->socketRef); - CFRelease(redisRunLoop->socketRef); - } - free(redisRunLoop); - } - return REDIS_ERR; -} - -static void redisMacOSAddRead(void *privdata) { - RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata; - CFSocketEnableCallBacks(redisRunLoop->socketRef, kCFSocketReadCallBack); -} - -static void redisMacOSDelRead(void *privdata) { - RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata; - CFSocketDisableCallBacks(redisRunLoop->socketRef, kCFSocketReadCallBack); -} - -static void redisMacOSAddWrite(void *privdata) { - RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata; - CFSocketEnableCallBacks(redisRunLoop->socketRef, kCFSocketWriteCallBack); -} - -static void redisMacOSDelWrite(void *privdata) { - RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata; - CFSocketDisableCallBacks(redisRunLoop->socketRef, kCFSocketWriteCallBack); -} - -static void redisMacOSCleanup(void *privdata) { - RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata; - freeRedisRunLoop(redisRunLoop); -} - -static void redisMacOSAsyncCallback(CFSocketRef __unused s, CFSocketCallBackType callbackType, CFDataRef __unused address, const void __unused *data, void *info) { - redisAsyncContext* context = (redisAsyncContext*) info; - - switch (callbackType) { - case kCFSocketReadCallBack: - redisAsyncHandleRead(context); - break; - - case kCFSocketWriteCallBack: - redisAsyncHandleWrite(context); - break; - - default: - break; - } -} - -static int redisMacOSAttach(redisAsyncContext *redisAsyncCtx, CFRunLoopRef runLoop) { - redisContext *redisCtx = &(redisAsyncCtx->c); - - /* Nothing should be attached when something is already attached */ - if( redisAsyncCtx->ev.data != NULL ) return REDIS_ERR; - - RedisRunLoop* redisRunLoop = (RedisRunLoop*) calloc(1, sizeof(RedisRunLoop)); - if( !redisRunLoop ) return REDIS_ERR; - - /* Setup redis stuff */ - redisRunLoop->context = redisAsyncCtx; - - redisAsyncCtx->ev.addRead = redisMacOSAddRead; - redisAsyncCtx->ev.delRead = redisMacOSDelRead; - redisAsyncCtx->ev.addWrite = redisMacOSAddWrite; - redisAsyncCtx->ev.delWrite = redisMacOSDelWrite; - redisAsyncCtx->ev.cleanup = redisMacOSCleanup; - redisAsyncCtx->ev.data = redisRunLoop; - - /* Initialize and install read/write events */ - CFSocketContext socketCtx = { 0, redisAsyncCtx, NULL, NULL, NULL }; - - redisRunLoop->socketRef = CFSocketCreateWithNative(NULL, redisCtx->fd, - kCFSocketReadCallBack | kCFSocketWriteCallBack, - redisMacOSAsyncCallback, - &socketCtx); - if( !redisRunLoop->socketRef ) return freeRedisRunLoop(redisRunLoop); - - redisRunLoop->sourceRef = CFSocketCreateRunLoopSource(NULL, redisRunLoop->socketRef, 0); - if( !redisRunLoop->sourceRef ) return freeRedisRunLoop(redisRunLoop); - - CFRunLoopAddSource(runLoop, redisRunLoop->sourceRef, kCFRunLoopDefaultMode); - - return REDIS_OK; -} - -#endif - diff --git a/third-party/hiredis/adapters/qt.h b/third-party/hiredis/adapters/qt.h deleted file mode 100644 index 5cc02e6ce..000000000 --- a/third-party/hiredis/adapters/qt.h +++ /dev/null @@ -1,135 +0,0 @@ -/*- - * Copyright (C) 2014 Pietro Cerutti - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifndef __HIREDIS_QT_H__ -#define __HIREDIS_QT_H__ -#include -#include "../async.h" - -static void RedisQtAddRead(void *); -static void RedisQtDelRead(void *); -static void RedisQtAddWrite(void *); -static void RedisQtDelWrite(void *); -static void RedisQtCleanup(void *); - -class RedisQtAdapter : public QObject { - - Q_OBJECT - - friend - void RedisQtAddRead(void * adapter) { - RedisQtAdapter * a = static_cast(adapter); - a->addRead(); - } - - friend - void RedisQtDelRead(void * adapter) { - RedisQtAdapter * a = static_cast(adapter); - a->delRead(); - } - - friend - void RedisQtAddWrite(void * adapter) { - RedisQtAdapter * a = static_cast(adapter); - a->addWrite(); - } - - friend - void RedisQtDelWrite(void * adapter) { - RedisQtAdapter * a = static_cast(adapter); - a->delWrite(); - } - - friend - void RedisQtCleanup(void * adapter) { - RedisQtAdapter * a = static_cast(adapter); - a->cleanup(); - } - - public: - RedisQtAdapter(QObject * parent = 0) - : QObject(parent), m_ctx(0), m_read(0), m_write(0) { } - - ~RedisQtAdapter() { - if (m_ctx != 0) { - m_ctx->ev.data = NULL; - } - } - - int setContext(redisAsyncContext * ac) { - if (ac->ev.data != NULL) { - return REDIS_ERR; - } - m_ctx = ac; - m_ctx->ev.data = this; - m_ctx->ev.addRead = RedisQtAddRead; - m_ctx->ev.delRead = RedisQtDelRead; - m_ctx->ev.addWrite = RedisQtAddWrite; - m_ctx->ev.delWrite = RedisQtDelWrite; - m_ctx->ev.cleanup = RedisQtCleanup; - return REDIS_OK; - } - - private: - void addRead() { - if (m_read) return; - m_read = new QSocketNotifier(m_ctx->c.fd, QSocketNotifier::Read, 0); - connect(m_read, SIGNAL(activated(int)), this, SLOT(read())); - } - - void delRead() { - if (!m_read) return; - delete m_read; - m_read = 0; - } - - void addWrite() { - if (m_write) return; - m_write = new QSocketNotifier(m_ctx->c.fd, QSocketNotifier::Write, 0); - connect(m_write, SIGNAL(activated(int)), this, SLOT(write())); - } - - void delWrite() { - if (!m_write) return; - delete m_write; - m_write = 0; - } - - void cleanup() { - delRead(); - delWrite(); - } - - private slots: - void read() { redisAsyncHandleRead(m_ctx); } - void write() { redisAsyncHandleWrite(m_ctx); } - - private: - redisAsyncContext * m_ctx; - QSocketNotifier * m_read; - QSocketNotifier * m_write; -}; - -#endif /* !__HIREDIS_QT_H__ */ diff --git a/third-party/hiredis/async.c b/third-party/hiredis/async.c deleted file mode 100644 index acca29ac4..000000000 --- a/third-party/hiredis/async.c +++ /dev/null @@ -1,687 +0,0 @@ -/* - * Copyright (c) 2009-2011, Salvatore Sanfilippo - * Copyright (c) 2010-2011, Pieter Noordhuis - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#include "fmacros.h" -#include -#include -#include -#include -#include -#include -#include "async.h" -#include "net.h" -#include "dict.c" -#include "sds.h" - -#define _EL_ADD_READ(ctx) do { \ - if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \ - } while(0) -#define _EL_DEL_READ(ctx) do { \ - if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \ - } while(0) -#define _EL_ADD_WRITE(ctx) do { \ - if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \ - } while(0) -#define _EL_DEL_WRITE(ctx) do { \ - if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \ - } while(0) -#define _EL_CLEANUP(ctx) do { \ - if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \ - } while(0); - -/* Forward declaration of function in hiredis.c */ -int __redisAppendCommand(redisContext *c, const char *cmd, size_t len); - -/* Functions managing dictionary of callbacks for pub/sub. */ -static unsigned int callbackHash(const void *key) { - return dictGenHashFunction((const unsigned char *)key, - sdslen((const sds)key)); -} - -static void *callbackValDup(void *privdata, const void *src) { - ((void) privdata); - redisCallback *dup = malloc(sizeof(*dup)); - memcpy(dup,src,sizeof(*dup)); - return dup; -} - -static int callbackKeyCompare(void *privdata, const void *key1, const void *key2) { - int l1, l2; - ((void) privdata); - - l1 = sdslen((const sds)key1); - l2 = sdslen((const sds)key2); - if (l1 != l2) return 0; - return memcmp(key1,key2,l1) == 0; -} - -static void callbackKeyDestructor(void *privdata, void *key) { - ((void) privdata); - sdsfree((sds)key); -} - -static void callbackValDestructor(void *privdata, void *val) { - ((void) privdata); - free(val); -} - -static dictType callbackDict = { - callbackHash, - NULL, - callbackValDup, - callbackKeyCompare, - callbackKeyDestructor, - callbackValDestructor -}; - -static redisAsyncContext *redisAsyncInitialize(redisContext *c) { - redisAsyncContext *ac; - - ac = realloc(c,sizeof(redisAsyncContext)); - if (ac == NULL) - return NULL; - - c = &(ac->c); - - /* The regular connect functions will always set the flag REDIS_CONNECTED. - * For the async API, we want to wait until the first write event is - * received up before setting this flag, so reset it here. */ - c->flags &= ~REDIS_CONNECTED; - - ac->err = 0; - ac->errstr = NULL; - ac->data = NULL; - - ac->ev.data = NULL; - ac->ev.addRead = NULL; - ac->ev.delRead = NULL; - ac->ev.addWrite = NULL; - ac->ev.delWrite = NULL; - ac->ev.cleanup = NULL; - - ac->onConnect = NULL; - ac->onDisconnect = NULL; - - ac->replies.head = NULL; - ac->replies.tail = NULL; - ac->sub.invalid.head = NULL; - ac->sub.invalid.tail = NULL; - ac->sub.channels = dictCreate(&callbackDict,NULL); - ac->sub.patterns = dictCreate(&callbackDict,NULL); - return ac; -} - -/* We want the error field to be accessible directly instead of requiring - * an indirection to the redisContext struct. */ -static void __redisAsyncCopyError(redisAsyncContext *ac) { - if (!ac) - return; - - redisContext *c = &(ac->c); - ac->err = c->err; - ac->errstr = c->errstr; -} - -redisAsyncContext *redisAsyncConnect(const char *ip, int port) { - redisContext *c; - redisAsyncContext *ac; - - c = redisConnectNonBlock(ip,port); - if (c == NULL) - return NULL; - - ac = redisAsyncInitialize(c); - if (ac == NULL) { - redisFree(c); - return NULL; - } - - __redisAsyncCopyError(ac); - return ac; -} - -redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, - const char *source_addr) { - redisContext *c = redisConnectBindNonBlock(ip,port,source_addr); - redisAsyncContext *ac = redisAsyncInitialize(c); - __redisAsyncCopyError(ac); - return ac; -} - -redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, - const char *source_addr) { - redisContext *c = redisConnectBindNonBlockWithReuse(ip,port,source_addr); - redisAsyncContext *ac = redisAsyncInitialize(c); - __redisAsyncCopyError(ac); - return ac; -} - -redisAsyncContext *redisAsyncConnectUnix(const char *path) { - redisContext *c; - redisAsyncContext *ac; - - c = redisConnectUnixNonBlock(path); - if (c == NULL) - return NULL; - - ac = redisAsyncInitialize(c); - if (ac == NULL) { - redisFree(c); - return NULL; - } - - __redisAsyncCopyError(ac); - return ac; -} - -int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn) { - if (ac->onConnect == NULL) { - ac->onConnect = fn; - - /* The common way to detect an established connection is to wait for - * the first write event to be fired. This assumes the related event - * library functions are already set. */ - _EL_ADD_WRITE(ac); - return REDIS_OK; - } - return REDIS_ERR; -} - -int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn) { - if (ac->onDisconnect == NULL) { - ac->onDisconnect = fn; - return REDIS_OK; - } - return REDIS_ERR; -} - -/* Helper functions to push/shift callbacks */ -static int __redisPushCallback(redisCallbackList *list, redisCallback *source) { - redisCallback *cb; - - /* Copy callback from stack to heap */ - cb = malloc(sizeof(*cb)); - if (cb == NULL) - return REDIS_ERR_OOM; - - if (source != NULL) { - memcpy(cb,source,sizeof(*cb)); - cb->next = NULL; - } - - /* Store callback in list */ - if (list->head == NULL) - list->head = cb; - if (list->tail != NULL) - list->tail->next = cb; - list->tail = cb; - return REDIS_OK; -} - -static int __redisShiftCallback(redisCallbackList *list, redisCallback *target) { - redisCallback *cb = list->head; - if (cb != NULL) { - list->head = cb->next; - if (cb == list->tail) - list->tail = NULL; - - /* Copy callback from heap to stack */ - if (target != NULL) - memcpy(target,cb,sizeof(*cb)); - free(cb); - return REDIS_OK; - } - return REDIS_ERR; -} - -static void __redisRunCallback(redisAsyncContext *ac, redisCallback *cb, redisReply *reply) { - redisContext *c = &(ac->c); - if (cb->fn != NULL) { - c->flags |= REDIS_IN_CALLBACK; - cb->fn(ac,reply,cb->privdata); - c->flags &= ~REDIS_IN_CALLBACK; - } -} - -/* Helper function to free the context. */ -static void __redisAsyncFree(redisAsyncContext *ac) { - redisContext *c = &(ac->c); - redisCallback cb; - dictIterator *it; - dictEntry *de; - - /* Execute pending callbacks with NULL reply. */ - while (__redisShiftCallback(&ac->replies,&cb) == REDIS_OK) - __redisRunCallback(ac,&cb,NULL); - - /* Execute callbacks for invalid commands */ - while (__redisShiftCallback(&ac->sub.invalid,&cb) == REDIS_OK) - __redisRunCallback(ac,&cb,NULL); - - /* Run subscription callbacks callbacks with NULL reply */ - it = dictGetIterator(ac->sub.channels); - while ((de = dictNext(it)) != NULL) - __redisRunCallback(ac,dictGetEntryVal(de),NULL); - dictReleaseIterator(it); - dictRelease(ac->sub.channels); - - it = dictGetIterator(ac->sub.patterns); - while ((de = dictNext(it)) != NULL) - __redisRunCallback(ac,dictGetEntryVal(de),NULL); - dictReleaseIterator(it); - dictRelease(ac->sub.patterns); - - /* Signal event lib to clean up */ - _EL_CLEANUP(ac); - - /* Execute disconnect callback. When redisAsyncFree() initiated destroying - * this context, the status will always be REDIS_OK. */ - if (ac->onDisconnect && (c->flags & REDIS_CONNECTED)) { - if (c->flags & REDIS_FREEING) { - ac->onDisconnect(ac,REDIS_OK); - } else { - ac->onDisconnect(ac,(ac->err == 0) ? REDIS_OK : REDIS_ERR); - } - } - - /* Cleanup self */ - redisFree(c); -} - -/* Free the async context. When this function is called from a callback, - * control needs to be returned to redisProcessCallbacks() before actual - * free'ing. To do so, a flag is set on the context which is picked up by - * redisProcessCallbacks(). Otherwise, the context is immediately free'd. */ -void redisAsyncFree(redisAsyncContext *ac) { - redisContext *c = &(ac->c); - c->flags |= REDIS_FREEING; - if (!(c->flags & REDIS_IN_CALLBACK)) - __redisAsyncFree(ac); -} - -/* Helper function to make the disconnect happen and clean up. */ -static void __redisAsyncDisconnect(redisAsyncContext *ac) { - redisContext *c = &(ac->c); - - /* Make sure error is accessible if there is any */ - __redisAsyncCopyError(ac); - - if (ac->err == 0) { - /* For clean disconnects, there should be no pending callbacks. */ - assert(__redisShiftCallback(&ac->replies,NULL) == REDIS_ERR); - } else { - /* Disconnection is caused by an error, make sure that pending - * callbacks cannot call new commands. */ - c->flags |= REDIS_DISCONNECTING; - } - - /* For non-clean disconnects, __redisAsyncFree() will execute pending - * callbacks with a NULL-reply. */ - __redisAsyncFree(ac); -} - -/* Tries to do a clean disconnect from Redis, meaning it stops new commands - * from being issued, but tries to flush the output buffer and execute - * callbacks for all remaining replies. When this function is called from a - * callback, there might be more replies and we can safely defer disconnecting - * to redisProcessCallbacks(). Otherwise, we can only disconnect immediately - * when there are no pending callbacks. */ -void redisAsyncDisconnect(redisAsyncContext *ac) { - redisContext *c = &(ac->c); - c->flags |= REDIS_DISCONNECTING; - if (!(c->flags & REDIS_IN_CALLBACK) && ac->replies.head == NULL) - __redisAsyncDisconnect(ac); -} - -static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, redisCallback *dstcb) { - redisContext *c = &(ac->c); - dict *callbacks; - dictEntry *de; - int pvariant; - char *stype; - sds sname; - - /* Custom reply functions are not supported for pub/sub. This will fail - * very hard when they are used... */ - if (reply->type == REDIS_REPLY_ARRAY) { - assert(reply->elements >= 2); - assert(reply->element[0]->type == REDIS_REPLY_STRING); - stype = reply->element[0]->str; - pvariant = (tolower(stype[0]) == 'p') ? 1 : 0; - - if (pvariant) - callbacks = ac->sub.patterns; - else - callbacks = ac->sub.channels; - - /* Locate the right callback */ - assert(reply->element[1]->type == REDIS_REPLY_STRING); - sname = sdsnewlen(reply->element[1]->str,reply->element[1]->len); - de = dictFind(callbacks,sname); - if (de != NULL) { - memcpy(dstcb,dictGetEntryVal(de),sizeof(*dstcb)); - - /* If this is an unsubscribe message, remove it. */ - if (strcasecmp(stype+pvariant,"unsubscribe") == 0) { - dictDelete(callbacks,sname); - - /* If this was the last unsubscribe message, revert to - * non-subscribe mode. */ - assert(reply->element[2]->type == REDIS_REPLY_INTEGER); - if (reply->element[2]->integer == 0) - c->flags &= ~REDIS_SUBSCRIBED; - } - } - sdsfree(sname); - } else { - /* Shift callback for invalid commands. */ - __redisShiftCallback(&ac->sub.invalid,dstcb); - } - return REDIS_OK; -} - -void redisProcessCallbacks(redisAsyncContext *ac) { - redisContext *c = &(ac->c); - redisCallback cb = {NULL, NULL, NULL}; - void *reply = NULL; - int status; - - while((status = redisGetReply(c,&reply)) == REDIS_OK) { - if (reply == NULL) { - /* When the connection is being disconnected and there are - * no more replies, this is the cue to really disconnect. */ - if (c->flags & REDIS_DISCONNECTING && sdslen(c->obuf) == 0 - && ac->replies.head == NULL) { - __redisAsyncDisconnect(ac); - return; - } - - /* If monitor mode, repush callback */ - if(c->flags & REDIS_MONITORING) { - __redisPushCallback(&ac->replies,&cb); - } - - /* When the connection is not being disconnected, simply stop - * trying to get replies and wait for the next loop tick. */ - break; - } - - /* Even if the context is subscribed, pending regular callbacks will - * get a reply before pub/sub messages arrive. */ - if (__redisShiftCallback(&ac->replies,&cb) != REDIS_OK) { - /* - * A spontaneous reply in a not-subscribed context can be the error - * reply that is sent when a new connection exceeds the maximum - * number of allowed connections on the server side. - * - * This is seen as an error instead of a regular reply because the - * server closes the connection after sending it. - * - * To prevent the error from being overwritten by an EOF error the - * connection is closed here. See issue #43. - * - * Another possibility is that the server is loading its dataset. - * In this case we also want to close the connection, and have the - * user wait until the server is ready to take our request. - */ - if (((redisReply*)reply)->type == REDIS_REPLY_ERROR) { - c->err = REDIS_ERR_OTHER; - snprintf(c->errstr,sizeof(c->errstr),"%s",((redisReply*)reply)->str); - c->reader->fn->freeObject(reply); - __redisAsyncDisconnect(ac); - return; - } - /* No more regular callbacks and no errors, the context *must* be subscribed or monitoring. */ - assert((c->flags & REDIS_SUBSCRIBED || c->flags & REDIS_MONITORING)); - if(c->flags & REDIS_SUBSCRIBED) - __redisGetSubscribeCallback(ac,reply,&cb); - } - - if (cb.fn != NULL) { - __redisRunCallback(ac,&cb,reply); - c->reader->fn->freeObject(reply); - - /* Proceed with free'ing when redisAsyncFree() was called. */ - if (c->flags & REDIS_FREEING) { - __redisAsyncFree(ac); - return; - } - } else { - /* No callback for this reply. This can either be a NULL callback, - * or there were no callbacks to begin with. Either way, don't - * abort with an error, but simply ignore it because the client - * doesn't know what the server will spit out over the wire. */ - c->reader->fn->freeObject(reply); - } - } - - /* Disconnect when there was an error reading the reply */ - if (status != REDIS_OK) - __redisAsyncDisconnect(ac); -} - -/* Internal helper function to detect socket status the first time a read or - * write event fires. When connecting was not succesful, the connect callback - * is called with a REDIS_ERR status and the context is free'd. */ -static int __redisAsyncHandleConnect(redisAsyncContext *ac) { - redisContext *c = &(ac->c); - - if (redisCheckSocketError(c) == REDIS_ERR) { - /* Try again later when connect(2) is still in progress. */ - if (errno == EINPROGRESS) - return REDIS_OK; - - if (ac->onConnect) ac->onConnect(ac,REDIS_ERR); - __redisAsyncDisconnect(ac); - return REDIS_ERR; - } - - /* Mark context as connected. */ - c->flags |= REDIS_CONNECTED; - if (ac->onConnect) ac->onConnect(ac,REDIS_OK); - return REDIS_OK; -} - -/* This function should be called when the socket is readable. - * It processes all replies that can be read and executes their callbacks. - */ -void redisAsyncHandleRead(redisAsyncContext *ac) { - redisContext *c = &(ac->c); - - if (!(c->flags & REDIS_CONNECTED)) { - /* Abort connect was not successful. */ - if (__redisAsyncHandleConnect(ac) != REDIS_OK) - return; - /* Try again later when the context is still not connected. */ - if (!(c->flags & REDIS_CONNECTED)) - return; - } - - if (redisBufferRead(c) == REDIS_ERR) { - __redisAsyncDisconnect(ac); - } else { - /* Always re-schedule reads */ - _EL_ADD_READ(ac); - redisProcessCallbacks(ac); - } -} - -void redisAsyncHandleWrite(redisAsyncContext *ac) { - redisContext *c = &(ac->c); - int done = 0; - - if (!(c->flags & REDIS_CONNECTED)) { - /* Abort connect was not successful. */ - if (__redisAsyncHandleConnect(ac) != REDIS_OK) - return; - /* Try again later when the context is still not connected. */ - if (!(c->flags & REDIS_CONNECTED)) - return; - } - - if (redisBufferWrite(c,&done) == REDIS_ERR) { - __redisAsyncDisconnect(ac); - } else { - /* Continue writing when not done, stop writing otherwise */ - if (!done) - _EL_ADD_WRITE(ac); - else - _EL_DEL_WRITE(ac); - - /* Always schedule reads after writes */ - _EL_ADD_READ(ac); - } -} - -/* Sets a pointer to the first argument and its length starting at p. Returns - * the number of bytes to skip to get to the following argument. */ -static const char *nextArgument(const char *start, const char **str, size_t *len) { - const char *p = start; - if (p[0] != '$') { - p = strchr(p,'$'); - if (p == NULL) return NULL; - } - - *len = (int)strtol(p+1,NULL,10); - p = strchr(p,'\r'); - assert(p); - *str = p+2; - return p+2+(*len)+2; -} - -/* Helper function for the redisAsyncCommand* family of functions. Writes a - * formatted command to the output buffer and registers the provided callback - * function with the context. */ -static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len) { - redisContext *c = &(ac->c); - redisCallback cb; - int pvariant, hasnext; - const char *cstr, *astr; - size_t clen, alen; - const char *p; - sds sname; - int ret; - - /* Don't accept new commands when the connection is about to be closed. */ - if (c->flags & (REDIS_DISCONNECTING | REDIS_FREEING)) return REDIS_ERR; - - /* Setup callback */ - cb.fn = fn; - cb.privdata = privdata; - - /* Find out which command will be appended. */ - p = nextArgument(cmd,&cstr,&clen); - assert(p != NULL); - hasnext = (p[0] == '$'); - pvariant = (tolower(cstr[0]) == 'p') ? 1 : 0; - cstr += pvariant; - clen -= pvariant; - - if (hasnext && strncasecmp(cstr,"subscribe\r\n",11) == 0) { - c->flags |= REDIS_SUBSCRIBED; - - /* Add every channel/pattern to the list of subscription callbacks. */ - while ((p = nextArgument(p,&astr,&alen)) != NULL) { - sname = sdsnewlen(astr,alen); - if (pvariant) - ret = dictReplace(ac->sub.patterns,sname,&cb); - else - ret = dictReplace(ac->sub.channels,sname,&cb); - - if (ret == 0) sdsfree(sname); - } - } else if (strncasecmp(cstr,"unsubscribe\r\n",13) == 0) { - /* It is only useful to call (P)UNSUBSCRIBE when the context is - * subscribed to one or more channels or patterns. */ - if (!(c->flags & REDIS_SUBSCRIBED)) return REDIS_ERR; - - /* (P)UNSUBSCRIBE does not have its own response: every channel or - * pattern that is unsubscribed will receive a message. This means we - * should not append a callback function for this command. */ - } else if(strncasecmp(cstr,"monitor\r\n",9) == 0) { - /* Set monitor flag and push callback */ - c->flags |= REDIS_MONITORING; - __redisPushCallback(&ac->replies,&cb); - } else { - if (c->flags & REDIS_SUBSCRIBED) - /* This will likely result in an error reply, but it needs to be - * received and passed to the callback. */ - __redisPushCallback(&ac->sub.invalid,&cb); - else - __redisPushCallback(&ac->replies,&cb); - } - - __redisAppendCommand(c,cmd,len); - - /* Always schedule a write when the write buffer is non-empty */ - _EL_ADD_WRITE(ac); - - return REDIS_OK; -} - -int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap) { - char *cmd; - int len; - int status; - len = redisvFormatCommand(&cmd,format,ap); - - /* We don't want to pass -1 or -2 to future functions as a length. */ - if (len < 0) - return REDIS_ERR; - - status = __redisAsyncCommand(ac,fn,privdata,cmd,len); - free(cmd); - return status; -} - -int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, ...) { - va_list ap; - int status; - va_start(ap,format); - status = redisvAsyncCommand(ac,fn,privdata,format,ap); - va_end(ap); - return status; -} - -int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen) { - sds cmd; - int len; - int status; - len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen); - status = __redisAsyncCommand(ac,fn,privdata,cmd,len); - sdsfree(cmd); - return status; -} - -int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len) { - int status = __redisAsyncCommand(ac,fn,privdata,cmd,len); - return status; -} diff --git a/third-party/hiredis/async.h b/third-party/hiredis/async.h deleted file mode 100644 index 59cbf469b..000000000 --- a/third-party/hiredis/async.h +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2009-2011, Salvatore Sanfilippo - * Copyright (c) 2010-2011, Pieter Noordhuis - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __HIREDIS_ASYNC_H -#define __HIREDIS_ASYNC_H -#include "hiredis.h" - -#ifdef __cplusplus -extern "C" { -#endif - -struct redisAsyncContext; /* need forward declaration of redisAsyncContext */ -struct dict; /* dictionary header is included in async.c */ - -/* Reply callback prototype and container */ -typedef void (redisCallbackFn)(struct redisAsyncContext*, void*, void*); -typedef struct redisCallback { - struct redisCallback *next; /* simple singly linked list */ - redisCallbackFn *fn; - void *privdata; -} redisCallback; - -/* List of callbacks for either regular replies or pub/sub */ -typedef struct redisCallbackList { - redisCallback *head, *tail; -} redisCallbackList; - -/* Connection callback prototypes */ -typedef void (redisDisconnectCallback)(const struct redisAsyncContext*, int status); -typedef void (redisConnectCallback)(const struct redisAsyncContext*, int status); - -/* Context for an async connection to Redis */ -typedef struct redisAsyncContext { - /* Hold the regular context, so it can be realloc'ed. */ - redisContext c; - - /* Setup error flags so they can be used directly. */ - int err; - char *errstr; - - /* Not used by hiredis */ - void *data; - - /* Event library data and hooks */ - struct { - void *data; - - /* Hooks that are called when the library expects to start - * reading/writing. These functions should be idempotent. */ - void (*addRead)(void *privdata); - void (*delRead)(void *privdata); - void (*addWrite)(void *privdata); - void (*delWrite)(void *privdata); - void (*cleanup)(void *privdata); - } ev; - - /* Called when either the connection is terminated due to an error or per - * user request. The status is set accordingly (REDIS_OK, REDIS_ERR). */ - redisDisconnectCallback *onDisconnect; - - /* Called when the first write event was received. */ - redisConnectCallback *onConnect; - - /* Regular command callbacks */ - redisCallbackList replies; - - /* Subscription callbacks */ - struct { - redisCallbackList invalid; - struct dict *channels; - struct dict *patterns; - } sub; -} redisAsyncContext; - -/* Functions that proxy to hiredis */ -redisAsyncContext *redisAsyncConnect(const char *ip, int port); -redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr); -redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, - const char *source_addr); -redisAsyncContext *redisAsyncConnectUnix(const char *path); -int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn); -int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); -void redisAsyncDisconnect(redisAsyncContext *ac); -void redisAsyncFree(redisAsyncContext *ac); - -/* Handle read/write events */ -void redisAsyncHandleRead(redisAsyncContext *ac); -void redisAsyncHandleWrite(redisAsyncContext *ac); - -/* Command functions for an async context. Write the command to the - * output buffer and register the provided callback. */ -int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap); -int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, ...); -int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen); -int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/third-party/hiredis/dict.c b/third-party/hiredis/dict.c deleted file mode 100644 index 79b1041ca..000000000 --- a/third-party/hiredis/dict.c +++ /dev/null @@ -1,338 +0,0 @@ -/* Hash table implementation. - * - * This file implements in memory hash tables with insert/del/replace/find/ - * get-random-element operations. Hash tables will auto resize if needed - * tables of power of two in size are used, collisions are handled by - * chaining. See the source code for more information... :) - * - * Copyright (c) 2006-2010, Salvatore Sanfilippo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#include "fmacros.h" -#include -#include -#include -#include "dict.h" - -/* -------------------------- private prototypes ---------------------------- */ - -static int _dictExpandIfNeeded(dict *ht); -static unsigned long _dictNextPower(unsigned long size); -static int _dictKeyIndex(dict *ht, const void *key); -static int _dictInit(dict *ht, dictType *type, void *privDataPtr); - -/* -------------------------- hash functions -------------------------------- */ - -/* Generic hash function (a popular one from Bernstein). - * I tested a few and this was the best. */ -static unsigned int dictGenHashFunction(const unsigned char *buf, int len) { - unsigned int hash = 5381; - - while (len--) - hash = ((hash << 5) + hash) + (*buf++); /* hash * 33 + c */ - return hash; -} - -/* ----------------------------- API implementation ------------------------- */ - -/* Reset an hashtable already initialized with ht_init(). - * NOTE: This function should only called by ht_destroy(). */ -static void _dictReset(dict *ht) { - ht->table = NULL; - ht->size = 0; - ht->sizemask = 0; - ht->used = 0; -} - -/* Create a new hash table */ -static dict *dictCreate(dictType *type, void *privDataPtr) { - dict *ht = malloc(sizeof(*ht)); - _dictInit(ht,type,privDataPtr); - return ht; -} - -/* Initialize the hash table */ -static int _dictInit(dict *ht, dictType *type, void *privDataPtr) { - _dictReset(ht); - ht->type = type; - ht->privdata = privDataPtr; - return DICT_OK; -} - -/* Expand or create the hashtable */ -static int dictExpand(dict *ht, unsigned long size) { - dict n; /* the new hashtable */ - unsigned long realsize = _dictNextPower(size), i; - - /* the size is invalid if it is smaller than the number of - * elements already inside the hashtable */ - if (ht->used > size) - return DICT_ERR; - - _dictInit(&n, ht->type, ht->privdata); - n.size = realsize; - n.sizemask = realsize-1; - n.table = calloc(realsize,sizeof(dictEntry*)); - - /* Copy all the elements from the old to the new table: - * note that if the old hash table is empty ht->size is zero, - * so dictExpand just creates an hash table. */ - n.used = ht->used; - for (i = 0; i < ht->size && ht->used > 0; i++) { - dictEntry *he, *nextHe; - - if (ht->table[i] == NULL) continue; - - /* For each hash entry on this slot... */ - he = ht->table[i]; - while(he) { - unsigned int h; - - nextHe = he->next; - /* Get the new element index */ - h = dictHashKey(ht, he->key) & n.sizemask; - he->next = n.table[h]; - n.table[h] = he; - ht->used--; - /* Pass to the next element */ - he = nextHe; - } - } - assert(ht->used == 0); - free(ht->table); - - /* Remap the new hashtable in the old */ - *ht = n; - return DICT_OK; -} - -/* Add an element to the target hash table */ -static int dictAdd(dict *ht, void *key, void *val) { - int index; - dictEntry *entry; - - /* Get the index of the new element, or -1 if - * the element already exists. */ - if ((index = _dictKeyIndex(ht, key)) == -1) - return DICT_ERR; - - /* Allocates the memory and stores key */ - entry = malloc(sizeof(*entry)); - entry->next = ht->table[index]; - ht->table[index] = entry; - - /* Set the hash entry fields. */ - dictSetHashKey(ht, entry, key); - dictSetHashVal(ht, entry, val); - ht->used++; - return DICT_OK; -} - -/* Add an element, discarding the old if the key already exists. - * Return 1 if the key was added from scratch, 0 if there was already an - * element with such key and dictReplace() just performed a value update - * operation. */ -static int dictReplace(dict *ht, void *key, void *val) { - dictEntry *entry, auxentry; - - /* Try to add the element. If the key - * does not exists dictAdd will suceed. */ - if (dictAdd(ht, key, val) == DICT_OK) - return 1; - /* It already exists, get the entry */ - entry = dictFind(ht, key); - /* Free the old value and set the new one */ - /* Set the new value and free the old one. Note that it is important - * to do that in this order, as the value may just be exactly the same - * as the previous one. In this context, think to reference counting, - * you want to increment (set), and then decrement (free), and not the - * reverse. */ - auxentry = *entry; - dictSetHashVal(ht, entry, val); - dictFreeEntryVal(ht, &auxentry); - return 0; -} - -/* Search and remove an element */ -static int dictDelete(dict *ht, const void *key) { - unsigned int h; - dictEntry *de, *prevde; - - if (ht->size == 0) - return DICT_ERR; - h = dictHashKey(ht, key) & ht->sizemask; - de = ht->table[h]; - - prevde = NULL; - while(de) { - if (dictCompareHashKeys(ht,key,de->key)) { - /* Unlink the element from the list */ - if (prevde) - prevde->next = de->next; - else - ht->table[h] = de->next; - - dictFreeEntryKey(ht,de); - dictFreeEntryVal(ht,de); - free(de); - ht->used--; - return DICT_OK; - } - prevde = de; - de = de->next; - } - return DICT_ERR; /* not found */ -} - -/* Destroy an entire hash table */ -static int _dictClear(dict *ht) { - unsigned long i; - - /* Free all the elements */ - for (i = 0; i < ht->size && ht->used > 0; i++) { - dictEntry *he, *nextHe; - - if ((he = ht->table[i]) == NULL) continue; - while(he) { - nextHe = he->next; - dictFreeEntryKey(ht, he); - dictFreeEntryVal(ht, he); - free(he); - ht->used--; - he = nextHe; - } - } - /* Free the table and the allocated cache structure */ - free(ht->table); - /* Re-initialize the table */ - _dictReset(ht); - return DICT_OK; /* never fails */ -} - -/* Clear & Release the hash table */ -static void dictRelease(dict *ht) { - _dictClear(ht); - free(ht); -} - -static dictEntry *dictFind(dict *ht, const void *key) { - dictEntry *he; - unsigned int h; - - if (ht->size == 0) return NULL; - h = dictHashKey(ht, key) & ht->sizemask; - he = ht->table[h]; - while(he) { - if (dictCompareHashKeys(ht, key, he->key)) - return he; - he = he->next; - } - return NULL; -} - -static dictIterator *dictGetIterator(dict *ht) { - dictIterator *iter = malloc(sizeof(*iter)); - - iter->ht = ht; - iter->index = -1; - iter->entry = NULL; - iter->nextEntry = NULL; - return iter; -} - -static dictEntry *dictNext(dictIterator *iter) { - while (1) { - if (iter->entry == NULL) { - iter->index++; - if (iter->index >= - (signed)iter->ht->size) break; - iter->entry = iter->ht->table[iter->index]; - } else { - iter->entry = iter->nextEntry; - } - if (iter->entry) { - /* We need to save the 'next' here, the iterator user - * may delete the entry we are returning. */ - iter->nextEntry = iter->entry->next; - return iter->entry; - } - } - return NULL; -} - -static void dictReleaseIterator(dictIterator *iter) { - free(iter); -} - -/* ------------------------- private functions ------------------------------ */ - -/* Expand the hash table if needed */ -static int _dictExpandIfNeeded(dict *ht) { - /* If the hash table is empty expand it to the intial size, - * if the table is "full" dobule its size. */ - if (ht->size == 0) - return dictExpand(ht, DICT_HT_INITIAL_SIZE); - if (ht->used == ht->size) - return dictExpand(ht, ht->size*2); - return DICT_OK; -} - -/* Our hash table capability is a power of two */ -static unsigned long _dictNextPower(unsigned long size) { - unsigned long i = DICT_HT_INITIAL_SIZE; - - if (size >= LONG_MAX) return LONG_MAX; - while(1) { - if (i >= size) - return i; - i *= 2; - } -} - -/* Returns the index of a free slot that can be populated with - * an hash entry for the given 'key'. - * If the key already exists, -1 is returned. */ -static int _dictKeyIndex(dict *ht, const void *key) { - unsigned int h; - dictEntry *he; - - /* Expand the hashtable if needed */ - if (_dictExpandIfNeeded(ht) == DICT_ERR) - return -1; - /* Compute the key hash value */ - h = dictHashKey(ht, key) & ht->sizemask; - /* Search if this slot does not already contain the given key */ - he = ht->table[h]; - while(he) { - if (dictCompareHashKeys(ht, key, he->key)) - return -1; - he = he->next; - } - return h; -} - diff --git a/third-party/hiredis/dict.h b/third-party/hiredis/dict.h deleted file mode 100644 index 95fcd280e..000000000 --- a/third-party/hiredis/dict.h +++ /dev/null @@ -1,126 +0,0 @@ -/* Hash table implementation. - * - * This file implements in memory hash tables with insert/del/replace/find/ - * get-random-element operations. Hash tables will auto resize if needed - * tables of power of two in size are used, collisions are handled by - * chaining. See the source code for more information... :) - * - * Copyright (c) 2006-2010, Salvatore Sanfilippo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __DICT_H -#define __DICT_H - -#define DICT_OK 0 -#define DICT_ERR 1 - -/* Unused arguments generate annoying warnings... */ -#define DICT_NOTUSED(V) ((void) V) - -typedef struct dictEntry { - void *key; - void *val; - struct dictEntry *next; -} dictEntry; - -typedef struct dictType { - unsigned int (*hashFunction)(const void *key); - void *(*keyDup)(void *privdata, const void *key); - void *(*valDup)(void *privdata, const void *obj); - int (*keyCompare)(void *privdata, const void *key1, const void *key2); - void (*keyDestructor)(void *privdata, void *key); - void (*valDestructor)(void *privdata, void *obj); -} dictType; - -typedef struct dict { - dictEntry **table; - dictType *type; - unsigned long size; - unsigned long sizemask; - unsigned long used; - void *privdata; -} dict; - -typedef struct dictIterator { - dict *ht; - int index; - dictEntry *entry, *nextEntry; -} dictIterator; - -/* This is the initial size of every hash table */ -#define DICT_HT_INITIAL_SIZE 4 - -/* ------------------------------- Macros ------------------------------------*/ -#define dictFreeEntryVal(ht, entry) \ - if ((ht)->type->valDestructor) \ - (ht)->type->valDestructor((ht)->privdata, (entry)->val) - -#define dictSetHashVal(ht, entry, _val_) do { \ - if ((ht)->type->valDup) \ - entry->val = (ht)->type->valDup((ht)->privdata, _val_); \ - else \ - entry->val = (_val_); \ -} while(0) - -#define dictFreeEntryKey(ht, entry) \ - if ((ht)->type->keyDestructor) \ - (ht)->type->keyDestructor((ht)->privdata, (entry)->key) - -#define dictSetHashKey(ht, entry, _key_) do { \ - if ((ht)->type->keyDup) \ - entry->key = (ht)->type->keyDup((ht)->privdata, _key_); \ - else \ - entry->key = (_key_); \ -} while(0) - -#define dictCompareHashKeys(ht, key1, key2) \ - (((ht)->type->keyCompare) ? \ - (ht)->type->keyCompare((ht)->privdata, key1, key2) : \ - (key1) == (key2)) - -#define dictHashKey(ht, key) (ht)->type->hashFunction(key) - -#define dictGetEntryKey(he) ((he)->key) -#define dictGetEntryVal(he) ((he)->val) -#define dictSlots(ht) ((ht)->size) -#define dictSize(ht) ((ht)->used) - -/* API */ -static unsigned int dictGenHashFunction(const unsigned char *buf, int len); -static dict *dictCreate(dictType *type, void *privDataPtr); -static int dictExpand(dict *ht, unsigned long size); -static int dictAdd(dict *ht, void *key, void *val); -static int dictReplace(dict *ht, void *key, void *val); -static int dictDelete(dict *ht, const void *key); -static void dictRelease(dict *ht); -static dictEntry * dictFind(dict *ht, const void *key); -static dictIterator *dictGetIterator(dict *ht); -static dictEntry *dictNext(dictIterator *iter); -static void dictReleaseIterator(dictIterator *iter); - -#endif /* __DICT_H */ diff --git a/third-party/hiredis/examples/example-ae.c b/third-party/hiredis/examples/example-ae.c deleted file mode 100644 index 8efa7306a..000000000 --- a/third-party/hiredis/examples/example-ae.c +++ /dev/null @@ -1,62 +0,0 @@ -#include -#include -#include -#include - -#include -#include -#include - -/* Put event loop in the global scope, so it can be explicitly stopped */ -static aeEventLoop *loop; - -void getCallback(redisAsyncContext *c, void *r, void *privdata) { - redisReply *reply = r; - if (reply == NULL) return; - printf("argv[%s]: %s\n", (char*)privdata, reply->str); - - /* Disconnect after receiving the reply to GET */ - redisAsyncDisconnect(c); -} - -void connectCallback(const redisAsyncContext *c, int status) { - if (status != REDIS_OK) { - printf("Error: %s\n", c->errstr); - aeStop(loop); - return; - } - - printf("Connected...\n"); -} - -void disconnectCallback(const redisAsyncContext *c, int status) { - if (status != REDIS_OK) { - printf("Error: %s\n", c->errstr); - aeStop(loop); - return; - } - - printf("Disconnected...\n"); - aeStop(loop); -} - -int main (int argc, char **argv) { - signal(SIGPIPE, SIG_IGN); - - redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); - if (c->err) { - /* Let *c leak for now... */ - printf("Error: %s\n", c->errstr); - return 1; - } - - loop = aeCreateEventLoop(64); - redisAeAttach(loop, c); - redisAsyncSetConnectCallback(c,connectCallback); - redisAsyncSetDisconnectCallback(c,disconnectCallback); - redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1])); - redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); - aeMain(loop); - return 0; -} - diff --git a/third-party/hiredis/examples/example-glib.c b/third-party/hiredis/examples/example-glib.c deleted file mode 100644 index d6e10f8e8..000000000 --- a/third-party/hiredis/examples/example-glib.c +++ /dev/null @@ -1,73 +0,0 @@ -#include - -#include -#include -#include - -static GMainLoop *mainloop; - -static void -connect_cb (const redisAsyncContext *ac G_GNUC_UNUSED, - int status) -{ - if (status != REDIS_OK) { - g_printerr("Failed to connect: %s\n", ac->errstr); - g_main_loop_quit(mainloop); - } else { - g_printerr("Connected...\n"); - } -} - -static void -disconnect_cb (const redisAsyncContext *ac G_GNUC_UNUSED, - int status) -{ - if (status != REDIS_OK) { - g_error("Failed to disconnect: %s", ac->errstr); - } else { - g_printerr("Disconnected...\n"); - g_main_loop_quit(mainloop); - } -} - -static void -command_cb(redisAsyncContext *ac, - gpointer r, - gpointer user_data G_GNUC_UNUSED) -{ - redisReply *reply = r; - - if (reply) { - g_print("REPLY: %s\n", reply->str); - } - - redisAsyncDisconnect(ac); -} - -gint -main (gint argc G_GNUC_UNUSED, - gchar *argv[] G_GNUC_UNUSED) -{ - redisAsyncContext *ac; - GMainContext *context = NULL; - GSource *source; - - ac = redisAsyncConnect("127.0.0.1", 6379); - if (ac->err) { - g_printerr("%s\n", ac->errstr); - exit(EXIT_FAILURE); - } - - source = redis_source_new(ac); - mainloop = g_main_loop_new(context, FALSE); - g_source_attach(source, context); - - redisAsyncSetConnectCallback(ac, connect_cb); - redisAsyncSetDisconnectCallback(ac, disconnect_cb); - redisAsyncCommand(ac, command_cb, NULL, "SET key 1234"); - redisAsyncCommand(ac, command_cb, NULL, "GET key"); - - g_main_loop_run(mainloop); - - return EXIT_SUCCESS; -} diff --git a/third-party/hiredis/examples/example-ivykis.c b/third-party/hiredis/examples/example-ivykis.c deleted file mode 100644 index 67affcef3..000000000 --- a/third-party/hiredis/examples/example-ivykis.c +++ /dev/null @@ -1,58 +0,0 @@ -#include -#include -#include -#include - -#include -#include -#include - -void getCallback(redisAsyncContext *c, void *r, void *privdata) { - redisReply *reply = r; - if (reply == NULL) return; - printf("argv[%s]: %s\n", (char*)privdata, reply->str); - - /* Disconnect after receiving the reply to GET */ - redisAsyncDisconnect(c); -} - -void connectCallback(const redisAsyncContext *c, int status) { - if (status != REDIS_OK) { - printf("Error: %s\n", c->errstr); - return; - } - printf("Connected...\n"); -} - -void disconnectCallback(const redisAsyncContext *c, int status) { - if (status != REDIS_OK) { - printf("Error: %s\n", c->errstr); - return; - } - printf("Disconnected...\n"); -} - -int main (int argc, char **argv) { - signal(SIGPIPE, SIG_IGN); - - iv_init(); - - redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); - if (c->err) { - /* Let *c leak for now... */ - printf("Error: %s\n", c->errstr); - return 1; - } - - redisIvykisAttach(c); - redisAsyncSetConnectCallback(c,connectCallback); - redisAsyncSetDisconnectCallback(c,disconnectCallback); - redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1])); - redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); - - iv_main(); - - iv_deinit(); - - return 0; -} diff --git a/third-party/hiredis/examples/example-libev.c b/third-party/hiredis/examples/example-libev.c deleted file mode 100644 index cc8b166ec..000000000 --- a/third-party/hiredis/examples/example-libev.c +++ /dev/null @@ -1,52 +0,0 @@ -#include -#include -#include -#include - -#include -#include -#include - -void getCallback(redisAsyncContext *c, void *r, void *privdata) { - redisReply *reply = r; - if (reply == NULL) return; - printf("argv[%s]: %s\n", (char*)privdata, reply->str); - - /* Disconnect after receiving the reply to GET */ - redisAsyncDisconnect(c); -} - -void connectCallback(const redisAsyncContext *c, int status) { - if (status != REDIS_OK) { - printf("Error: %s\n", c->errstr); - return; - } - printf("Connected...\n"); -} - -void disconnectCallback(const redisAsyncContext *c, int status) { - if (status != REDIS_OK) { - printf("Error: %s\n", c->errstr); - return; - } - printf("Disconnected...\n"); -} - -int main (int argc, char **argv) { - signal(SIGPIPE, SIG_IGN); - - redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); - if (c->err) { - /* Let *c leak for now... */ - printf("Error: %s\n", c->errstr); - return 1; - } - - redisLibevAttach(EV_DEFAULT_ c); - redisAsyncSetConnectCallback(c,connectCallback); - redisAsyncSetDisconnectCallback(c,disconnectCallback); - redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1])); - redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); - ev_loop(EV_DEFAULT_ 0); - return 0; -} diff --git a/third-party/hiredis/examples/example-libevent.c b/third-party/hiredis/examples/example-libevent.c deleted file mode 100644 index d333c22b7..000000000 --- a/third-party/hiredis/examples/example-libevent.c +++ /dev/null @@ -1,53 +0,0 @@ -#include -#include -#include -#include - -#include -#include -#include - -void getCallback(redisAsyncContext *c, void *r, void *privdata) { - redisReply *reply = r; - if (reply == NULL) return; - printf("argv[%s]: %s\n", (char*)privdata, reply->str); - - /* Disconnect after receiving the reply to GET */ - redisAsyncDisconnect(c); -} - -void connectCallback(const redisAsyncContext *c, int status) { - if (status != REDIS_OK) { - printf("Error: %s\n", c->errstr); - return; - } - printf("Connected...\n"); -} - -void disconnectCallback(const redisAsyncContext *c, int status) { - if (status != REDIS_OK) { - printf("Error: %s\n", c->errstr); - return; - } - printf("Disconnected...\n"); -} - -int main (int argc, char **argv) { - signal(SIGPIPE, SIG_IGN); - struct event_base *base = event_base_new(); - - redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); - if (c->err) { - /* Let *c leak for now... */ - printf("Error: %s\n", c->errstr); - return 1; - } - - redisLibeventAttach(c,base); - redisAsyncSetConnectCallback(c,connectCallback); - redisAsyncSetDisconnectCallback(c,disconnectCallback); - redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1])); - redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); - event_base_dispatch(base); - return 0; -} diff --git a/third-party/hiredis/examples/example-libuv.c b/third-party/hiredis/examples/example-libuv.c deleted file mode 100644 index a5462d410..000000000 --- a/third-party/hiredis/examples/example-libuv.c +++ /dev/null @@ -1,53 +0,0 @@ -#include -#include -#include -#include - -#include -#include -#include - -void getCallback(redisAsyncContext *c, void *r, void *privdata) { - redisReply *reply = r; - if (reply == NULL) return; - printf("argv[%s]: %s\n", (char*)privdata, reply->str); - - /* Disconnect after receiving the reply to GET */ - redisAsyncDisconnect(c); -} - -void connectCallback(const redisAsyncContext *c, int status) { - if (status != REDIS_OK) { - printf("Error: %s\n", c->errstr); - return; - } - printf("Connected...\n"); -} - -void disconnectCallback(const redisAsyncContext *c, int status) { - if (status != REDIS_OK) { - printf("Error: %s\n", c->errstr); - return; - } - printf("Disconnected...\n"); -} - -int main (int argc, char **argv) { - signal(SIGPIPE, SIG_IGN); - uv_loop_t* loop = uv_default_loop(); - - redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); - if (c->err) { - /* Let *c leak for now... */ - printf("Error: %s\n", c->errstr); - return 1; - } - - redisLibuvAttach(c,loop); - redisAsyncSetConnectCallback(c,connectCallback); - redisAsyncSetDisconnectCallback(c,disconnectCallback); - redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1])); - redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); - uv_run(loop, UV_RUN_DEFAULT); - return 0; -} diff --git a/third-party/hiredis/examples/example-macosx.c b/third-party/hiredis/examples/example-macosx.c deleted file mode 100644 index bc84ed5ba..000000000 --- a/third-party/hiredis/examples/example-macosx.c +++ /dev/null @@ -1,66 +0,0 @@ -// -// Created by Дмитрий Бахвалов on 13.07.15. -// Copyright (c) 2015 Dmitry Bakhvalov. All rights reserved. -// - -#include - -#include -#include -#include - -void getCallback(redisAsyncContext *c, void *r, void *privdata) { - redisReply *reply = r; - if (reply == NULL) return; - printf("argv[%s]: %s\n", (char*)privdata, reply->str); - - /* Disconnect after receiving the reply to GET */ - redisAsyncDisconnect(c); -} - -void connectCallback(const redisAsyncContext *c, int status) { - if (status != REDIS_OK) { - printf("Error: %s\n", c->errstr); - return; - } - printf("Connected...\n"); -} - -void disconnectCallback(const redisAsyncContext *c, int status) { - if (status != REDIS_OK) { - printf("Error: %s\n", c->errstr); - return; - } - CFRunLoopStop(CFRunLoopGetCurrent()); - printf("Disconnected...\n"); -} - -int main (int argc, char **argv) { - signal(SIGPIPE, SIG_IGN); - - CFRunLoopRef loop = CFRunLoopGetCurrent(); - if( !loop ) { - printf("Error: Cannot get current run loop\n"); - return 1; - } - - redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); - if (c->err) { - /* Let *c leak for now... */ - printf("Error: %s\n", c->errstr); - return 1; - } - - redisMacOSAttach(c, loop); - - redisAsyncSetConnectCallback(c,connectCallback); - redisAsyncSetDisconnectCallback(c,disconnectCallback); - - redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1])); - redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); - - CFRunLoopRun(); - - return 0; -} - diff --git a/third-party/hiredis/examples/example-qt.cpp b/third-party/hiredis/examples/example-qt.cpp deleted file mode 100644 index f524c3f3d..000000000 --- a/third-party/hiredis/examples/example-qt.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include -using namespace std; - -#include -#include - -#include "example-qt.h" - -void getCallback(redisAsyncContext *, void * r, void * privdata) { - - redisReply * reply = static_cast(r); - ExampleQt * ex = static_cast(privdata); - if (reply == nullptr || ex == nullptr) return; - - cout << "key: " << reply->str << endl; - - ex->finish(); -} - -void ExampleQt::run() { - - m_ctx = redisAsyncConnect("localhost", 6379); - - if (m_ctx->err) { - cerr << "Error: " << m_ctx->errstr << endl; - redisAsyncFree(m_ctx); - emit finished(); - } - - m_adapter.setContext(m_ctx); - - redisAsyncCommand(m_ctx, NULL, NULL, "SET key %s", m_value); - redisAsyncCommand(m_ctx, getCallback, this, "GET key"); -} - -int main (int argc, char **argv) { - - QCoreApplication app(argc, argv); - - ExampleQt example(argv[argc-1]); - - QObject::connect(&example, SIGNAL(finished()), &app, SLOT(quit())); - QTimer::singleShot(0, &example, SLOT(run())); - - return app.exec(); -} diff --git a/third-party/hiredis/examples/example-qt.h b/third-party/hiredis/examples/example-qt.h deleted file mode 100644 index 374f47666..000000000 --- a/third-party/hiredis/examples/example-qt.h +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef __HIREDIS_EXAMPLE_QT_H -#define __HIREDIS_EXAMPLE_QT_H - -#include - -class ExampleQt : public QObject { - - Q_OBJECT - - public: - ExampleQt(const char * value, QObject * parent = 0) - : QObject(parent), m_value(value) {} - - signals: - void finished(); - - public slots: - void run(); - - private: - void finish() { emit finished(); } - - private: - const char * m_value; - redisAsyncContext * m_ctx; - RedisQtAdapter m_adapter; - - friend - void getCallback(redisAsyncContext *, void *, void *); -}; - -#endif /* !__HIREDIS_EXAMPLE_QT_H */ diff --git a/third-party/hiredis/examples/example.c b/third-party/hiredis/examples/example.c deleted file mode 100644 index 25226a807..000000000 --- a/third-party/hiredis/examples/example.c +++ /dev/null @@ -1,78 +0,0 @@ -#include -#include -#include - -#include - -int main(int argc, char **argv) { - unsigned int j; - redisContext *c; - redisReply *reply; - const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; - int port = (argc > 2) ? atoi(argv[2]) : 6379; - - struct timeval timeout = { 1, 500000 }; // 1.5 seconds - c = redisConnectWithTimeout(hostname, port, timeout); - if (c == NULL || c->err) { - if (c) { - printf("Connection error: %s\n", c->errstr); - redisFree(c); - } else { - printf("Connection error: can't allocate redis context\n"); - } - exit(1); - } - - /* PING server */ - reply = redisCommand(c,"PING"); - printf("PING: %s\n", reply->str); - freeReplyObject(reply); - - /* Set a key */ - reply = redisCommand(c,"SET %s %s", "foo", "hello world"); - printf("SET: %s\n", reply->str); - freeReplyObject(reply); - - /* Set a key using binary safe API */ - reply = redisCommand(c,"SET %b %b", "bar", (size_t) 3, "hello", (size_t) 5); - printf("SET (binary API): %s\n", reply->str); - freeReplyObject(reply); - - /* Try a GET and two INCR */ - reply = redisCommand(c,"GET foo"); - printf("GET foo: %s\n", reply->str); - freeReplyObject(reply); - - reply = redisCommand(c,"INCR counter"); - printf("INCR counter: %lld\n", reply->integer); - freeReplyObject(reply); - /* again ... */ - reply = redisCommand(c,"INCR counter"); - printf("INCR counter: %lld\n", reply->integer); - freeReplyObject(reply); - - /* Create a list of numbers, from 0 to 9 */ - reply = redisCommand(c,"DEL mylist"); - freeReplyObject(reply); - for (j = 0; j < 10; j++) { - char buf[64]; - - snprintf(buf,64,"%d",j); - reply = redisCommand(c,"LPUSH mylist element-%s", buf); - freeReplyObject(reply); - } - - /* Let's check what we have inside the list */ - reply = redisCommand(c,"LRANGE mylist 0 -1"); - if (reply->type == REDIS_REPLY_ARRAY) { - for (j = 0; j < reply->elements; j++) { - printf("%u) %s\n", j, reply->element[j]->str); - } - } - freeReplyObject(reply); - - /* Disconnects and frees the context */ - redisFree(c); - - return 0; -} diff --git a/third-party/hiredis/fmacros.h b/third-party/hiredis/fmacros.h deleted file mode 100644 index 19d7b2193..000000000 --- a/third-party/hiredis/fmacros.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef __HIREDIS_FMACRO_H -#define __HIREDIS_FMACRO_H - -#if defined(__linux__) -#define _BSD_SOURCE -#define _DEFAULT_SOURCE -#endif - -#if defined(__sun__) -#define _POSIX_C_SOURCE 200112L -#elif defined(__linux__) || defined(__OpenBSD__) || defined(__NetBSD__) -#define _XOPEN_SOURCE 600 -#else -#define _XOPEN_SOURCE -#endif - -#if __APPLE__ && __MACH__ -#define _OSX -#endif - -#endif diff --git a/third-party/hiredis/hiredis.c b/third-party/hiredis/hiredis.c deleted file mode 100644 index 73d0251bc..000000000 --- a/third-party/hiredis/hiredis.c +++ /dev/null @@ -1,1021 +0,0 @@ -/* - * Copyright (c) 2009-2011, Salvatore Sanfilippo - * Copyright (c) 2010-2014, Pieter Noordhuis - * Copyright (c) 2015, Matt Stancliff , - * Jan-Erik Rediger - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#include "fmacros.h" -#include -#include -#include -#include -#include -#include - -#include "hiredis.h" -#include "net.h" -#include "sds.h" - -static redisReply *createReplyObject(int type); -static void *createStringObject(const redisReadTask *task, char *str, size_t len); -static void *createArrayObject(const redisReadTask *task, int elements); -static void *createIntegerObject(const redisReadTask *task, long long value); -static void *createNilObject(const redisReadTask *task); - -/* Default set of functions to build the reply. Keep in mind that such a - * function returning NULL is interpreted as OOM. */ -static redisReplyObjectFunctions defaultFunctions = { - createStringObject, - createArrayObject, - createIntegerObject, - createNilObject, - freeReplyObject -}; - -/* Create a reply object */ -static redisReply *createReplyObject(int type) { - redisReply *r = calloc(1,sizeof(*r)); - - if (r == NULL) - return NULL; - - r->type = type; - return r; -} - -/* Free a reply object */ -void freeReplyObject(void *reply) { - redisReply *r = reply; - size_t j; - - if (r == NULL) - return; - - switch(r->type) { - case REDIS_REPLY_INTEGER: - break; /* Nothing to free */ - case REDIS_REPLY_ARRAY: - if (r->element != NULL) { - for (j = 0; j < r->elements; j++) - if (r->element[j] != NULL) - freeReplyObject(r->element[j]); - free(r->element); - } - break; - case REDIS_REPLY_ERROR: - case REDIS_REPLY_STATUS: - case REDIS_REPLY_STRING: - if (r->str != NULL) - free(r->str); - break; - } - free(r); -} - -static void *createStringObject(const redisReadTask *task, char *str, size_t len) { - redisReply *r, *parent; - char *buf; - - r = createReplyObject(task->type); - if (r == NULL) - return NULL; - - buf = malloc(len+1); - if (buf == NULL) { - freeReplyObject(r); - return NULL; - } - - assert(task->type == REDIS_REPLY_ERROR || - task->type == REDIS_REPLY_STATUS || - task->type == REDIS_REPLY_STRING); - - /* Copy string value */ - memcpy(buf,str,len); - buf[len] = '\0'; - r->str = buf; - r->len = len; - - if (task->parent) { - parent = task->parent->obj; - assert(parent->type == REDIS_REPLY_ARRAY); - parent->element[task->idx] = r; - } - return r; -} - -static void *createArrayObject(const redisReadTask *task, int elements) { - redisReply *r, *parent; - - r = createReplyObject(REDIS_REPLY_ARRAY); - if (r == NULL) - return NULL; - - if (elements > 0) { - r->element = calloc(elements,sizeof(redisReply*)); - if (r->element == NULL) { - freeReplyObject(r); - return NULL; - } - } - - r->elements = elements; - - if (task->parent) { - parent = task->parent->obj; - assert(parent->type == REDIS_REPLY_ARRAY); - parent->element[task->idx] = r; - } - return r; -} - -static void *createIntegerObject(const redisReadTask *task, long long value) { - redisReply *r, *parent; - - r = createReplyObject(REDIS_REPLY_INTEGER); - if (r == NULL) - return NULL; - - r->integer = value; - - if (task->parent) { - parent = task->parent->obj; - assert(parent->type == REDIS_REPLY_ARRAY); - parent->element[task->idx] = r; - } - return r; -} - -static void *createNilObject(const redisReadTask *task) { - redisReply *r, *parent; - - r = createReplyObject(REDIS_REPLY_NIL); - if (r == NULL) - return NULL; - - if (task->parent) { - parent = task->parent->obj; - assert(parent->type == REDIS_REPLY_ARRAY); - parent->element[task->idx] = r; - } - return r; -} - -/* Return the number of digits of 'v' when converted to string in radix 10. - * Implementation borrowed from link in redis/src/util.c:string2ll(). */ -static uint32_t countDigits(uint64_t v) { - uint32_t result = 1; - for (;;) { - if (v < 10) return result; - if (v < 100) return result + 1; - if (v < 1000) return result + 2; - if (v < 10000) return result + 3; - v /= 10000U; - result += 4; - } -} - -/* Helper that calculates the bulk length given a certain string length. */ -static size_t bulklen(size_t len) { - return 1+countDigits(len)+2+len+2; -} - -int redisvFormatCommand(char **target, const char *format, va_list ap) { - const char *c = format; - char *cmd = NULL; /* final command */ - int pos; /* position in final command */ - sds curarg, newarg; /* current argument */ - int touched = 0; /* was the current argument touched? */ - char **curargv = NULL, **newargv = NULL; - int argc = 0; - int totlen = 0; - int error_type = 0; /* 0 = no error; -1 = memory error; -2 = format error */ - int j; - - /* Abort if there is not target to set */ - if (target == NULL) - return -1; - - /* Build the command string accordingly to protocol */ - curarg = sdsempty(); - if (curarg == NULL) - return -1; - - while(*c != '\0') { - if (*c != '%' || c[1] == '\0') { - if (*c == ' ') { - if (touched) { - newargv = realloc(curargv,sizeof(char*)*(argc+1)); - if (newargv == NULL) goto memory_err; - curargv = newargv; - curargv[argc++] = curarg; - totlen += bulklen(sdslen(curarg)); - - /* curarg is put in argv so it can be overwritten. */ - curarg = sdsempty(); - if (curarg == NULL) goto memory_err; - touched = 0; - } - } else { - newarg = sdscatlen(curarg,c,1); - if (newarg == NULL) goto memory_err; - curarg = newarg; - touched = 1; - } - } else { - char *arg; - size_t size; - - /* Set newarg so it can be checked even if it is not touched. */ - newarg = curarg; - - switch(c[1]) { - case 's': - arg = va_arg(ap,char*); - size = strlen(arg); - if (size > 0) - newarg = sdscatlen(curarg,arg,size); - break; - case 'b': - arg = va_arg(ap,char*); - size = va_arg(ap,size_t); - if (size > 0) - newarg = sdscatlen(curarg,arg,size); - break; - case '%': - newarg = sdscat(curarg,"%"); - break; - default: - /* Try to detect printf format */ - { - static const char intfmts[] = "diouxX"; - static const char flags[] = "#0-+ "; - char _format[16]; - const char *_p = c+1; - size_t _l = 0; - va_list _cpy; - - /* Flags */ - while (*_p != '\0' && strchr(flags,*_p) != NULL) _p++; - - /* Field width */ - while (*_p != '\0' && isdigit(*_p)) _p++; - - /* Precision */ - if (*_p == '.') { - _p++; - while (*_p != '\0' && isdigit(*_p)) _p++; - } - - /* Copy va_list before consuming with va_arg */ - va_copy(_cpy,ap); - - /* Integer conversion (without modifiers) */ - if (strchr(intfmts,*_p) != NULL) { - va_arg(ap,int); - goto fmt_valid; - } - - /* Double conversion (without modifiers) */ - if (strchr("eEfFgGaA",*_p) != NULL) { - va_arg(ap,double); - goto fmt_valid; - } - - /* Size: char */ - if (_p[0] == 'h' && _p[1] == 'h') { - _p += 2; - if (*_p != '\0' && strchr(intfmts,*_p) != NULL) { - va_arg(ap,int); /* char gets promoted to int */ - goto fmt_valid; - } - goto fmt_invalid; - } - - /* Size: short */ - if (_p[0] == 'h') { - _p += 1; - if (*_p != '\0' && strchr(intfmts,*_p) != NULL) { - va_arg(ap,int); /* short gets promoted to int */ - goto fmt_valid; - } - goto fmt_invalid; - } - - /* Size: long long */ - if (_p[0] == 'l' && _p[1] == 'l') { - _p += 2; - if (*_p != '\0' && strchr(intfmts,*_p) != NULL) { - va_arg(ap,long long); - goto fmt_valid; - } - goto fmt_invalid; - } - - /* Size: long */ - if (_p[0] == 'l') { - _p += 1; - if (*_p != '\0' && strchr(intfmts,*_p) != NULL) { - va_arg(ap,long); - goto fmt_valid; - } - goto fmt_invalid; - } - - fmt_invalid: - va_end(_cpy); - goto format_err; - - fmt_valid: - _l = (_p+1)-c; - if (_l < sizeof(_format)-2) { - memcpy(_format,c,_l); - _format[_l] = '\0'; - newarg = sdscatvprintf(curarg,_format,_cpy); - - /* Update current position (note: outer blocks - * increment c twice so compensate here) */ - c = _p-1; - } - - va_end(_cpy); - break; - } - } - - if (newarg == NULL) goto memory_err; - curarg = newarg; - - touched = 1; - c++; - } - c++; - } - - /* Add the last argument if needed */ - if (touched) { - newargv = realloc(curargv,sizeof(char*)*(argc+1)); - if (newargv == NULL) goto memory_err; - curargv = newargv; - curargv[argc++] = curarg; - totlen += bulklen(sdslen(curarg)); - } else { - sdsfree(curarg); - } - - /* Clear curarg because it was put in curargv or was free'd. */ - curarg = NULL; - - /* Add bytes needed to hold multi bulk count */ - totlen += 1+countDigits(argc)+2; - - /* Build the command at protocol level */ - cmd = malloc(totlen+1); - if (cmd == NULL) goto memory_err; - - pos = sprintf(cmd,"*%d\r\n",argc); - for (j = 0; j < argc; j++) { - pos += sprintf(cmd+pos,"$%zu\r\n",sdslen(curargv[j])); - memcpy(cmd+pos,curargv[j],sdslen(curargv[j])); - pos += sdslen(curargv[j]); - sdsfree(curargv[j]); - cmd[pos++] = '\r'; - cmd[pos++] = '\n'; - } - assert(pos == totlen); - cmd[pos] = '\0'; - - free(curargv); - *target = cmd; - return totlen; - -format_err: - error_type = -2; - goto cleanup; - -memory_err: - error_type = -1; - goto cleanup; - -cleanup: - if (curargv) { - while(argc--) - sdsfree(curargv[argc]); - free(curargv); - } - - sdsfree(curarg); - - /* No need to check cmd since it is the last statement that can fail, - * but do it anyway to be as defensive as possible. */ - if (cmd != NULL) - free(cmd); - - return error_type; -} - -/* Format a command according to the Redis protocol. This function - * takes a format similar to printf: - * - * %s represents a C null terminated string you want to interpolate - * %b represents a binary safe string - * - * When using %b you need to provide both the pointer to the string - * and the length in bytes as a size_t. Examples: - * - * len = redisFormatCommand(target, "GET %s", mykey); - * len = redisFormatCommand(target, "SET %s %b", mykey, myval, myvallen); - */ -int redisFormatCommand(char **target, const char *format, ...) { - va_list ap; - int len; - va_start(ap,format); - len = redisvFormatCommand(target,format,ap); - va_end(ap); - - /* The API says "-1" means bad result, but we now also return "-2" in some - * cases. Force the return value to always be -1. */ - if (len < 0) - len = -1; - - return len; -} - -/* Format a command according to the Redis protocol using an sds string and - * sdscatfmt for the processing of arguments. This function takes the - * number of arguments, an array with arguments and an array with their - * lengths. If the latter is set to NULL, strlen will be used to compute the - * argument lengths. - */ -int redisFormatSdsCommandArgv(sds *target, int argc, const char **argv, - const size_t *argvlen) -{ - sds cmd; - unsigned long long totlen; - int j; - size_t len; - - /* Abort on a NULL target */ - if (target == NULL) - return -1; - - /* Calculate our total size */ - totlen = 1+countDigits(argc)+2; - for (j = 0; j < argc; j++) { - len = argvlen ? argvlen[j] : strlen(argv[j]); - totlen += bulklen(len); - } - - /* Use an SDS string for command construction */ - cmd = sdsempty(); - if (cmd == NULL) - return -1; - - /* We already know how much storage we need */ - cmd = sdsMakeRoomFor(cmd, totlen); - if (cmd == NULL) - return -1; - - /* Construct command */ - cmd = sdscatfmt(cmd, "*%i\r\n", argc); - for (j=0; j < argc; j++) { - len = argvlen ? argvlen[j] : strlen(argv[j]); - cmd = sdscatfmt(cmd, "$%T\r\n", len); - cmd = sdscatlen(cmd, argv[j], len); - cmd = sdscatlen(cmd, "\r\n", sizeof("\r\n")-1); - } - - assert(sdslen(cmd)==totlen); - - *target = cmd; - return totlen; -} - -void redisFreeSdsCommand(sds cmd) { - sdsfree(cmd); -} - -/* Format a command according to the Redis protocol. This function takes the - * number of arguments, an array with arguments and an array with their - * lengths. If the latter is set to NULL, strlen will be used to compute the - * argument lengths. - */ -int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen) { - char *cmd = NULL; /* final command */ - int pos; /* position in final command */ - size_t len; - int totlen, j; - - /* Abort on a NULL target */ - if (target == NULL) - return -1; - - /* Calculate number of bytes needed for the command */ - totlen = 1+countDigits(argc)+2; - for (j = 0; j < argc; j++) { - len = argvlen ? argvlen[j] : strlen(argv[j]); - totlen += bulklen(len); - } - - /* Build the command at protocol level */ - cmd = malloc(totlen+1); - if (cmd == NULL) - return -1; - - pos = sprintf(cmd,"*%d\r\n",argc); - for (j = 0; j < argc; j++) { - len = argvlen ? argvlen[j] : strlen(argv[j]); - pos += sprintf(cmd+pos,"$%zu\r\n",len); - memcpy(cmd+pos,argv[j],len); - pos += len; - cmd[pos++] = '\r'; - cmd[pos++] = '\n'; - } - assert(pos == totlen); - cmd[pos] = '\0'; - - *target = cmd; - return totlen; -} - -void redisFreeCommand(char *cmd) { - free(cmd); -} - -void __redisSetError(redisContext *c, int type, const char *str) { - size_t len; - - c->err = type; - if (str != NULL) { - len = strlen(str); - len = len < (sizeof(c->errstr)-1) ? len : (sizeof(c->errstr)-1); - memcpy(c->errstr,str,len); - c->errstr[len] = '\0'; - } else { - /* Only REDIS_ERR_IO may lack a description! */ - assert(type == REDIS_ERR_IO); - __redis_strerror_r(errno, c->errstr, sizeof(c->errstr)); - } -} - -redisReader *redisReaderCreate(void) { - return redisReaderCreateWithFunctions(&defaultFunctions); -} - -static redisContext *redisContextInit(void) { - redisContext *c; - - c = calloc(1,sizeof(redisContext)); - if (c == NULL) - return NULL; - - c->err = 0; - c->errstr[0] = '\0'; - c->obuf = sdsempty(); - c->reader = redisReaderCreate(); - c->tcp.host = NULL; - c->tcp.source_addr = NULL; - c->unix_sock.path = NULL; - c->timeout = NULL; - - if (c->obuf == NULL || c->reader == NULL) { - redisFree(c); - return NULL; - } - - return c; -} - -void redisFree(redisContext *c) { - if (c == NULL) - return; - if (c->fd > 0) - close(c->fd); - if (c->obuf != NULL) - sdsfree(c->obuf); - if (c->reader != NULL) - redisReaderFree(c->reader); - if (c->tcp.host) - free(c->tcp.host); - if (c->tcp.source_addr) - free(c->tcp.source_addr); - if (c->unix_sock.path) - free(c->unix_sock.path); - if (c->timeout) - free(c->timeout); - free(c); -} - -int redisFreeKeepFd(redisContext *c) { - int fd = c->fd; - c->fd = -1; - redisFree(c); - return fd; -} - -int redisReconnect(redisContext *c) { - c->err = 0; - memset(c->errstr, '\0', strlen(c->errstr)); - - if (c->fd > 0) { - close(c->fd); - } - - sdsfree(c->obuf); - redisReaderFree(c->reader); - - c->obuf = sdsempty(); - c->reader = redisReaderCreate(); - - if (c->connection_type == REDIS_CONN_TCP) { - return redisContextConnectBindTcp(c, c->tcp.host, c->tcp.port, - c->timeout, c->tcp.source_addr); - } else if (c->connection_type == REDIS_CONN_UNIX) { - return redisContextConnectUnix(c, c->unix_sock.path, c->timeout); - } else { - /* Something bad happened here and shouldn't have. There isn't - enough information in the context to reconnect. */ - __redisSetError(c,REDIS_ERR_OTHER,"Not enough information to reconnect"); - } - - return REDIS_ERR; -} - -/* Connect to a Redis instance. On error the field error in the returned - * context will be set to the return value of the error function. - * When no set of reply functions is given, the default set will be used. */ -redisContext *redisConnect(const char *ip, int port) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,NULL); - return c; -} - -redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,&tv); - return c; -} - -redisContext *redisConnectNonBlock(const char *ip, int port) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags &= ~REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,NULL); - return c; -} - -redisContext *redisConnectBindNonBlock(const char *ip, int port, - const char *source_addr) { - redisContext *c = redisContextInit(); - c->flags &= ~REDIS_BLOCK; - redisContextConnectBindTcp(c,ip,port,NULL,source_addr); - return c; -} - -redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, - const char *source_addr) { - redisContext *c = redisContextInit(); - c->flags &= ~REDIS_BLOCK; - c->flags |= REDIS_REUSEADDR; - redisContextConnectBindTcp(c,ip,port,NULL,source_addr); - return c; -} - -redisContext *redisConnectUnix(const char *path) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectUnix(c,path,NULL); - return c; -} - -redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectUnix(c,path,&tv); - return c; -} - -redisContext *redisConnectUnixNonBlock(const char *path) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags &= ~REDIS_BLOCK; - redisContextConnectUnix(c,path,NULL); - return c; -} - -redisContext *redisConnectFd(int fd) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->fd = fd; - c->flags |= REDIS_BLOCK | REDIS_CONNECTED; - return c; -} - -/* Set read/write timeout on a blocking socket. */ -int redisSetTimeout(redisContext *c, const struct timeval tv) { - if (c->flags & REDIS_BLOCK) - return redisContextSetTimeout(c,tv); - return REDIS_ERR; -} - -/* Enable connection KeepAlive. */ -int redisEnableKeepAlive(redisContext *c) { - if (redisKeepAlive(c, REDIS_KEEPALIVE_INTERVAL) != REDIS_OK) - return REDIS_ERR; - return REDIS_OK; -} - -/* Use this function to handle a read event on the descriptor. It will try - * and read some bytes from the socket and feed them to the reply parser. - * - * After this function is called, you may use redisContextReadReply to - * see if there is a reply available. */ -int redisBufferRead(redisContext *c) { - char buf[1024*16]; - int nread; - - /* Return early when the context has seen an error. */ - if (c->err) - return REDIS_ERR; - - nread = read(c->fd,buf,sizeof(buf)); - if (nread == -1) { - if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { - /* Try again later */ - } else { - __redisSetError(c,REDIS_ERR_IO,NULL); - return REDIS_ERR; - } - } else if (nread == 0) { - __redisSetError(c,REDIS_ERR_EOF,"Server closed the connection"); - return REDIS_ERR; - } else { - if (redisReaderFeed(c->reader,buf,nread) != REDIS_OK) { - __redisSetError(c,c->reader->err,c->reader->errstr); - return REDIS_ERR; - } - } - return REDIS_OK; -} - -/* Write the output buffer to the socket. - * - * Returns REDIS_OK when the buffer is empty, or (a part of) the buffer was - * succesfully written to the socket. When the buffer is empty after the - * write operation, "done" is set to 1 (if given). - * - * Returns REDIS_ERR if an error occured trying to write and sets - * c->errstr to hold the appropriate error string. - */ -int redisBufferWrite(redisContext *c, int *done) { - int nwritten; - - /* Return early when the context has seen an error. */ - if (c->err) - return REDIS_ERR; - - if (sdslen(c->obuf) > 0) { - nwritten = write(c->fd,c->obuf,sdslen(c->obuf)); - if (nwritten == -1) { - if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { - /* Try again later */ - } else { - __redisSetError(c,REDIS_ERR_IO,NULL); - return REDIS_ERR; - } - } else if (nwritten > 0) { - if (nwritten == (signed)sdslen(c->obuf)) { - sdsfree(c->obuf); - c->obuf = sdsempty(); - } else { - sdsrange(c->obuf,nwritten,-1); - } - } - } - if (done != NULL) *done = (sdslen(c->obuf) == 0); - return REDIS_OK; -} - -/* Internal helper function to try and get a reply from the reader, - * or set an error in the context otherwise. */ -int redisGetReplyFromReader(redisContext *c, void **reply) { - if (redisReaderGetReply(c->reader,reply) == REDIS_ERR) { - __redisSetError(c,c->reader->err,c->reader->errstr); - return REDIS_ERR; - } - return REDIS_OK; -} - -int redisGetReply(redisContext *c, void **reply) { - int wdone = 0; - void *aux = NULL; - - /* Try to read pending replies */ - if (redisGetReplyFromReader(c,&aux) == REDIS_ERR) - return REDIS_ERR; - - /* For the blocking context, flush output buffer and read reply */ - if (aux == NULL && c->flags & REDIS_BLOCK) { - /* Write until done */ - do { - if (redisBufferWrite(c,&wdone) == REDIS_ERR) - return REDIS_ERR; - } while (!wdone); - - /* Read until there is a reply */ - do { - if (redisBufferRead(c) == REDIS_ERR) - return REDIS_ERR; - if (redisGetReplyFromReader(c,&aux) == REDIS_ERR) - return REDIS_ERR; - } while (aux == NULL); - } - - /* Set reply object */ - if (reply != NULL) *reply = aux; - return REDIS_OK; -} - - -/* Helper function for the redisAppendCommand* family of functions. - * - * Write a formatted command to the output buffer. When this family - * is used, you need to call redisGetReply yourself to retrieve - * the reply (or replies in pub/sub). - */ -int __redisAppendCommand(redisContext *c, const char *cmd, size_t len) { - sds newbuf; - - newbuf = sdscatlen(c->obuf,cmd,len); - if (newbuf == NULL) { - __redisSetError(c,REDIS_ERR_OOM,"Out of memory"); - return REDIS_ERR; - } - - c->obuf = newbuf; - return REDIS_OK; -} - -int redisAppendFormattedCommand(redisContext *c, const char *cmd, size_t len) { - - if (__redisAppendCommand(c, cmd, len) != REDIS_OK) { - return REDIS_ERR; - } - - return REDIS_OK; -} - -int redisvAppendCommand(redisContext *c, const char *format, va_list ap) { - char *cmd; - int len; - - len = redisvFormatCommand(&cmd,format,ap); - if (len == -1) { - __redisSetError(c,REDIS_ERR_OOM,"Out of memory"); - return REDIS_ERR; - } else if (len == -2) { - __redisSetError(c,REDIS_ERR_OTHER,"Invalid format string"); - return REDIS_ERR; - } - - if (__redisAppendCommand(c,cmd,len) != REDIS_OK) { - free(cmd); - return REDIS_ERR; - } - - free(cmd); - return REDIS_OK; -} - -int redisAppendCommand(redisContext *c, const char *format, ...) { - va_list ap; - int ret; - - va_start(ap,format); - ret = redisvAppendCommand(c,format,ap); - va_end(ap); - return ret; -} - -int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) { - sds cmd; - int len; - - len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen); - if (len == -1) { - __redisSetError(c,REDIS_ERR_OOM,"Out of memory"); - return REDIS_ERR; - } - - if (__redisAppendCommand(c,cmd,len) != REDIS_OK) { - sdsfree(cmd); - return REDIS_ERR; - } - - sdsfree(cmd); - return REDIS_OK; -} - -/* Helper function for the redisCommand* family of functions. - * - * Write a formatted command to the output buffer. If the given context is - * blocking, immediately read the reply into the "reply" pointer. When the - * context is non-blocking, the "reply" pointer will not be used and the - * command is simply appended to the write buffer. - * - * Returns the reply when a reply was succesfully retrieved. Returns NULL - * otherwise. When NULL is returned in a blocking context, the error field - * in the context will be set. - */ -static void *__redisBlockForReply(redisContext *c) { - void *reply; - - if (c->flags & REDIS_BLOCK) { - if (redisGetReply(c,&reply) != REDIS_OK) - return NULL; - return reply; - } - return NULL; -} - -void *redisvCommand(redisContext *c, const char *format, va_list ap) { - if (redisvAppendCommand(c,format,ap) != REDIS_OK) - return NULL; - return __redisBlockForReply(c); -} - -void *redisCommand(redisContext *c, const char *format, ...) { - va_list ap; - void *reply = NULL; - va_start(ap,format); - reply = redisvCommand(c,format,ap); - va_end(ap); - return reply; -} - -void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) { - if (redisAppendCommandArgv(c,argc,argv,argvlen) != REDIS_OK) - return NULL; - return __redisBlockForReply(c); -} diff --git a/third-party/hiredis/hiredis.h b/third-party/hiredis/hiredis.h deleted file mode 100644 index fe267b9b3..000000000 --- a/third-party/hiredis/hiredis.h +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Copyright (c) 2009-2011, Salvatore Sanfilippo - * Copyright (c) 2010-2014, Pieter Noordhuis - * Copyright (c) 2015, Matt Stancliff , - * Jan-Erik Rediger - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __HIREDIS_H -#define __HIREDIS_H -#include "read.h" -#include /* for va_list */ -#include /* for struct timeval */ -#include /* uintXX_t, etc */ -#include "sds.h" /* for sds */ - -#define HIREDIS_MAJOR 0 -#define HIREDIS_MINOR 13 -#define HIREDIS_PATCH 3 -#define HIREDIS_SONAME 0.13 - -/* Connection type can be blocking or non-blocking and is set in the - * least significant bit of the flags field in redisContext. */ -#define REDIS_BLOCK 0x1 - -/* Connection may be disconnected before being free'd. The second bit - * in the flags field is set when the context is connected. */ -#define REDIS_CONNECTED 0x2 - -/* The async API might try to disconnect cleanly and flush the output - * buffer and read all subsequent replies before disconnecting. - * This flag means no new commands can come in and the connection - * should be terminated once all replies have been read. */ -#define REDIS_DISCONNECTING 0x4 - -/* Flag specific to the async API which means that the context should be clean - * up as soon as possible. */ -#define REDIS_FREEING 0x8 - -/* Flag that is set when an async callback is executed. */ -#define REDIS_IN_CALLBACK 0x10 - -/* Flag that is set when the async context has one or more subscriptions. */ -#define REDIS_SUBSCRIBED 0x20 - -/* Flag that is set when monitor mode is active */ -#define REDIS_MONITORING 0x40 - -/* Flag that is set when we should set SO_REUSEADDR before calling bind() */ -#define REDIS_REUSEADDR 0x80 - -#define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */ - -/* number of times we retry to connect in the case of EADDRNOTAVAIL and - * SO_REUSEADDR is being used. */ -#define REDIS_CONNECT_RETRIES 10 - -/* strerror_r has two completely different prototypes and behaviors - * depending on system issues, so we need to operate on the error buffer - * differently depending on which strerror_r we're using. */ -#ifndef _GNU_SOURCE -/* "regular" POSIX strerror_r that does the right thing. */ -#define __redis_strerror_r(errno, buf, len) \ - do { \ - strerror_r((errno), (buf), (len)); \ - } while (0) -#else -/* "bad" GNU strerror_r we need to clean up after. */ -#define __redis_strerror_r(errno, buf, len) \ - do { \ - char *err_str = strerror_r((errno), (buf), (len)); \ - /* If return value _isn't_ the start of the buffer we passed in, \ - * then GNU strerror_r returned an internal static buffer and we \ - * need to copy the result into our private buffer. */ \ - if (err_str != (buf)) { \ - buf[(len)] = '\0'; \ - strncat((buf), err_str, ((len) - 1)); \ - } \ - } while (0) -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -/* This is the reply object returned by redisCommand() */ -typedef struct redisReply { - int type; /* REDIS_REPLY_* */ - long long integer; /* The integer when type is REDIS_REPLY_INTEGER */ - int len; /* Length of string */ - char *str; /* Used for both REDIS_REPLY_ERROR and REDIS_REPLY_STRING */ - size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */ - struct redisReply **element; /* elements vector for REDIS_REPLY_ARRAY */ -} redisReply; - -redisReader *redisReaderCreate(void); - -/* Function to free the reply objects hiredis returns by default. */ -void freeReplyObject(void *reply); - -/* Functions to format a command according to the protocol. */ -int redisvFormatCommand(char **target, const char *format, va_list ap); -int redisFormatCommand(char **target, const char *format, ...); -int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen); -int redisFormatSdsCommandArgv(sds *target, int argc, const char ** argv, const size_t *argvlen); -void redisFreeCommand(char *cmd); -void redisFreeSdsCommand(sds cmd); - -enum redisConnectionType { - REDIS_CONN_TCP, - REDIS_CONN_UNIX, -}; - -/* Context for a connection to Redis */ -typedef struct redisContext { - int err; /* Error flags, 0 when there is no error */ - char errstr[128]; /* String representation of error when applicable */ - int fd; - int flags; - char *obuf; /* Write buffer */ - redisReader *reader; /* Protocol reader */ - - enum redisConnectionType connection_type; - struct timeval *timeout; - - struct { - char *host; - char *source_addr; - int port; - } tcp; - - struct { - char *path; - } unix_sock; - -} redisContext; - -redisContext *redisConnect(const char *ip, int port); -redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv); -redisContext *redisConnectNonBlock(const char *ip, int port); -redisContext *redisConnectBindNonBlock(const char *ip, int port, - const char *source_addr); -redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, - const char *source_addr); -redisContext *redisConnectUnix(const char *path); -redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv); -redisContext *redisConnectUnixNonBlock(const char *path); -redisContext *redisConnectFd(int fd); - -/** - * Reconnect the given context using the saved information. - * - * This re-uses the exact same connect options as in the initial connection. - * host, ip (or path), timeout and bind address are reused, - * flags are used unmodified from the existing context. - * - * Returns REDIS_OK on successfull connect or REDIS_ERR otherwise. - */ -int redisReconnect(redisContext *c); - -int redisSetTimeout(redisContext *c, const struct timeval tv); -int redisEnableKeepAlive(redisContext *c); -void redisFree(redisContext *c); -int redisFreeKeepFd(redisContext *c); -int redisBufferRead(redisContext *c); -int redisBufferWrite(redisContext *c, int *done); - -/* In a blocking context, this function first checks if there are unconsumed - * replies to return and returns one if so. Otherwise, it flushes the output - * buffer to the socket and reads until it has a reply. In a non-blocking - * context, it will return unconsumed replies until there are no more. */ -int redisGetReply(redisContext *c, void **reply); -int redisGetReplyFromReader(redisContext *c, void **reply); - -/* Write a formatted command to the output buffer. Use these functions in blocking mode - * to get a pipeline of commands. */ -int redisAppendFormattedCommand(redisContext *c, const char *cmd, size_t len); - -/* Write a command to the output buffer. Use these functions in blocking mode - * to get a pipeline of commands. */ -int redisvAppendCommand(redisContext *c, const char *format, va_list ap); -int redisAppendCommand(redisContext *c, const char *format, ...); -int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen); - -/* Issue a command to Redis. In a blocking context, it is identical to calling - * redisAppendCommand, followed by redisGetReply. The function will return - * NULL if there was an error in performing the request, otherwise it will - * return the reply. In a non-blocking context, it is identical to calling - * only redisAppendCommand and will always return NULL. */ -void *redisvCommand(redisContext *c, const char *format, va_list ap); -void *redisCommand(redisContext *c, const char *format, ...); -void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/third-party/hiredis/net.c b/third-party/hiredis/net.c deleted file mode 100644 index 60a2dc754..000000000 --- a/third-party/hiredis/net.c +++ /dev/null @@ -1,458 +0,0 @@ -/* Extracted from anet.c to work properly with Hiredis error reporting. - * - * Copyright (c) 2009-2011, Salvatore Sanfilippo - * Copyright (c) 2010-2014, Pieter Noordhuis - * Copyright (c) 2015, Matt Stancliff , - * Jan-Erik Rediger - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#include "fmacros.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "net.h" -#include "sds.h" - -/* Defined in hiredis.c */ -void __redisSetError(redisContext *c, int type, const char *str); - -static void redisContextCloseFd(redisContext *c) { - if (c && c->fd >= 0) { - close(c->fd); - c->fd = -1; - } -} - -static void __redisSetErrorFromErrno(redisContext *c, int type, const char *prefix) { - char buf[128] = { 0 }; - size_t len = 0; - - if (prefix != NULL) - len = snprintf(buf,sizeof(buf),"%s: ",prefix); - __redis_strerror_r(errno, (char *)(buf + len), sizeof(buf) - len); - __redisSetError(c,type,buf); -} - -static int redisSetReuseAddr(redisContext *c) { - int on = 1; - if (setsockopt(c->fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) { - __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); - return REDIS_ERR; - } - return REDIS_OK; -} - -static int redisCreateSocket(redisContext *c, int type) { - int s; - if ((s = socket(type, SOCK_STREAM, 0)) == -1) { - __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - return REDIS_ERR; - } - c->fd = s; - if (type == AF_INET) { - if (redisSetReuseAddr(c) == REDIS_ERR) { - return REDIS_ERR; - } - } - return REDIS_OK; -} - -static int redisSetBlocking(redisContext *c, int blocking) { - int flags; - - /* Set the socket nonblocking. - * Note that fcntl(2) for F_GETFL and F_SETFL can't be - * interrupted by a signal. */ - if ((flags = fcntl(c->fd, F_GETFL)) == -1) { - __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_GETFL)"); - redisContextCloseFd(c); - return REDIS_ERR; - } - - if (blocking) - flags &= ~O_NONBLOCK; - else - flags |= O_NONBLOCK; - - if (fcntl(c->fd, F_SETFL, flags) == -1) { - __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_SETFL)"); - redisContextCloseFd(c); - return REDIS_ERR; - } - return REDIS_OK; -} - -int redisKeepAlive(redisContext *c, int interval) { - int val = 1; - int fd = c->fd; - - if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1){ - __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); - return REDIS_ERR; - } - - val = interval; - -#ifdef _OSX - if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE, &val, sizeof(val)) < 0) { - __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); - return REDIS_ERR; - } -#else -#if defined(__GLIBC__) && !defined(__FreeBSD_kernel__) - val = interval; - if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &val, sizeof(val)) < 0) { - __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); - return REDIS_ERR; - } - - val = interval/3; - if (val == 0) val = 1; - if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &val, sizeof(val)) < 0) { - __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); - return REDIS_ERR; - } - - val = 3; - if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &val, sizeof(val)) < 0) { - __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); - return REDIS_ERR; - } -#endif -#endif - - return REDIS_OK; -} - -static int redisSetTcpNoDelay(redisContext *c) { - int yes = 1; - if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) { - __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(TCP_NODELAY)"); - redisContextCloseFd(c); - return REDIS_ERR; - } - return REDIS_OK; -} - -#define __MAX_MSEC (((LONG_MAX) - 999) / 1000) - -static int redisContextWaitReady(redisContext *c, const struct timeval *timeout) { - struct pollfd wfd[1]; - long msec; - - msec = -1; - wfd[0].fd = c->fd; - wfd[0].events = POLLOUT; - - /* Only use timeout when not NULL. */ - if (timeout != NULL) { - if (timeout->tv_usec > 1000000 || timeout->tv_sec > __MAX_MSEC) { - __redisSetErrorFromErrno(c, REDIS_ERR_IO, NULL); - redisContextCloseFd(c); - return REDIS_ERR; - } - - msec = (timeout->tv_sec * 1000) + ((timeout->tv_usec + 999) / 1000); - - if (msec < 0 || msec > INT_MAX) { - msec = INT_MAX; - } - } - - if (errno == EINPROGRESS) { - int res; - - if ((res = poll(wfd, 1, msec)) == -1) { - __redisSetErrorFromErrno(c, REDIS_ERR_IO, "poll(2)"); - redisContextCloseFd(c); - return REDIS_ERR; - } else if (res == 0) { - errno = ETIMEDOUT; - __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); - return REDIS_ERR; - } - - if (redisCheckSocketError(c) != REDIS_OK) - return REDIS_ERR; - - return REDIS_OK; - } - - __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); - return REDIS_ERR; -} - -int redisCheckSocketError(redisContext *c) { - int err = 0; - socklen_t errlen = sizeof(err); - - if (getsockopt(c->fd, SOL_SOCKET, SO_ERROR, &err, &errlen) == -1) { - __redisSetErrorFromErrno(c,REDIS_ERR_IO,"getsockopt(SO_ERROR)"); - return REDIS_ERR; - } - - if (err) { - errno = err; - __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - return REDIS_ERR; - } - - return REDIS_OK; -} - -int redisContextSetTimeout(redisContext *c, const struct timeval tv) { - if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,&tv,sizeof(tv)) == -1) { - __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_RCVTIMEO)"); - return REDIS_ERR; - } - if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,&tv,sizeof(tv)) == -1) { - __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_SNDTIMEO)"); - return REDIS_ERR; - } - return REDIS_OK; -} - -static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, - const struct timeval *timeout, - const char *source_addr) { - int s, rv, n; - char _port[6]; /* strlen("65535"); */ - struct addrinfo hints, *servinfo, *bservinfo, *p, *b; - int blocking = (c->flags & REDIS_BLOCK); - int reuseaddr = (c->flags & REDIS_REUSEADDR); - int reuses = 0; - - c->connection_type = REDIS_CONN_TCP; - c->tcp.port = port; - - /* We need to take possession of the passed parameters - * to make them reusable for a reconnect. - * We also carefully check we don't free data we already own, - * as in the case of the reconnect method. - * - * This is a bit ugly, but atleast it works and doesn't leak memory. - **/ - if (c->tcp.host != addr) { - if (c->tcp.host) - free(c->tcp.host); - - c->tcp.host = strdup(addr); - } - - if (timeout) { - if (c->timeout != timeout) { - if (c->timeout == NULL) - c->timeout = malloc(sizeof(struct timeval)); - - memcpy(c->timeout, timeout, sizeof(struct timeval)); - } - } else { - if (c->timeout) - free(c->timeout); - c->timeout = NULL; - } - - if (source_addr == NULL) { - free(c->tcp.source_addr); - c->tcp.source_addr = NULL; - } else if (c->tcp.source_addr != source_addr) { - free(c->tcp.source_addr); - c->tcp.source_addr = strdup(source_addr); - } - - snprintf(_port, 6, "%d", port); - memset(&hints,0,sizeof(hints)); - hints.ai_family = AF_INET; - hints.ai_socktype = SOCK_STREAM; - - /* Try with IPv6 if no IPv4 address was found. We do it in this order since - * in a Redis client you can't afford to test if you have IPv6 connectivity - * as this would add latency to every connect. Otherwise a more sensible - * route could be: Use IPv6 if both addresses are available and there is IPv6 - * connectivity. */ - if ((rv = getaddrinfo(c->tcp.host,_port,&hints,&servinfo)) != 0) { - hints.ai_family = AF_INET6; - if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) { - __redisSetError(c,REDIS_ERR_OTHER,gai_strerror(rv)); - return REDIS_ERR; - } - } - for (p = servinfo; p != NULL; p = p->ai_next) { -addrretry: - if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) - continue; - - c->fd = s; - if (redisSetBlocking(c,0) != REDIS_OK) - goto error; - if (c->tcp.source_addr) { - int bound = 0; - /* Using getaddrinfo saves us from self-determining IPv4 vs IPv6 */ - if ((rv = getaddrinfo(c->tcp.source_addr, NULL, &hints, &bservinfo)) != 0) { - char buf[128]; - snprintf(buf,sizeof(buf),"Can't get addr: %s",gai_strerror(rv)); - __redisSetError(c,REDIS_ERR_OTHER,buf); - goto error; - } - - if (reuseaddr) { - n = 1; - if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char*) &n, - sizeof(n)) < 0) { - goto error; - } - } - - for (b = bservinfo; b != NULL; b = b->ai_next) { - if (bind(s,b->ai_addr,b->ai_addrlen) != -1) { - bound = 1; - break; - } - } - freeaddrinfo(bservinfo); - if (!bound) { - char buf[128]; - snprintf(buf,sizeof(buf),"Can't bind socket: %s",strerror(errno)); - __redisSetError(c,REDIS_ERR_OTHER,buf); - goto error; - } - } - if (connect(s,p->ai_addr,p->ai_addrlen) == -1) { - if (errno == EHOSTUNREACH) { - redisContextCloseFd(c); - continue; - } else if (errno == EINPROGRESS && !blocking) { - /* This is ok. */ - } else if (errno == EADDRNOTAVAIL && reuseaddr) { - if (++reuses >= REDIS_CONNECT_RETRIES) { - goto error; - } else { - goto addrretry; - } - } else { - if (redisContextWaitReady(c,c->timeout) != REDIS_OK) - goto error; - } - } - if (blocking && redisSetBlocking(c,1) != REDIS_OK) - goto error; - if (redisSetTcpNoDelay(c) != REDIS_OK) - goto error; - - c->flags |= REDIS_CONNECTED; - rv = REDIS_OK; - goto end; - } - if (p == NULL) { - char buf[128]; - snprintf(buf,sizeof(buf),"Can't create socket: %s",strerror(errno)); - __redisSetError(c,REDIS_ERR_OTHER,buf); - goto error; - } - -error: - rv = REDIS_ERR; -end: - freeaddrinfo(servinfo); - return rv; // Need to return REDIS_OK if alright -} - -int redisContextConnectTcp(redisContext *c, const char *addr, int port, - const struct timeval *timeout) { - return _redisContextConnectTcp(c, addr, port, timeout, NULL); -} - -int redisContextConnectBindTcp(redisContext *c, const char *addr, int port, - const struct timeval *timeout, - const char *source_addr) { - return _redisContextConnectTcp(c, addr, port, timeout, source_addr); -} - -int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout) { - int blocking = (c->flags & REDIS_BLOCK); - struct sockaddr_un sa; - - if (redisCreateSocket(c,AF_LOCAL) < 0) - return REDIS_ERR; - if (redisSetBlocking(c,0) != REDIS_OK) - return REDIS_ERR; - - c->connection_type = REDIS_CONN_UNIX; - if (c->unix_sock.path != path) - c->unix_sock.path = strdup(path); - - if (timeout) { - if (c->timeout != timeout) { - if (c->timeout == NULL) - c->timeout = malloc(sizeof(struct timeval)); - - memcpy(c->timeout, timeout, sizeof(struct timeval)); - } - } else { - if (c->timeout) - free(c->timeout); - c->timeout = NULL; - } - - sa.sun_family = AF_LOCAL; - strncpy(sa.sun_path,path,sizeof(sa.sun_path)-1); - if (connect(c->fd, (struct sockaddr*)&sa, sizeof(sa)) == -1) { - if (errno == EINPROGRESS && !blocking) { - /* This is ok. */ - } else { - if (redisContextWaitReady(c,c->timeout) != REDIS_OK) - return REDIS_ERR; - } - } - - /* Reset socket to be blocking after connect(2). */ - if (blocking && redisSetBlocking(c,1) != REDIS_OK) - return REDIS_ERR; - - c->flags |= REDIS_CONNECTED; - return REDIS_OK; -} diff --git a/third-party/hiredis/net.h b/third-party/hiredis/net.h deleted file mode 100644 index 2f1a0bf85..000000000 --- a/third-party/hiredis/net.h +++ /dev/null @@ -1,53 +0,0 @@ -/* Extracted from anet.c to work properly with Hiredis error reporting. - * - * Copyright (c) 2009-2011, Salvatore Sanfilippo - * Copyright (c) 2010-2014, Pieter Noordhuis - * Copyright (c) 2015, Matt Stancliff , - * Jan-Erik Rediger - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __NET_H -#define __NET_H - -#include "hiredis.h" - -#if defined(__sun) -#define AF_LOCAL AF_UNIX -#endif - -int redisCheckSocketError(redisContext *c); -int redisContextSetTimeout(redisContext *c, const struct timeval tv); -int redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout); -int redisContextConnectBindTcp(redisContext *c, const char *addr, int port, - const struct timeval *timeout, - const char *source_addr); -int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout); -int redisKeepAlive(redisContext *c, int interval); - -#endif diff --git a/third-party/hiredis/read.c b/third-party/hiredis/read.c deleted file mode 100644 index df1a467a9..000000000 --- a/third-party/hiredis/read.c +++ /dev/null @@ -1,525 +0,0 @@ -/* - * Copyright (c) 2009-2011, Salvatore Sanfilippo - * Copyright (c) 2010-2011, Pieter Noordhuis - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - - -#include "fmacros.h" -#include -#include -#ifndef _MSC_VER -#include -#endif -#include -#include -#include - -#include "read.h" -#include "sds.h" - -static void __redisReaderSetError(redisReader *r, int type, const char *str) { - size_t len; - - if (r->reply != NULL && r->fn && r->fn->freeObject) { - r->fn->freeObject(r->reply); - r->reply = NULL; - } - - /* Clear input buffer on errors. */ - if (r->buf != NULL) { - sdsfree(r->buf); - r->buf = NULL; - r->pos = r->len = 0; - } - - /* Reset task stack. */ - r->ridx = -1; - - /* Set error. */ - r->err = type; - len = strlen(str); - len = len < (sizeof(r->errstr)-1) ? len : (sizeof(r->errstr)-1); - memcpy(r->errstr,str,len); - r->errstr[len] = '\0'; -} - -static size_t chrtos(char *buf, size_t size, char byte) { - size_t len = 0; - - switch(byte) { - case '\\': - case '"': - len = snprintf(buf,size,"\"\\%c\"",byte); - break; - case '\n': len = snprintf(buf,size,"\"\\n\""); break; - case '\r': len = snprintf(buf,size,"\"\\r\""); break; - case '\t': len = snprintf(buf,size,"\"\\t\""); break; - case '\a': len = snprintf(buf,size,"\"\\a\""); break; - case '\b': len = snprintf(buf,size,"\"\\b\""); break; - default: - if (isprint(byte)) - len = snprintf(buf,size,"\"%c\"",byte); - else - len = snprintf(buf,size,"\"\\x%02x\"",(unsigned char)byte); - break; - } - - return len; -} - -static void __redisReaderSetErrorProtocolByte(redisReader *r, char byte) { - char cbuf[8], sbuf[128]; - - chrtos(cbuf,sizeof(cbuf),byte); - snprintf(sbuf,sizeof(sbuf), - "Protocol error, got %s as reply type byte", cbuf); - __redisReaderSetError(r,REDIS_ERR_PROTOCOL,sbuf); -} - -static void __redisReaderSetErrorOOM(redisReader *r) { - __redisReaderSetError(r,REDIS_ERR_OOM,"Out of memory"); -} - -static char *readBytes(redisReader *r, unsigned int bytes) { - char *p; - if (r->len-r->pos >= bytes) { - p = r->buf+r->pos; - r->pos += bytes; - return p; - } - return NULL; -} - -/* Find pointer to \r\n. */ -static char *seekNewline(char *s, size_t len) { - int pos = 0; - int _len = len-1; - - /* Position should be < len-1 because the character at "pos" should be - * followed by a \n. Note that strchr cannot be used because it doesn't - * allow to search a limited length and the buffer that is being searched - * might not have a trailing NULL character. */ - while (pos < _len) { - while(pos < _len && s[pos] != '\r') pos++; - if (s[pos] != '\r') { - /* Not found. */ - return NULL; - } else { - if (s[pos+1] == '\n') { - /* Found. */ - return s+pos; - } else { - /* Continue searching. */ - pos++; - } - } - } - return NULL; -} - -/* Read a long long value starting at *s, under the assumption that it will be - * terminated by \r\n. Ambiguously returns -1 for unexpected input. */ -static long long readLongLong(char *s) { - long long v = 0; - int dec, mult = 1; - char c; - - if (*s == '-') { - mult = -1; - s++; - } else if (*s == '+') { - mult = 1; - s++; - } - - while ((c = *(s++)) != '\r') { - dec = c - '0'; - if (dec >= 0 && dec < 10) { - v *= 10; - v += dec; - } else { - /* Should not happen... */ - return -1; - } - } - - return mult*v; -} - -static char *readLine(redisReader *r, int *_len) { - char *p, *s; - int len; - - p = r->buf+r->pos; - s = seekNewline(p,(r->len-r->pos)); - if (s != NULL) { - len = s-(r->buf+r->pos); - r->pos += len+2; /* skip \r\n */ - if (_len) *_len = len; - return p; - } - return NULL; -} - -static void moveToNextTask(redisReader *r) { - redisReadTask *cur, *prv; - while (r->ridx >= 0) { - /* Return a.s.a.p. when the stack is now empty. */ - if (r->ridx == 0) { - r->ridx--; - return; - } - - cur = &(r->rstack[r->ridx]); - prv = &(r->rstack[r->ridx-1]); - assert(prv->type == REDIS_REPLY_ARRAY); - if (cur->idx == prv->elements-1) { - r->ridx--; - } else { - /* Reset the type because the next item can be anything */ - assert(cur->idx < prv->elements); - cur->type = -1; - cur->elements = -1; - cur->idx++; - return; - } - } -} - -static int processLineItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); - void *obj; - char *p; - int len; - - if ((p = readLine(r,&len)) != NULL) { - if (cur->type == REDIS_REPLY_INTEGER) { - if (r->fn && r->fn->createInteger) - obj = r->fn->createInteger(cur,readLongLong(p)); - else - obj = (void*)REDIS_REPLY_INTEGER; - } else { - /* Type will be error or status. */ - if (r->fn && r->fn->createString) - obj = r->fn->createString(cur,p,len); - else - obj = (void*)(size_t)(cur->type); - } - - if (obj == NULL) { - __redisReaderSetErrorOOM(r); - return REDIS_ERR; - } - - /* Set reply if this is the root object. */ - if (r->ridx == 0) r->reply = obj; - moveToNextTask(r); - return REDIS_OK; - } - - return REDIS_ERR; -} - -static int processBulkItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); - void *obj = NULL; - char *p, *s; - long len; - unsigned long bytelen; - int success = 0; - - p = r->buf+r->pos; - s = seekNewline(p,r->len-r->pos); - if (s != NULL) { - p = r->buf+r->pos; - bytelen = s-(r->buf+r->pos)+2; /* include \r\n */ - len = readLongLong(p); - - if (len < 0) { - /* The nil object can always be created. */ - if (r->fn && r->fn->createNil) - obj = r->fn->createNil(cur); - else - obj = (void*)REDIS_REPLY_NIL; - success = 1; - } else { - /* Only continue when the buffer contains the entire bulk item. */ - bytelen += len+2; /* include \r\n */ - if (r->pos+bytelen <= r->len) { - if (r->fn && r->fn->createString) - obj = r->fn->createString(cur,s+2,len); - else - obj = (void*)REDIS_REPLY_STRING; - success = 1; - } - } - - /* Proceed when obj was created. */ - if (success) { - if (obj == NULL) { - __redisReaderSetErrorOOM(r); - return REDIS_ERR; - } - - r->pos += bytelen; - - /* Set reply if this is the root object. */ - if (r->ridx == 0) r->reply = obj; - moveToNextTask(r); - return REDIS_OK; - } - } - - return REDIS_ERR; -} - -static int processMultiBulkItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); - void *obj; - char *p; - long elements; - int root = 0; - - /* Set error for nested multi bulks with depth > 7 */ - if (r->ridx == 8) { - __redisReaderSetError(r,REDIS_ERR_PROTOCOL, - "No support for nested multi bulk replies with depth > 7"); - return REDIS_ERR; - } - - if ((p = readLine(r,NULL)) != NULL) { - elements = readLongLong(p); - root = (r->ridx == 0); - - if (elements == -1) { - if (r->fn && r->fn->createNil) - obj = r->fn->createNil(cur); - else - obj = (void*)REDIS_REPLY_NIL; - - if (obj == NULL) { - __redisReaderSetErrorOOM(r); - return REDIS_ERR; - } - - moveToNextTask(r); - } else { - if (r->fn && r->fn->createArray) - obj = r->fn->createArray(cur,elements); - else - obj = (void*)REDIS_REPLY_ARRAY; - - if (obj == NULL) { - __redisReaderSetErrorOOM(r); - return REDIS_ERR; - } - - /* Modify task stack when there are more than 0 elements. */ - if (elements > 0) { - cur->elements = elements; - cur->obj = obj; - r->ridx++; - r->rstack[r->ridx].type = -1; - r->rstack[r->ridx].elements = -1; - r->rstack[r->ridx].idx = 0; - r->rstack[r->ridx].obj = NULL; - r->rstack[r->ridx].parent = cur; - r->rstack[r->ridx].privdata = r->privdata; - } else { - moveToNextTask(r); - } - } - - /* Set reply if this is the root object. */ - if (root) r->reply = obj; - return REDIS_OK; - } - - return REDIS_ERR; -} - -static int processItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); - char *p; - - /* check if we need to read type */ - if (cur->type < 0) { - if ((p = readBytes(r,1)) != NULL) { - switch (p[0]) { - case '-': - cur->type = REDIS_REPLY_ERROR; - break; - case '+': - cur->type = REDIS_REPLY_STATUS; - break; - case ':': - cur->type = REDIS_REPLY_INTEGER; - break; - case '$': - cur->type = REDIS_REPLY_STRING; - break; - case '*': - cur->type = REDIS_REPLY_ARRAY; - break; - default: - __redisReaderSetErrorProtocolByte(r,*p); - return REDIS_ERR; - } - } else { - /* could not consume 1 byte */ - return REDIS_ERR; - } - } - - /* process typed item */ - switch(cur->type) { - case REDIS_REPLY_ERROR: - case REDIS_REPLY_STATUS: - case REDIS_REPLY_INTEGER: - return processLineItem(r); - case REDIS_REPLY_STRING: - return processBulkItem(r); - case REDIS_REPLY_ARRAY: - return processMultiBulkItem(r); - default: - assert(NULL); - return REDIS_ERR; /* Avoid warning. */ - } -} - -redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn) { - redisReader *r; - - r = calloc(sizeof(redisReader),1); - if (r == NULL) - return NULL; - - r->err = 0; - r->errstr[0] = '\0'; - r->fn = fn; - r->buf = sdsempty(); - r->maxbuf = REDIS_READER_MAX_BUF; - if (r->buf == NULL) { - free(r); - return NULL; - } - - r->ridx = -1; - return r; -} - -void redisReaderFree(redisReader *r) { - if (r->reply != NULL && r->fn && r->fn->freeObject) - r->fn->freeObject(r->reply); - if (r->buf != NULL) - sdsfree(r->buf); - free(r); -} - -int redisReaderFeed(redisReader *r, const char *buf, size_t len) { - sds newbuf; - - /* Return early when this reader is in an erroneous state. */ - if (r->err) - return REDIS_ERR; - - /* Copy the provided buffer. */ - if (buf != NULL && len >= 1) { - /* Destroy internal buffer when it is empty and is quite large. */ - if (r->len == 0 && r->maxbuf != 0 && sdsavail(r->buf) > r->maxbuf) { - sdsfree(r->buf); - r->buf = sdsempty(); - r->pos = 0; - - /* r->buf should not be NULL since we just free'd a larger one. */ - assert(r->buf != NULL); - } - - newbuf = sdscatlen(r->buf,buf,len); - if (newbuf == NULL) { - __redisReaderSetErrorOOM(r); - return REDIS_ERR; - } - - r->buf = newbuf; - r->len = sdslen(r->buf); - } - - return REDIS_OK; -} - -int redisReaderGetReply(redisReader *r, void **reply) { - /* Default target pointer to NULL. */ - if (reply != NULL) - *reply = NULL; - - /* Return early when this reader is in an erroneous state. */ - if (r->err) - return REDIS_ERR; - - /* When the buffer is empty, there will never be a reply. */ - if (r->len == 0) - return REDIS_OK; - - /* Set first item to process when the stack is empty. */ - if (r->ridx == -1) { - r->rstack[0].type = -1; - r->rstack[0].elements = -1; - r->rstack[0].idx = -1; - r->rstack[0].obj = NULL; - r->rstack[0].parent = NULL; - r->rstack[0].privdata = r->privdata; - r->ridx = 0; - } - - /* Process items in reply. */ - while (r->ridx >= 0) - if (processItem(r) != REDIS_OK) - break; - - /* Return ASAP when an error occurred. */ - if (r->err) - return REDIS_ERR; - - /* Discard part of the buffer when we've consumed at least 1k, to avoid - * doing unnecessary calls to memmove() in sds.c. */ - if (r->pos >= 1024) { - sdsrange(r->buf,r->pos,-1); - r->pos = 0; - r->len = sdslen(r->buf); - } - - /* Emit a reply when there is one. */ - if (r->ridx == -1) { - if (reply != NULL) - *reply = r->reply; - r->reply = NULL; - } - return REDIS_OK; -} diff --git a/third-party/hiredis/read.h b/third-party/hiredis/read.h deleted file mode 100644 index 635a872cd..000000000 --- a/third-party/hiredis/read.h +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (c) 2009-2011, Salvatore Sanfilippo - * Copyright (c) 2010-2011, Pieter Noordhuis - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - - -#ifndef __HIREDIS_READ_H -#define __HIREDIS_READ_H -#include /* for size_t */ - -#define REDIS_ERR -1 -#define REDIS_OK 0 - -/* When an error occurs, the err flag in a context is set to hold the type of - * error that occured. REDIS_ERR_IO means there was an I/O error and you - * should use the "errno" variable to find out what is wrong. - * For other values, the "errstr" field will hold a description. */ -#define REDIS_ERR_IO 1 /* Error in read or write */ -#define REDIS_ERR_EOF 3 /* End of file */ -#define REDIS_ERR_PROTOCOL 4 /* Protocol error */ -#define REDIS_ERR_OOM 5 /* Out of memory */ -#define REDIS_ERR_OTHER 2 /* Everything else... */ - -#define REDIS_REPLY_STRING 1 -#define REDIS_REPLY_ARRAY 2 -#define REDIS_REPLY_INTEGER 3 -#define REDIS_REPLY_NIL 4 -#define REDIS_REPLY_STATUS 5 -#define REDIS_REPLY_ERROR 6 - -#define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */ - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct redisReadTask { - int type; - int elements; /* number of elements in multibulk container */ - int idx; /* index in parent (array) object */ - void *obj; /* holds user-generated value for a read task */ - struct redisReadTask *parent; /* parent task */ - void *privdata; /* user-settable arbitrary field */ -} redisReadTask; - -typedef struct redisReplyObjectFunctions { - void *(*createString)(const redisReadTask*, char*, size_t); - void *(*createArray)(const redisReadTask*, int); - void *(*createInteger)(const redisReadTask*, long long); - void *(*createNil)(const redisReadTask*); - void (*freeObject)(void*); -} redisReplyObjectFunctions; - -typedef struct redisReader { - int err; /* Error flags, 0 when there is no error */ - char errstr[128]; /* String representation of error when applicable */ - - char *buf; /* Read buffer */ - size_t pos; /* Buffer cursor */ - size_t len; /* Buffer length */ - size_t maxbuf; /* Max length of unused buffer */ - - redisReadTask rstack[9]; - int ridx; /* Index of current read task */ - void *reply; /* Temporary reply pointer */ - - redisReplyObjectFunctions *fn; - void *privdata; -} redisReader; - -/* Public API for the protocol parser. */ -redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn); -void redisReaderFree(redisReader *r); -int redisReaderFeed(redisReader *r, const char *buf, size_t len); -int redisReaderGetReply(redisReader *r, void **reply); - -/* Backwards compatibility, can be removed on big version bump. */ -#define redisReplyReaderCreate redisReaderCreate -#define redisReplyReaderFree redisReaderFree -#define redisReplyReaderFeed redisReaderFeed -#define redisReplyReaderGetReply redisReaderGetReply -#define redisReplyReaderSetPrivdata(_r, _p) (int)(((redisReader*)(_r))->privdata = (_p)) -#define redisReplyReaderGetObject(_r) (((redisReader*)(_r))->reply) -#define redisReplyReaderGetError(_r) (((redisReader*)(_r))->errstr) - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/third-party/hiredis/sds.c b/third-party/hiredis/sds.c deleted file mode 100644 index 5d55b7792..000000000 --- a/third-party/hiredis/sds.c +++ /dev/null @@ -1,1095 +0,0 @@ -/* SDS (Simple Dynamic Strings), A C dynamic strings library. - * - * Copyright (c) 2006-2014, Salvatore Sanfilippo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include - -#include "sds.h" - -/* Create a new sds string with the content specified by the 'init' pointer - * and 'initlen'. - * If NULL is used for 'init' the string is initialized with zero bytes. - * - * The string is always null-termined (all the sds strings are, always) so - * even if you create an sds string with: - * - * mystring = sdsnewlen("abc",3"); - * - * You can print the string with printf() as there is an implicit \0 at the - * end of the string. However the string is binary safe and can contain - * \0 characters in the middle, as the length is stored in the sds header. */ -sds sdsnewlen(const void *init, size_t initlen) { - struct sdshdr *sh; - - if (init) { - sh = malloc(sizeof *sh+initlen+1); - } else { - sh = calloc(sizeof *sh+initlen+1,1); - } - if (sh == NULL) return NULL; - sh->len = initlen; - sh->free = 0; - if (initlen && init) - memcpy(sh->buf, init, initlen); - sh->buf[initlen] = '\0'; - return (char*)sh->buf; -} - -/* Create an empty (zero length) sds string. Even in this case the string - * always has an implicit null term. */ -sds sdsempty(void) { - return sdsnewlen("",0); -} - -/* Create a new sds string starting from a null termined C string. */ -sds sdsnew(const char *init) { - size_t initlen = (init == NULL) ? 0 : strlen(init); - return sdsnewlen(init, initlen); -} - -/* Duplicate an sds string. */ -sds sdsdup(const sds s) { - return sdsnewlen(s, sdslen(s)); -} - -/* Free an sds string. No operation is performed if 's' is NULL. */ -void sdsfree(sds s) { - if (s == NULL) return; - free(s-sizeof(struct sdshdr)); -} - -/* Set the sds string length to the length as obtained with strlen(), so - * considering as content only up to the first null term character. - * - * This function is useful when the sds string is hacked manually in some - * way, like in the following example: - * - * s = sdsnew("foobar"); - * s[2] = '\0'; - * sdsupdatelen(s); - * printf("%d\n", sdslen(s)); - * - * The output will be "2", but if we comment out the call to sdsupdatelen() - * the output will be "6" as the string was modified but the logical length - * remains 6 bytes. */ -void sdsupdatelen(sds s) { - struct sdshdr *sh = (void*) (s-sizeof *sh); - int reallen = strlen(s); - sh->free += (sh->len-reallen); - sh->len = reallen; -} - -/* Modify an sds string on-place to make it empty (zero length). - * However all the existing buffer is not discarded but set as free space - * so that next append operations will not require allocations up to the - * number of bytes previously available. */ -void sdsclear(sds s) { - struct sdshdr *sh = (void*) (s-sizeof *sh); - sh->free += sh->len; - sh->len = 0; - sh->buf[0] = '\0'; -} - -/* Enlarge the free space at the end of the sds string so that the caller - * is sure that after calling this function can overwrite up to addlen - * bytes after the end of the string, plus one more byte for nul term. - * - * Note: this does not change the *length* of the sds string as returned - * by sdslen(), but only the free buffer space we have. */ -sds sdsMakeRoomFor(sds s, size_t addlen) { - struct sdshdr *sh, *newsh; - size_t free = sdsavail(s); - size_t len, newlen; - - if (free >= addlen) return s; - len = sdslen(s); - sh = (void*) (s-sizeof *sh); - newlen = (len+addlen); - if (newlen < SDS_MAX_PREALLOC) - newlen *= 2; - else - newlen += SDS_MAX_PREALLOC; - newsh = realloc(sh, sizeof *newsh+newlen+1); - if (newsh == NULL) return NULL; - - newsh->free = newlen - len; - return newsh->buf; -} - -/* Reallocate the sds string so that it has no free space at the end. The - * contained string remains not altered, but next concatenation operations - * will require a reallocation. - * - * After the call, the passed sds string is no longer valid and all the - * references must be substituted with the new pointer returned by the call. */ -sds sdsRemoveFreeSpace(sds s) { - struct sdshdr *sh; - - sh = (void*) (s-sizeof *sh); - sh = realloc(sh, sizeof *sh+sh->len+1); - sh->free = 0; - return sh->buf; -} - -/* Return the total size of the allocation of the specifed sds string, - * including: - * 1) The sds header before the pointer. - * 2) The string. - * 3) The free buffer at the end if any. - * 4) The implicit null term. - */ -size_t sdsAllocSize(sds s) { - struct sdshdr *sh = (void*) (s-sizeof *sh); - - return sizeof(*sh)+sh->len+sh->free+1; -} - -/* Increment the sds length and decrements the left free space at the - * end of the string according to 'incr'. Also set the null term - * in the new end of the string. - * - * This function is used in order to fix the string length after the - * user calls sdsMakeRoomFor(), writes something after the end of - * the current string, and finally needs to set the new length. - * - * Note: it is possible to use a negative increment in order to - * right-trim the string. - * - * Usage example: - * - * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the - * following schema, to cat bytes coming from the kernel to the end of an - * sds string without copying into an intermediate buffer: - * - * oldlen = sdslen(s); - * s = sdsMakeRoomFor(s, BUFFER_SIZE); - * nread = read(fd, s+oldlen, BUFFER_SIZE); - * ... check for nread <= 0 and handle it ... - * sdsIncrLen(s, nread); - */ -void sdsIncrLen(sds s, int incr) { - struct sdshdr *sh = (void*) (s-sizeof *sh); - - assert(sh->free >= incr); - sh->len += incr; - sh->free -= incr; - assert(sh->free >= 0); - s[sh->len] = '\0'; -} - -/* Grow the sds to have the specified length. Bytes that were not part of - * the original length of the sds will be set to zero. - * - * if the specified length is smaller than the current length, no operation - * is performed. */ -sds sdsgrowzero(sds s, size_t len) { - struct sdshdr *sh = (void*) (s-sizeof *sh); - size_t totlen, curlen = sh->len; - - if (len <= curlen) return s; - s = sdsMakeRoomFor(s,len-curlen); - if (s == NULL) return NULL; - - /* Make sure added region doesn't contain garbage */ - sh = (void*)(s-sizeof *sh); - memset(s+curlen,0,(len-curlen+1)); /* also set trailing \0 byte */ - totlen = sh->len+sh->free; - sh->len = len; - sh->free = totlen-sh->len; - return s; -} - -/* Append the specified binary-safe string pointed by 't' of 'len' bytes to the - * end of the specified sds string 's'. - * - * After the call, the passed sds string is no longer valid and all the - * references must be substituted with the new pointer returned by the call. */ -sds sdscatlen(sds s, const void *t, size_t len) { - struct sdshdr *sh; - size_t curlen = sdslen(s); - - s = sdsMakeRoomFor(s,len); - if (s == NULL) return NULL; - sh = (void*) (s-sizeof *sh); - memcpy(s+curlen, t, len); - sh->len = curlen+len; - sh->free = sh->free-len; - s[curlen+len] = '\0'; - return s; -} - -/* Append the specified null termianted C string to the sds string 's'. - * - * After the call, the passed sds string is no longer valid and all the - * references must be substituted with the new pointer returned by the call. */ -sds sdscat(sds s, const char *t) { - return sdscatlen(s, t, strlen(t)); -} - -/* Append the specified sds 't' to the existing sds 's'. - * - * After the call, the modified sds string is no longer valid and all the - * references must be substituted with the new pointer returned by the call. */ -sds sdscatsds(sds s, const sds t) { - return sdscatlen(s, t, sdslen(t)); -} - -/* Destructively modify the sds string 's' to hold the specified binary - * safe string pointed by 't' of length 'len' bytes. */ -sds sdscpylen(sds s, const char *t, size_t len) { - struct sdshdr *sh = (void*) (s-sizeof *sh); - size_t totlen = sh->free+sh->len; - - if (totlen < len) { - s = sdsMakeRoomFor(s,len-sh->len); - if (s == NULL) return NULL; - sh = (void*) (s-sizeof *sh); - totlen = sh->free+sh->len; - } - memcpy(s, t, len); - s[len] = '\0'; - sh->len = len; - sh->free = totlen-len; - return s; -} - -/* Like sdscpylen() but 't' must be a null-termined string so that the length - * of the string is obtained with strlen(). */ -sds sdscpy(sds s, const char *t) { - return sdscpylen(s, t, strlen(t)); -} - -/* Helper for sdscatlonglong() doing the actual number -> string - * conversion. 's' must point to a string with room for at least - * SDS_LLSTR_SIZE bytes. - * - * The function returns the lenght of the null-terminated string - * representation stored at 's'. */ -#define SDS_LLSTR_SIZE 21 -int sdsll2str(char *s, long long value) { - char *p, aux; - unsigned long long v; - size_t l; - - /* Generate the string representation, this method produces - * an reversed string. */ - v = (value < 0) ? -value : value; - p = s; - do { - *p++ = '0'+(v%10); - v /= 10; - } while(v); - if (value < 0) *p++ = '-'; - - /* Compute length and add null term. */ - l = p-s; - *p = '\0'; - - /* Reverse the string. */ - p--; - while(s < p) { - aux = *s; - *s = *p; - *p = aux; - s++; - p--; - } - return l; -} - -/* Identical sdsll2str(), but for unsigned long long type. */ -int sdsull2str(char *s, unsigned long long v) { - char *p, aux; - size_t l; - - /* Generate the string representation, this method produces - * an reversed string. */ - p = s; - do { - *p++ = '0'+(v%10); - v /= 10; - } while(v); - - /* Compute length and add null term. */ - l = p-s; - *p = '\0'; - - /* Reverse the string. */ - p--; - while(s < p) { - aux = *s; - *s = *p; - *p = aux; - s++; - p--; - } - return l; -} - -/* Like sdscatpritf() but gets va_list instead of being variadic. */ -sds sdscatvprintf(sds s, const char *fmt, va_list ap) { - va_list cpy; - char *buf, *t; - size_t buflen = 16; - - while(1) { - buf = malloc(buflen); - if (buf == NULL) return NULL; - buf[buflen-2] = '\0'; - va_copy(cpy,ap); - vsnprintf(buf, buflen, fmt, cpy); - if (buf[buflen-2] != '\0') { - free(buf); - buflen *= 2; - continue; - } - break; - } - t = sdscat(s, buf); - free(buf); - return t; -} - -/* Append to the sds string 's' a string obtained using printf-alike format - * specifier. - * - * After the call, the modified sds string is no longer valid and all the - * references must be substituted with the new pointer returned by the call. - * - * Example: - * - * s = sdsnew("Sum is: "); - * s = sdscatprintf(s,"%d+%d = %d",a,b,a+b); - * - * Often you need to create a string from scratch with the printf-alike - * format. When this is the need, just use sdsempty() as the target string: - * - * s = sdscatprintf(sdsempty(), "... your format ...", args); - */ -sds sdscatprintf(sds s, const char *fmt, ...) { - va_list ap; - char *t; - va_start(ap, fmt); - t = sdscatvprintf(s,fmt,ap); - va_end(ap); - return t; -} - -/* This function is similar to sdscatprintf, but much faster as it does - * not rely on sprintf() family functions implemented by the libc that - * are often very slow. Moreover directly handling the sds string as - * new data is concatenated provides a performance improvement. - * - * However this function only handles an incompatible subset of printf-alike - * format specifiers: - * - * %s - C String - * %S - SDS string - * %i - signed int - * %I - 64 bit signed integer (long long, int64_t) - * %u - unsigned int - * %U - 64 bit unsigned integer (unsigned long long, uint64_t) - * %T - A size_t variable. - * %% - Verbatim "%" character. - */ -sds sdscatfmt(sds s, char const *fmt, ...) { - struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); - size_t initlen = sdslen(s); - const char *f = fmt; - int i; - va_list ap; - - va_start(ap,fmt); - f = fmt; /* Next format specifier byte to process. */ - i = initlen; /* Position of the next byte to write to dest str. */ - while(*f) { - char next, *str; - int l; - long long num; - unsigned long long unum; - - /* Make sure there is always space for at least 1 char. */ - if (sh->free == 0) { - s = sdsMakeRoomFor(s,1); - sh = (void*) (s-(sizeof(struct sdshdr))); - } - - switch(*f) { - case '%': - next = *(f+1); - f++; - switch(next) { - case 's': - case 'S': - str = va_arg(ap,char*); - l = (next == 's') ? strlen(str) : sdslen(str); - if (sh->free < l) { - s = sdsMakeRoomFor(s,l); - sh = (void*) (s-(sizeof(struct sdshdr))); - } - memcpy(s+i,str,l); - sh->len += l; - sh->free -= l; - i += l; - break; - case 'i': - case 'I': - if (next == 'i') - num = va_arg(ap,int); - else - num = va_arg(ap,long long); - { - char buf[SDS_LLSTR_SIZE]; - l = sdsll2str(buf,num); - if (sh->free < l) { - s = sdsMakeRoomFor(s,l); - sh = (void*) (s-(sizeof(struct sdshdr))); - } - memcpy(s+i,buf,l); - sh->len += l; - sh->free -= l; - i += l; - } - break; - case 'u': - case 'U': - case 'T': - if (next == 'u') - unum = va_arg(ap,unsigned int); - else if(next == 'U') - unum = va_arg(ap,unsigned long long); - else - unum = (unsigned long long)va_arg(ap,size_t); - { - char buf[SDS_LLSTR_SIZE]; - l = sdsull2str(buf,unum); - if (sh->free < l) { - s = sdsMakeRoomFor(s,l); - sh = (void*) (s-(sizeof(struct sdshdr))); - } - memcpy(s+i,buf,l); - sh->len += l; - sh->free -= l; - i += l; - } - break; - default: /* Handle %% and generally %. */ - s[i++] = next; - sh->len += 1; - sh->free -= 1; - break; - } - break; - default: - s[i++] = *f; - sh->len += 1; - sh->free -= 1; - break; - } - f++; - } - va_end(ap); - - /* Add null-term */ - s[i] = '\0'; - return s; -} - - -/* Remove the part of the string from left and from right composed just of - * contiguous characters found in 'cset', that is a null terminted C string. - * - * After the call, the modified sds string is no longer valid and all the - * references must be substituted with the new pointer returned by the call. - * - * Example: - * - * s = sdsnew("AA...AA.a.aa.aHelloWorld :::"); - * s = sdstrim(s,"A. :"); - * printf("%s\n", s); - * - * Output will be just "Hello World". - */ -void sdstrim(sds s, const char *cset) { - struct sdshdr *sh = (void*) (s-sizeof *sh); - char *start, *end, *sp, *ep; - size_t len; - - sp = start = s; - ep = end = s+sdslen(s)-1; - while(sp <= end && strchr(cset, *sp)) sp++; - while(ep > start && strchr(cset, *ep)) ep--; - len = (sp > ep) ? 0 : ((ep-sp)+1); - if (sh->buf != sp) memmove(sh->buf, sp, len); - sh->buf[len] = '\0'; - sh->free = sh->free+(sh->len-len); - sh->len = len; -} - -/* Turn the string into a smaller (or equal) string containing only the - * substring specified by the 'start' and 'end' indexes. - * - * start and end can be negative, where -1 means the last character of the - * string, -2 the penultimate character, and so forth. - * - * The interval is inclusive, so the start and end characters will be part - * of the resulting string. - * - * The string is modified in-place. - * - * Example: - * - * s = sdsnew("Hello World"); - * sdsrange(s,1,-1); => "ello World" - */ -void sdsrange(sds s, int start, int end) { - struct sdshdr *sh = (void*) (s-sizeof *sh); - size_t newlen, len = sdslen(s); - - if (len == 0) return; - if (start < 0) { - start = len+start; - if (start < 0) start = 0; - } - if (end < 0) { - end = len+end; - if (end < 0) end = 0; - } - newlen = (start > end) ? 0 : (end-start)+1; - if (newlen != 0) { - if (start >= (signed)len) { - newlen = 0; - } else if (end >= (signed)len) { - end = len-1; - newlen = (start > end) ? 0 : (end-start)+1; - } - } else { - start = 0; - } - if (start && newlen) memmove(sh->buf, sh->buf+start, newlen); - sh->buf[newlen] = 0; - sh->free = sh->free+(sh->len-newlen); - sh->len = newlen; -} - -/* Apply tolower() to every character of the sds string 's'. */ -void sdstolower(sds s) { - int len = sdslen(s), j; - - for (j = 0; j < len; j++) s[j] = tolower(s[j]); -} - -/* Apply toupper() to every character of the sds string 's'. */ -void sdstoupper(sds s) { - int len = sdslen(s), j; - - for (j = 0; j < len; j++) s[j] = toupper(s[j]); -} - -/* Compare two sds strings s1 and s2 with memcmp(). - * - * Return value: - * - * 1 if s1 > s2. - * -1 if s1 < s2. - * 0 if s1 and s2 are exactly the same binary string. - * - * If two strings share exactly the same prefix, but one of the two has - * additional characters, the longer string is considered to be greater than - * the smaller one. */ -int sdscmp(const sds s1, const sds s2) { - size_t l1, l2, minlen; - int cmp; - - l1 = sdslen(s1); - l2 = sdslen(s2); - minlen = (l1 < l2) ? l1 : l2; - cmp = memcmp(s1,s2,minlen); - if (cmp == 0) return l1-l2; - return cmp; -} - -/* Split 's' with separator in 'sep'. An array - * of sds strings is returned. *count will be set - * by reference to the number of tokens returned. - * - * On out of memory, zero length string, zero length - * separator, NULL is returned. - * - * Note that 'sep' is able to split a string using - * a multi-character separator. For example - * sdssplit("foo_-_bar","_-_"); will return two - * elements "foo" and "bar". - * - * This version of the function is binary-safe but - * requires length arguments. sdssplit() is just the - * same function but for zero-terminated strings. - */ -sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) { - int elements = 0, slots = 5, start = 0, j; - sds *tokens; - - if (seplen < 1 || len < 0) return NULL; - - tokens = malloc(sizeof(sds)*slots); - if (tokens == NULL) return NULL; - - if (len == 0) { - *count = 0; - return tokens; - } - for (j = 0; j < (len-(seplen-1)); j++) { - /* make sure there is room for the next element and the final one */ - if (slots < elements+2) { - sds *newtokens; - - slots *= 2; - newtokens = realloc(tokens,sizeof(sds)*slots); - if (newtokens == NULL) goto cleanup; - tokens = newtokens; - } - /* search the separator */ - if ((seplen == 1 && *(s+j) == sep[0]) || (memcmp(s+j,sep,seplen) == 0)) { - tokens[elements] = sdsnewlen(s+start,j-start); - if (tokens[elements] == NULL) goto cleanup; - elements++; - start = j+seplen; - j = j+seplen-1; /* skip the separator */ - } - } - /* Add the final element. We are sure there is room in the tokens array. */ - tokens[elements] = sdsnewlen(s+start,len-start); - if (tokens[elements] == NULL) goto cleanup; - elements++; - *count = elements; - return tokens; - -cleanup: - { - int i; - for (i = 0; i < elements; i++) sdsfree(tokens[i]); - free(tokens); - *count = 0; - return NULL; - } -} - -/* Free the result returned by sdssplitlen(), or do nothing if 'tokens' is NULL. */ -void sdsfreesplitres(sds *tokens, int count) { - if (!tokens) return; - while(count--) - sdsfree(tokens[count]); - free(tokens); -} - -/* Create an sds string from a long long value. It is much faster than: - * - * sdscatprintf(sdsempty(),"%lld\n", value); - */ -sds sdsfromlonglong(long long value) { - char buf[32], *p; - unsigned long long v; - - v = (value < 0) ? -value : value; - p = buf+31; /* point to the last character */ - do { - *p-- = '0'+(v%10); - v /= 10; - } while(v); - if (value < 0) *p-- = '-'; - p++; - return sdsnewlen(p,32-(p-buf)); -} - -/* Append to the sds string "s" an escaped string representation where - * all the non-printable characters (tested with isprint()) are turned into - * escapes in the form "\n\r\a...." or "\x". - * - * After the call, the modified sds string is no longer valid and all the - * references must be substituted with the new pointer returned by the call. */ -sds sdscatrepr(sds s, const char *p, size_t len) { - s = sdscatlen(s,"\"",1); - while(len--) { - switch(*p) { - case '\\': - case '"': - s = sdscatprintf(s,"\\%c",*p); - break; - case '\n': s = sdscatlen(s,"\\n",2); break; - case '\r': s = sdscatlen(s,"\\r",2); break; - case '\t': s = sdscatlen(s,"\\t",2); break; - case '\a': s = sdscatlen(s,"\\a",2); break; - case '\b': s = sdscatlen(s,"\\b",2); break; - default: - if (isprint(*p)) - s = sdscatprintf(s,"%c",*p); - else - s = sdscatprintf(s,"\\x%02x",(unsigned char)*p); - break; - } - p++; - } - return sdscatlen(s,"\"",1); -} - -/* Helper function for sdssplitargs() that returns non zero if 'c' - * is a valid hex digit. */ -int is_hex_digit(char c) { - return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || - (c >= 'A' && c <= 'F'); -} - -/* Helper function for sdssplitargs() that converts a hex digit into an - * integer from 0 to 15 */ -int hex_digit_to_int(char c) { - switch(c) { - case '0': return 0; - case '1': return 1; - case '2': return 2; - case '3': return 3; - case '4': return 4; - case '5': return 5; - case '6': return 6; - case '7': return 7; - case '8': return 8; - case '9': return 9; - case 'a': case 'A': return 10; - case 'b': case 'B': return 11; - case 'c': case 'C': return 12; - case 'd': case 'D': return 13; - case 'e': case 'E': return 14; - case 'f': case 'F': return 15; - default: return 0; - } -} - -/* Split a line into arguments, where every argument can be in the - * following programming-language REPL-alike form: - * - * foo bar "newline are supported\n" and "\xff\x00otherstuff" - * - * The number of arguments is stored into *argc, and an array - * of sds is returned. - * - * The caller should free the resulting array of sds strings with - * sdsfreesplitres(). - * - * Note that sdscatrepr() is able to convert back a string into - * a quoted string in the same format sdssplitargs() is able to parse. - * - * The function returns the allocated tokens on success, even when the - * input string is empty, or NULL if the input contains unbalanced - * quotes or closed quotes followed by non space characters - * as in: "foo"bar or "foo' - */ -sds *sdssplitargs(const char *line, int *argc) { - const char *p = line; - char *current = NULL; - char **vector = NULL; - - *argc = 0; - while(1) { - /* skip blanks */ - while(*p && isspace(*p)) p++; - if (*p) { - /* get a token */ - int inq=0; /* set to 1 if we are in "quotes" */ - int insq=0; /* set to 1 if we are in 'single quotes' */ - int done=0; - - if (current == NULL) current = sdsempty(); - while(!done) { - if (inq) { - if (*p == '\\' && *(p+1) == 'x' && - is_hex_digit(*(p+2)) && - is_hex_digit(*(p+3))) - { - unsigned char byte; - - byte = (hex_digit_to_int(*(p+2))*16)+ - hex_digit_to_int(*(p+3)); - current = sdscatlen(current,(char*)&byte,1); - p += 3; - } else if (*p == '\\' && *(p+1)) { - char c; - - p++; - switch(*p) { - case 'n': c = '\n'; break; - case 'r': c = '\r'; break; - case 't': c = '\t'; break; - case 'b': c = '\b'; break; - case 'a': c = '\a'; break; - default: c = *p; break; - } - current = sdscatlen(current,&c,1); - } else if (*p == '"') { - /* closing quote must be followed by a space or - * nothing at all. */ - if (*(p+1) && !isspace(*(p+1))) goto err; - done=1; - } else if (!*p) { - /* unterminated quotes */ - goto err; - } else { - current = sdscatlen(current,p,1); - } - } else if (insq) { - if (*p == '\\' && *(p+1) == '\'') { - p++; - current = sdscatlen(current,"'",1); - } else if (*p == '\'') { - /* closing quote must be followed by a space or - * nothing at all. */ - if (*(p+1) && !isspace(*(p+1))) goto err; - done=1; - } else if (!*p) { - /* unterminated quotes */ - goto err; - } else { - current = sdscatlen(current,p,1); - } - } else { - switch(*p) { - case ' ': - case '\n': - case '\r': - case '\t': - case '\0': - done=1; - break; - case '"': - inq=1; - break; - case '\'': - insq=1; - break; - default: - current = sdscatlen(current,p,1); - break; - } - } - if (*p) p++; - } - /* add the token to the vector */ - vector = realloc(vector,((*argc)+1)*sizeof(char*)); - vector[*argc] = current; - (*argc)++; - current = NULL; - } else { - /* Even on empty input string return something not NULL. */ - if (vector == NULL) vector = malloc(sizeof(void*)); - return vector; - } - } - -err: - while((*argc)--) - sdsfree(vector[*argc]); - free(vector); - if (current) sdsfree(current); - *argc = 0; - return NULL; -} - -/* Modify the string substituting all the occurrences of the set of - * characters specified in the 'from' string to the corresponding character - * in the 'to' array. - * - * For instance: sdsmapchars(mystring, "ho", "01", 2) - * will have the effect of turning the string "hello" into "0ell1". - * - * The function returns the sds string pointer, that is always the same - * as the input pointer since no resize is needed. */ -sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) { - size_t j, i, l = sdslen(s); - - for (j = 0; j < l; j++) { - for (i = 0; i < setlen; i++) { - if (s[j] == from[i]) { - s[j] = to[i]; - break; - } - } - } - return s; -} - -/* Join an array of C strings using the specified separator (also a C string). - * Returns the result as an sds string. */ -sds sdsjoin(char **argv, int argc, char *sep, size_t seplen) { - sds join = sdsempty(); - int j; - - for (j = 0; j < argc; j++) { - join = sdscat(join, argv[j]); - if (j != argc-1) join = sdscatlen(join,sep,seplen); - } - return join; -} - -/* Like sdsjoin, but joins an array of SDS strings. */ -sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) { - sds join = sdsempty(); - int j; - - for (j = 0; j < argc; j++) { - join = sdscatsds(join, argv[j]); - if (j != argc-1) join = sdscatlen(join,sep,seplen); - } - return join; -} - -#ifdef SDS_TEST_MAIN -#include -#include "testhelp.h" - -int main(void) { - { - struct sdshdr *sh; - sds x = sdsnew("foo"), y; - - test_cond("Create a string and obtain the length", - sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0) - - sdsfree(x); - x = sdsnewlen("foo",2); - test_cond("Create a string with specified length", - sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0) - - x = sdscat(x,"bar"); - test_cond("Strings concatenation", - sdslen(x) == 5 && memcmp(x,"fobar\0",6) == 0); - - x = sdscpy(x,"a"); - test_cond("sdscpy() against an originally longer string", - sdslen(x) == 1 && memcmp(x,"a\0",2) == 0) - - x = sdscpy(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk"); - test_cond("sdscpy() against an originally shorter string", - sdslen(x) == 33 && - memcmp(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk\0",33) == 0) - - sdsfree(x); - x = sdscatprintf(sdsempty(),"%d",123); - test_cond("sdscatprintf() seems working in the base case", - sdslen(x) == 3 && memcmp(x,"123\0",4) ==0) - - sdsfree(x); - x = sdsnew("xxciaoyyy"); - sdstrim(x,"xy"); - test_cond("sdstrim() correctly trims characters", - sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0) - - y = sdsdup(x); - sdsrange(y,1,1); - test_cond("sdsrange(...,1,1)", - sdslen(y) == 1 && memcmp(y,"i\0",2) == 0) - - sdsfree(y); - y = sdsdup(x); - sdsrange(y,1,-1); - test_cond("sdsrange(...,1,-1)", - sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) - - sdsfree(y); - y = sdsdup(x); - sdsrange(y,-2,-1); - test_cond("sdsrange(...,-2,-1)", - sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0) - - sdsfree(y); - y = sdsdup(x); - sdsrange(y,2,1); - test_cond("sdsrange(...,2,1)", - sdslen(y) == 0 && memcmp(y,"\0",1) == 0) - - sdsfree(y); - y = sdsdup(x); - sdsrange(y,1,100); - test_cond("sdsrange(...,1,100)", - sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) - - sdsfree(y); - y = sdsdup(x); - sdsrange(y,100,100); - test_cond("sdsrange(...,100,100)", - sdslen(y) == 0 && memcmp(y,"\0",1) == 0) - - sdsfree(y); - sdsfree(x); - x = sdsnew("foo"); - y = sdsnew("foa"); - test_cond("sdscmp(foo,foa)", sdscmp(x,y) > 0) - - sdsfree(y); - sdsfree(x); - x = sdsnew("bar"); - y = sdsnew("bar"); - test_cond("sdscmp(bar,bar)", sdscmp(x,y) == 0) - - sdsfree(y); - sdsfree(x); - x = sdsnew("aar"); - y = sdsnew("bar"); - test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0) - - sdsfree(y); - sdsfree(x); - x = sdsnewlen("\a\n\0foo\r",7); - y = sdscatrepr(sdsempty(),x,sdslen(x)); - test_cond("sdscatrepr(...data...)", - memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0) - - { - int oldfree; - - sdsfree(x); - x = sdsnew("0"); - sh = (void*) (x-(sizeof(struct sdshdr))); - test_cond("sdsnew() free/len buffers", sh->len == 1 && sh->free == 0); - x = sdsMakeRoomFor(x,1); - sh = (void*) (x-(sizeof(struct sdshdr))); - test_cond("sdsMakeRoomFor()", sh->len == 1 && sh->free > 0); - oldfree = sh->free; - x[1] = '1'; - sdsIncrLen(x,1); - test_cond("sdsIncrLen() -- content", x[0] == '0' && x[1] == '1'); - test_cond("sdsIncrLen() -- len", sh->len == 2); - test_cond("sdsIncrLen() -- free", sh->free == oldfree-1); - } - } - test_report() - return 0; -} -#endif diff --git a/third-party/hiredis/sds.h b/third-party/hiredis/sds.h deleted file mode 100644 index 19a2abd31..000000000 --- a/third-party/hiredis/sds.h +++ /dev/null @@ -1,105 +0,0 @@ -/* SDS (Simple Dynamic Strings), A C dynamic strings library. - * - * Copyright (c) 2006-2014, Salvatore Sanfilippo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __SDS_H -#define __SDS_H - -#define SDS_MAX_PREALLOC (1024*1024) - -#include -#include -#ifdef _MSC_VER -#include "win32.h" -#endif - -typedef char *sds; - -struct sdshdr { - int len; - int free; - char buf[]; -}; - -static inline size_t sdslen(const sds s) { - struct sdshdr *sh = (struct sdshdr *)(s-sizeof *sh); - return sh->len; -} - -static inline size_t sdsavail(const sds s) { - struct sdshdr *sh = (struct sdshdr *)(s-sizeof *sh); - return sh->free; -} - -sds sdsnewlen(const void *init, size_t initlen); -sds sdsnew(const char *init); -sds sdsempty(void); -size_t sdslen(const sds s); -sds sdsdup(const sds s); -void sdsfree(sds s); -size_t sdsavail(const sds s); -sds sdsgrowzero(sds s, size_t len); -sds sdscatlen(sds s, const void *t, size_t len); -sds sdscat(sds s, const char *t); -sds sdscatsds(sds s, const sds t); -sds sdscpylen(sds s, const char *t, size_t len); -sds sdscpy(sds s, const char *t); - -sds sdscatvprintf(sds s, const char *fmt, va_list ap); -#ifdef __GNUC__ -sds sdscatprintf(sds s, const char *fmt, ...) - __attribute__((format(printf, 2, 3))); -#else -sds sdscatprintf(sds s, const char *fmt, ...); -#endif - -sds sdscatfmt(sds s, char const *fmt, ...); -void sdstrim(sds s, const char *cset); -void sdsrange(sds s, int start, int end); -void sdsupdatelen(sds s); -void sdsclear(sds s); -int sdscmp(const sds s1, const sds s2); -sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count); -void sdsfreesplitres(sds *tokens, int count); -void sdstolower(sds s); -void sdstoupper(sds s); -sds sdsfromlonglong(long long value); -sds sdscatrepr(sds s, const char *p, size_t len); -sds *sdssplitargs(const char *line, int *argc); -sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen); -sds sdsjoin(char **argv, int argc, char *sep, size_t seplen); -sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen); - -/* Low level functions exposed to the user API */ -sds sdsMakeRoomFor(sds s, size_t addlen); -void sdsIncrLen(sds s, int incr); -sds sdsRemoveFreeSpace(sds s); -size_t sdsAllocSize(sds s); - -#endif diff --git a/third-party/hiredis/test.c b/third-party/hiredis/test.c deleted file mode 100644 index d5e82170d..000000000 --- a/third-party/hiredis/test.c +++ /dev/null @@ -1,807 +0,0 @@ -#include "fmacros.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "hiredis.h" -#include "net.h" - -enum connection_type { - CONN_TCP, - CONN_UNIX, - CONN_FD -}; - -struct config { - enum connection_type type; - - struct { - const char *host; - int port; - struct timeval timeout; - } tcp; - - struct { - const char *path; - } unix; -}; - -/* The following lines make up our testing "framework" :) */ -static int tests = 0, fails = 0; -#define test(_s) { printf("#%02d ", ++tests); printf(_s); } -#define test_cond(_c) if(_c) printf("\033[0;32mPASSED\033[0;0m\n"); else {printf("\033[0;31mFAILED\033[0;0m\n"); fails++;} - -static long long usec(void) { - struct timeval tv; - gettimeofday(&tv,NULL); - return (((long long)tv.tv_sec)*1000000)+tv.tv_usec; -} - -/* The assert() calls below have side effects, so we need assert() - * even if we are compiling without asserts (-DNDEBUG). */ -#ifdef NDEBUG -#undef assert -#define assert(e) (void)(e) -#endif - -static redisContext *select_database(redisContext *c) { - redisReply *reply; - - /* Switch to DB 9 for testing, now that we know we can chat. */ - reply = redisCommand(c,"SELECT 9"); - assert(reply != NULL); - freeReplyObject(reply); - - /* Make sure the DB is emtpy */ - reply = redisCommand(c,"DBSIZE"); - assert(reply != NULL); - if (reply->type == REDIS_REPLY_INTEGER && reply->integer == 0) { - /* Awesome, DB 9 is empty and we can continue. */ - freeReplyObject(reply); - } else { - printf("Database #9 is not empty, test can not continue\n"); - exit(1); - } - - return c; -} - -static int disconnect(redisContext *c, int keep_fd) { - redisReply *reply; - - /* Make sure we're on DB 9. */ - reply = redisCommand(c,"SELECT 9"); - assert(reply != NULL); - freeReplyObject(reply); - reply = redisCommand(c,"FLUSHDB"); - assert(reply != NULL); - freeReplyObject(reply); - - /* Free the context as well, but keep the fd if requested. */ - if (keep_fd) - return redisFreeKeepFd(c); - redisFree(c); - return -1; -} - -static redisContext *connect(struct config config) { - redisContext *c = NULL; - - if (config.type == CONN_TCP) { - c = redisConnect(config.tcp.host, config.tcp.port); - } else if (config.type == CONN_UNIX) { - c = redisConnectUnix(config.unix.path); - } else if (config.type == CONN_FD) { - /* Create a dummy connection just to get an fd to inherit */ - redisContext *dummy_ctx = redisConnectUnix(config.unix.path); - if (dummy_ctx) { - int fd = disconnect(dummy_ctx, 1); - printf("Connecting to inherited fd %d\n", fd); - c = redisConnectFd(fd); - } - } else { - assert(NULL); - } - - if (c == NULL) { - printf("Connection error: can't allocate redis context\n"); - exit(1); - } else if (c->err) { - printf("Connection error: %s\n", c->errstr); - redisFree(c); - exit(1); - } - - return select_database(c); -} - -static void test_format_commands(void) { - char *cmd; - int len; - - test("Format command without interpolation: "); - len = redisFormatCommand(&cmd,"SET foo bar"); - test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && - len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - free(cmd); - - test("Format command with %%s string interpolation: "); - len = redisFormatCommand(&cmd,"SET %s %s","foo","bar"); - test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && - len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - free(cmd); - - test("Format command with %%s and an empty string: "); - len = redisFormatCommand(&cmd,"SET %s %s","foo",""); - test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$0\r\n\r\n",len) == 0 && - len == 4+4+(3+2)+4+(3+2)+4+(0+2)); - free(cmd); - - test("Format command with an empty string in between proper interpolations: "); - len = redisFormatCommand(&cmd,"SET %s %s","","foo"); - test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$0\r\n\r\n$3\r\nfoo\r\n",len) == 0 && - len == 4+4+(3+2)+4+(0+2)+4+(3+2)); - free(cmd); - - test("Format command with %%b string interpolation: "); - len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"b\0r",(size_t)3); - test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nb\0r\r\n",len) == 0 && - len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - free(cmd); - - test("Format command with %%b and an empty string: "); - len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"",(size_t)0); - test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$0\r\n\r\n",len) == 0 && - len == 4+4+(3+2)+4+(3+2)+4+(0+2)); - free(cmd); - - test("Format command with literal %%: "); - len = redisFormatCommand(&cmd,"SET %% %%"); - test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$1\r\n%\r\n$1\r\n%\r\n",len) == 0 && - len == 4+4+(3+2)+4+(1+2)+4+(1+2)); - free(cmd); - - /* Vararg width depends on the type. These tests make sure that the - * width is correctly determined using the format and subsequent varargs - * can correctly be interpolated. */ -#define INTEGER_WIDTH_TEST(fmt, type) do { \ - type value = 123; \ - test("Format command with printf-delegation (" #type "): "); \ - len = redisFormatCommand(&cmd,"key:%08" fmt " str:%s", value, "hello"); \ - test_cond(strncmp(cmd,"*2\r\n$12\r\nkey:00000123\r\n$9\r\nstr:hello\r\n",len) == 0 && \ - len == 4+5+(12+2)+4+(9+2)); \ - free(cmd); \ -} while(0) - -#define FLOAT_WIDTH_TEST(type) do { \ - type value = 123.0; \ - test("Format command with printf-delegation (" #type "): "); \ - len = redisFormatCommand(&cmd,"key:%08.3f str:%s", value, "hello"); \ - test_cond(strncmp(cmd,"*2\r\n$12\r\nkey:0123.000\r\n$9\r\nstr:hello\r\n",len) == 0 && \ - len == 4+5+(12+2)+4+(9+2)); \ - free(cmd); \ -} while(0) - - INTEGER_WIDTH_TEST("d", int); - INTEGER_WIDTH_TEST("hhd", char); - INTEGER_WIDTH_TEST("hd", short); - INTEGER_WIDTH_TEST("ld", long); - INTEGER_WIDTH_TEST("lld", long long); - INTEGER_WIDTH_TEST("u", unsigned int); - INTEGER_WIDTH_TEST("hhu", unsigned char); - INTEGER_WIDTH_TEST("hu", unsigned short); - INTEGER_WIDTH_TEST("lu", unsigned long); - INTEGER_WIDTH_TEST("llu", unsigned long long); - FLOAT_WIDTH_TEST(float); - FLOAT_WIDTH_TEST(double); - - test("Format command with invalid printf format: "); - len = redisFormatCommand(&cmd,"key:%08p %b",(void*)1234,"foo",(size_t)3); - test_cond(len == -1); - - const char *argv[3]; - argv[0] = "SET"; - argv[1] = "foo\0xxx"; - argv[2] = "bar"; - size_t lens[3] = { 3, 7, 3 }; - int argc = 3; - - test("Format command by passing argc/argv without lengths: "); - len = redisFormatCommandArgv(&cmd,argc,argv,NULL); - test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && - len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - free(cmd); - - test("Format command by passing argc/argv with lengths: "); - len = redisFormatCommandArgv(&cmd,argc,argv,lens); - test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 && - len == 4+4+(3+2)+4+(7+2)+4+(3+2)); - free(cmd); -} - -static void test_append_formatted_commands(struct config config) { - redisContext *c; - redisReply *reply; - char *cmd; - int len; - - c = connect(config); - - test("Append format command: "); - - len = redisFormatCommand(&cmd, "SET foo bar"); - - test_cond(redisAppendFormattedCommand(c, cmd, len) == REDIS_OK); - - assert(redisGetReply(c, (void*)&reply) == REDIS_OK); - - free(cmd); - freeReplyObject(reply); - - disconnect(c, 0); -} - -static void test_reply_reader(void) { - redisReader *reader; - void *reply; - int ret; - int i; - - test("Error handling in reply parser: "); - reader = redisReaderCreate(); - redisReaderFeed(reader,(char*)"@foo\r\n",6); - ret = redisReaderGetReply(reader,NULL); - test_cond(ret == REDIS_ERR && - strcasecmp(reader->errstr,"Protocol error, got \"@\" as reply type byte") == 0); - redisReaderFree(reader); - - /* when the reply already contains multiple items, they must be free'd - * on an error. valgrind will bark when this doesn't happen. */ - test("Memory cleanup in reply parser: "); - reader = redisReaderCreate(); - redisReaderFeed(reader,(char*)"*2\r\n",4); - redisReaderFeed(reader,(char*)"$5\r\nhello\r\n",11); - redisReaderFeed(reader,(char*)"@foo\r\n",6); - ret = redisReaderGetReply(reader,NULL); - test_cond(ret == REDIS_ERR && - strcasecmp(reader->errstr,"Protocol error, got \"@\" as reply type byte") == 0); - redisReaderFree(reader); - - test("Set error on nested multi bulks with depth > 7: "); - reader = redisReaderCreate(); - - for (i = 0; i < 9; i++) { - redisReaderFeed(reader,(char*)"*1\r\n",4); - } - - ret = redisReaderGetReply(reader,NULL); - test_cond(ret == REDIS_ERR && - strncasecmp(reader->errstr,"No support for",14) == 0); - redisReaderFree(reader); - - test("Works with NULL functions for reply: "); - reader = redisReaderCreate(); - reader->fn = NULL; - redisReaderFeed(reader,(char*)"+OK\r\n",5); - ret = redisReaderGetReply(reader,&reply); - test_cond(ret == REDIS_OK && reply == (void*)REDIS_REPLY_STATUS); - redisReaderFree(reader); - - test("Works when a single newline (\\r\\n) covers two calls to feed: "); - reader = redisReaderCreate(); - reader->fn = NULL; - redisReaderFeed(reader,(char*)"+OK\r",4); - ret = redisReaderGetReply(reader,&reply); - assert(ret == REDIS_OK && reply == NULL); - redisReaderFeed(reader,(char*)"\n",1); - ret = redisReaderGetReply(reader,&reply); - test_cond(ret == REDIS_OK && reply == (void*)REDIS_REPLY_STATUS); - redisReaderFree(reader); - - test("Don't reset state after protocol error: "); - reader = redisReaderCreate(); - reader->fn = NULL; - redisReaderFeed(reader,(char*)"x",1); - ret = redisReaderGetReply(reader,&reply); - assert(ret == REDIS_ERR); - ret = redisReaderGetReply(reader,&reply); - test_cond(ret == REDIS_ERR && reply == NULL); - redisReaderFree(reader); - - /* Regression test for issue #45 on GitHub. */ - test("Don't do empty allocation for empty multi bulk: "); - reader = redisReaderCreate(); - redisReaderFeed(reader,(char*)"*0\r\n",4); - ret = redisReaderGetReply(reader,&reply); - test_cond(ret == REDIS_OK && - ((redisReply*)reply)->type == REDIS_REPLY_ARRAY && - ((redisReply*)reply)->elements == 0); - freeReplyObject(reply); - redisReaderFree(reader); -} - -static void test_free_null(void) { - void *redisContext = NULL; - void *reply = NULL; - - test("Don't fail when redisFree is passed a NULL value: "); - redisFree(redisContext); - test_cond(redisContext == NULL); - - test("Don't fail when freeReplyObject is passed a NULL value: "); - freeReplyObject(reply); - test_cond(reply == NULL); -} - -static void test_blocking_connection_errors(void) { - redisContext *c; - - test("Returns error when host cannot be resolved: "); - c = redisConnect((char*)"idontexist.test", 6379); - test_cond(c->err == REDIS_ERR_OTHER && - (strcmp(c->errstr,"Name or service not known") == 0 || - strcmp(c->errstr,"Can't resolve: idontexist.test") == 0 || - strcmp(c->errstr,"nodename nor servname provided, or not known") == 0 || - strcmp(c->errstr,"No address associated with hostname") == 0 || - strcmp(c->errstr,"Temporary failure in name resolution") == 0 || - strcmp(c->errstr,"hostname nor servname provided, or not known") == 0 || - strcmp(c->errstr,"no address associated with name") == 0)); - redisFree(c); - - test("Returns error when the port is not open: "); - c = redisConnect((char*)"localhost", 1); - test_cond(c->err == REDIS_ERR_IO && - strcmp(c->errstr,"Connection refused") == 0); - redisFree(c); - - test("Returns error when the unix socket path doesn't accept connections: "); - c = redisConnectUnix((char*)"/tmp/idontexist.sock"); - test_cond(c->err == REDIS_ERR_IO); /* Don't care about the message... */ - redisFree(c); -} - -static void test_blocking_connection(struct config config) { - redisContext *c; - redisReply *reply; - - c = connect(config); - - test("Is able to deliver commands: "); - reply = redisCommand(c,"PING"); - test_cond(reply->type == REDIS_REPLY_STATUS && - strcasecmp(reply->str,"pong") == 0) - freeReplyObject(reply); - - test("Is a able to send commands verbatim: "); - reply = redisCommand(c,"SET foo bar"); - test_cond (reply->type == REDIS_REPLY_STATUS && - strcasecmp(reply->str,"ok") == 0) - freeReplyObject(reply); - - test("%%s String interpolation works: "); - reply = redisCommand(c,"SET %s %s","foo","hello world"); - freeReplyObject(reply); - reply = redisCommand(c,"GET foo"); - test_cond(reply->type == REDIS_REPLY_STRING && - strcmp(reply->str,"hello world") == 0); - freeReplyObject(reply); - - test("%%b String interpolation works: "); - reply = redisCommand(c,"SET %b %b","foo",(size_t)3,"hello\x00world",(size_t)11); - freeReplyObject(reply); - reply = redisCommand(c,"GET foo"); - test_cond(reply->type == REDIS_REPLY_STRING && - memcmp(reply->str,"hello\x00world",11) == 0) - - test("Binary reply length is correct: "); - test_cond(reply->len == 11) - freeReplyObject(reply); - - test("Can parse nil replies: "); - reply = redisCommand(c,"GET nokey"); - test_cond(reply->type == REDIS_REPLY_NIL) - freeReplyObject(reply); - - /* test 7 */ - test("Can parse integer replies: "); - reply = redisCommand(c,"INCR mycounter"); - test_cond(reply->type == REDIS_REPLY_INTEGER && reply->integer == 1) - freeReplyObject(reply); - - test("Can parse multi bulk replies: "); - freeReplyObject(redisCommand(c,"LPUSH mylist foo")); - freeReplyObject(redisCommand(c,"LPUSH mylist bar")); - reply = redisCommand(c,"LRANGE mylist 0 -1"); - test_cond(reply->type == REDIS_REPLY_ARRAY && - reply->elements == 2 && - !memcmp(reply->element[0]->str,"bar",3) && - !memcmp(reply->element[1]->str,"foo",3)) - freeReplyObject(reply); - - /* m/e with multi bulk reply *before* other reply. - * specifically test ordering of reply items to parse. */ - test("Can handle nested multi bulk replies: "); - freeReplyObject(redisCommand(c,"MULTI")); - freeReplyObject(redisCommand(c,"LRANGE mylist 0 -1")); - freeReplyObject(redisCommand(c,"PING")); - reply = (redisCommand(c,"EXEC")); - test_cond(reply->type == REDIS_REPLY_ARRAY && - reply->elements == 2 && - reply->element[0]->type == REDIS_REPLY_ARRAY && - reply->element[0]->elements == 2 && - !memcmp(reply->element[0]->element[0]->str,"bar",3) && - !memcmp(reply->element[0]->element[1]->str,"foo",3) && - reply->element[1]->type == REDIS_REPLY_STATUS && - strcasecmp(reply->element[1]->str,"pong") == 0); - freeReplyObject(reply); - - disconnect(c, 0); -} - -static void test_blocking_connection_timeouts(struct config config) { - redisContext *c; - redisReply *reply; - ssize_t s; - const char *cmd = "DEBUG SLEEP 3\r\n"; - struct timeval tv; - - c = connect(config); - test("Successfully completes a command when the timeout is not exceeded: "); - reply = redisCommand(c,"SET foo fast"); - freeReplyObject(reply); - tv.tv_sec = 0; - tv.tv_usec = 10000; - redisSetTimeout(c, tv); - reply = redisCommand(c, "GET foo"); - test_cond(reply != NULL && reply->type == REDIS_REPLY_STRING && memcmp(reply->str, "fast", 4) == 0); - freeReplyObject(reply); - disconnect(c, 0); - - c = connect(config); - test("Does not return a reply when the command times out: "); - s = write(c->fd, cmd, strlen(cmd)); - tv.tv_sec = 0; - tv.tv_usec = 10000; - redisSetTimeout(c, tv); - reply = redisCommand(c, "GET foo"); - test_cond(s > 0 && reply == NULL && c->err == REDIS_ERR_IO && strcmp(c->errstr, "Resource temporarily unavailable") == 0); - freeReplyObject(reply); - - test("Reconnect properly reconnects after a timeout: "); - redisReconnect(c); - reply = redisCommand(c, "PING"); - test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); - freeReplyObject(reply); - - test("Reconnect properly uses owned parameters: "); - config.tcp.host = "foo"; - config.unix.path = "foo"; - redisReconnect(c); - reply = redisCommand(c, "PING"); - test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); - freeReplyObject(reply); - - disconnect(c, 0); -} - -static void test_blocking_io_errors(struct config config) { - redisContext *c; - redisReply *reply; - void *_reply; - int major, minor; - - /* Connect to target given by config. */ - c = connect(config); - { - /* Find out Redis version to determine the path for the next test */ - const char *field = "redis_version:"; - char *p, *eptr; - - reply = redisCommand(c,"INFO"); - p = strstr(reply->str,field); - major = strtol(p+strlen(field),&eptr,10); - p = eptr+1; /* char next to the first "." */ - minor = strtol(p,&eptr,10); - freeReplyObject(reply); - } - - test("Returns I/O error when the connection is lost: "); - reply = redisCommand(c,"QUIT"); - if (major > 2 || (major == 2 && minor > 0)) { - /* > 2.0 returns OK on QUIT and read() should be issued once more - * to know the descriptor is at EOF. */ - test_cond(strcasecmp(reply->str,"OK") == 0 && - redisGetReply(c,&_reply) == REDIS_ERR); - freeReplyObject(reply); - } else { - test_cond(reply == NULL); - } - - /* On 2.0, QUIT will cause the connection to be closed immediately and - * the read(2) for the reply on QUIT will set the error to EOF. - * On >2.0, QUIT will return with OK and another read(2) needed to be - * issued to find out the socket was closed by the server. In both - * conditions, the error will be set to EOF. */ - assert(c->err == REDIS_ERR_EOF && - strcmp(c->errstr,"Server closed the connection") == 0); - redisFree(c); - - c = connect(config); - test("Returns I/O error on socket timeout: "); - struct timeval tv = { 0, 1000 }; - assert(redisSetTimeout(c,tv) == REDIS_OK); - test_cond(redisGetReply(c,&_reply) == REDIS_ERR && - c->err == REDIS_ERR_IO && errno == EAGAIN); - redisFree(c); -} - -static void test_invalid_timeout_errors(struct config config) { - redisContext *c; - - test("Set error when an invalid timeout usec value is given to redisConnectWithTimeout: "); - - config.tcp.timeout.tv_sec = 0; - config.tcp.timeout.tv_usec = 10000001; - - c = redisConnectWithTimeout(config.tcp.host, config.tcp.port, config.tcp.timeout); - - test_cond(c->err == REDIS_ERR_IO); - redisFree(c); - - test("Set error when an invalid timeout sec value is given to redisConnectWithTimeout: "); - - config.tcp.timeout.tv_sec = (((LONG_MAX) - 999) / 1000) + 1; - config.tcp.timeout.tv_usec = 0; - - c = redisConnectWithTimeout(config.tcp.host, config.tcp.port, config.tcp.timeout); - - test_cond(c->err == REDIS_ERR_IO); - redisFree(c); -} - -static void test_throughput(struct config config) { - redisContext *c = connect(config); - redisReply **replies; - int i, num; - long long t1, t2; - - test("Throughput:\n"); - for (i = 0; i < 500; i++) - freeReplyObject(redisCommand(c,"LPUSH mylist foo")); - - num = 1000; - replies = malloc(sizeof(redisReply*)*num); - t1 = usec(); - for (i = 0; i < num; i++) { - replies[i] = redisCommand(c,"PING"); - assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_STATUS); - } - t2 = usec(); - for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); - printf("\t(%dx PING: %.3fs)\n", num, (t2-t1)/1000000.0); - - replies = malloc(sizeof(redisReply*)*num); - t1 = usec(); - for (i = 0; i < num; i++) { - replies[i] = redisCommand(c,"LRANGE mylist 0 499"); - assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_ARRAY); - assert(replies[i] != NULL && replies[i]->elements == 500); - } - t2 = usec(); - for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); - printf("\t(%dx LRANGE with 500 elements: %.3fs)\n", num, (t2-t1)/1000000.0); - - num = 10000; - replies = malloc(sizeof(redisReply*)*num); - for (i = 0; i < num; i++) - redisAppendCommand(c,"PING"); - t1 = usec(); - for (i = 0; i < num; i++) { - assert(redisGetReply(c, (void*)&replies[i]) == REDIS_OK); - assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_STATUS); - } - t2 = usec(); - for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); - printf("\t(%dx PING (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0); - - replies = malloc(sizeof(redisReply*)*num); - for (i = 0; i < num; i++) - redisAppendCommand(c,"LRANGE mylist 0 499"); - t1 = usec(); - for (i = 0; i < num; i++) { - assert(redisGetReply(c, (void*)&replies[i]) == REDIS_OK); - assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_ARRAY); - assert(replies[i] != NULL && replies[i]->elements == 500); - } - t2 = usec(); - for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); - printf("\t(%dx LRANGE with 500 elements (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0); - - disconnect(c, 0); -} - -// static long __test_callback_flags = 0; -// static void __test_callback(redisContext *c, void *privdata) { -// ((void)c); -// /* Shift to detect execution order */ -// __test_callback_flags <<= 8; -// __test_callback_flags |= (long)privdata; -// } -// -// static void __test_reply_callback(redisContext *c, redisReply *reply, void *privdata) { -// ((void)c); -// /* Shift to detect execution order */ -// __test_callback_flags <<= 8; -// __test_callback_flags |= (long)privdata; -// if (reply) freeReplyObject(reply); -// } -// -// static redisContext *__connect_nonblock() { -// /* Reset callback flags */ -// __test_callback_flags = 0; -// return redisConnectNonBlock("127.0.0.1", port, NULL); -// } -// -// static void test_nonblocking_connection() { -// redisContext *c; -// int wdone = 0; -// -// test("Calls command callback when command is issued: "); -// c = __connect_nonblock(); -// redisSetCommandCallback(c,__test_callback,(void*)1); -// redisCommand(c,"PING"); -// test_cond(__test_callback_flags == 1); -// redisFree(c); -// -// test("Calls disconnect callback on redisDisconnect: "); -// c = __connect_nonblock(); -// redisSetDisconnectCallback(c,__test_callback,(void*)2); -// redisDisconnect(c); -// test_cond(__test_callback_flags == 2); -// redisFree(c); -// -// test("Calls disconnect callback and free callback on redisFree: "); -// c = __connect_nonblock(); -// redisSetDisconnectCallback(c,__test_callback,(void*)2); -// redisSetFreeCallback(c,__test_callback,(void*)4); -// redisFree(c); -// test_cond(__test_callback_flags == ((2 << 8) | 4)); -// -// test("redisBufferWrite against empty write buffer: "); -// c = __connect_nonblock(); -// test_cond(redisBufferWrite(c,&wdone) == REDIS_OK && wdone == 1); -// redisFree(c); -// -// test("redisBufferWrite against not yet connected fd: "); -// c = __connect_nonblock(); -// redisCommand(c,"PING"); -// test_cond(redisBufferWrite(c,NULL) == REDIS_ERR && -// strncmp(c->error,"write:",6) == 0); -// redisFree(c); -// -// test("redisBufferWrite against closed fd: "); -// c = __connect_nonblock(); -// redisCommand(c,"PING"); -// redisDisconnect(c); -// test_cond(redisBufferWrite(c,NULL) == REDIS_ERR && -// strncmp(c->error,"write:",6) == 0); -// redisFree(c); -// -// test("Process callbacks in the right sequence: "); -// c = __connect_nonblock(); -// redisCommandWithCallback(c,__test_reply_callback,(void*)1,"PING"); -// redisCommandWithCallback(c,__test_reply_callback,(void*)2,"PING"); -// redisCommandWithCallback(c,__test_reply_callback,(void*)3,"PING"); -// -// /* Write output buffer */ -// wdone = 0; -// while(!wdone) { -// usleep(500); -// redisBufferWrite(c,&wdone); -// } -// -// /* Read until at least one callback is executed (the 3 replies will -// * arrive in a single packet, causing all callbacks to be executed in -// * a single pass). */ -// while(__test_callback_flags == 0) { -// assert(redisBufferRead(c) == REDIS_OK); -// redisProcessCallbacks(c); -// } -// test_cond(__test_callback_flags == 0x010203); -// redisFree(c); -// -// test("redisDisconnect executes pending callbacks with NULL reply: "); -// c = __connect_nonblock(); -// redisSetDisconnectCallback(c,__test_callback,(void*)1); -// redisCommandWithCallback(c,__test_reply_callback,(void*)2,"PING"); -// redisDisconnect(c); -// test_cond(__test_callback_flags == 0x0201); -// redisFree(c); -// } - -int main(int argc, char **argv) { - struct config cfg = { - .tcp = { - .host = "127.0.0.1", - .port = 6379 - }, - .unix = { - .path = "/tmp/redis.sock" - } - }; - int throughput = 1; - int test_inherit_fd = 1; - - /* Ignore broken pipe signal (for I/O error tests). */ - signal(SIGPIPE, SIG_IGN); - - /* Parse command line options. */ - argv++; argc--; - while (argc) { - if (argc >= 2 && !strcmp(argv[0],"-h")) { - argv++; argc--; - cfg.tcp.host = argv[0]; - } else if (argc >= 2 && !strcmp(argv[0],"-p")) { - argv++; argc--; - cfg.tcp.port = atoi(argv[0]); - } else if (argc >= 2 && !strcmp(argv[0],"-s")) { - argv++; argc--; - cfg.unix.path = argv[0]; - } else if (argc >= 1 && !strcmp(argv[0],"--skip-throughput")) { - throughput = 0; - } else if (argc >= 1 && !strcmp(argv[0],"--skip-inherit-fd")) { - test_inherit_fd = 0; - } else { - fprintf(stderr, "Invalid argument: %s\n", argv[0]); - exit(1); - } - argv++; argc--; - } - - test_format_commands(); - test_reply_reader(); - test_blocking_connection_errors(); - test_free_null(); - - printf("\nTesting against TCP connection (%s:%d):\n", cfg.tcp.host, cfg.tcp.port); - cfg.type = CONN_TCP; - test_blocking_connection(cfg); - test_blocking_connection_timeouts(cfg); - test_blocking_io_errors(cfg); - test_invalid_timeout_errors(cfg); - test_append_formatted_commands(cfg); - if (throughput) test_throughput(cfg); - - printf("\nTesting against Unix socket connection (%s):\n", cfg.unix.path); - cfg.type = CONN_UNIX; - test_blocking_connection(cfg); - test_blocking_connection_timeouts(cfg); - test_blocking_io_errors(cfg); - if (throughput) test_throughput(cfg); - - if (test_inherit_fd) { - printf("\nTesting against inherited fd (%s):\n", cfg.unix.path); - cfg.type = CONN_FD; - test_blocking_connection(cfg); - } - - - if (fails) { - printf("*** %d TESTS FAILED ***\n", fails); - return 1; - } - - printf("ALL TESTS PASSED\n"); - return 0; -} diff --git a/third-party/hiredis/win32.h b/third-party/hiredis/win32.h deleted file mode 100644 index 1a27c18f2..000000000 --- a/third-party/hiredis/win32.h +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef _WIN32_HELPER_INCLUDE -#define _WIN32_HELPER_INCLUDE -#ifdef _MSC_VER - -#ifndef inline -#define inline __inline -#endif - -#ifndef va_copy -#define va_copy(d,s) ((d) = (s)) -#endif - -#ifndef snprintf -#define snprintf c99_snprintf - -__inline int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap) -{ - int count = -1; - - if (size != 0) - count = _vsnprintf_s(str, size, _TRUNCATE, format, ap); - if (count == -1) - count = _vscprintf(format, ap); - - return count; -} - -__inline int c99_snprintf(char* str, size_t size, const char* format, ...) -{ - int count; - va_list ap; - - va_start(ap, format); - count = c99_vsnprintf(str, size, format, ap); - va_end(ap); - - return count; -} -#endif - -#endif -#endif \ No newline at end of file From fa5c9ac9858c8b62386b1faa874a7250de6a9fee Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 7 Aug 2019 13:26:05 +0200 Subject: [PATCH 159/219] RedisConnection: don't read the response before the request has been written refs #50 --- lib/redis/redisconnection.cpp | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index 080b64c4c..86187d2fc 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -206,6 +206,7 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) if (!m_Queues.FireAndForgetQuery.empty()) { auto item (std::move(m_Queues.FireAndForgetQuery.front())); m_Queues.FireAndForgetQuery.pop(); + WriteOne(item, yc); if (m_Queues.FutureResponseActions.empty() || m_Queues.FutureResponseActions.back().Action != ResponseAction::Ignore) { m_Queues.FutureResponseActions.emplace(FutureResponseAction{1, ResponseAction::Ignore}); @@ -215,14 +216,16 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) m_QueuedReads.Set(); writtenAll = false; - - WriteOne(item, yc); } if (!m_Queues.FireAndForgetQueries.empty()) { auto item (std::move(m_Queues.FireAndForgetQueries.front())); m_Queues.FireAndForgetQueries.pop(); + for (auto& query : item) { + WriteOne(query, yc); + } + if (m_Queues.FutureResponseActions.empty() || m_Queues.FutureResponseActions.back().Action != ResponseAction::Ignore) { m_Queues.FutureResponseActions.emplace(FutureResponseAction{item.size(), ResponseAction::Ignore}); } else { @@ -231,15 +234,13 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) m_QueuedReads.Set(); writtenAll = false; - - for (auto& query : item) { - WriteOne(query, yc); - } } if (!m_Queues.GetResultOfQuery.empty()) { auto item (std::move(m_Queues.GetResultOfQuery.front())); m_Queues.GetResultOfQuery.pop(); + WriteOne(item.first, yc); + m_Queues.ReplyPromises.emplace(std::move(item.second)); if (m_Queues.FutureResponseActions.empty() || m_Queues.FutureResponseActions.back().Action != ResponseAction::Deliver) { @@ -250,33 +251,25 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) m_QueuedReads.Set(); writtenAll = false; - - WriteOne(item.first, yc); } if (!m_Queues.GetResultsOfQueries.empty()) { auto item (std::move(m_Queues.GetResultsOfQueries.front())); m_Queues.GetResultsOfQueries.pop(); + + for (auto& query : item.first) { + WriteOne(query, yc); + } + m_Queues.RepliesPromises.emplace(std::move(item.second)); m_Queues.FutureResponseActions.emplace(FutureResponseAction{item.first.size(), ResponseAction::DeliverBulk}); m_QueuedReads.Set(); writtenAll = false; - - for (auto& query : item.first) { - WriteOne(query, yc); - } } } while (!writtenAll); m_QueuedWrites.Clear(); - - if (m_Path.IsEmpty()) { - m_TcpConn->async_flush(yc); - } else { - m_UnixConn->async_flush(yc); - } - } } @@ -293,7 +286,9 @@ void RedisConnection::WriteOne(RedisConnection::Query& query, asio::yield_contex { if (m_Path.IsEmpty()) { WriteRESP(*m_TcpConn, query, yc); + m_TcpConn->async_flush(yc); } else { WriteRESP(*m_UnixConn, query, yc); + m_UnixConn->async_flush(yc); } } From 06d88477bd06cdd8fce0f7128c38cc670536a21f Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 7 Aug 2019 14:15:01 +0200 Subject: [PATCH 160/219] RedisConnection: forward I/O errors to async-ly waiting requestors refs #50 --- lib/redis/redisconnection.cpp | 44 +++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index 86187d2fc..22c3f43da 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -169,7 +170,18 @@ void RedisConnection::ReadLoop(asio::yield_context& yc) auto promise (std::move(m_Queues.ReplyPromises.front())); m_Queues.ReplyPromises.pop(); - promise.set_value(ReadOne(yc)); + Reply reply; + + try { + reply = ReadOne(yc); + } catch (const boost::coroutines::detail::forced_unwind&) { + throw; + } catch (...) { + promise.set_exception(std::current_exception()); + throw; + } + + promise.set_value(std::move(reply)); } break; case ResponseAction::DeliverBulk: @@ -181,7 +193,14 @@ void RedisConnection::ReadLoop(asio::yield_context& yc) replies.reserve(item.Amount); for (auto i (item.Amount); i; --i) { - replies.emplace_back(ReadOne(yc)); + try { + replies.emplace_back(ReadOne(yc)); + } catch (const boost::coroutines::detail::forced_unwind&) { + throw; + } catch (...) { + promise.set_exception(std::current_exception()); + throw; + } } promise.set_value(std::move(replies)); @@ -239,7 +258,15 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) if (!m_Queues.GetResultOfQuery.empty()) { auto item (std::move(m_Queues.GetResultOfQuery.front())); m_Queues.GetResultOfQuery.pop(); - WriteOne(item.first, yc); + + try { + WriteOne(item.first, yc); + } catch (const boost::coroutines::detail::forced_unwind&) { + throw; + } catch (...) { + item.second.set_exception(std::current_exception()); + throw; + } m_Queues.ReplyPromises.emplace(std::move(item.second)); @@ -257,8 +284,15 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) auto item (std::move(m_Queues.GetResultsOfQueries.front())); m_Queues.GetResultsOfQueries.pop(); - for (auto& query : item.first) { - WriteOne(query, yc); + try { + for (auto& query : item.first) { + WriteOne(query, yc); + } + } catch (const boost::coroutines::detail::forced_unwind&) { + throw; + } catch (...) { + item.second.set_exception(std::current_exception()); + throw; } m_Queues.RepliesPromises.emplace(std::move(item.second)); From 430c769371faa8d85c9fb3e8d069d8c714b935b9 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 8 Aug 2019 17:42:31 +0200 Subject: [PATCH 161/219] RedisConnection: handle errors as expected refs #50 --- lib/redis/redisconnection.cpp | 83 +++++++++++++++++++++-------------- lib/redis/redisconnection.hpp | 65 +++++++++++++++++++++++++-- 2 files changed, 111 insertions(+), 37 deletions(-) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index 22c3f43da..c710920d1 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -46,12 +46,19 @@ RedisConnection::RedisConnection(const String host, const int port, const String RedisConnection::RedisConnection(boost::asio::io_context& io, String host, int port, String path, String password, int db) : m_Host(std::move(host)), m_Port(port), m_Path(std::move(path)), m_Password(std::move(password)), m_DbIndex(db), - m_Connecting(false), m_Connected(false), m_Strand(io), m_QueuedWrites(io), m_QueuedReads(io) + m_Connecting(false), m_Connected(false), m_Started(false), m_Strand(io), m_QueuedWrites(io), m_QueuedReads(io) { } void RedisConnection::Start() { + if (!m_Started.exchange(true)) { + Ptr keepAlive (this); + + asio::spawn(m_Strand, [this, keepAlive](asio::yield_context yc) { ReadLoop(yc); }); + asio::spawn(m_Strand, [this, keepAlive](asio::yield_context yc) { WriteLoop(yc); }); + } + if (!m_Connecting.exchange(true)) { Ptr keepAlive (this); @@ -117,28 +124,19 @@ RedisConnection::Replies RedisConnection::GetResultsOfQueries(RedisConnection::Q void RedisConnection::Connect(asio::yield_context& yc) { - Defer notConnecting ([this]() { - if (!m_Connected.load()) { - m_Connecting.store(false); - } - }); + Defer notConnecting ([this]() { m_Connecting.store(m_Connected.load()); }); Log(LogInformation, "RedisWriter", "Trying to connect to Redis server (async)"); try { if (m_Path.IsEmpty()) { - m_TcpConn = decltype(m_TcpConn)(new TcpConn(m_Strand.context())); - icinga::Connect(m_TcpConn->next_layer(), m_Host, Convert::ToString(m_Port), yc); + decltype(m_TcpConn) conn (new TcpConn(m_Strand.context())); + icinga::Connect(conn->next_layer(), m_Host, Convert::ToString(m_Port), yc); + m_TcpConn = std::move(conn); } else { - m_UnixConn = decltype(m_UnixConn)(new UnixConn(m_Strand.context())); - m_UnixConn->next_layer().async_connect(Unix::endpoint(m_Path.CStr()), yc); - } - - { - Ptr keepAlive (this); - - asio::spawn(m_Strand, [this, keepAlive](asio::yield_context yc) { ReadLoop(yc); }); - asio::spawn(m_Strand, [this, keepAlive](asio::yield_context yc) { WriteLoop(yc); }); + decltype(m_UnixConn) conn (new UnixConn(m_Strand.context())); + conn->next_layer().async_connect(Unix::endpoint(m_Path.CStr()), yc); + m_UnixConn = std::move(conn); } m_Connected.store(true); @@ -155,14 +153,20 @@ void RedisConnection::ReadLoop(asio::yield_context& yc) for (;;) { m_QueuedReads.Wait(yc); - do { + while (!m_Queues.FutureResponseActions.empty()) { auto item (std::move(m_Queues.FutureResponseActions.front())); m_Queues.FutureResponseActions.pop(); switch (item.Action) { case ResponseAction::Ignore: - for (auto i (item.Amount); i; --i) { - ReadOne(yc); + try { + for (auto i (item.Amount); i; --i) { + ReadOne(yc); + } + } catch (const boost::coroutines::detail::forced_unwind&) { + throw; + } catch (...) { + continue; } break; case ResponseAction::Deliver: @@ -178,7 +182,7 @@ void RedisConnection::ReadLoop(asio::yield_context& yc) throw; } catch (...) { promise.set_exception(std::current_exception()); - throw; + continue; } promise.set_value(std::move(reply)); @@ -199,14 +203,14 @@ void RedisConnection::ReadLoop(asio::yield_context& yc) throw; } catch (...) { promise.set_exception(std::current_exception()); - throw; + continue; } } promise.set_value(std::move(replies)); } } - } while (!m_Queues.FutureResponseActions.empty()); + } m_QueuedReads.Clear(); } @@ -225,7 +229,14 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) if (!m_Queues.FireAndForgetQuery.empty()) { auto item (std::move(m_Queues.FireAndForgetQuery.front())); m_Queues.FireAndForgetQuery.pop(); - WriteOne(item, yc); + + try { + WriteOne(item, yc); + } catch (const boost::coroutines::detail::forced_unwind&) { + throw; + } catch (...) { + continue; + } if (m_Queues.FutureResponseActions.empty() || m_Queues.FutureResponseActions.back().Action != ResponseAction::Ignore) { m_Queues.FutureResponseActions.emplace(FutureResponseAction{1, ResponseAction::Ignore}); @@ -241,8 +252,14 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) auto item (std::move(m_Queues.FireAndForgetQueries.front())); m_Queues.FireAndForgetQueries.pop(); - for (auto& query : item) { - WriteOne(query, yc); + try { + for (auto& query : item) { + WriteOne(query, yc); + } + } catch (const boost::coroutines::detail::forced_unwind&) { + throw; + } catch (...) { + continue; } if (m_Queues.FutureResponseActions.empty() || m_Queues.FutureResponseActions.back().Action != ResponseAction::Ignore) { @@ -265,7 +282,7 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) throw; } catch (...) { item.second.set_exception(std::current_exception()); - throw; + continue; } m_Queues.ReplyPromises.emplace(std::move(item.second)); @@ -292,7 +309,7 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) throw; } catch (...) { item.second.set_exception(std::current_exception()); - throw; + continue; } m_Queues.RepliesPromises.emplace(std::move(item.second)); @@ -310,19 +327,17 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) RedisConnection::Reply RedisConnection::ReadOne(boost::asio::yield_context& yc) { if (m_Path.IsEmpty()) { - return ReadRESP(*m_TcpConn, yc); + return ReadOne(m_TcpConn, yc); } else { - return ReadRESP(*m_UnixConn, yc); + return ReadOne(m_UnixConn, yc); } } void RedisConnection::WriteOne(RedisConnection::Query& query, asio::yield_context& yc) { if (m_Path.IsEmpty()) { - WriteRESP(*m_TcpConn, query, yc); - m_TcpConn->async_flush(yc); + WriteOne(m_TcpConn, query, yc); } else { - WriteRESP(*m_UnixConn, query, yc); - m_UnixConn->async_flush(yc); + WriteOne(m_UnixConn, query, yc); } } diff --git a/lib/redis/redisconnection.hpp b/lib/redis/redisconnection.hpp index b6b5c005d..a39c152e6 100644 --- a/lib/redis/redisconnection.hpp +++ b/lib/redis/redisconnection.hpp @@ -117,6 +117,12 @@ namespace icinga Reply ReadOne(boost::asio::yield_context& yc); void WriteOne(Query& query, boost::asio::yield_context& yc); + template + Reply ReadOne(StreamPtr& stream, boost::asio::yield_context& yc); + + template + void WriteOne(StreamPtr& stream, Query& query, boost::asio::yield_context& yc); + String m_Path; String m_Host; int m_Port; @@ -124,9 +130,9 @@ namespace icinga int m_DbIndex; boost::asio::io_context::strand m_Strand; - std::unique_ptr m_TcpConn; - std::unique_ptr m_UnixConn; - Atomic m_Connecting, m_Connected; + std::shared_ptr m_TcpConn; + std::shared_ptr m_UnixConn; + Atomic m_Connecting, m_Connected, m_Started; struct { std::queue FireAndForgetQuery; @@ -159,6 +165,14 @@ private: String m_Message; }; +class RedisDisconnected : public std::runtime_error +{ +public: + inline RedisDisconnected() : runtime_error("") + { + } +}; + class RedisProtocolError : public std::runtime_error { protected: @@ -200,6 +214,51 @@ private: std::vector m_What; }; +template +RedisConnection::Reply RedisConnection::ReadOne(StreamPtr& stream, boost::asio::yield_context& yc) +{ + if (!stream) { + throw RedisDisconnected(); + } + + auto strm (stream); + + try { + return ReadRESP(*strm, yc); + } catch (const boost::coroutines::detail::forced_unwind&) { + throw; + } catch (...) { + if (m_Connecting.exchange(false)) { + m_Connected.store(false); + stream = nullptr; + } + throw; + } +} + +template +void RedisConnection::WriteOne(StreamPtr& stream, RedisConnection::Query& query, boost::asio::yield_context& yc) +{ + if (!stream) { + throw RedisDisconnected(); + } + + auto strm (stream); + + try { + WriteRESP(*strm, query, yc); + strm->async_flush(yc); + } catch (const boost::coroutines::detail::forced_unwind&) { + throw; + } catch (...) { + if (m_Connecting.exchange(false)) { + m_Connected.store(false); + stream = nullptr; + } + throw; + } +} + template Value RedisConnection::ReadRESP(AsyncReadStream& stream, boost::asio::yield_context& yc) { From 3b61b3d37a4e64085859ab232abd7818c0e5f2f9 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 3 Sep 2019 11:54:31 +0200 Subject: [PATCH 162/219] RedisConnection: log messages refs #50 --- lib/redis/redisconnection.cpp | 61 +++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index c710920d1..b31cb6656 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -70,8 +70,28 @@ bool RedisConnection::IsConnected() { return m_Connected.load(); } +static inline +void LogQuery(RedisConnection::Query& query, Log& msg) +{ + int i = 0; + + for (auto& arg : query) { + if (++i == 8) { + msg << " ..."; + break; + } + + msg << " '" << arg << '\''; + } +} + void RedisConnection::FireAndForgetQuery(RedisConnection::Query query) { + { + Log msg (LogNotice, "RedisWriter", "Firing and forgetting query:"); + LogQuery(query, msg); + } + auto item (std::make_shared(std::move(query))); asio::post(m_Strand, [this, item]() { @@ -82,6 +102,11 @@ void RedisConnection::FireAndForgetQuery(RedisConnection::Query query) void RedisConnection::FireAndForgetQueries(RedisConnection::Queries queries) { + for (auto& query : queries) { + Log msg (LogNotice, "RedisWriter", "Firing and forgetting query:"); + LogQuery(query, msg); + } + auto item (std::make_shared(std::move(queries))); asio::post(m_Strand, [this, item]() { @@ -92,6 +117,11 @@ void RedisConnection::FireAndForgetQueries(RedisConnection::Queries queries) RedisConnection::Reply RedisConnection::GetResultOfQuery(RedisConnection::Query query) { + { + Log msg (LogNotice, "RedisWriter", "Executing query:"); + LogQuery(query, msg); + } + std::promise promise; auto future (promise.get_future()); auto item (std::make_shared(std::move(query), std::move(promise))); @@ -108,6 +138,11 @@ RedisConnection::Reply RedisConnection::GetResultOfQuery(RedisConnection::Query RedisConnection::Replies RedisConnection::GetResultsOfQueries(RedisConnection::Queries queries) { + for (auto& query : queries) { + Log msg (LogNotice, "RedisWriter", "Executing query:"); + LogQuery(query, msg); + } + std::promise promise; auto future (promise.get_future()); auto item (std::make_shared(std::move(queries), std::move(promise))); @@ -140,6 +175,8 @@ void RedisConnection::Connect(asio::yield_context& yc) } m_Connected.store(true); + + Log(LogInformation, "RedisWriter", "Connected to Redis server"); } catch (const boost::coroutines::detail::forced_unwind&) { throw; } catch (const std::exception& ex) { @@ -165,7 +202,13 @@ void RedisConnection::ReadLoop(asio::yield_context& yc) } } catch (const boost::coroutines::detail::forced_unwind&) { throw; + } catch (const std::exception& ex) { + Log(LogCritical, "RedisWriter") + << "Error during receiving the response to a query which has been fired and forgotten: " << ex.what(); + continue; } catch (...) { + Log(LogCritical, "RedisWriter") + << "Error during receiving the response to a query which has been fired and forgotten"; continue; } break; @@ -234,7 +277,15 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) WriteOne(item, yc); } catch (const boost::coroutines::detail::forced_unwind&) { throw; + } catch (const std::exception& ex) { + Log msg (LogCritical, "RedisWriter", "Error during sending query"); + LogQuery(item, msg); + msg << " which has been fired and forgotten: " << ex.what(); + continue; } catch (...) { + Log msg (LogCritical, "RedisWriter", "Error during sending query"); + LogQuery(item, msg); + msg << " which has been fired and forgotten"; continue; } @@ -251,14 +302,24 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) if (!m_Queues.FireAndForgetQueries.empty()) { auto item (std::move(m_Queues.FireAndForgetQueries.front())); m_Queues.FireAndForgetQueries.pop(); + size_t i = 0; try { for (auto& query : item) { WriteOne(query, yc); + ++i; } } catch (const boost::coroutines::detail::forced_unwind&) { throw; + } catch (const std::exception& ex) { + Log msg (LogCritical, "RedisWriter", "Error during sending query"); + LogQuery(item[i], msg); + msg << " which has been fired and forgotten: " << ex.what(); + continue; } catch (...) { + Log msg (LogCritical, "RedisWriter", "Error during sending query"); + LogQuery(item[i], msg); + msg << " which has been fired and forgotten"; continue; } From 1d3109458dd3957613cebfd1c1427584421cf27b Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 3 Sep 2019 15:12:12 +0200 Subject: [PATCH 163/219] RedisConnection: merge write queues refs #50 --- lib/redis/redisconnection.cpp | 49 ++++++++++++++--------------------- lib/redis/redisconnection.hpp | 13 +++++++--- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index b31cb6656..e855cea71 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -92,10 +92,10 @@ void RedisConnection::FireAndForgetQuery(RedisConnection::Query query) LogQuery(query, msg); } - auto item (std::make_shared(std::move(query))); + auto item (std::make_shared(std::move(query))); asio::post(m_Strand, [this, item]() { - m_Queues.FireAndForgetQuery.emplace(std::move(*item)); + m_Queues.Writes.emplace(WriteQueueItem{item, nullptr, nullptr, nullptr}); m_QueuedWrites.Set(); }); } @@ -107,10 +107,10 @@ void RedisConnection::FireAndForgetQueries(RedisConnection::Queries queries) LogQuery(query, msg); } - auto item (std::make_shared(std::move(queries))); + auto item (std::make_shared(std::move(queries))); asio::post(m_Strand, [this, item]() { - m_Queues.FireAndForgetQueries.emplace(std::move(*item)); + m_Queues.Writes.emplace(WriteQueueItem{nullptr, item, nullptr, nullptr}); m_QueuedWrites.Set(); }); } @@ -124,10 +124,10 @@ RedisConnection::Reply RedisConnection::GetResultOfQuery(RedisConnection::Query std::promise promise; auto future (promise.get_future()); - auto item (std::make_shared(std::move(query), std::move(promise))); + auto item (std::make_shared(std::move(query), std::move(promise))); asio::post(m_Strand, [this, item]() { - m_Queues.GetResultOfQuery.emplace(std::move(*item)); + m_Queues.Writes.emplace(WriteQueueItem{nullptr, nullptr, item, nullptr}); m_QueuedWrites.Set(); }); @@ -145,10 +145,10 @@ RedisConnection::Replies RedisConnection::GetResultsOfQueries(RedisConnection::Q std::promise promise; auto future (promise.get_future()); - auto item (std::make_shared(std::move(queries), std::move(promise))); + auto item (std::make_shared(std::move(queries), std::move(promise))); asio::post(m_Strand, [this, item]() { - m_Queues.GetResultsOfQueries.emplace(std::move(*item)); + m_Queues.Writes.emplace(WriteQueueItem{nullptr, nullptr, nullptr, item}); m_QueuedWrites.Set(); }); @@ -264,14 +264,12 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) for (;;) { m_QueuedWrites.Wait(yc); - bool writtenAll = true; + while (!m_Queues.Writes.empty()) { + auto next (std::move(m_Queues.Writes.front())); + m_Queues.Writes.pop(); - do { - writtenAll = true; - - if (!m_Queues.FireAndForgetQuery.empty()) { - auto item (std::move(m_Queues.FireAndForgetQuery.front())); - m_Queues.FireAndForgetQuery.pop(); + if (next.FireAndForgetQuery) { + auto& item (*next.FireAndForgetQuery); try { WriteOne(item, yc); @@ -296,12 +294,10 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) } m_QueuedReads.Set(); - writtenAll = false; } - if (!m_Queues.FireAndForgetQueries.empty()) { - auto item (std::move(m_Queues.FireAndForgetQueries.front())); - m_Queues.FireAndForgetQueries.pop(); + if (next.FireAndForgetQueries) { + auto& item (*next.FireAndForgetQueries); size_t i = 0; try { @@ -330,12 +326,10 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) } m_QueuedReads.Set(); - writtenAll = false; } - if (!m_Queues.GetResultOfQuery.empty()) { - auto item (std::move(m_Queues.GetResultOfQuery.front())); - m_Queues.GetResultOfQuery.pop(); + if (next.GetResultOfQuery) { + auto& item (*next.GetResultOfQuery); try { WriteOne(item.first, yc); @@ -355,12 +349,10 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) } m_QueuedReads.Set(); - writtenAll = false; } - if (!m_Queues.GetResultsOfQueries.empty()) { - auto item (std::move(m_Queues.GetResultsOfQueries.front())); - m_Queues.GetResultsOfQueries.pop(); + if (next.GetResultsOfQueries) { + auto& item (*next.GetResultsOfQueries); try { for (auto& query : item.first) { @@ -377,9 +369,8 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) m_Queues.FutureResponseActions.emplace(FutureResponseAction{item.first.size(), ResponseAction::DeliverBulk}); m_QueuedReads.Set(); - writtenAll = false; } - } while (!writtenAll); + } m_QueuedWrites.Clear(); } diff --git a/lib/redis/redisconnection.hpp b/lib/redis/redisconnection.hpp index a39c152e6..9ef6b5566 100644 --- a/lib/redis/redisconnection.hpp +++ b/lib/redis/redisconnection.hpp @@ -134,11 +134,16 @@ namespace icinga std::shared_ptr m_UnixConn; Atomic m_Connecting, m_Connected, m_Started; + struct WriteQueueItem + { + std::shared_ptr FireAndForgetQuery; + std::shared_ptr FireAndForgetQueries; + std::shared_ptr>> GetResultOfQuery; + std::shared_ptr>> GetResultsOfQueries; + }; + struct { - std::queue FireAndForgetQuery; - std::queue FireAndForgetQueries; - std::queue>> GetResultOfQuery; - std::queue>> GetResultsOfQueries; + std::queue Writes; std::queue> ReplyPromises; std::queue> RepliesPromises; std::queue FutureResponseActions; From d9ff921934fda442ed09abf1364e193126f8fdb5 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 9 Aug 2019 15:11:39 +0200 Subject: [PATCH 164/219] RedisWriter: publish the current config dump state via icinga:dump refs #53 --- lib/redis/rediswriter-objects.cpp | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index e02d33bc6..549b9a556 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -50,6 +50,19 @@ using namespace icinga; +static const char * const l_LuaResetDump = R"EOF( + +local id = redis.call('XADD', KEYS[1], '*', 'type', '*', 'state', 'wip') + +local xr = redis.call('XRANGE', KEYS[1], '-', '+') +for i = 1, #xr - 1 do + redis.call('XDEL', KEYS[1], xr[i][1]) +end + +return id + +)EOF"; + INITIALIZE_ONCE(&RedisWriter::ConfigStaticInitialize); void RedisWriter::ConfigStaticInitialize() @@ -109,6 +122,8 @@ void RedisWriter::UpdateAllConfigObjects() } } + m_Rcon->FireAndForgetQuery({"EVAL", l_LuaResetDump, "1", "icinga:dump"}); + const std::vector globalKeys = { m_PrefixConfigObject + "customvar", m_PrefixConfigObject + "action_url", @@ -187,8 +202,6 @@ void RedisWriter::UpdateAllConfigObjects() m_Rcon->FireAndForgetQueries(std::move(transaction)); } - m_Rcon->FireAndForgetQuery({"PUBLISH", "icinga:config:dump", lcType}); - Log(LogNotice, "RedisWriter") << "Dumped " << bulkCounter << " objects of type " << type.second; }); @@ -202,6 +215,8 @@ void RedisWriter::UpdateAllConfigObjects() } } } + + m_Rcon->FireAndForgetQuery({"XADD", "icinga:dump", "*", "type", lcType, "state", "done"}); }); upq.Join(); @@ -219,6 +234,8 @@ void RedisWriter::UpdateAllConfigObjects() } } + m_Rcon->FireAndForgetQuery({"XADD", "icinga:dump", "*", "type", "*", "state", "done"}); + Log(LogInformation, "RedisWriter") << "Initial config/status dump finished in " << Utility::GetTime() - startTime << " seconds."; } From fe49141490e8c6f76fbb146d678419aed23b0cde Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 17 Sep 2019 10:44:59 +0200 Subject: [PATCH 165/219] RedisWriter#SendStatusUpdate(): ensure valid UTF-8 output --- lib/redis/rediswriter-objects.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 549b9a556..3d40997b6 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -43,6 +43,7 @@ #include "base/convert.hpp" #include "base/array.hpp" #include "base/exception.hpp" +#include "base/utility.hpp" #include #include #include @@ -1024,7 +1025,7 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) ObjectLock olock(objectAttrs); for (const Dictionary::Pair& kv : objectAttrs) { streamadd.emplace_back(kv.first); - streamadd.emplace_back(kv.second); + streamadd.emplace_back(Utility::ValidateUTF8(kv.second)); } m_Rcon->FireAndForgetQuery(std::move(streamadd)); From 322f3fbb0da8738b485ab6d52e2cd9dbe8940f9b Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 17 Sep 2019 11:20:52 +0200 Subject: [PATCH 166/219] RedisWriter: Fix state sync prefix --- lib/redis/rediswriter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index 7670537e1..eaa2609a9 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -46,7 +46,7 @@ RedisWriter::RedisWriter() m_PrefixConfigObject = "icinga:config:"; m_PrefixConfigCheckSum = "icinga:checksum:"; - m_PrefixStateObject = "icinga:state:object:"; + m_PrefixStateObject = "icinga:config:state:"; } /** From b0cd306b61eec8460ff50ee896e341944a825171 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 18 Sep 2019 12:04:59 +0200 Subject: [PATCH 167/219] RedisWriter#InsertObjectDependencies(): PUBLISH icinga:config:update ... for all dependencies --- lib/redis/rediswriter-objects.cpp | 191 +++++++++++++++++++++++------- lib/redis/rediswriter.hpp | 7 +- 2 files changed, 154 insertions(+), 44 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 3d40997b6..e1d8191ea 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -145,7 +145,7 @@ void RedisWriter::UpdateAllConfigObjects() upqObjectType.SetName("RedisWriter:ConfigDump:" + lcType); upqObjectType.ParallelFor(objectChunks, [this, &type, &lcType](decltype(objectChunks)::const_reference chunk) { - std::map> statements; + std::map> hMSets, publishes; std::vector states = {"HMSET", m_PrefixStateObject + lcType}; std::vector > transaction = {{"MULTI"}}; @@ -156,7 +156,7 @@ void RedisWriter::UpdateAllConfigObjects() if (lcType != GetLowerCaseTypeNameDB(object)) continue; - CreateConfigUpdate(object, lcType, statements, false); + CreateConfigUpdate(object, lcType, hMSets, publishes, false); // Write out inital state for checkables if (dumpState) { @@ -166,7 +166,7 @@ void RedisWriter::UpdateAllConfigObjects() bulkCounter++; if (!(bulkCounter % 100)) { - for (auto& kv : statements) { + for (auto& kv : hMSets) { if (!kv.second.empty()) { kv.second.insert(kv.second.begin(), {"HMSET", kv.first}); transaction.emplace_back(std::move(kv.second)); @@ -178,7 +178,21 @@ void RedisWriter::UpdateAllConfigObjects() states = {"HMSET", m_PrefixStateObject + lcType}; } - statements = decltype(statements)(); + for (auto& kv : publishes) { + for (auto& message : kv.second) { + std::vector publish; + + publish.reserve(3); + publish.emplace_back("PUBLISH"); + publish.emplace_back(kv.first); + publish.emplace_back(std::move(message)); + + transaction.emplace_back(std::move(publish)); + } + } + + hMSets = decltype(hMSets)(); + publishes = decltype(publishes)(); if (transaction.size() > 1) { transaction.push_back({"EXEC"}); @@ -188,7 +202,7 @@ void RedisWriter::UpdateAllConfigObjects() } } - for (auto& kv : statements) { + for (auto& kv : hMSets) { if (!kv.second.empty()) { kv.second.insert(kv.second.begin(), {"HMSET", kv.first}); transaction.emplace_back(std::move(kv.second)); @@ -198,6 +212,19 @@ void RedisWriter::UpdateAllConfigObjects() if (states.size() > 2) transaction.emplace_back(std::move(states)); + for (auto& kv : publishes) { + for (auto& message : kv.second) { + std::vector publish; + + publish.reserve(3); + publish.emplace_back("PUBLISH"); + publish.emplace_back(kv.first); + publish.emplace_back(std::move(message)); + + transaction.emplace_back(std::move(publish)); + } + } + if (transaction.size() > 1) { transaction.push_back({"EXEC"}); m_Rcon->FireAndForgetQueries(std::move(transaction)); @@ -313,18 +340,20 @@ static ConfigObject::Ptr GetObjectByName(const String& name) return ConfigObject::GetObject(name); } -void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map >& statements) +void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map>& hMSets, + std::map>& publishes, bool runtimeUpdate) { String objectKey = GetObjectIdentifier(object); CustomVarObject::Ptr customVarObject = dynamic_pointer_cast(object); String envId = CalculateCheckSumString(GetEnvironment()); + auto* configUpdates (runtimeUpdate ? &publishes["icinga:config:update"] : nullptr); if (customVarObject) { auto vars(SerializeVars(customVarObject)); if (vars) { - auto& typeCvs (statements[m_PrefixConfigObject + typeName + ":customvar"]); - auto& allCvs (statements[m_PrefixConfigObject + "customvar"]); - auto& cvChksms (statements[m_PrefixConfigCheckSum + typeName + ":customvar"]); + auto& typeCvs (hMSets[m_PrefixConfigObject + typeName + ":customvar"]); + auto& allCvs (hMSets[m_PrefixConfigObject + "customvar"]); + auto& cvChksms (hMSets[m_PrefixConfigCheckSum + typeName + ":customvar"]); cvChksms.emplace_back(objectKey); cvChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", CalculateCheckSumVars(customVarObject)}}))); @@ -337,9 +366,18 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons for (auto& kv : vars) { allCvs.emplace_back(kv.first); allCvs.emplace_back(JsonEncode(kv.second)); + + if (configUpdates) { + configUpdates->emplace_back("customvar:" + kv.first); + } + String id = CalculateCheckSumArray(new Array({envId, kv.first, objectKey})); typeCvs.emplace_back(id); typeCvs.emplace_back(JsonEncode(new Dictionary({{"object_id", objectKey}, {"env_id", envId}, {"customvar_id", kv.first}}))); + + if (configUpdates) { + configUpdates->emplace_back(typeName + ":customvar:" + id); + } } } } @@ -352,19 +390,31 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons String notesUrl = checkable->GetNotesUrl(); String iconImage = checkable->GetIconImage(); if (!actionUrl.IsEmpty()) { - auto& actionUrls (statements[m_PrefixConfigObject + "action_url"]); + auto& actionUrls (hMSets[m_PrefixConfigObject + "action_url"]); actionUrls.emplace_back(CalculateCheckSumArray(new Array({envId, actionUrl}))); actionUrls.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"action_url", actionUrl}}))); + + if (configUpdates) { + configUpdates->emplace_back("action_url:" + actionUrls.at(actionUrls.size() - 2u)); + } } if (!notesUrl.IsEmpty()) { - auto& notesUrls (statements[m_PrefixConfigObject + "notes_url"]); + auto& notesUrls (hMSets[m_PrefixConfigObject + "notes_url"]); notesUrls.emplace_back(CalculateCheckSumArray(new Array({envId, notesUrl}))); notesUrls.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"notes_url", notesUrl}}))); + + if (configUpdates) { + configUpdates->emplace_back("notes_url:" + notesUrls.at(notesUrls.size() - 2u)); + } } if (!iconImage.IsEmpty()) { - auto& iconImages (statements[m_PrefixConfigObject + "icon_image"]); + auto& iconImages (hMSets[m_PrefixConfigObject + "icon_image"]); iconImages.emplace_back(CalculateCheckSumArray(new Array({envId, iconImage}))); iconImages.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"icon_image", iconImage}}))); + + if (configUpdates) { + configUpdates->emplace_back("icon_image:" + iconImages.at(iconImages.size() - 2u)); + } } Host::Ptr host; @@ -387,8 +437,8 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons groupIds->Reserve(groups->GetLength()); - auto& members (statements[m_PrefixConfigObject + typeName + ":groupmember"]); - auto& memberChksms (statements[m_PrefixConfigCheckSum + typeName + ":groupmember"]); + auto& members (hMSets[m_PrefixConfigObject + typeName + ":groupmember"]); + auto& memberChksms (hMSets[m_PrefixConfigCheckSum + typeName + ":groupmember"]); for (auto& group : groups) { String groupId = GetObjectIdentifier((*getGroup)(group)); @@ -396,6 +446,10 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons members.emplace_back(id); members.emplace_back(JsonEncode(new Dictionary({{"object_id", objectKey}, {"env_id", envId}, {"group_id", groupId}}))); + if (configUpdates) { + configUpdates->emplace_back(typeName + ":groupmember:" + id); + } + groupIds->Add(groupId); } @@ -413,8 +467,8 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons if (ranges) { ObjectLock rangesLock(ranges); Array::Ptr rangeIds(new Array); - auto& typeRanges (statements[m_PrefixConfigObject + typeName + ":range"]); - auto& rangeChksms (statements[m_PrefixConfigCheckSum + typeName + ":range"]); + auto& typeRanges (hMSets[m_PrefixConfigObject + typeName + ":range"]); + auto& rangeChksms (hMSets[m_PrefixConfigCheckSum + typeName + ":range"]); rangeIds->Reserve(ranges->GetLength()); @@ -425,6 +479,10 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons String id = CalculateCheckSumArray(new Array({envId, rangeId, objectKey})); typeRanges.emplace_back(id); typeRanges.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"timeperiod_id", objectKey}, {"range_key", kv.first}, {"range_value", kv.second}}))); + + if (configUpdates) { + configUpdates->emplace_back(typeName + ":range:" + id); + } } rangeChksms.emplace_back(objectKey); @@ -444,8 +502,8 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons includeChecksums->Reserve(includes->GetLength()); - auto& includs (statements[m_PrefixConfigObject + typeName + ":override:include"]); - auto& includeChksms (statements[m_PrefixConfigCheckSum + typeName + ":override:include"]); + auto& includs (hMSets[m_PrefixConfigObject + typeName + ":override:include"]); + auto& includeChksms (hMSets[m_PrefixConfigCheckSum + typeName + ":override:include"]); for (auto include : includes) { String includeId = GetObjectIdentifier((*getInclude)(include.Get())); includeChecksums->Add(includeId); @@ -453,6 +511,10 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons String id = CalculateCheckSumArray(new Array({envId, includeId, objectKey})); includs.emplace_back(id); includs.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"timeperiod_id", objectKey}, {"include_id", includeId}}))); + + if (configUpdates) { + configUpdates->emplace_back(typeName + ":override:include:" + id); + } } includeChksms.emplace_back(objectKey); @@ -471,8 +533,8 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons excludeChecksums->Reserve(excludes->GetLength()); - auto& excluds (statements[m_PrefixConfigObject + typeName + ":override:exclude"]); - auto& excludeChksms (statements[m_PrefixConfigCheckSum + typeName + ":override:exclude"]); + auto& excluds (hMSets[m_PrefixConfigObject + typeName + ":override:exclude"]); + auto& excludeChksms (hMSets[m_PrefixConfigCheckSum + typeName + ":override:exclude"]); for (auto exclude : excludes) { String excludeId = GetObjectIdentifier((*getExclude)(exclude.Get())); @@ -481,6 +543,10 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons String id = CalculateCheckSumArray(new Array({envId, excludeId, objectKey})); excluds.emplace_back(id); excluds.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"timeperiod_id", objectKey}, {"exclude_id", excludeId}}))); + + if (configUpdates) { + configUpdates->emplace_back(typeName + ":override:exclude:" + id); + } } excludeChksms.emplace_back(objectKey); @@ -497,14 +563,19 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons parents->Reserve(parentsRaw.size()); - auto& parnts (statements[m_PrefixConfigObject + typeName + ":parent"]); - auto& parentChksms (statements[m_PrefixConfigCheckSum + typeName + ":parent"]); + auto& parnts (hMSets[m_PrefixConfigObject + typeName + ":parent"]); + auto& parentChksms (hMSets[m_PrefixConfigCheckSum + typeName + ":parent"]); for (auto& parent : parentsRaw) { String parentId = GetObjectIdentifier(parent); String id = CalculateCheckSumArray(new Array({envId, parentId, objectKey})); parnts.emplace_back(id); parnts.emplace_back(JsonEncode(new Dictionary({{"zone_id", objectKey}, {"env_id", envId}, {"parent_id", parentId}}))); + + if (configUpdates) { + configUpdates->emplace_back(typeName + ":parent:" + id); + } + parents->Add(GetObjectIdentifier(parent)); } @@ -529,14 +600,19 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons groupIds->Reserve(groups->GetLength()); - auto& members (statements[m_PrefixConfigObject + typeName + ":groupmember"]); - auto& memberChksms (statements[m_PrefixConfigCheckSum + typeName + ":groupmember"]); + auto& members (hMSets[m_PrefixConfigObject + typeName + ":groupmember"]); + auto& memberChksms (hMSets[m_PrefixConfigCheckSum + typeName + ":groupmember"]); for (auto& group : groups) { String groupId = GetObjectIdentifier((*getGroup)(group)); String id = CalculateCheckSumArray(new Array({envId, groupId, objectKey})); members.emplace_back(id); members.emplace_back(JsonEncode(new Dictionary({{"user_id", objectKey}, {"env_id", envId}, {"group_id", groupId}}))); + + if (configUpdates) { + configUpdates->emplace_back(typeName + ":groupmember:" + id); + } + groupIds->Add(groupId); } @@ -558,14 +634,19 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons userIds->Reserve(users.size()); - auto& usrs (statements[m_PrefixConfigObject + typeName + ":user"]); - auto& userChksms (statements[m_PrefixConfigCheckSum + typeName + ":user"]); + auto& usrs (hMSets[m_PrefixConfigObject + typeName + ":user"]); + auto& userChksms (hMSets[m_PrefixConfigCheckSum + typeName + ":user"]); for (auto& user : users) { String userId = GetObjectIdentifier(user); String id = CalculateCheckSumArray(new Array({envId, userId, objectKey})); usrs.emplace_back(id); usrs.emplace_back(JsonEncode(new Dictionary({{"notification_id", objectKey}, {"env_id", envId}, {"user_id", userId}}))); + + if (configUpdates) { + configUpdates->emplace_back(typeName + ":user:" + id); + } + userIds->Add(userId); } @@ -574,14 +655,19 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons usergroupIds->Reserve(usergroups.size()); - auto& groups (statements[m_PrefixConfigObject + typeName + ":usergroup"]); - auto& groupChksms (statements[m_PrefixConfigCheckSum + typeName + ":usergroup"]); + auto& groups (hMSets[m_PrefixConfigObject + typeName + ":usergroup"]); + auto& groupChksms (hMSets[m_PrefixConfigCheckSum + typeName + ":usergroup"]); for (auto& usergroup : usergroups) { String usergroupId = GetObjectIdentifier(usergroup); String id = CalculateCheckSumArray(new Array({envId, usergroupId, objectKey})); groups.emplace_back(id); groups.emplace_back(JsonEncode(new Dictionary({{"notification_id", objectKey}, {"env_id", envId}, {"usergroup_id", usergroupId}}))); + + if (configUpdates) { + configUpdates->emplace_back(typeName + ":usergroup:" + id); + } + usergroupIds->Add(usergroupId); } @@ -597,8 +683,8 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons Dictionary::Ptr arguments = command->GetArguments(); if (arguments) { ObjectLock argumentsLock(arguments); - auto& typeArgs (statements[m_PrefixConfigObject + typeName + ":argument"]); - auto& argChksms (statements[m_PrefixConfigCheckSum + typeName + ":argument"]); + auto& typeArgs (hMSets[m_PrefixConfigObject + typeName + ":argument"]); + auto& argChksms (hMSets[m_PrefixConfigCheckSum + typeName + ":argument"]); for (auto& kv : arguments) { Dictionary::Ptr values; @@ -620,6 +706,11 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons typeArgs.emplace_back(id); typeArgs.emplace_back(JsonEncode(values)); + + if (configUpdates) { + configUpdates->emplace_back(typeName + ":argument:" + id); + } + argChksms.emplace_back(id); argChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", HashValue(kv.second)}}))); } @@ -629,8 +720,8 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons if (envvars) { ObjectLock envvarsLock(envvars); Array::Ptr envvarIds(new Array); - auto& typeVars (statements[m_PrefixConfigObject + typeName + ":envvar"]); - auto& varChksms (statements[m_PrefixConfigCheckSum + typeName + ":envvar"]); + auto& typeVars (hMSets[m_PrefixConfigObject + typeName + ":envvar"]); + auto& varChksms (hMSets[m_PrefixConfigCheckSum + typeName + ":envvar"]); envvarIds->Reserve(envvars->GetLength()); @@ -654,6 +745,11 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons typeVars.emplace_back(id); typeVars.emplace_back(JsonEncode(values)); + + if (configUpdates) { + configUpdates->emplace_back(typeName + ":envvar:" + id); + } + varChksms.emplace_back(id); varChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", HashValue(kv.second)}}))); } @@ -678,10 +774,10 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtime String typeName = GetLowerCaseTypeNameDB(object); - std::map> statements; + std::map> hMSets, publishes; std::vector states = {"HMSET", m_PrefixStateObject + typeName}; - CreateConfigUpdate(object, typeName, statements, runtimeUpdate); + CreateConfigUpdate(object, typeName, hMSets, publishes, runtimeUpdate); Checkable::Ptr checkable = dynamic_pointer_cast(object); if (checkable) { m_Rcon->FireAndForgetQuery({"HSET", m_PrefixStateObject + typeName, @@ -690,13 +786,26 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtime std::vector > transaction = {{"MULTI"}}; - for (auto& kv : statements) { + for (auto& kv : hMSets) { if (!kv.second.empty()) { kv.second.insert(kv.second.begin(), {"HMSET", kv.first}); transaction.emplace_back(std::move(kv.second)); } } + for (auto& kv : publishes) { + for (auto& message : kv.second) { + std::vector publish; + + publish.reserve(3); + publish.emplace_back("PUBLISH"); + publish.emplace_back(kv.first); + publish.emplace_back(std::move(message)); + + transaction.emplace_back(std::move(publish)); + } + } + if (transaction.size() > 1) { transaction.push_back({"EXEC"}); m_Rcon->FireAndForgetQueries(std::move(transaction)); @@ -952,8 +1061,8 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr * icinga:config:object:downtime) need to be prepended. There is nothing to indicate success or failure. */ void -RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, const String typeName, std::map >& statements, - bool runtimeUpdate) +RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, const String typeName, std::map>& hMSets, + std::map>& publishes, bool runtimeUpdate) { /* TODO: This isn't essentially correct as we don't keep track of config objects ourselves. This would avoid duplicated config updates at startup. if (!runtimeUpdate && m_ConfigDumpInProgress) @@ -969,11 +1078,11 @@ RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, const String ty if (!PrepareObject(object, attr, chksm)) return; - InsertObjectDependencies(object, typeName, statements); + InsertObjectDependencies(object, typeName, hMSets, publishes, runtimeUpdate); String objectKey = GetObjectIdentifier(object); - auto& attrs (statements[m_PrefixConfigObject + typeName]); - auto& chksms (statements[m_PrefixConfigCheckSum + typeName]); + auto& attrs (hMSets[m_PrefixConfigObject + typeName]); + auto& chksms (hMSets[m_PrefixConfigCheckSum + typeName]); attrs.emplace_back(objectKey); attrs.emplace_back(JsonEncode(attr)); @@ -983,7 +1092,7 @@ RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, const String ty /* Send an update event to subscribers. */ if (runtimeUpdate) { - m_Rcon->FireAndForgetQuery({"PUBLISH", "icinga:config:update", typeName + ":" + objectKey}); + publishes["icinga:config:update"].emplace_back(typeName + ":" + objectKey); } } diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 240af965d..7de8635df 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -73,11 +73,12 @@ private: std::vector>> ChunkObjects(std::vector> objects, size_t chunkSize); void DeleteKeys(const std::vector& keys); std::vector GetTypeObjectKeys(const String& type); - void InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map >& statements); + void InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map>& hMSets, + std::map>& publishes, bool runtimeUpdate); void UpdateState(const Checkable::Ptr& checkable); void SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate); - void CreateConfigUpdate(const ConfigObject::Ptr& object, const String type, std::map >& statements, - bool runtimeUpdate); + void CreateConfigUpdate(const ConfigObject::Ptr& object, const String type, std::map>& hMSets, + std::map>& publishes, bool runtimeUpdate); void SendConfigDelete(const ConfigObject::Ptr& object); void SendStatusUpdate(const ConfigObject::Ptr& object); std::vector UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride); From 07515d444615ee485e87118fb5fbbae431be6b7c Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Thu, 19 Sep 2019 07:36:27 +0200 Subject: [PATCH 168/219] Fix RedisWriter not clearing "icinga:config:state:*" keys on initial dump --- lib/redis/rediswriter-objects.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index e1d8191ea..ed86a2896 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -309,6 +309,7 @@ std::vector RedisWriter::GetTypeObjectKeys(const String& type) if (type == "host" || type == "service" || type == "user") { keys.emplace_back(m_PrefixConfigObject + type + ":groupmember"); keys.emplace_back(m_PrefixConfigCheckSum + type + ":groupmember"); + keys.emplace_back(m_PrefixStateObject + type); } else if (type == "timeperiod") { keys.emplace_back(m_PrefixConfigObject + type + ":override:include"); keys.emplace_back(m_PrefixConfigCheckSum + type + ":override:include"); From 1566c4f9acb641176a92e91c55f35a697be9069a Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Thu, 19 Sep 2019 08:06:56 +0200 Subject: [PATCH 169/219] Fix runtime update of state not being published --- lib/redis/rediswriter-objects.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index ed86a2896..4f44ef3b4 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -781,8 +781,9 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtime CreateConfigUpdate(object, typeName, hMSets, publishes, runtimeUpdate); Checkable::Ptr checkable = dynamic_pointer_cast(object); if (checkable) { - m_Rcon->FireAndForgetQuery({"HSET", m_PrefixStateObject + typeName, - GetObjectIdentifier(checkable), JsonEncode(SerializeState(checkable))}); + String objectKey = GetObjectIdentifier(object); + m_Rcon->FireAndForgetQuery({"HSET", m_PrefixStateObject + typeName, objectKey, JsonEncode(SerializeState(checkable))}); + publishes["icinga:config:update"].emplace_back("state:" + typeName + ":" + objectKey); } std::vector > transaction = {{"MULTI"}}; From 9f077e82164e6bfcd6b19231e710cb91a28a4b1a Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 25 Sep 2019 11:03:33 +0200 Subject: [PATCH 170/219] RedisWriter: Streamline JSON key names --- lib/redis/rediswriter-objects.cpp | 38 +++++++++++++++---------------- lib/redis/rediswriter-utility.cpp | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 4f44ef3b4..c0730f159 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -374,7 +374,7 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons String id = CalculateCheckSumArray(new Array({envId, kv.first, objectKey})); typeCvs.emplace_back(id); - typeCvs.emplace_back(JsonEncode(new Dictionary({{"object_id", objectKey}, {"env_id", envId}, {"customvar_id", kv.first}}))); + typeCvs.emplace_back(JsonEncode(new Dictionary({{"object_id", objectKey}, {"environment_id", envId}, {"customvar_id", kv.first}}))); if (configUpdates) { configUpdates->emplace_back(typeName + ":customvar:" + id); @@ -393,7 +393,7 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons if (!actionUrl.IsEmpty()) { auto& actionUrls (hMSets[m_PrefixConfigObject + "action_url"]); actionUrls.emplace_back(CalculateCheckSumArray(new Array({envId, actionUrl}))); - actionUrls.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"action_url", actionUrl}}))); + actionUrls.emplace_back(JsonEncode(new Dictionary({{"environment_id", envId}, {"action_url", actionUrl}}))); if (configUpdates) { configUpdates->emplace_back("action_url:" + actionUrls.at(actionUrls.size() - 2u)); @@ -402,7 +402,7 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons if (!notesUrl.IsEmpty()) { auto& notesUrls (hMSets[m_PrefixConfigObject + "notes_url"]); notesUrls.emplace_back(CalculateCheckSumArray(new Array({envId, notesUrl}))); - notesUrls.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"notes_url", notesUrl}}))); + notesUrls.emplace_back(JsonEncode(new Dictionary({{"environment_id", envId}, {"notes_url", notesUrl}}))); if (configUpdates) { configUpdates->emplace_back("notes_url:" + notesUrls.at(notesUrls.size() - 2u)); @@ -411,7 +411,7 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons if (!iconImage.IsEmpty()) { auto& iconImages (hMSets[m_PrefixConfigObject + "icon_image"]); iconImages.emplace_back(CalculateCheckSumArray(new Array({envId, iconImage}))); - iconImages.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"icon_image", iconImage}}))); + iconImages.emplace_back(JsonEncode(new Dictionary({{"environment_id", envId}, {"icon_image", iconImage}}))); if (configUpdates) { configUpdates->emplace_back("icon_image:" + iconImages.at(iconImages.size() - 2u)); @@ -445,7 +445,7 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons String groupId = GetObjectIdentifier((*getGroup)(group)); String id = CalculateCheckSumArray(new Array({envId, groupId, objectKey})); members.emplace_back(id); - members.emplace_back(JsonEncode(new Dictionary({{"object_id", objectKey}, {"env_id", envId}, {"group_id", groupId}}))); + members.emplace_back(JsonEncode(new Dictionary({{"object_id", objectKey}, {"environment_id", envId}, {"group_id", groupId}}))); if (configUpdates) { configUpdates->emplace_back(typeName + ":groupmember:" + id); @@ -479,7 +479,7 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons String id = CalculateCheckSumArray(new Array({envId, rangeId, objectKey})); typeRanges.emplace_back(id); - typeRanges.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"timeperiod_id", objectKey}, {"range_key", kv.first}, {"range_value", kv.second}}))); + typeRanges.emplace_back(JsonEncode(new Dictionary({{"environment_id", envId}, {"timeperiod_id", objectKey}, {"range_key", kv.first}, {"range_value", kv.second}}))); if (configUpdates) { configUpdates->emplace_back(typeName + ":range:" + id); @@ -511,7 +511,7 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons String id = CalculateCheckSumArray(new Array({envId, includeId, objectKey})); includs.emplace_back(id); - includs.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"timeperiod_id", objectKey}, {"include_id", includeId}}))); + includs.emplace_back(JsonEncode(new Dictionary({{"environment_id", envId}, {"timeperiod_id", objectKey}, {"include_id", includeId}}))); if (configUpdates) { configUpdates->emplace_back(typeName + ":override:include:" + id); @@ -543,7 +543,7 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons String id = CalculateCheckSumArray(new Array({envId, excludeId, objectKey})); excluds.emplace_back(id); - excluds.emplace_back(JsonEncode(new Dictionary({{"env_id", envId}, {"timeperiod_id", objectKey}, {"exclude_id", excludeId}}))); + excluds.emplace_back(JsonEncode(new Dictionary({{"environment_id", envId}, {"timeperiod_id", objectKey}, {"exclude_id", excludeId}}))); if (configUpdates) { configUpdates->emplace_back(typeName + ":override:exclude:" + id); @@ -571,7 +571,7 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons String parentId = GetObjectIdentifier(parent); String id = CalculateCheckSumArray(new Array({envId, parentId, objectKey})); parnts.emplace_back(id); - parnts.emplace_back(JsonEncode(new Dictionary({{"zone_id", objectKey}, {"env_id", envId}, {"parent_id", parentId}}))); + parnts.emplace_back(JsonEncode(new Dictionary({{"zone_id", objectKey}, {"environment_id", envId}, {"parent_id", parentId}}))); if (configUpdates) { configUpdates->emplace_back(typeName + ":parent:" + id); @@ -608,7 +608,7 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons String groupId = GetObjectIdentifier((*getGroup)(group)); String id = CalculateCheckSumArray(new Array({envId, groupId, objectKey})); members.emplace_back(id); - members.emplace_back(JsonEncode(new Dictionary({{"user_id", objectKey}, {"env_id", envId}, {"group_id", groupId}}))); + members.emplace_back(JsonEncode(new Dictionary({{"user_id", objectKey}, {"environment_id", envId}, {"group_id", groupId}}))); if (configUpdates) { configUpdates->emplace_back(typeName + ":groupmember:" + id); @@ -642,7 +642,7 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons String userId = GetObjectIdentifier(user); String id = CalculateCheckSumArray(new Array({envId, userId, objectKey})); usrs.emplace_back(id); - usrs.emplace_back(JsonEncode(new Dictionary({{"notification_id", objectKey}, {"env_id", envId}, {"user_id", userId}}))); + usrs.emplace_back(JsonEncode(new Dictionary({{"notification_id", objectKey}, {"environment_id", envId}, {"user_id", userId}}))); if (configUpdates) { configUpdates->emplace_back(typeName + ":user:" + id); @@ -663,7 +663,7 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons String usergroupId = GetObjectIdentifier(usergroup); String id = CalculateCheckSumArray(new Array({envId, usergroupId, objectKey})); groups.emplace_back(id); - groups.emplace_back(JsonEncode(new Dictionary({{"notification_id", objectKey}, {"env_id", envId}, {"usergroup_id", usergroupId}}))); + groups.emplace_back(JsonEncode(new Dictionary({{"notification_id", objectKey}, {"environment_id", envId}, {"usergroup_id", usergroupId}}))); if (configUpdates) { configUpdates->emplace_back(typeName + ":usergroup:" + id); @@ -701,7 +701,7 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons values->Set("value", JsonEncode(values->Get("value"))); values->Set("command_id", objectKey); values->Set("argument_key", kv.first); - values->Set("env_id", envId); + values->Set("environment_id", envId); String id = HashValue(objectKey + kv.first + envId); @@ -740,7 +740,7 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons values->Set("value", JsonEncode(values->Get("value"))); values->Set("command_id", objectKey); values->Set("envvar_key", kv.first); - values->Set("env_id", envId); + values->Set("environment_id", envId); String id = HashValue(objectKey + kv.first + envId); @@ -819,7 +819,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtime bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& attributes, Dictionary::Ptr& checksums) { attributes->Set("name_checksum", CalculateCheckSumString(object->GetName())); - attributes->Set("env_id", CalculateCheckSumString(GetEnvironment())); + attributes->Set("environment_id", CalculateCheckSumString(GetEnvironment())); attributes->Set("name", object->GetName()); Zone::Ptr ObjectsZone = static_pointer_cast(object->GetZone()); @@ -880,7 +880,7 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr TimePeriod::Ptr timePeriod = checkable->GetCheckPeriod(); if (timePeriod) { - attributes->Set("check_period_id", GetObjectIdentifier(timePeriod)); + attributes->Set("check_timeperiod_id", GetObjectIdentifier(timePeriod)); attributes->Set("check_period", timePeriod->GetName()); } @@ -931,7 +931,7 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr attributes->Set("types", user->GetTypes()); if (user->GetPeriod()) - attributes->Set("period_id", GetObjectIdentifier(user->GetPeriod())); + attributes->Set("timeperiod_id", GetObjectIdentifier(user->GetPeriod())); return true; } @@ -960,7 +960,7 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr TimePeriod::Ptr timeperiod = notification->GetPeriod(); if (timeperiod) - attributes->Set("period_id", GetObjectIdentifier(timeperiod)); + attributes->Set("timeperiod_id", GetObjectIdentifier(timeperiod)); if (notification->GetTimes()) { attributes->Set("times_begin", notification->GetTimes()->Get("begin")); @@ -1152,7 +1152,7 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) tie(host, service) = GetHostService(checkable); attrs->Set("id", GetObjectIdentifier(checkable));; - attrs->Set("env_id", CalculateCheckSumString(GetEnvironment())); + attrs->Set("environment_id", CalculateCheckSumString(GetEnvironment())); attrs->Set("state_type", checkable->GetStateType()); // TODO: last_hard/soft_state should be "previous". diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index cb65f1d2e..e6171ef4d 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -186,7 +186,7 @@ Dictionary::Ptr RedisWriter::SerializeVars(const CustomVarObject::Ptr& object) res->Set( SHA1(PackObject((Array::Ptr)new Array({env, kv.first, kv.second}))), (Dictionary::Ptr)new Dictionary({ - {"env_id", envChecksum}, + {"environment_id", envChecksum}, {"name_checksum", SHA1(kv.first)}, {"name", kv.first}, {"value", JsonEncode(kv.second)}, From 9a3ab8c547f472c27baaa0fdededb6cb507795c9 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 25 Sep 2019 12:06:50 +0200 Subject: [PATCH 171/219] Adjust IoEngine usage --- lib/redis/redisconnection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index e855cea71..a3248c8c0 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -40,7 +40,7 @@ using namespace icinga; namespace asio = boost::asio; RedisConnection::RedisConnection(const String host, const int port, const String path, const String password, const int db) : - RedisConnection(IoEngine::Get().GetIoService(), host, port, path, password, db) + RedisConnection(IoEngine::Get().GetIoContext(), host, port, path, password, db) { } From 21a900cd9dd7fdfea71f9ab01ca1a3e0a04631df Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 25 Sep 2019 12:00:34 +0200 Subject: [PATCH 172/219] RedisWriter#SerializeState(): serialize also Checkable#next_update refs #54 --- lib/redis/rediswriter-objects.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index c0730f159..049d1eb37 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -1232,6 +1232,7 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) attrs->Set("last_update", Utility::GetTime()); attrs->Set("last_state_change", checkable->GetLastStateChange()); attrs->Set("next_check", checkable->GetNextCheck()); + attrs->Set("next_update", checkable->GetNextUpdate()); return attrs; } From 33bdc8b5c3493486ad58a55ba7ed8d3f464430f9 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Mon, 30 Sep 2019 14:32:53 +0200 Subject: [PATCH 173/219] RedisWriter: Rename check_period to check_timeperiod --- lib/redis/rediswriter-objects.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 049d1eb37..632fe4a5f 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -881,7 +881,7 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr TimePeriod::Ptr timePeriod = checkable->GetCheckPeriod(); if (timePeriod) { attributes->Set("check_timeperiod_id", GetObjectIdentifier(timePeriod)); - attributes->Set("check_period", timePeriod->GetName()); + attributes->Set("check_timeperiod", timePeriod->GetName()); } EventCommand::Ptr eventCommand = checkable->GetEventCommand(); From d0165773d274df87308d3b18a6864dff6a9ed077 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 2 Oct 2019 12:01:51 +0200 Subject: [PATCH 174/219] RedisWriter: populate icinga:history:stream:*:notification --- lib/redis/rediswriter-objects.cpp | 63 +++++++++++++++++++++++++++++++ lib/redis/rediswriter.hpp | 11 ++++++ 2 files changed, 74 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 632fe4a5f..0bd32ab58 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -46,6 +46,7 @@ #include "base/utility.hpp" #include #include +#include #include #include @@ -92,6 +93,14 @@ void RedisWriter::ConfigStaticInitialize() Downtime::OnDowntimeTriggered.connect(&RedisWriter::DowntimeChangedHandler); /* fixed/flexible downtime end */ Downtime::OnDowntimeRemoved.connect(&RedisWriter::DowntimeChangedHandler); + + Checkable::OnNotificationSentToAllUsers.connect([]( + const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set& users, + const NotificationType& type, const CheckResult::Ptr& cr, const String& author, const String& text, + const MessageOrigin::Ptr& + ) { + RedisWriter::NotificationSentToAllUsersHandler(notification, checkable, users, type, cr, author, text); + }); } void RedisWriter::UpdateAllConfigObjects() @@ -1142,6 +1151,41 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) m_Rcon->FireAndForgetQuery(std::move(streamadd)); } +void RedisWriter::SendSentNotification( + const Notification::Ptr& notification, const Checkable::Ptr& checkable, size_t users, + NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text +) +{ + if (!m_Rcon || !m_Rcon->IsConnected()) + return; + + auto service (dynamic_pointer_cast(checkable)); + auto output (cr->GetOutput()); + auto pos (output.Find("\n")); + String shortOutput, longOutput; + + if (pos == String::NPos) { + shortOutput = std::move(output); + } else { + shortOutput = output.SubStr(0, pos); + longOutput = output.SubStr(pos + 1u); + } + + m_Rcon->FireAndForgetQuery({ + "XADD", service ? "icinga:history:stream:service:notification" : "icinga:history:stream:host:notification", "*", + "id", Utility::NewUniqueID(), + "environment_id", SHA1(GetEnvironment()), + service ? "service_id" : "host_id", GetObjectIdentifier(checkable), + "notification_id", GetObjectIdentifier(notification), + "type", Convert::ToString(type), + "send_time", Convert::ToString(Utility::GetTime()), + "state", Convert::ToString(cr->GetState()), + "output", Utility::ValidateUTF8(std::move(shortOutput)), + "long_output", Utility::ValidateUTF8(std::move(longOutput)), + "users_notified", Convert::ToString(users) + }); +} + Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) { Dictionary::Ptr attrs = new Dictionary(); @@ -1313,3 +1357,22 @@ void RedisWriter::DowntimeChangedHandler(const Downtime::Ptr& downtime) { StateChangeHandler(downtime->GetCheckable()); } + +void RedisWriter::NotificationSentToAllUsersHandler( + const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set& users, + NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text +) +{ + auto rws (ConfigType::GetObjectsByType()); + + if (!rws.empty()) { + auto usersAmount (users.size()); + auto authorAndText (std::make_shared>(author, text)); + + for (auto& rw : rws) { + rw->m_WorkQueue.Enqueue([rw, notification, checkable, usersAmount, type, cr, authorAndText]() { + rw->SendSentNotification(notification, checkable, usersAmount, type, cr, authorAndText->first, authorAndText->second); + }); + } + } +} diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 7de8635df..a8208612b 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -81,6 +81,12 @@ private: std::map>& publishes, bool runtimeUpdate); void SendConfigDelete(const ConfigObject::Ptr& object); void SendStatusUpdate(const ConfigObject::Ptr& object); + + void SendSentNotification( + const Notification::Ptr& notification, const Checkable::Ptr& checkable, size_t users, + NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text + ); + std::vector UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride); Dictionary::Ptr SerializeState(const Checkable::Ptr& checkable); @@ -110,6 +116,11 @@ private: static void VersionChangedHandler(const ConfigObject::Ptr& object); static void DowntimeChangedHandler(const Downtime::Ptr& downtime); + static void NotificationSentToAllUsersHandler( + const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set& users, + NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text + ); + void AssertOnWorkQueue(); void ExceptionHandler(boost::exception_ptr exp); From feeae9d518087708c424ea99b11fc8a61d8bb2db Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 2 Oct 2019 13:14:31 +0200 Subject: [PATCH 175/219] RedisWriter: populate icinga:history:stream:*:state --- lib/redis/rediswriter-objects.cpp | 67 ++++++++++++++++++++++--------- lib/redis/rediswriter.hpp | 3 +- 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 0bd32ab58..b3d48767d 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -70,8 +70,8 @@ INITIALIZE_ONCE(&RedisWriter::ConfigStaticInitialize); void RedisWriter::ConfigStaticInitialize() { /* triggered in ProcessCheckResult(), requires UpdateNextCheck() to be called before */ - Checkable::OnStateChange.connect([](const Checkable::Ptr& checkable, const CheckResult::Ptr&, StateType, const MessageOrigin::Ptr&) { - RedisWriter::StateChangeHandler(checkable); + Checkable::OnStateChange.connect([](const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, StateType type, const MessageOrigin::Ptr&) { + RedisWriter::StateChangeHandler(checkable, cr, type); }); /* triggered when acknowledged host/service goes back to ok and when the acknowledgement gets deleted */ @@ -103,6 +103,19 @@ void RedisWriter::ConfigStaticInitialize() }); } +static std::pair SplitOutput(String output) +{ + String longOutput; + auto pos (output.Find("\n")); + + if (pos != String::NPos) { + longOutput = output.SubStr(pos + 1u); + output.erase(output.Begin() + pos, output.End()); + } + + return {std::move(output), std::move(longOutput)}; +} + void RedisWriter::UpdateAllConfigObjects() { double startTime = Utility::GetTime(); @@ -1119,7 +1132,7 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) }); } -void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) +void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type) { if (!m_Rcon || !m_Rcon->IsConnected()) return; @@ -1149,6 +1162,26 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object) } m_Rcon->FireAndForgetQuery(std::move(streamadd)); + + auto output (SplitOutput(cr->GetOutput())); + + m_Rcon->FireAndForgetQuery({ + "XADD", service ? "icinga:history:stream:service:state" : "icinga:history:stream:host:state", "*", + "id", Utility::NewUniqueID(), + "environment_id", SHA1(GetEnvironment()), + service ? "service_id" : "host_id", GetObjectIdentifier(checkable), + "change_time", Convert::ToString(cr->GetExecutionEnd()), + "state_type", Convert::ToString(type), + "soft_state", Convert::ToString(cr->GetState()), + "hard_state", Convert::ToString(service ? service->GetLastHardState() : host->GetLastHardState()), + "attempt", Convert::ToString(checkable->GetCheckAttempt()), + // TODO: last_hard/soft_state should be "previous". + "last_soft_state", Convert::ToString(cr->GetState()), + "last_hard_state", Convert::ToString(service ? service->GetLastHardState() : host->GetLastHardState()), + "output", Utility::ValidateUTF8(std::move(output.first)), + "long_output", Utility::ValidateUTF8(std::move(output.second)), + "max_check_attempts", Convert::ToString(checkable->GetMaxCheckAttempts()) + }); } void RedisWriter::SendSentNotification( @@ -1160,16 +1193,7 @@ void RedisWriter::SendSentNotification( return; auto service (dynamic_pointer_cast(checkable)); - auto output (cr->GetOutput()); - auto pos (output.Find("\n")); - String shortOutput, longOutput; - - if (pos == String::NPos) { - shortOutput = std::move(output); - } else { - shortOutput = output.SubStr(0, pos); - longOutput = output.SubStr(pos + 1u); - } + auto output (SplitOutput(cr->GetOutput())); m_Rcon->FireAndForgetQuery({ "XADD", service ? "icinga:history:stream:service:notification" : "icinga:history:stream:host:notification", "*", @@ -1180,8 +1204,8 @@ void RedisWriter::SendSentNotification( "type", Convert::ToString(type), "send_time", Convert::ToString(Utility::GetTime()), "state", Convert::ToString(cr->GetState()), - "output", Utility::ValidateUTF8(std::move(shortOutput)), - "long_output", Utility::ValidateUTF8(std::move(longOutput)), + "output", Utility::ValidateUTF8(std::move(output.first)), + "long_output", Utility::ValidateUTF8(std::move(output.second)), "users_notified", Convert::ToString(users) }); } @@ -1324,12 +1348,19 @@ RedisWriter::UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, //m_Rcon->FireAndForgetQuery({"HSET", keyPrefix + typeName, GetObjectIdentifier(object), JsonEncode(attrs)}); } -void RedisWriter::StateChangeHandler(const ConfigObject::Ptr &object) +void RedisWriter::StateChangeHandler(const ConfigObject::Ptr& object) { - Type::Ptr type = object->GetReflectionType(); + auto checkable (dynamic_pointer_cast(object)); + if (checkable) { + RedisWriter::StateChangeHandler(object, checkable->GetLastCheckResult(), checkable->GetStateType()); + } +} + +void RedisWriter::StateChangeHandler(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type) +{ for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { - rw->m_WorkQueue.Enqueue([rw, object]() { rw->SendStatusUpdate(object); }); + rw->m_WorkQueue.Enqueue([rw, object, cr, type]() { rw->SendStatusUpdate(object, cr, type); }); } } diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index a8208612b..3cab086e6 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -80,7 +80,7 @@ private: void CreateConfigUpdate(const ConfigObject::Ptr& object, const String type, std::map>& hMSets, std::map>& publishes, bool runtimeUpdate); void SendConfigDelete(const ConfigObject::Ptr& object); - void SendStatusUpdate(const ConfigObject::Ptr& object); + void SendStatusUpdate(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type); void SendSentNotification( const Notification::Ptr& notification, const Checkable::Ptr& checkable, size_t users, @@ -113,6 +113,7 @@ private: static bool PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& attributes, Dictionary::Ptr& checkSums); static void StateChangeHandler(const ConfigObject::Ptr& object); + static void StateChangeHandler(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type); static void VersionChangedHandler(const ConfigObject::Ptr& object); static void DowntimeChangedHandler(const Downtime::Ptr& downtime); From 6c536c97e8e6dc25909b7e8cdd494de8e8210446 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 2 Oct 2019 14:32:10 +0200 Subject: [PATCH 176/219] RedisWriter: populate icinga:history:stream:*:downtime --- lib/redis/rediswriter-objects.cpp | 108 ++++++++++++++++++++++++++++-- lib/redis/rediswriter.hpp | 6 +- 2 files changed, 109 insertions(+), 5 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index b3d48767d..5fd494bbb 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -88,11 +88,11 @@ void RedisWriter::ConfigStaticInitialize() }); /* fixed downtime start */ - Downtime::OnDowntimeStarted.connect(&RedisWriter::DowntimeChangedHandler); + Downtime::OnDowntimeStarted.connect(&RedisWriter::DowntimeStartedHandler); /* flexible downtime start */ - Downtime::OnDowntimeTriggered.connect(&RedisWriter::DowntimeChangedHandler); + Downtime::OnDowntimeTriggered.connect(&RedisWriter::DowntimeStartedHandler); /* fixed/flexible downtime end */ - Downtime::OnDowntimeRemoved.connect(&RedisWriter::DowntimeChangedHandler); + Downtime::OnDowntimeRemoved.connect(&RedisWriter::DowntimeRemovedHandler); Checkable::OnNotificationSentToAllUsers.connect([]( const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set& users, @@ -1210,6 +1210,93 @@ void RedisWriter::SendSentNotification( }); } +void RedisWriter::SendStartedDowntime(const Downtime::Ptr& downtime) +{ + auto service (dynamic_pointer_cast(downtime->GetCheckable())); + auto triggeredBy (Downtime::GetByName(downtime->GetTriggeredBy())); + + std::vector xAdd ({ + "XADD", service ? "icinga:history:stream:service:downtime" : "icinga:history:stream:host:downtime", "*", + "downtime_id", GetObjectIdentifier(downtime), + "environment_id", SHA1(GetEnvironment()), + service ? "service_id" : "host_id", GetObjectIdentifier(downtime->GetCheckable()), + "entry_time", Convert::ToString(downtime->GetEntryTime()), + "author", Utility::ValidateUTF8(downtime->GetAuthor()), + "comment", Utility::ValidateUTF8(downtime->GetComment()), + "is_fixed", Convert::ToString((unsigned short)downtime->GetFixed()), + "duration", Convert::ToString(downtime->GetDuration()), + "scheduled_start_time", Convert::ToString(downtime->GetStartTime()), + "scheduled_end_time", Convert::ToString(downtime->GetEndTime()), + "was_started", "1", + "was_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()), + "is_in_effect", Convert::ToString((unsigned short)downtime->IsInEffect()), + "trigger_time", Convert::ToString(downtime->GetTriggerTime()) + }); + + if (triggeredBy) { + xAdd.emplace_back("triggered_by_id"); + xAdd.emplace_back(GetObjectIdentifier(triggeredBy)); + } + + if (downtime->GetFixed()) { + xAdd.emplace_back("actual_start_time"); + xAdd.emplace_back(Convert::ToString(downtime->GetStartTime())); + xAdd.emplace_back("actual_end_time"); + xAdd.emplace_back(Convert::ToString(downtime->GetEndTime())); + } else { + xAdd.emplace_back("actual_start_time"); + xAdd.emplace_back(Convert::ToString(downtime->GetTriggerTime())); + xAdd.emplace_back("actual_end_time"); + xAdd.emplace_back(Convert::ToString(downtime->GetTriggerTime() + downtime->GetDuration())); + } + + m_Rcon->FireAndForgetQuery(std::move(xAdd)); +} + +void RedisWriter::SendRemovedDowntime(const Downtime::Ptr& downtime) +{ + auto service (dynamic_pointer_cast(downtime->GetCheckable())); + auto triggeredBy (Downtime::GetByName(downtime->GetTriggeredBy())); + + std::vector xAdd ({ + "XADD", service ? "icinga:history:stream:service:downtime" : "icinga:history:stream:host:downtime", "*", + "downtime_id", GetObjectIdentifier(downtime), + "environment_id", SHA1(GetEnvironment()), + service ? "service_id" : "host_id", GetObjectIdentifier(downtime->GetCheckable()), + "entry_time", Convert::ToString(downtime->GetEntryTime()), + "author", Utility::ValidateUTF8(downtime->GetAuthor()), + "comment", Utility::ValidateUTF8(downtime->GetComment()), + "is_fixed", Convert::ToString((unsigned short)downtime->GetFixed()), + "duration", Convert::ToString(downtime->GetDuration()), + "scheduled_start_time", Convert::ToString(downtime->GetStartTime()), + "scheduled_end_time", Convert::ToString(downtime->GetEndTime()), + "was_started", "1", + "was_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()), + "is_in_effect", Convert::ToString((unsigned short)downtime->IsInEffect()), + "trigger_time", Convert::ToString(downtime->GetTriggerTime()), + "deletion_time", Convert::ToString(Utility::GetTime()) + }); + + if (triggeredBy) { + xAdd.emplace_back("triggered_by_id"); + xAdd.emplace_back(GetObjectIdentifier(triggeredBy)); + } + + if (downtime->GetFixed()) { + xAdd.emplace_back("actual_start_time"); + xAdd.emplace_back(Convert::ToString(downtime->GetStartTime())); + xAdd.emplace_back("actual_end_time"); + xAdd.emplace_back(Convert::ToString(downtime->GetEndTime())); + } else { + xAdd.emplace_back("actual_start_time"); + xAdd.emplace_back(Convert::ToString(downtime->GetTriggerTime())); + xAdd.emplace_back("actual_end_time"); + xAdd.emplace_back(Convert::ToString(downtime->GetTriggerTime() + downtime->GetDuration())); + } + + m_Rcon->FireAndForgetQuery(std::move(xAdd)); +} + Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) { Dictionary::Ptr attrs = new Dictionary(); @@ -1384,9 +1471,22 @@ void RedisWriter::VersionChangedHandler(const ConfigObject::Ptr& object) } } -void RedisWriter::DowntimeChangedHandler(const Downtime::Ptr& downtime) +void RedisWriter::DowntimeStartedHandler(const Downtime::Ptr& downtime) { StateChangeHandler(downtime->GetCheckable()); + + for (auto& rw : ConfigType::GetObjectsByType()) { + rw->m_WorkQueue.Enqueue([rw, downtime]() { rw->SendStartedDowntime(downtime); }); + } +} + +void RedisWriter::DowntimeRemovedHandler(const Downtime::Ptr& downtime) +{ + StateChangeHandler(downtime->GetCheckable()); + + for (auto& rw : ConfigType::GetObjectsByType()) { + rw->m_WorkQueue.Enqueue([rw, downtime]() { rw->SendRemovedDowntime(downtime); }); + } } void RedisWriter::NotificationSentToAllUsersHandler( diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 3cab086e6..34ac32233 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -87,6 +87,9 @@ private: NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text ); + void SendStartedDowntime(const Downtime::Ptr& downtime); + void SendRemovedDowntime(const Downtime::Ptr& downtime); + std::vector UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride); Dictionary::Ptr SerializeState(const Checkable::Ptr& checkable); @@ -115,7 +118,8 @@ private: static void StateChangeHandler(const ConfigObject::Ptr& object); static void StateChangeHandler(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type); static void VersionChangedHandler(const ConfigObject::Ptr& object); - static void DowntimeChangedHandler(const Downtime::Ptr& downtime); + static void DowntimeStartedHandler(const Downtime::Ptr& downtime); + static void DowntimeRemovedHandler(const Downtime::Ptr& downtime); static void NotificationSentToAllUsersHandler( const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set& users, From 81873248e145f7f4a99da04e07ce4d549f15a255 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 2 Oct 2019 15:53:52 +0200 Subject: [PATCH 177/219] RedisWriter: populate icinga:history:stream:*:comment --- lib/redis/rediswriter-objects.cpp | 54 +++++++++++++++++++++++++++++++ lib/redis/rediswriter.hpp | 5 +++ 2 files changed, 59 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 5fd494bbb..9617c98ee 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -101,6 +101,9 @@ void RedisWriter::ConfigStaticInitialize() ) { RedisWriter::NotificationSentToAllUsersHandler(notification, checkable, users, type, cr, author, text); }); + + Comment::OnCommentAdded.connect(&RedisWriter::CommentAddedHandler); + Comment::OnCommentRemoved.connect(&RedisWriter::CommentRemovedHandler); } static std::pair SplitOutput(String output) @@ -1297,6 +1300,43 @@ void RedisWriter::SendRemovedDowntime(const Downtime::Ptr& downtime) m_Rcon->FireAndForgetQuery(std::move(xAdd)); } +void RedisWriter::SendAddedComment(const Comment::Ptr& comment) +{ + auto service (dynamic_pointer_cast(comment->GetCheckable())); + + m_Rcon->FireAndForgetQuery({ + "XADD", service ? "icinga:history:stream:service:comment" : "icinga:history:stream:host:comment", "*", + "comment_id", GetObjectIdentifier(comment), + "environment_id", SHA1(GetEnvironment()), + service ? "service_id" : "host_id", GetObjectIdentifier(comment->GetCheckable()), + "entry_time", Convert::ToString(comment->GetEntryTime()), + "author", Utility::ValidateUTF8(comment->GetAuthor()), + "comment", Utility::ValidateUTF8(comment->GetText()), + "entry_type", Convert::ToString(comment->GetEntryType()), + "is_persistent", Convert::ToString((unsigned short)comment->GetPersistent()), + "expire_time", Convert::ToString(comment->GetExpireTime()) + }); +} + +void RedisWriter::SendRemovedComment(const Comment::Ptr& comment) +{ + auto service (dynamic_pointer_cast(comment->GetCheckable())); + + m_Rcon->FireAndForgetQuery({ + "XADD", service ? "icinga:history:stream:service:comment" : "icinga:history:stream:host:comment", "*", + "comment_id", GetObjectIdentifier(comment), + "environment_id", SHA1(GetEnvironment()), + service ? "service_id" : "host_id", GetObjectIdentifier(comment->GetCheckable()), + "entry_time", Convert::ToString(comment->GetEntryTime()), + "author", Utility::ValidateUTF8(comment->GetAuthor()), + "comment", Utility::ValidateUTF8(comment->GetText()), + "entry_type", Convert::ToString(comment->GetEntryType()), + "is_persistent", Convert::ToString((unsigned short)comment->GetPersistent()), + "expire_time", Convert::ToString(comment->GetExpireTime()), + "deletion_time", Convert::ToString(Utility::GetTime()) + }); +} + Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) { Dictionary::Ptr attrs = new Dictionary(); @@ -1507,3 +1547,17 @@ void RedisWriter::NotificationSentToAllUsersHandler( } } } + +void RedisWriter::CommentAddedHandler(const Comment::Ptr& comment) +{ + for (auto& rw : ConfigType::GetObjectsByType()) { + rw->m_WorkQueue.Enqueue([rw, comment]() { rw->SendAddedComment(comment); }); + } +} + +void RedisWriter::CommentRemovedHandler(const Comment::Ptr& comment) +{ + for (auto& rw : ConfigType::GetObjectsByType()) { + rw->m_WorkQueue.Enqueue([rw, comment]() { rw->SendRemovedComment(comment); }); + } +} diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 34ac32233..4300c165a 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -89,6 +89,8 @@ private: void SendStartedDowntime(const Downtime::Ptr& downtime); void SendRemovedDowntime(const Downtime::Ptr& downtime); + void SendAddedComment(const Comment::Ptr& comment); + void SendRemovedComment(const Comment::Ptr& comment); std::vector UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride); Dictionary::Ptr SerializeState(const Checkable::Ptr& checkable); @@ -126,6 +128,9 @@ private: NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text ); + static void CommentAddedHandler(const Comment::Ptr& comment); + static void CommentRemovedHandler(const Comment::Ptr& comment); + void AssertOnWorkQueue(); void ExceptionHandler(boost::exception_ptr exp); From fb6d31fcde30530abdd7cb6563317624645f652c Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 2 Oct 2019 16:20:12 +0200 Subject: [PATCH 178/219] RedisWriter: populate icinga:history:stream:*:flapping --- lib/redis/rediswriter-objects.cpp | 28 ++++++++++++++++++++++++++++ lib/redis/rediswriter.hpp | 2 ++ 2 files changed, 30 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 9617c98ee..a4ab277ff 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -104,6 +104,8 @@ void RedisWriter::ConfigStaticInitialize() Comment::OnCommentAdded.connect(&RedisWriter::CommentAddedHandler); Comment::OnCommentRemoved.connect(&RedisWriter::CommentRemovedHandler); + + Checkable::OnFlappingChanged.connect(&RedisWriter::FlappingChangedHandler); } static std::pair SplitOutput(String output) @@ -1337,6 +1339,25 @@ void RedisWriter::SendRemovedComment(const Comment::Ptr& comment) }); } +void RedisWriter::SendFlappingChanged(const Checkable::Ptr& checkable, const Value& value) +{ + if (!m_Rcon || !m_Rcon->IsConnected()) + return; + + auto service (dynamic_pointer_cast(checkable)); + + m_Rcon->FireAndForgetQuery({ + "XADD", service ? "icinga:history:stream:service:flapping" : "icinga:history:stream:host:flapping", "*", + "id", Utility::NewUniqueID(), + "environment_id", SHA1(GetEnvironment()), + service ? "service_id" : "host_id", GetObjectIdentifier(checkable), + value.ToBool() ? "start_time" : "end_time", Convert::ToString(Utility::GetTime()), + "percent_state_change", Convert::ToString(checkable->GetFlappingCurrent()), + "flapping_threshold_low", Convert::ToString(checkable->GetFlappingThresholdLow()), + "flapping_threshold_high", Convert::ToString(checkable->GetFlappingThresholdHigh()) + }); +} + Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) { Dictionary::Ptr attrs = new Dictionary(); @@ -1561,3 +1582,10 @@ void RedisWriter::CommentRemovedHandler(const Comment::Ptr& comment) rw->m_WorkQueue.Enqueue([rw, comment]() { rw->SendRemovedComment(comment); }); } } + +void RedisWriter::FlappingChangedHandler(const Checkable::Ptr& checkable, const Value& value) +{ + for (auto& rw : ConfigType::GetObjectsByType()) { + rw->m_WorkQueue.Enqueue([rw, checkable, value]() { rw->SendFlappingChanged(checkable, value); }); + } +} diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 4300c165a..e44965b67 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -91,6 +91,7 @@ private: void SendRemovedDowntime(const Downtime::Ptr& downtime); void SendAddedComment(const Comment::Ptr& comment); void SendRemovedComment(const Comment::Ptr& comment); + void SendFlappingChanged(const Checkable::Ptr& checkable, const Value& value); std::vector UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride); Dictionary::Ptr SerializeState(const Checkable::Ptr& checkable); @@ -130,6 +131,7 @@ private: static void CommentAddedHandler(const Comment::Ptr& comment); static void CommentRemovedHandler(const Comment::Ptr& comment); + static void FlappingChangedHandler(const Checkable::Ptr& checkable, const Value& value); void AssertOnWorkQueue(); From 5edf3623c5c2fe5968ab35bbc9f78abc07a49dfb Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Tue, 8 Oct 2019 16:15:45 +0200 Subject: [PATCH 179/219] RedisWriter-History: Add event_id and event_type to history entries --- lib/redis/rediswriter-objects.cpp | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index a4ab277ff..362619bd2 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -1185,7 +1185,9 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, const CheckR "last_hard_state", Convert::ToString(service ? service->GetLastHardState() : host->GetLastHardState()), "output", Utility::ValidateUTF8(std::move(output.first)), "long_output", Utility::ValidateUTF8(std::move(output.second)), - "max_check_attempts", Convert::ToString(checkable->GetMaxCheckAttempts()) + "max_check_attempts", Convert::ToString(checkable->GetMaxCheckAttempts()), + "event_id", Utility::NewUniqueID(), + "event_type", "state" }); } @@ -1211,7 +1213,9 @@ void RedisWriter::SendSentNotification( "state", Convert::ToString(cr->GetState()), "output", Utility::ValidateUTF8(std::move(output.first)), "long_output", Utility::ValidateUTF8(std::move(output.second)), - "users_notified", Convert::ToString(users) + "users_notified", Convert::ToString(users), + "event_id", Utility::NewUniqueID(), + "event_type", "notification" }); } @@ -1235,7 +1239,9 @@ void RedisWriter::SendStartedDowntime(const Downtime::Ptr& downtime) "was_started", "1", "was_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()), "is_in_effect", Convert::ToString((unsigned short)downtime->IsInEffect()), - "trigger_time", Convert::ToString(downtime->GetTriggerTime()) + "trigger_time", Convert::ToString(downtime->GetTriggerTime()), + "event_id", Utility::NewUniqueID(), + "event_type", "downtime_start" }); if (triggeredBy) { @@ -1279,7 +1285,9 @@ void RedisWriter::SendRemovedDowntime(const Downtime::Ptr& downtime) "was_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()), "is_in_effect", Convert::ToString((unsigned short)downtime->IsInEffect()), "trigger_time", Convert::ToString(downtime->GetTriggerTime()), - "deletion_time", Convert::ToString(Utility::GetTime()) + "deletion_time", Convert::ToString(Utility::GetTime()), + "event_id", Utility::NewUniqueID(), + "event_type", "downtime_end" }); if (triggeredBy) { @@ -1316,7 +1324,9 @@ void RedisWriter::SendAddedComment(const Comment::Ptr& comment) "comment", Utility::ValidateUTF8(comment->GetText()), "entry_type", Convert::ToString(comment->GetEntryType()), "is_persistent", Convert::ToString((unsigned short)comment->GetPersistent()), - "expire_time", Convert::ToString(comment->GetExpireTime()) + "expire_time", Convert::ToString(comment->GetExpireTime()), + "event_id", Utility::NewUniqueID(), + "event_type", "comment_add" }); } @@ -1335,7 +1345,9 @@ void RedisWriter::SendRemovedComment(const Comment::Ptr& comment) "entry_type", Convert::ToString(comment->GetEntryType()), "is_persistent", Convert::ToString((unsigned short)comment->GetPersistent()), "expire_time", Convert::ToString(comment->GetExpireTime()), - "deletion_time", Convert::ToString(Utility::GetTime()) + "deletion_time", Convert::ToString(Utility::GetTime()), + "event_id", Utility::NewUniqueID(), + "event_type", "comment_remove" }); } @@ -1354,7 +1366,9 @@ void RedisWriter::SendFlappingChanged(const Checkable::Ptr& checkable, const Val value.ToBool() ? "start_time" : "end_time", Convert::ToString(Utility::GetTime()), "percent_state_change", Convert::ToString(checkable->GetFlappingCurrent()), "flapping_threshold_low", Convert::ToString(checkable->GetFlappingThresholdLow()), - "flapping_threshold_high", Convert::ToString(checkable->GetFlappingThresholdHigh()) + "flapping_threshold_high", Convert::ToString(checkable->GetFlappingThresholdHigh()), + "event_id", Utility::NewUniqueID(), + "event_type", value.ToBool() ? "flapping_start" : "flapping_end" }); } From 6f0822cbe8f8613cc1a5d3747359834b3db5aaa2 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 9 Oct 2019 14:08:03 +0200 Subject: [PATCH 180/219] RedisWriter: Use ms instead of seconds for timestamps --- lib/redis/rediswriter-objects.cpp | 82 +++++++++++++++---------------- lib/redis/rediswriter-utility.cpp | 4 ++ lib/redis/rediswriter.hpp | 1 + 3 files changed, 46 insertions(+), 41 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 362619bd2..6e0c26fc6 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -1007,9 +1007,9 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr attributes->Set("author", comment->GetAuthor()); attributes->Set("text", comment->GetText()); attributes->Set("entry_type", comment->GetEntryType()); - attributes->Set("entry_time", comment->GetEntryTime()); + attributes->Set("entry_time", comment->GetEntryTime() * 1000); attributes->Set("is_persistent", comment->GetPersistent()); - attributes->Set("expire_time", comment->GetExpireTime()); + attributes->Set("expire_time", comment->GetExpireTime() * 1000); Host::Ptr host; Service::Ptr service; @@ -1027,14 +1027,14 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr attributes->Set("author", downtime->GetAuthor()); attributes->Set("comment", downtime->GetComment()); - attributes->Set("entry_time", downtime->GetEntryTime()); - attributes->Set("scheduled_start_time", downtime->GetStartTime()); - attributes->Set("scheduled_end_time", downtime->GetEndTime()); - attributes->Set("duration", downtime->GetDuration()); + attributes->Set("entry_time", downtime->GetEntryTime() * 1000); + attributes->Set("scheduled_start_time", downtime->GetStartTime() * 1000); + attributes->Set("scheduled_end_time", downtime->GetEndTime() * 1000); + attributes->Set("duration", downtime->GetDuration() * 1000); attributes->Set("is_fixed", downtime->GetFixed()); attributes->Set("is_in_effect", downtime->IsInEffect()); if (downtime->IsInEffect()) - attributes->Set("actual_start_time", downtime->GetTriggerTime()); + attributes->Set("actual_start_time", downtime->GetTriggerTime() * 1000); Host::Ptr host; Service::Ptr service; @@ -1175,7 +1175,7 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, const CheckR "id", Utility::NewUniqueID(), "environment_id", SHA1(GetEnvironment()), service ? "service_id" : "host_id", GetObjectIdentifier(checkable), - "change_time", Convert::ToString(cr->GetExecutionEnd()), + "change_time", Convert::ToString(TimestampToMilliseconds(cr->GetExecutionEnd())), "state_type", Convert::ToString(type), "soft_state", Convert::ToString(cr->GetState()), "hard_state", Convert::ToString(service ? service->GetLastHardState() : host->GetLastHardState()), @@ -1209,7 +1209,7 @@ void RedisWriter::SendSentNotification( service ? "service_id" : "host_id", GetObjectIdentifier(checkable), "notification_id", GetObjectIdentifier(notification), "type", Convert::ToString(type), - "send_time", Convert::ToString(Utility::GetTime()), + "send_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), "state", Convert::ToString(cr->GetState()), "output", Utility::ValidateUTF8(std::move(output.first)), "long_output", Utility::ValidateUTF8(std::move(output.second)), @@ -1229,17 +1229,17 @@ void RedisWriter::SendStartedDowntime(const Downtime::Ptr& downtime) "downtime_id", GetObjectIdentifier(downtime), "environment_id", SHA1(GetEnvironment()), service ? "service_id" : "host_id", GetObjectIdentifier(downtime->GetCheckable()), - "entry_time", Convert::ToString(downtime->GetEntryTime()), + "entry_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEntryTime())), "author", Utility::ValidateUTF8(downtime->GetAuthor()), "comment", Utility::ValidateUTF8(downtime->GetComment()), "is_fixed", Convert::ToString((unsigned short)downtime->GetFixed()), "duration", Convert::ToString(downtime->GetDuration()), - "scheduled_start_time", Convert::ToString(downtime->GetStartTime()), - "scheduled_end_time", Convert::ToString(downtime->GetEndTime()), + "scheduled_start_time", Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime())), + "scheduled_end_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())), "was_started", "1", "was_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()), "is_in_effect", Convert::ToString((unsigned short)downtime->IsInEffect()), - "trigger_time", Convert::ToString(downtime->GetTriggerTime()), + "trigger_time", Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime())), "event_id", Utility::NewUniqueID(), "event_type", "downtime_start" }); @@ -1251,14 +1251,14 @@ void RedisWriter::SendStartedDowntime(const Downtime::Ptr& downtime) if (downtime->GetFixed()) { xAdd.emplace_back("actual_start_time"); - xAdd.emplace_back(Convert::ToString(downtime->GetStartTime())); + xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime()))); xAdd.emplace_back("actual_end_time"); - xAdd.emplace_back(Convert::ToString(downtime->GetEndTime())); + xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime()))); } else { xAdd.emplace_back("actual_start_time"); - xAdd.emplace_back(Convert::ToString(downtime->GetTriggerTime())); + xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime()))); xAdd.emplace_back("actual_end_time"); - xAdd.emplace_back(Convert::ToString(downtime->GetTriggerTime() + downtime->GetDuration())); + xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime() + downtime->GetDuration()))); } m_Rcon->FireAndForgetQuery(std::move(xAdd)); @@ -1274,18 +1274,18 @@ void RedisWriter::SendRemovedDowntime(const Downtime::Ptr& downtime) "downtime_id", GetObjectIdentifier(downtime), "environment_id", SHA1(GetEnvironment()), service ? "service_id" : "host_id", GetObjectIdentifier(downtime->GetCheckable()), - "entry_time", Convert::ToString(downtime->GetEntryTime()), + "entry_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEntryTime())), "author", Utility::ValidateUTF8(downtime->GetAuthor()), "comment", Utility::ValidateUTF8(downtime->GetComment()), "is_fixed", Convert::ToString((unsigned short)downtime->GetFixed()), "duration", Convert::ToString(downtime->GetDuration()), - "scheduled_start_time", Convert::ToString(downtime->GetStartTime()), - "scheduled_end_time", Convert::ToString(downtime->GetEndTime()), + "scheduled_start_time", Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime())), + "scheduled_end_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())), "was_started", "1", "was_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()), "is_in_effect", Convert::ToString((unsigned short)downtime->IsInEffect()), - "trigger_time", Convert::ToString(downtime->GetTriggerTime()), - "deletion_time", Convert::ToString(Utility::GetTime()), + "trigger_time", Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime())), + "deletion_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), "event_id", Utility::NewUniqueID(), "event_type", "downtime_end" }); @@ -1297,14 +1297,14 @@ void RedisWriter::SendRemovedDowntime(const Downtime::Ptr& downtime) if (downtime->GetFixed()) { xAdd.emplace_back("actual_start_time"); - xAdd.emplace_back(Convert::ToString(downtime->GetStartTime())); + xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime()))); xAdd.emplace_back("actual_end_time"); - xAdd.emplace_back(Convert::ToString(downtime->GetEndTime())); + xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime()))); } else { xAdd.emplace_back("actual_start_time"); - xAdd.emplace_back(Convert::ToString(downtime->GetTriggerTime())); + xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime()))); xAdd.emplace_back("actual_end_time"); - xAdd.emplace_back(Convert::ToString(downtime->GetTriggerTime() + downtime->GetDuration())); + xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime() + downtime->GetDuration()))); } m_Rcon->FireAndForgetQuery(std::move(xAdd)); @@ -1319,12 +1319,12 @@ void RedisWriter::SendAddedComment(const Comment::Ptr& comment) "comment_id", GetObjectIdentifier(comment), "environment_id", SHA1(GetEnvironment()), service ? "service_id" : "host_id", GetObjectIdentifier(comment->GetCheckable()), - "entry_time", Convert::ToString(comment->GetEntryTime()), + "entry_time", Convert::ToString(TimestampToMilliseconds(comment->GetEntryTime())), "author", Utility::ValidateUTF8(comment->GetAuthor()), "comment", Utility::ValidateUTF8(comment->GetText()), "entry_type", Convert::ToString(comment->GetEntryType()), "is_persistent", Convert::ToString((unsigned short)comment->GetPersistent()), - "expire_time", Convert::ToString(comment->GetExpireTime()), + "expire_time", Convert::ToString(TimestampToMilliseconds(comment->GetExpireTime())), "event_id", Utility::NewUniqueID(), "event_type", "comment_add" }); @@ -1339,13 +1339,13 @@ void RedisWriter::SendRemovedComment(const Comment::Ptr& comment) "comment_id", GetObjectIdentifier(comment), "environment_id", SHA1(GetEnvironment()), service ? "service_id" : "host_id", GetObjectIdentifier(comment->GetCheckable()), - "entry_time", Convert::ToString(comment->GetEntryTime()), + "entry_time", Convert::ToString(TimestampToMilliseconds(comment->GetEntryTime())), "author", Utility::ValidateUTF8(comment->GetAuthor()), "comment", Utility::ValidateUTF8(comment->GetText()), "entry_type", Convert::ToString(comment->GetEntryType()), "is_persistent", Convert::ToString((unsigned short)comment->GetPersistent()), - "expire_time", Convert::ToString(comment->GetExpireTime()), - "deletion_time", Convert::ToString(Utility::GetTime()), + "expire_time", Convert::ToString(TimestampToMilliseconds(comment->GetExpireTime())), + "deletion_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), "event_id", Utility::NewUniqueID(), "event_type", "comment_remove" }); @@ -1363,7 +1363,7 @@ void RedisWriter::SendFlappingChanged(const Checkable::Ptr& checkable, const Val "id", Utility::NewUniqueID(), "environment_id", SHA1(GetEnvironment()), service ? "service_id" : "host_id", GetObjectIdentifier(checkable), - value.ToBool() ? "start_time" : "end_time", Convert::ToString(Utility::GetTime()), + value.ToBool() ? "start_time" : "end_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), "percent_state_change", Convert::ToString(checkable->GetFlappingCurrent()), "flapping_threshold_low", Convert::ToString(checkable->GetFlappingThresholdLow()), "flapping_threshold_high", Convert::ToString(checkable->GetFlappingThresholdHigh()), @@ -1425,8 +1425,8 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) if (!cr->GetCommand().IsEmpty()) attrs->Set("commandline", FormatCommandLine(cr->GetCommand())); - attrs->Set("execution_time", cr->CalculateExecutionTime()); - attrs->Set("latency", cr->CalculateLatency()); + attrs->Set("execution_time", TimestampToMilliseconds(cr->CalculateExecutionTime())); + attrs->Set("latency", TimestampToMilliseconds(cr->CalculateLatency())); } bool isProblem = !checkable->IsStateOK(checkable->GetStateRaw()); @@ -1455,14 +1455,14 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) attrs->Set("in_downtime", checkable->IsInDowntime()); if (checkable->GetCheckTimeout().IsEmpty()) - attrs->Set("check_timeout",checkable->GetCheckCommand()->GetTimeout()); + attrs->Set("check_timeout",checkable->GetCheckCommand()->GetTimeout() * 1000); else - attrs->Set("check_timeout", checkable->GetCheckTimeout()); + attrs->Set("check_timeout", TimestampToMilliseconds(checkable->GetCheckTimeout())); - attrs->Set("last_update", Utility::GetTime()); - attrs->Set("last_state_change", checkable->GetLastStateChange()); - attrs->Set("next_check", checkable->GetNextCheck()); - attrs->Set("next_update", checkable->GetNextUpdate()); + attrs->Set("last_update", TimestampToMilliseconds(Utility::GetTime())); + attrs->Set("last_state_change", TimestampToMilliseconds(checkable->GetLastStateChange())); + attrs->Set("next_check", TimestampToMilliseconds(checkable->GetNextCheck())); + attrs->Set("next_update", TimestampToMilliseconds(checkable->GetNextUpdate())); return attrs; } @@ -1497,7 +1497,7 @@ RedisWriter::UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, Downtime::Ptr downtime = dynamic_pointer_cast(object); if (downtime) { attrs->Set("in_effect", Serialize(downtime->IsInEffect())); - attrs->Set("trigger_time", Serialize(downtime->GetTriggerTime())); + attrs->Set("trigger_time", Serialize(TimestampToMilliseconds(downtime->GetTriggerTime()))); } diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index e6171ef4d..04cd275d9 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -277,3 +277,7 @@ String RedisWriter::GetLowerCaseTypeNameDB(const ConfigObject::Ptr& obj) return typeName; } + +long long RedisWriter::TimestampToMilliseconds(double timestamp) { + return static_cast(timestamp * 1000); +} diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index e44965b67..cf54ea1b6 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -102,6 +102,7 @@ private: /* utilities */ static String FormatCheckSumBinary(const String& str); static String FormatCommandLine(const Value& commandLine); + static long long TimestampToMilliseconds(double timestamp); static String GetObjectIdentifier(const ConfigObject::Ptr& object); static String GetEnvironment(); From 042ed8b9e91d1b12ff752b9d823a6c6389a7cad1 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 9 Oct 2019 15:28:23 +0200 Subject: [PATCH 181/219] RedisWriter: Add downtime schedule history event --- lib/redis/rediswriter-objects.cpp | 56 ++++++++++++++++++++++++++++++- lib/redis/rediswriter.hpp | 2 ++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 6e0c26fc6..cd9856b2c 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -87,11 +87,13 @@ void RedisWriter::ConfigStaticInitialize() RedisWriter::VersionChangedHandler(object); }); + /* fixed/flexible downtime add */ + Downtime::OnDowntimeAdded.connect(&RedisWriter::DowntimeAddedHandler); /* fixed downtime start */ Downtime::OnDowntimeStarted.connect(&RedisWriter::DowntimeStartedHandler); /* flexible downtime start */ Downtime::OnDowntimeTriggered.connect(&RedisWriter::DowntimeStartedHandler); - /* fixed/flexible downtime end */ + /* fixed/flexible downtime end or remove */ Downtime::OnDowntimeRemoved.connect(&RedisWriter::DowntimeRemovedHandler); Checkable::OnNotificationSentToAllUsers.connect([]( @@ -1219,6 +1221,51 @@ void RedisWriter::SendSentNotification( }); } +void RedisWriter::SendAddedDowntime(const Downtime::Ptr& downtime) +{ + auto service (dynamic_pointer_cast(downtime->GetCheckable())); + auto triggeredBy (Downtime::GetByName(downtime->GetTriggeredBy())); + + std::vector xAdd ({ + "XADD", service ? "icinga:history:stream:service:downtime" : "icinga:history:stream:host:downtime", "*", + "downtime_id", GetObjectIdentifier(downtime), + "environment_id", SHA1(GetEnvironment()), + service ? "service_id" : "host_id", GetObjectIdentifier(downtime->GetCheckable()), + "entry_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEntryTime())), + "author", Utility::ValidateUTF8(downtime->GetAuthor()), + "comment", Utility::ValidateUTF8(downtime->GetComment()), + "is_fixed", Convert::ToString((unsigned short)downtime->GetFixed()), + "duration", Convert::ToString(downtime->GetDuration()), + "scheduled_start_time", Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime())), + "scheduled_end_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())), + "was_started", "0", + "was_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()), + "is_in_effect", Convert::ToString((unsigned short)downtime->IsInEffect()), + "trigger_time", Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime())), + "event_id", Utility::NewUniqueID(), + "event_type", "downtime_schedule" + }); + + if (triggeredBy) { + xAdd.emplace_back("triggered_by_id"); + xAdd.emplace_back(GetObjectIdentifier(triggeredBy)); + } + + if (downtime->GetFixed()) { + xAdd.emplace_back("actual_start_time"); + xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime()))); + xAdd.emplace_back("actual_end_time"); + xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime()))); + } else { + xAdd.emplace_back("actual_start_time"); + xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime()))); + xAdd.emplace_back("actual_end_time"); + xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime() + downtime->GetDuration()))); + } + + m_Rcon->FireAndForgetQuery(std::move(xAdd)); +} + void RedisWriter::SendStartedDowntime(const Downtime::Ptr& downtime) { auto service (dynamic_pointer_cast(downtime->GetCheckable())); @@ -1546,6 +1593,13 @@ void RedisWriter::VersionChangedHandler(const ConfigObject::Ptr& object) } } +void RedisWriter::DowntimeAddedHandler(const Downtime::Ptr& downtime) +{ + for (auto& rw : ConfigType::GetObjectsByType()) { + rw->m_WorkQueue.Enqueue([rw, downtime]() { rw->SendAddedDowntime(downtime); }); + } +} + void RedisWriter::DowntimeStartedHandler(const Downtime::Ptr& downtime) { StateChangeHandler(downtime->GetCheckable()); diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index cf54ea1b6..666f9b1b7 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -87,6 +87,7 @@ private: NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text ); + void SendAddedDowntime(const Downtime::Ptr& downtime); void SendStartedDowntime(const Downtime::Ptr& downtime); void SendRemovedDowntime(const Downtime::Ptr& downtime); void SendAddedComment(const Comment::Ptr& comment); @@ -122,6 +123,7 @@ private: static void StateChangeHandler(const ConfigObject::Ptr& object); static void StateChangeHandler(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type); static void VersionChangedHandler(const ConfigObject::Ptr& object); + static void DowntimeAddedHandler(const Downtime::Ptr& downtime); static void DowntimeStartedHandler(const Downtime::Ptr& downtime); static void DowntimeRemovedHandler(const Downtime::Ptr& downtime); From 846f3270544479447e075d622f90a8fc22ab68d7 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Thu, 10 Oct 2019 13:04:35 +0200 Subject: [PATCH 182/219] RedisWriter: Use TimestampToMilliseconds() instead of multiplying inline --- lib/redis/rediswriter-objects.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index cd9856b2c..5949dee95 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -1009,9 +1009,9 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr attributes->Set("author", comment->GetAuthor()); attributes->Set("text", comment->GetText()); attributes->Set("entry_type", comment->GetEntryType()); - attributes->Set("entry_time", comment->GetEntryTime() * 1000); + attributes->Set("entry_time", TimestampToMilliseconds(comment->GetEntryTime())); attributes->Set("is_persistent", comment->GetPersistent()); - attributes->Set("expire_time", comment->GetExpireTime() * 1000); + attributes->Set("expire_time", TimestampToMilliseconds(comment->GetExpireTime())); Host::Ptr host; Service::Ptr service; @@ -1029,14 +1029,14 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr attributes->Set("author", downtime->GetAuthor()); attributes->Set("comment", downtime->GetComment()); - attributes->Set("entry_time", downtime->GetEntryTime() * 1000); - attributes->Set("scheduled_start_time", downtime->GetStartTime() * 1000); - attributes->Set("scheduled_end_time", downtime->GetEndTime() * 1000); - attributes->Set("duration", downtime->GetDuration() * 1000); + attributes->Set("entry_time", TimestampToMilliseconds(downtime->GetEntryTime())); + attributes->Set("scheduled_start_time", TimestampToMilliseconds(downtime->GetStartTime())); + attributes->Set("scheduled_end_time", TimestampToMilliseconds(downtime->GetEndTime())); + attributes->Set("duration", TimestampToMilliseconds(downtime->GetDuration())); attributes->Set("is_fixed", downtime->GetFixed()); attributes->Set("is_in_effect", downtime->IsInEffect()); if (downtime->IsInEffect()) - attributes->Set("actual_start_time", downtime->GetTriggerTime() * 1000); + attributes->Set("actual_start_time", TimestampToMilliseconds(downtime->GetTriggerTime())); Host::Ptr host; Service::Ptr service; @@ -1502,7 +1502,7 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) attrs->Set("in_downtime", checkable->IsInDowntime()); if (checkable->GetCheckTimeout().IsEmpty()) - attrs->Set("check_timeout",checkable->GetCheckCommand()->GetTimeout() * 1000); + attrs->Set("check_timeout", TimestampToMilliseconds(checkable->GetCheckCommand()->GetTimeout())); else attrs->Set("check_timeout", TimestampToMilliseconds(checkable->GetCheckTimeout())); From 52fb723d91a53c5b95c2a24eb96f077739ced41c Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Thu, 10 Oct 2019 13:05:16 +0200 Subject: [PATCH 183/219] RedisWriter: Do not split up comments/downtimes into host/service keys --- lib/redis/rediswriter-objects.cpp | 27 ++++++++++++++------------- lib/redis/rediswriter-utility.cpp | 23 +---------------------- 2 files changed, 15 insertions(+), 35 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 5949dee95..8e5117010 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -140,16 +140,7 @@ void RedisWriter::UpdateAllConfigObjects() continue; String lcType(type->GetName().ToLower()); - - if (lcType == "downtime") { - types.emplace_back(ctype, "hostdowntime"); - types.emplace_back(ctype, "servicedowntime"); - } else if (lcType == "comment") { - types.emplace_back(ctype, "hostcomment"); - types.emplace_back(ctype, "servicecomment"); - } else { - types.emplace_back(ctype, lcType); - } + types.emplace_back(ctype, lcType); } m_Rcon->FireAndForgetQuery({"EVAL", l_LuaResetDump, "1", "icinga:dump"}); @@ -1016,10 +1007,15 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr Host::Ptr host; Service::Ptr service; tie(host, service) = GetHostService(comment->GetCheckable()); - if (service) + if (service) { + attributes->Set("object_type", "service"); attributes->Set("service_id", GetObjectIdentifier(service)); - else + attributes->Set("host_id", "00000000000000000000000000000000"); + } else { + attributes->Set("object_type", "host"); attributes->Set("host_id", GetObjectIdentifier(host)); + attributes->Set("service_id", "00000000000000000000000000000000"); + } return true; } @@ -1043,9 +1039,14 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr tie(host, service) = GetHostService(downtime->GetCheckable()); if (service) { + attributes->Set("object_type", "service"); attributes->Set("service_id", GetObjectIdentifier(service)); - } else + attributes->Set("host_id", "00000000000000000000000000000000"); + } else { + attributes->Set("object_type", "host"); attributes->Set("host_id", GetObjectIdentifier(host)); + attributes->Set("service_id", "00000000000000000000000000000000"); + } return true; } diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index 04cd275d9..f2a4a5281 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -254,28 +254,7 @@ String RedisWriter::HashValue(const Value& value, const std::set& proper String RedisWriter::GetLowerCaseTypeNameDB(const ConfigObject::Ptr& obj) { - String typeName = obj->GetReflectionType()->GetName().ToLower(); - if (typeName == "downtime") { - Downtime::Ptr downtime = dynamic_pointer_cast(obj); - Host::Ptr host; - Service::Ptr service; - tie(host, service) = GetHostService(downtime->GetCheckable()); - if (service) - typeName = "servicedowntime"; - else - typeName = "hostdowntime"; - } else if (typeName == "comment") { - Comment::Ptr comment = dynamic_pointer_cast(obj); - Host::Ptr host; - Service::Ptr service; - tie(host, service) = GetHostService(comment->GetCheckable()); - if (service) - typeName = "servicecomment"; - else - typeName = "hostcomment"; - } - - return typeName; + return obj->GetReflectionType()->GetName().ToLower(); } long long RedisWriter::TimestampToMilliseconds(double timestamp) { From 82898b31aecbd532cd6779ecf6ba206e6278d35f Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 21 Oct 2019 16:52:48 +0200 Subject: [PATCH 184/219] RedisWriter: fix missing m_Rcon!=null checks --- lib/redis/rediswriter-objects.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 8e5117010..b0f47fc40 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -1224,6 +1224,9 @@ void RedisWriter::SendSentNotification( void RedisWriter::SendAddedDowntime(const Downtime::Ptr& downtime) { + if (!m_Rcon || !m_Rcon->IsConnected()) + return; + auto service (dynamic_pointer_cast(downtime->GetCheckable())); auto triggeredBy (Downtime::GetByName(downtime->GetTriggeredBy())); @@ -1269,6 +1272,9 @@ void RedisWriter::SendAddedDowntime(const Downtime::Ptr& downtime) void RedisWriter::SendStartedDowntime(const Downtime::Ptr& downtime) { + if (!m_Rcon || !m_Rcon->IsConnected()) + return; + auto service (dynamic_pointer_cast(downtime->GetCheckable())); auto triggeredBy (Downtime::GetByName(downtime->GetTriggeredBy())); @@ -1314,6 +1320,9 @@ void RedisWriter::SendStartedDowntime(const Downtime::Ptr& downtime) void RedisWriter::SendRemovedDowntime(const Downtime::Ptr& downtime) { + if (!m_Rcon || !m_Rcon->IsConnected()) + return; + auto service (dynamic_pointer_cast(downtime->GetCheckable())); auto triggeredBy (Downtime::GetByName(downtime->GetTriggeredBy())); @@ -1360,6 +1369,9 @@ void RedisWriter::SendRemovedDowntime(const Downtime::Ptr& downtime) void RedisWriter::SendAddedComment(const Comment::Ptr& comment) { + if (!m_Rcon || !m_Rcon->IsConnected()) + return; + auto service (dynamic_pointer_cast(comment->GetCheckable())); m_Rcon->FireAndForgetQuery({ @@ -1380,6 +1392,9 @@ void RedisWriter::SendAddedComment(const Comment::Ptr& comment) void RedisWriter::SendRemovedComment(const Comment::Ptr& comment) { + if (!m_Rcon || !m_Rcon->IsConnected()) + return; + auto service (dynamic_pointer_cast(comment->GetCheckable())); m_Rcon->FireAndForgetQuery({ From 747521fa84ab7d03b38e39487be77d900fa98e8d Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 21 Oct 2019 16:49:53 +0200 Subject: [PATCH 185/219] RedisWriter#SendStatusUpdate(): handle pending checkable --- lib/redis/rediswriter-objects.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index b0f47fc40..fe13e274a 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -1171,20 +1171,20 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, const CheckR m_Rcon->FireAndForgetQuery(std::move(streamadd)); - auto output (SplitOutput(cr->GetOutput())); + auto output (SplitOutput(cr ? cr->GetOutput() : "")); m_Rcon->FireAndForgetQuery({ "XADD", service ? "icinga:history:stream:service:state" : "icinga:history:stream:host:state", "*", "id", Utility::NewUniqueID(), "environment_id", SHA1(GetEnvironment()), service ? "service_id" : "host_id", GetObjectIdentifier(checkable), - "change_time", Convert::ToString(TimestampToMilliseconds(cr->GetExecutionEnd())), + "change_time", Convert::ToString(TimestampToMilliseconds(cr ? cr->GetExecutionEnd() : Utility::GetTime())), "state_type", Convert::ToString(type), - "soft_state", Convert::ToString(cr->GetState()), + "soft_state", Convert::ToString(cr ? cr->GetState() : 99), "hard_state", Convert::ToString(service ? service->GetLastHardState() : host->GetLastHardState()), "attempt", Convert::ToString(checkable->GetCheckAttempt()), // TODO: last_hard/soft_state should be "previous". - "last_soft_state", Convert::ToString(cr->GetState()), + "last_soft_state", Convert::ToString(cr ? cr->GetState() : 99), "last_hard_state", Convert::ToString(service ? service->GetLastHardState() : host->GetLastHardState()), "output", Utility::ValidateUTF8(std::move(output.first)), "long_output", Utility::ValidateUTF8(std::move(output.second)), From 9d0e402d92a051b558b1ce021e51ac076ec86cac Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 21 Oct 2019 15:22:09 +0200 Subject: [PATCH 186/219] RedisWriter#SendStartedDowntime(): send also a config update --- lib/redis/rediswriter-objects.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index fe13e274a..f26d3d292 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -1275,6 +1275,8 @@ void RedisWriter::SendStartedDowntime(const Downtime::Ptr& downtime) if (!m_Rcon || !m_Rcon->IsConnected()) return; + SendConfigUpdate(downtime, true); + auto service (dynamic_pointer_cast(downtime->GetCheckable())); auto triggeredBy (Downtime::GetByName(downtime->GetTriggeredBy())); From c89f354a8417d252a56307f25eefb0f503e46004 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 21 Oct 2019 15:50:51 +0200 Subject: [PATCH 187/219] Replace is_fixed with is_flexible --- lib/redis/rediswriter-objects.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index f26d3d292..cb7b772df 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -1029,7 +1029,7 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr attributes->Set("scheduled_start_time", TimestampToMilliseconds(downtime->GetStartTime())); attributes->Set("scheduled_end_time", TimestampToMilliseconds(downtime->GetEndTime())); attributes->Set("duration", TimestampToMilliseconds(downtime->GetDuration())); - attributes->Set("is_fixed", downtime->GetFixed()); + attributes->Set("is_flexible", !downtime->GetFixed()); attributes->Set("is_in_effect", downtime->IsInEffect()); if (downtime->IsInEffect()) attributes->Set("actual_start_time", TimestampToMilliseconds(downtime->GetTriggerTime())); @@ -1238,7 +1238,7 @@ void RedisWriter::SendAddedDowntime(const Downtime::Ptr& downtime) "entry_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEntryTime())), "author", Utility::ValidateUTF8(downtime->GetAuthor()), "comment", Utility::ValidateUTF8(downtime->GetComment()), - "is_fixed", Convert::ToString((unsigned short)downtime->GetFixed()), + "is_flexible", Convert::ToString((unsigned short)!downtime->GetFixed()), "duration", Convert::ToString(downtime->GetDuration()), "scheduled_start_time", Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime())), "scheduled_end_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())), @@ -1288,7 +1288,7 @@ void RedisWriter::SendStartedDowntime(const Downtime::Ptr& downtime) "entry_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEntryTime())), "author", Utility::ValidateUTF8(downtime->GetAuthor()), "comment", Utility::ValidateUTF8(downtime->GetComment()), - "is_fixed", Convert::ToString((unsigned short)downtime->GetFixed()), + "is_flexible", Convert::ToString((unsigned short)!downtime->GetFixed()), "duration", Convert::ToString(downtime->GetDuration()), "scheduled_start_time", Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime())), "scheduled_end_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())), @@ -1336,7 +1336,7 @@ void RedisWriter::SendRemovedDowntime(const Downtime::Ptr& downtime) "entry_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEntryTime())), "author", Utility::ValidateUTF8(downtime->GetAuthor()), "comment", Utility::ValidateUTF8(downtime->GetComment()), - "is_fixed", Convert::ToString((unsigned short)downtime->GetFixed()), + "is_flexible", Convert::ToString((unsigned short)!downtime->GetFixed()), "duration", Convert::ToString(downtime->GetDuration()), "scheduled_start_time", Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime())), "scheduled_end_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())), From 0510e0cad0517b1eafbbc8b3e79bd843808b8519 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Mon, 21 Oct 2019 16:06:34 +0200 Subject: [PATCH 188/219] Replace duration with flexible_duration --- lib/redis/rediswriter-objects.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index cb7b772df..136992bdf 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -1028,7 +1028,7 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr attributes->Set("entry_time", TimestampToMilliseconds(downtime->GetEntryTime())); attributes->Set("scheduled_start_time", TimestampToMilliseconds(downtime->GetStartTime())); attributes->Set("scheduled_end_time", TimestampToMilliseconds(downtime->GetEndTime())); - attributes->Set("duration", TimestampToMilliseconds(downtime->GetDuration())); + attributes->Set("flexible_duration", TimestampToMilliseconds(downtime->GetDuration())); attributes->Set("is_flexible", !downtime->GetFixed()); attributes->Set("is_in_effect", downtime->IsInEffect()); if (downtime->IsInEffect()) @@ -1239,7 +1239,7 @@ void RedisWriter::SendAddedDowntime(const Downtime::Ptr& downtime) "author", Utility::ValidateUTF8(downtime->GetAuthor()), "comment", Utility::ValidateUTF8(downtime->GetComment()), "is_flexible", Convert::ToString((unsigned short)!downtime->GetFixed()), - "duration", Convert::ToString(downtime->GetDuration()), + "flexible_duration", Convert::ToString(downtime->GetDuration()), "scheduled_start_time", Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime())), "scheduled_end_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())), "was_started", "0", @@ -1289,7 +1289,7 @@ void RedisWriter::SendStartedDowntime(const Downtime::Ptr& downtime) "author", Utility::ValidateUTF8(downtime->GetAuthor()), "comment", Utility::ValidateUTF8(downtime->GetComment()), "is_flexible", Convert::ToString((unsigned short)!downtime->GetFixed()), - "duration", Convert::ToString(downtime->GetDuration()), + "flexible_duration", Convert::ToString(downtime->GetDuration()), "scheduled_start_time", Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime())), "scheduled_end_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())), "was_started", "1", @@ -1337,7 +1337,7 @@ void RedisWriter::SendRemovedDowntime(const Downtime::Ptr& downtime) "author", Utility::ValidateUTF8(downtime->GetAuthor()), "comment", Utility::ValidateUTF8(downtime->GetComment()), "is_flexible", Convert::ToString((unsigned short)!downtime->GetFixed()), - "duration", Convert::ToString(downtime->GetDuration()), + "flexible_duration", Convert::ToString(downtime->GetDuration()), "scheduled_start_time", Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime())), "scheduled_end_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())), "was_started", "1", From e6043e908adffa41fe2cf5de48b0e586f914eebd Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 22 Oct 2019 15:10:00 +0200 Subject: [PATCH 189/219] RedisWriter: adjust icinga:history:stream:*:flapping schema --- lib/redis/rediswriter-objects.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 136992bdf..1f709f87d 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -1428,7 +1428,8 @@ void RedisWriter::SendFlappingChanged(const Checkable::Ptr& checkable, const Val "id", Utility::NewUniqueID(), "environment_id", SHA1(GetEnvironment()), service ? "service_id" : "host_id", GetObjectIdentifier(checkable), - value.ToBool() ? "start_time" : "end_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), + "change_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), + "change_type", value.ToBool() ? "start" : "end", "percent_state_change", Convert::ToString(checkable->GetFlappingCurrent()), "flapping_threshold_low", Convert::ToString(checkable->GetFlappingThresholdLow()), "flapping_threshold_high", Convert::ToString(checkable->GetFlappingThresholdHigh()), From f314489d230e56119fef5bb83914f0b1f8ee1f7e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 22 Oct 2019 18:14:09 +0200 Subject: [PATCH 190/219] Merge host and service histories --- lib/redis/rediswriter-objects.cpp | 155 +++++++++++++++++++++++++----- 1 file changed, 129 insertions(+), 26 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 1f709f87d..e51753272 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -1173,11 +1173,10 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, const CheckR auto output (SplitOutput(cr ? cr->GetOutput() : "")); - m_Rcon->FireAndForgetQuery({ - "XADD", service ? "icinga:history:stream:service:state" : "icinga:history:stream:host:state", "*", + std::vector xAdd ({ + "XADD", "icinga:history:stream:state", "*", "id", Utility::NewUniqueID(), "environment_id", SHA1(GetEnvironment()), - service ? "service_id" : "host_id", GetObjectIdentifier(checkable), "change_time", Convert::ToString(TimestampToMilliseconds(cr ? cr->GetExecutionEnd() : Utility::GetTime())), "state_type", Convert::ToString(type), "soft_state", Convert::ToString(cr ? cr->GetState() : 99), @@ -1192,6 +1191,20 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, const CheckR "event_id", Utility::NewUniqueID(), "event_type", "state" }); + + if (service) { + xAdd.emplace_back("object_type"); + xAdd.emplace_back("service"); + xAdd.emplace_back("service_id"); + xAdd.emplace_back(GetObjectIdentifier(checkable)); + } else { + xAdd.emplace_back("object_type"); + xAdd.emplace_back("host"); + xAdd.emplace_back("host_id"); + xAdd.emplace_back(GetObjectIdentifier(checkable)); + } + + m_Rcon->FireAndForgetQuery(std::move(xAdd)); } void RedisWriter::SendSentNotification( @@ -1205,11 +1218,10 @@ void RedisWriter::SendSentNotification( auto service (dynamic_pointer_cast(checkable)); auto output (SplitOutput(cr->GetOutput())); - m_Rcon->FireAndForgetQuery({ - "XADD", service ? "icinga:history:stream:service:notification" : "icinga:history:stream:host:notification", "*", + std::vector xAdd ({ + "XADD", "icinga:history:stream:notification", "*", "id", Utility::NewUniqueID(), "environment_id", SHA1(GetEnvironment()), - service ? "service_id" : "host_id", GetObjectIdentifier(checkable), "notification_id", GetObjectIdentifier(notification), "type", Convert::ToString(type), "send_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), @@ -1220,6 +1232,20 @@ void RedisWriter::SendSentNotification( "event_id", Utility::NewUniqueID(), "event_type", "notification" }); + + if (service) { + xAdd.emplace_back("object_type"); + xAdd.emplace_back("service"); + xAdd.emplace_back("service_id"); + xAdd.emplace_back(GetObjectIdentifier(checkable)); + } else { + xAdd.emplace_back("object_type"); + xAdd.emplace_back("host"); + xAdd.emplace_back("host_id"); + xAdd.emplace_back(GetObjectIdentifier(checkable)); + } + + m_Rcon->FireAndForgetQuery(std::move(xAdd)); } void RedisWriter::SendAddedDowntime(const Downtime::Ptr& downtime) @@ -1227,14 +1253,14 @@ void RedisWriter::SendAddedDowntime(const Downtime::Ptr& downtime) if (!m_Rcon || !m_Rcon->IsConnected()) return; - auto service (dynamic_pointer_cast(downtime->GetCheckable())); + auto checkable (downtime->GetCheckable()); + auto service (dynamic_pointer_cast(checkable)); auto triggeredBy (Downtime::GetByName(downtime->GetTriggeredBy())); std::vector xAdd ({ - "XADD", service ? "icinga:history:stream:service:downtime" : "icinga:history:stream:host:downtime", "*", + "XADD", "icinga:history:stream:downtime", "*", "downtime_id", GetObjectIdentifier(downtime), "environment_id", SHA1(GetEnvironment()), - service ? "service_id" : "host_id", GetObjectIdentifier(downtime->GetCheckable()), "entry_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEntryTime())), "author", Utility::ValidateUTF8(downtime->GetAuthor()), "comment", Utility::ValidateUTF8(downtime->GetComment()), @@ -1250,6 +1276,18 @@ void RedisWriter::SendAddedDowntime(const Downtime::Ptr& downtime) "event_type", "downtime_schedule" }); + if (service) { + xAdd.emplace_back("object_type"); + xAdd.emplace_back("service"); + xAdd.emplace_back("service_id"); + xAdd.emplace_back(GetObjectIdentifier(checkable)); + } else { + xAdd.emplace_back("object_type"); + xAdd.emplace_back("host"); + xAdd.emplace_back("host_id"); + xAdd.emplace_back(GetObjectIdentifier(checkable)); + } + if (triggeredBy) { xAdd.emplace_back("triggered_by_id"); xAdd.emplace_back(GetObjectIdentifier(triggeredBy)); @@ -1277,14 +1315,14 @@ void RedisWriter::SendStartedDowntime(const Downtime::Ptr& downtime) SendConfigUpdate(downtime, true); - auto service (dynamic_pointer_cast(downtime->GetCheckable())); + auto checkable (downtime->GetCheckable()); + auto service (dynamic_pointer_cast(checkable)); auto triggeredBy (Downtime::GetByName(downtime->GetTriggeredBy())); std::vector xAdd ({ - "XADD", service ? "icinga:history:stream:service:downtime" : "icinga:history:stream:host:downtime", "*", + "XADD", "icinga:history:stream:downtime", "*", "downtime_id", GetObjectIdentifier(downtime), "environment_id", SHA1(GetEnvironment()), - service ? "service_id" : "host_id", GetObjectIdentifier(downtime->GetCheckable()), "entry_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEntryTime())), "author", Utility::ValidateUTF8(downtime->GetAuthor()), "comment", Utility::ValidateUTF8(downtime->GetComment()), @@ -1300,6 +1338,18 @@ void RedisWriter::SendStartedDowntime(const Downtime::Ptr& downtime) "event_type", "downtime_start" }); + if (service) { + xAdd.emplace_back("object_type"); + xAdd.emplace_back("service"); + xAdd.emplace_back("service_id"); + xAdd.emplace_back(GetObjectIdentifier(checkable)); + } else { + xAdd.emplace_back("object_type"); + xAdd.emplace_back("host"); + xAdd.emplace_back("host_id"); + xAdd.emplace_back(GetObjectIdentifier(checkable)); + } + if (triggeredBy) { xAdd.emplace_back("triggered_by_id"); xAdd.emplace_back(GetObjectIdentifier(triggeredBy)); @@ -1325,14 +1375,14 @@ void RedisWriter::SendRemovedDowntime(const Downtime::Ptr& downtime) if (!m_Rcon || !m_Rcon->IsConnected()) return; - auto service (dynamic_pointer_cast(downtime->GetCheckable())); + auto checkable (downtime->GetCheckable()); + auto service (dynamic_pointer_cast(checkable)); auto triggeredBy (Downtime::GetByName(downtime->GetTriggeredBy())); std::vector xAdd ({ - "XADD", service ? "icinga:history:stream:service:downtime" : "icinga:history:stream:host:downtime", "*", + "XADD", "icinga:history:stream:downtime", "*", "downtime_id", GetObjectIdentifier(downtime), "environment_id", SHA1(GetEnvironment()), - service ? "service_id" : "host_id", GetObjectIdentifier(downtime->GetCheckable()), "entry_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEntryTime())), "author", Utility::ValidateUTF8(downtime->GetAuthor()), "comment", Utility::ValidateUTF8(downtime->GetComment()), @@ -1349,6 +1399,18 @@ void RedisWriter::SendRemovedDowntime(const Downtime::Ptr& downtime) "event_type", "downtime_end" }); + if (service) { + xAdd.emplace_back("object_type"); + xAdd.emplace_back("service"); + xAdd.emplace_back("service_id"); + xAdd.emplace_back(GetObjectIdentifier(checkable)); + } else { + xAdd.emplace_back("object_type"); + xAdd.emplace_back("host"); + xAdd.emplace_back("host_id"); + xAdd.emplace_back(GetObjectIdentifier(checkable)); + } + if (triggeredBy) { xAdd.emplace_back("triggered_by_id"); xAdd.emplace_back(GetObjectIdentifier(triggeredBy)); @@ -1374,13 +1436,13 @@ void RedisWriter::SendAddedComment(const Comment::Ptr& comment) if (!m_Rcon || !m_Rcon->IsConnected()) return; - auto service (dynamic_pointer_cast(comment->GetCheckable())); + auto checkable (comment->GetCheckable()); + auto service (dynamic_pointer_cast(checkable)); - m_Rcon->FireAndForgetQuery({ - "XADD", service ? "icinga:history:stream:service:comment" : "icinga:history:stream:host:comment", "*", + std::vector xAdd ({ + "XADD", "icinga:history:stream:comment", "*", "comment_id", GetObjectIdentifier(comment), "environment_id", SHA1(GetEnvironment()), - service ? "service_id" : "host_id", GetObjectIdentifier(comment->GetCheckable()), "entry_time", Convert::ToString(TimestampToMilliseconds(comment->GetEntryTime())), "author", Utility::ValidateUTF8(comment->GetAuthor()), "comment", Utility::ValidateUTF8(comment->GetText()), @@ -1390,6 +1452,20 @@ void RedisWriter::SendAddedComment(const Comment::Ptr& comment) "event_id", Utility::NewUniqueID(), "event_type", "comment_add" }); + + if (service) { + xAdd.emplace_back("object_type"); + xAdd.emplace_back("service"); + xAdd.emplace_back("service_id"); + xAdd.emplace_back(GetObjectIdentifier(checkable)); + } else { + xAdd.emplace_back("object_type"); + xAdd.emplace_back("host"); + xAdd.emplace_back("host_id"); + xAdd.emplace_back(GetObjectIdentifier(checkable)); + } + + m_Rcon->FireAndForgetQuery(std::move(xAdd)); } void RedisWriter::SendRemovedComment(const Comment::Ptr& comment) @@ -1397,13 +1473,13 @@ void RedisWriter::SendRemovedComment(const Comment::Ptr& comment) if (!m_Rcon || !m_Rcon->IsConnected()) return; - auto service (dynamic_pointer_cast(comment->GetCheckable())); + auto checkable (comment->GetCheckable()); + auto service (dynamic_pointer_cast(checkable)); - m_Rcon->FireAndForgetQuery({ - "XADD", service ? "icinga:history:stream:service:comment" : "icinga:history:stream:host:comment", "*", + std::vector xAdd ({ + "XADD", "icinga:history:stream:comment", "*", "comment_id", GetObjectIdentifier(comment), "environment_id", SHA1(GetEnvironment()), - service ? "service_id" : "host_id", GetObjectIdentifier(comment->GetCheckable()), "entry_time", Convert::ToString(TimestampToMilliseconds(comment->GetEntryTime())), "author", Utility::ValidateUTF8(comment->GetAuthor()), "comment", Utility::ValidateUTF8(comment->GetText()), @@ -1414,6 +1490,20 @@ void RedisWriter::SendRemovedComment(const Comment::Ptr& comment) "event_id", Utility::NewUniqueID(), "event_type", "comment_remove" }); + + if (service) { + xAdd.emplace_back("object_type"); + xAdd.emplace_back("service"); + xAdd.emplace_back("service_id"); + xAdd.emplace_back(GetObjectIdentifier(checkable)); + } else { + xAdd.emplace_back("object_type"); + xAdd.emplace_back("host"); + xAdd.emplace_back("host_id"); + xAdd.emplace_back(GetObjectIdentifier(checkable)); + } + + m_Rcon->FireAndForgetQuery(std::move(xAdd)); } void RedisWriter::SendFlappingChanged(const Checkable::Ptr& checkable, const Value& value) @@ -1423,11 +1513,10 @@ void RedisWriter::SendFlappingChanged(const Checkable::Ptr& checkable, const Val auto service (dynamic_pointer_cast(checkable)); - m_Rcon->FireAndForgetQuery({ - "XADD", service ? "icinga:history:stream:service:flapping" : "icinga:history:stream:host:flapping", "*", + std::vector xAdd ({ + "XADD", "icinga:history:stream:flapping", "*", "id", Utility::NewUniqueID(), "environment_id", SHA1(GetEnvironment()), - service ? "service_id" : "host_id", GetObjectIdentifier(checkable), "change_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), "change_type", value.ToBool() ? "start" : "end", "percent_state_change", Convert::ToString(checkable->GetFlappingCurrent()), @@ -1436,6 +1525,20 @@ void RedisWriter::SendFlappingChanged(const Checkable::Ptr& checkable, const Val "event_id", Utility::NewUniqueID(), "event_type", value.ToBool() ? "flapping_start" : "flapping_end" }); + + if (service) { + xAdd.emplace_back("object_type"); + xAdd.emplace_back("service"); + xAdd.emplace_back("service_id"); + xAdd.emplace_back(GetObjectIdentifier(checkable)); + } else { + xAdd.emplace_back("object_type"); + xAdd.emplace_back("host"); + xAdd.emplace_back("host_id"); + xAdd.emplace_back(GetObjectIdentifier(checkable)); + } + + m_Rcon->FireAndForgetQuery(std::move(xAdd)); } Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) From 8613365f1c8c96cf40ca4fe1c60c94dbdf37f017 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 16 Oct 2019 14:42:38 +0200 Subject: [PATCH 191/219] RedisWriter: add icinga:{state:stream:*,history:stream:*:state}#check_source --- lib/redis/rediswriter-objects.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index e51753272..d6396b9e1 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -1187,6 +1187,7 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, const CheckR "last_hard_state", Convert::ToString(service ? service->GetLastHardState() : host->GetLastHardState()), "output", Utility::ValidateUTF8(std::move(output.first)), "long_output", Utility::ValidateUTF8(std::move(output.second)), + "check_source", cr->GetCheckSource(), "max_check_attempts", Convert::ToString(checkable->GetMaxCheckAttempts()), "event_id", Utility::NewUniqueID(), "event_type", "state" @@ -1596,6 +1597,7 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) attrs->Set("commandline", FormatCommandLine(cr->GetCommand())); attrs->Set("execution_time", TimestampToMilliseconds(cr->CalculateExecutionTime())); attrs->Set("latency", TimestampToMilliseconds(cr->CalculateLatency())); + attrs->Set("check_source", cr->GetCheckSource()); } bool isProblem = !checkable->IsStateOK(checkable->GetStateRaw()); From 4a7a8cb6f8560bb21354cd9beab58f829e354aa1 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 23 Oct 2019 10:54:56 +0200 Subject: [PATCH 192/219] RedisWriter: add icinga:history:stream:*#endpoint_id --- lib/redis/rediswriter-objects.cpp | 56 +++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index d6396b9e1..86d05de30 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -1205,6 +1205,13 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, const CheckR xAdd.emplace_back(GetObjectIdentifier(checkable)); } + auto endpoint (Endpoint::GetLocalEndpoint()); + + if (endpoint) { + xAdd.emplace_back("endpoint_id"); + xAdd.emplace_back(GetObjectIdentifier(endpoint)); + } + m_Rcon->FireAndForgetQuery(std::move(xAdd)); } @@ -1246,6 +1253,13 @@ void RedisWriter::SendSentNotification( xAdd.emplace_back(GetObjectIdentifier(checkable)); } + auto endpoint (Endpoint::GetLocalEndpoint()); + + if (endpoint) { + xAdd.emplace_back("endpoint_id"); + xAdd.emplace_back(GetObjectIdentifier(endpoint)); + } + m_Rcon->FireAndForgetQuery(std::move(xAdd)); } @@ -1306,6 +1320,13 @@ void RedisWriter::SendAddedDowntime(const Downtime::Ptr& downtime) xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime() + downtime->GetDuration()))); } + auto endpoint (Endpoint::GetLocalEndpoint()); + + if (endpoint) { + xAdd.emplace_back("endpoint_id"); + xAdd.emplace_back(GetObjectIdentifier(endpoint)); + } + m_Rcon->FireAndForgetQuery(std::move(xAdd)); } @@ -1368,6 +1389,13 @@ void RedisWriter::SendStartedDowntime(const Downtime::Ptr& downtime) xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime() + downtime->GetDuration()))); } + auto endpoint (Endpoint::GetLocalEndpoint()); + + if (endpoint) { + xAdd.emplace_back("endpoint_id"); + xAdd.emplace_back(GetObjectIdentifier(endpoint)); + } + m_Rcon->FireAndForgetQuery(std::move(xAdd)); } @@ -1429,6 +1457,13 @@ void RedisWriter::SendRemovedDowntime(const Downtime::Ptr& downtime) xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime() + downtime->GetDuration()))); } + auto endpoint (Endpoint::GetLocalEndpoint()); + + if (endpoint) { + xAdd.emplace_back("endpoint_id"); + xAdd.emplace_back(GetObjectIdentifier(endpoint)); + } + m_Rcon->FireAndForgetQuery(std::move(xAdd)); } @@ -1466,6 +1501,13 @@ void RedisWriter::SendAddedComment(const Comment::Ptr& comment) xAdd.emplace_back(GetObjectIdentifier(checkable)); } + auto endpoint (Endpoint::GetLocalEndpoint()); + + if (endpoint) { + xAdd.emplace_back("endpoint_id"); + xAdd.emplace_back(GetObjectIdentifier(endpoint)); + } + m_Rcon->FireAndForgetQuery(std::move(xAdd)); } @@ -1504,6 +1546,13 @@ void RedisWriter::SendRemovedComment(const Comment::Ptr& comment) xAdd.emplace_back(GetObjectIdentifier(checkable)); } + auto endpoint (Endpoint::GetLocalEndpoint()); + + if (endpoint) { + xAdd.emplace_back("endpoint_id"); + xAdd.emplace_back(GetObjectIdentifier(endpoint)); + } + m_Rcon->FireAndForgetQuery(std::move(xAdd)); } @@ -1539,6 +1588,13 @@ void RedisWriter::SendFlappingChanged(const Checkable::Ptr& checkable, const Val xAdd.emplace_back(GetObjectIdentifier(checkable)); } + auto endpoint (Endpoint::GetLocalEndpoint()); + + if (endpoint) { + xAdd.emplace_back("endpoint_id"); + xAdd.emplace_back(GetObjectIdentifier(endpoint)); + } + m_Rcon->FireAndForgetQuery(std::move(xAdd)); } From 746a48e2ca4333cf1f1fccb8ecee1f1fddd60cfd Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 23 Oct 2019 15:22:54 +0200 Subject: [PATCH 193/219] RedisWriter: add icinga:history:stream:{state,notification}#previous_hard_state --- lib/icinga/checkable-check.cpp | 1 + lib/icinga/checkable.ti | 3 +++ lib/redis/rediswriter-objects.cpp | 14 ++++++++++++++ 3 files changed, 18 insertions(+) diff --git a/lib/icinga/checkable-check.cpp b/lib/icinga/checkable-check.cpp index 02bc8fc91..94940b29f 100644 --- a/lib/icinga/checkable-check.cpp +++ b/lib/icinga/checkable-check.cpp @@ -300,6 +300,7 @@ void Checkable::ProcessCheckResult(const CheckResult::Ptr& cr, const MessageOrig if (hardChange || is_volatile) { SetLastHardStateRaw(new_state); SetLastHardStateChange(now); + SetLastHardStatesRaw(GetLastHardStatesRaw() / 100u + new_state * 100u); } if (!IsStateOK(new_state)) diff --git a/lib/icinga/checkable.ti b/lib/icinga/checkable.ti index a920d2589..07a946258 100644 --- a/lib/icinga/checkable.ti +++ b/lib/icinga/checkable.ti @@ -105,6 +105,9 @@ abstract class Checkable : CustomVarObject [state, enum, no_user_view, no_user_modify] ServiceState last_hard_state_raw { default {{{ return ServiceUnknown; }}} }; + [state, no_user_view, no_user_modify] "unsigned short" last_hard_states_raw { + default {{{ return /* current */ 99 * 100 + /* previous */ 99; }}} + }; [state, enum] StateType last_state_type { default {{{ return StateTypeSoft; }}} }; diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 86d05de30..c4907fb32 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -1140,6 +1140,18 @@ void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) }); } +static +unsigned short GetPreviousHardState(const Checkable::Ptr& checkable, const Service::Ptr& service) +{ + auto phs (checkable->GetLastHardStatesRaw() % 100u); + + if (service) { + return phs; + } else { + return phs == 99 ? phs : Host::CalculateState(ServiceState(phs)); + } +} + void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type) { if (!m_Rcon || !m_Rcon->IsConnected()) @@ -1185,6 +1197,7 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, const CheckR // TODO: last_hard/soft_state should be "previous". "last_soft_state", Convert::ToString(cr ? cr->GetState() : 99), "last_hard_state", Convert::ToString(service ? service->GetLastHardState() : host->GetLastHardState()), + "previous_hard_state", Convert::ToString(GetPreviousHardState(checkable, service)), "output", Utility::ValidateUTF8(std::move(output.first)), "long_output", Utility::ValidateUTF8(std::move(output.second)), "check_source", cr->GetCheckSource(), @@ -1234,6 +1247,7 @@ void RedisWriter::SendSentNotification( "type", Convert::ToString(type), "send_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), "state", Convert::ToString(cr->GetState()), + "previous_hard_state", Convert::ToString(GetPreviousHardState(checkable, service)), "output", Utility::ValidateUTF8(std::move(output.first)), "long_output", Utility::ValidateUTF8(std::move(output.second)), "users_notified", Convert::ToString(users), From badcb5662ccc5f89526cd329d54ee768f46fd790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Aleksandrovi=C4=8D=20Klimov?= Date: Tue, 29 Oct 2019 16:20:16 +0100 Subject: [PATCH 194/219] RedisWriter: add icinga:state:stream:*#previous_hard_state --- lib/redis/rediswriter-objects.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index c4907fb32..db31205bf 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -1638,6 +1638,7 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) attrs->Set("severity", host->GetSeverity()); } + attrs->Set("previous_hard_state", Convert::ToString(GetPreviousHardState(checkable, service))); attrs->Set("check_attempt", checkable->GetCheckAttempt()); attrs->Set("is_active", checkable->IsActive()); From 617a4a2e92cc30e9ce02a5bcaa90d305f5bec9e6 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 23 Oct 2019 15:57:30 +0200 Subject: [PATCH 195/219] RedisWriter: drop icinga:history:stream:notification#{,long_}output and add #{author,text} --- lib/redis/rediswriter-objects.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index db31205bf..db0e2f5de 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -1237,7 +1237,6 @@ void RedisWriter::SendSentNotification( return; auto service (dynamic_pointer_cast(checkable)); - auto output (SplitOutput(cr->GetOutput())); std::vector xAdd ({ "XADD", "icinga:history:stream:notification", "*", @@ -1248,8 +1247,8 @@ void RedisWriter::SendSentNotification( "send_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), "state", Convert::ToString(cr->GetState()), "previous_hard_state", Convert::ToString(GetPreviousHardState(checkable, service)), - "output", Utility::ValidateUTF8(std::move(output.first)), - "long_output", Utility::ValidateUTF8(std::move(output.second)), + "author", Utility::ValidateUTF8(author), + "text", Utility::ValidateUTF8(text), "users_notified", Convert::ToString(users), "event_id", Utility::NewUniqueID(), "event_type", "notification" From c7e99972943cb6e7c97be5e9a3f9f96d80ce0829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Aleksandrovi=C4=8D=20Klimov?= Date: Tue, 29 Oct 2019 18:06:52 +0100 Subject: [PATCH 196/219] RedisWriter#SerializeState(): don't stringify previous_hard_state --- lib/redis/rediswriter-objects.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index db0e2f5de..6cb41a5e3 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -1637,7 +1637,7 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) attrs->Set("severity", host->GetSeverity()); } - attrs->Set("previous_hard_state", Convert::ToString(GetPreviousHardState(checkable, service))); + attrs->Set("previous_hard_state", GetPreviousHardState(checkable, service)); attrs->Set("check_attempt", checkable->GetCheckAttempt()); attrs->Set("is_active", checkable->IsActive()); From f51454af6561d8960ea32fbb1b11940c0ee0f253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Aleksandrovi=C4=8D=20Klimov?= Date: Tue, 29 Oct 2019 17:32:29 +0100 Subject: [PATCH 197/219] git ls-files -z |xargs -0 perl -pi -e 's/RedisWriter/IcingaDB/g' --- etc/icinga2/features-available/redis.conf | 2 +- lib/icinga/compatutility.cpp | 4 +- lib/redis/redisconnection.cpp | 26 ++--- lib/redis/rediswriter-objects.cpp | 120 +++++++++++----------- lib/redis/rediswriter-stats.cpp | 2 +- lib/redis/rediswriter-utility.cpp | 28 ++--- lib/redis/rediswriter.cpp | 66 ++++++------ lib/redis/rediswriter.hpp | 8 +- lib/redis/rediswriter.ti | 2 +- 9 files changed, 129 insertions(+), 129 deletions(-) diff --git a/etc/icinga2/features-available/redis.conf b/etc/icinga2/features-available/redis.conf index bcf781ad2..4be873ef3 100644 --- a/etc/icinga2/features-available/redis.conf +++ b/etc/icinga2/features-available/redis.conf @@ -7,7 +7,7 @@ library "redis" -object RedisWriter "redis" { +object IcingaDB "redis" { //host = "127.0.0.1" //port = 6379 //password = "xxx" diff --git a/lib/icinga/compatutility.cpp b/lib/icinga/compatutility.cpp index 5945ebfaf..40c01f397 100644 --- a/lib/icinga/compatutility.cpp +++ b/lib/icinga/compatutility.cpp @@ -249,7 +249,7 @@ std::set CompatUtility::GetCheckableNotificationUserGroups(const return usergroups; } -/* Used in DB IDO, StatusDataWriter, Livestatus, CompatLogger, GelfWriter, RedisWriter. */ +/* Used in DB IDO, StatusDataWriter, Livestatus, CompatLogger, GelfWriter, IcingaDB. */ String CompatUtility::GetCheckResultOutput(const CheckResult::Ptr& cr) { if (!cr) @@ -264,7 +264,7 @@ String CompatUtility::GetCheckResultOutput(const CheckResult::Ptr& cr) return raw_output.SubStr(0, line_end); } -/* Used in DB IDO, StatusDataWriter and Livestatus, RedisWriter. */ +/* Used in DB IDO, StatusDataWriter and Livestatus, IcingaDB. */ String CompatUtility::GetCheckResultLongOutput(const CheckResult::Ptr& cr) { if (!cr) diff --git a/lib/redis/redisconnection.cpp b/lib/redis/redisconnection.cpp index a3248c8c0..7445a2a8b 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/redis/redisconnection.cpp @@ -88,7 +88,7 @@ void LogQuery(RedisConnection::Query& query, Log& msg) void RedisConnection::FireAndForgetQuery(RedisConnection::Query query) { { - Log msg (LogNotice, "RedisWriter", "Firing and forgetting query:"); + Log msg (LogNotice, "IcingaDB", "Firing and forgetting query:"); LogQuery(query, msg); } @@ -103,7 +103,7 @@ void RedisConnection::FireAndForgetQuery(RedisConnection::Query query) void RedisConnection::FireAndForgetQueries(RedisConnection::Queries queries) { for (auto& query : queries) { - Log msg (LogNotice, "RedisWriter", "Firing and forgetting query:"); + Log msg (LogNotice, "IcingaDB", "Firing and forgetting query:"); LogQuery(query, msg); } @@ -118,7 +118,7 @@ void RedisConnection::FireAndForgetQueries(RedisConnection::Queries queries) RedisConnection::Reply RedisConnection::GetResultOfQuery(RedisConnection::Query query) { { - Log msg (LogNotice, "RedisWriter", "Executing query:"); + Log msg (LogNotice, "IcingaDB", "Executing query:"); LogQuery(query, msg); } @@ -139,7 +139,7 @@ RedisConnection::Reply RedisConnection::GetResultOfQuery(RedisConnection::Query RedisConnection::Replies RedisConnection::GetResultsOfQueries(RedisConnection::Queries queries) { for (auto& query : queries) { - Log msg (LogNotice, "RedisWriter", "Executing query:"); + Log msg (LogNotice, "IcingaDB", "Executing query:"); LogQuery(query, msg); } @@ -161,7 +161,7 @@ void RedisConnection::Connect(asio::yield_context& yc) { Defer notConnecting ([this]() { m_Connecting.store(m_Connected.load()); }); - Log(LogInformation, "RedisWriter", "Trying to connect to Redis server (async)"); + Log(LogInformation, "IcingaDB", "Trying to connect to Redis server (async)"); try { if (m_Path.IsEmpty()) { @@ -176,11 +176,11 @@ void RedisConnection::Connect(asio::yield_context& yc) m_Connected.store(true); - Log(LogInformation, "RedisWriter", "Connected to Redis server"); + Log(LogInformation, "IcingaDB", "Connected to Redis server"); } catch (const boost::coroutines::detail::forced_unwind&) { throw; } catch (const std::exception& ex) { - Log(LogCritical, "RedisWriter") + Log(LogCritical, "IcingaDB") << "Cannot connect to " << m_Host << ":" << m_Port << ": " << ex.what(); } } @@ -203,11 +203,11 @@ void RedisConnection::ReadLoop(asio::yield_context& yc) } catch (const boost::coroutines::detail::forced_unwind&) { throw; } catch (const std::exception& ex) { - Log(LogCritical, "RedisWriter") + Log(LogCritical, "IcingaDB") << "Error during receiving the response to a query which has been fired and forgotten: " << ex.what(); continue; } catch (...) { - Log(LogCritical, "RedisWriter") + Log(LogCritical, "IcingaDB") << "Error during receiving the response to a query which has been fired and forgotten"; continue; } @@ -276,12 +276,12 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) } catch (const boost::coroutines::detail::forced_unwind&) { throw; } catch (const std::exception& ex) { - Log msg (LogCritical, "RedisWriter", "Error during sending query"); + Log msg (LogCritical, "IcingaDB", "Error during sending query"); LogQuery(item, msg); msg << " which has been fired and forgotten: " << ex.what(); continue; } catch (...) { - Log msg (LogCritical, "RedisWriter", "Error during sending query"); + Log msg (LogCritical, "IcingaDB", "Error during sending query"); LogQuery(item, msg); msg << " which has been fired and forgotten"; continue; @@ -308,12 +308,12 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) } catch (const boost::coroutines::detail::forced_unwind&) { throw; } catch (const std::exception& ex) { - Log msg (LogCritical, "RedisWriter", "Error during sending query"); + Log msg (LogCritical, "IcingaDB", "Error during sending query"); LogQuery(item[i], msg); msg << " which has been fired and forgotten: " << ex.what(); continue; } catch (...) { - Log msg (LogCritical, "RedisWriter", "Error during sending query"); + Log msg (LogCritical, "IcingaDB", "Error during sending query"); LogQuery(item[i], msg); msg << " which has been fired and forgotten"; continue; diff --git a/lib/redis/rediswriter-objects.cpp b/lib/redis/rediswriter-objects.cpp index 6cb41a5e3..b6f9424f4 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/redis/rediswriter-objects.cpp @@ -65,49 +65,49 @@ return id )EOF"; -INITIALIZE_ONCE(&RedisWriter::ConfigStaticInitialize); +INITIALIZE_ONCE(&IcingaDB::ConfigStaticInitialize); -void RedisWriter::ConfigStaticInitialize() +void IcingaDB::ConfigStaticInitialize() { /* triggered in ProcessCheckResult(), requires UpdateNextCheck() to be called before */ Checkable::OnStateChange.connect([](const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, StateType type, const MessageOrigin::Ptr&) { - RedisWriter::StateChangeHandler(checkable, cr, type); + IcingaDB::StateChangeHandler(checkable, cr, type); }); /* triggered when acknowledged host/service goes back to ok and when the acknowledgement gets deleted */ Checkable::OnAcknowledgementCleared.connect([](const Checkable::Ptr& checkable, const MessageOrigin::Ptr&) { - RedisWriter::StateChangeHandler(checkable); + IcingaDB::StateChangeHandler(checkable); }); /* triggered on create, update and delete objects */ ConfigObject::OnActiveChanged.connect([](const ConfigObject::Ptr& object, const Value&) { - RedisWriter::VersionChangedHandler(object); + IcingaDB::VersionChangedHandler(object); }); ConfigObject::OnVersionChanged.connect([](const ConfigObject::Ptr& object, const Value&) { - RedisWriter::VersionChangedHandler(object); + IcingaDB::VersionChangedHandler(object); }); /* fixed/flexible downtime add */ - Downtime::OnDowntimeAdded.connect(&RedisWriter::DowntimeAddedHandler); + Downtime::OnDowntimeAdded.connect(&IcingaDB::DowntimeAddedHandler); /* fixed downtime start */ - Downtime::OnDowntimeStarted.connect(&RedisWriter::DowntimeStartedHandler); + Downtime::OnDowntimeStarted.connect(&IcingaDB::DowntimeStartedHandler); /* flexible downtime start */ - Downtime::OnDowntimeTriggered.connect(&RedisWriter::DowntimeStartedHandler); + Downtime::OnDowntimeTriggered.connect(&IcingaDB::DowntimeStartedHandler); /* fixed/flexible downtime end or remove */ - Downtime::OnDowntimeRemoved.connect(&RedisWriter::DowntimeRemovedHandler); + Downtime::OnDowntimeRemoved.connect(&IcingaDB::DowntimeRemovedHandler); Checkable::OnNotificationSentToAllUsers.connect([]( const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set& users, const NotificationType& type, const CheckResult::Ptr& cr, const String& author, const String& text, const MessageOrigin::Ptr& ) { - RedisWriter::NotificationSentToAllUsersHandler(notification, checkable, users, type, cr, author, text); + IcingaDB::NotificationSentToAllUsersHandler(notification, checkable, users, type, cr, author, text); }); - Comment::OnCommentAdded.connect(&RedisWriter::CommentAddedHandler); - Comment::OnCommentRemoved.connect(&RedisWriter::CommentRemovedHandler); + Comment::OnCommentAdded.connect(&IcingaDB::CommentAddedHandler); + Comment::OnCommentRemoved.connect(&IcingaDB::CommentRemovedHandler); - Checkable::OnFlappingChanged.connect(&RedisWriter::FlappingChangedHandler); + Checkable::OnFlappingChanged.connect(&IcingaDB::FlappingChangedHandler); } static std::pair SplitOutput(String output) @@ -123,13 +123,13 @@ static std::pair SplitOutput(String output) return {std::move(output), std::move(longOutput)}; } -void RedisWriter::UpdateAllConfigObjects() +void IcingaDB::UpdateAllConfigObjects() { double startTime = Utility::GetTime(); // Use a Workqueue to pack objects in parallel WorkQueue upq(25000, Configuration::Concurrency); - upq.SetName("RedisWriter:ConfigDump"); + upq.SetName("IcingaDB:ConfigDump"); typedef std::pair TypePair; std::vector types; @@ -162,7 +162,7 @@ void RedisWriter::UpdateAllConfigObjects() auto objectChunks (ChunkObjects(type.first->GetObjects(), 500)); WorkQueue upqObjectType(25000, Configuration::Concurrency); - upqObjectType.SetName("RedisWriter:ConfigDump:" + lcType); + upqObjectType.SetName("IcingaDB:ConfigDump:" + lcType); upqObjectType.ParallelFor(objectChunks, [this, &type, &lcType](decltype(objectChunks)::const_reference chunk) { std::map> hMSets, publishes; @@ -250,7 +250,7 @@ void RedisWriter::UpdateAllConfigObjects() m_Rcon->FireAndForgetQueries(std::move(transaction)); } - Log(LogNotice, "RedisWriter") + Log(LogNotice, "IcingaDB") << "Dumped " << bulkCounter << " objects of type " << type.second; }); @@ -276,7 +276,7 @@ void RedisWriter::UpdateAllConfigObjects() boost::rethrow_exception(exc); } } catch(const std::exception& e) { - Log(LogCritical, "RedisWriter") + Log(LogCritical, "IcingaDB") << "Exception during ConfigDump: " << e.what(); } } @@ -284,11 +284,11 @@ void RedisWriter::UpdateAllConfigObjects() m_Rcon->FireAndForgetQuery({"XADD", "icinga:dump", "*", "type", "*", "state", "done"}); - Log(LogInformation, "RedisWriter") + Log(LogInformation, "IcingaDB") << "Initial config/status dump finished in " << Utility::GetTime() - startTime << " seconds."; } -std::vector>> RedisWriter::ChunkObjects(std::vector> objects, size_t chunkSize) { +std::vector>> IcingaDB::ChunkObjects(std::vector> objects, size_t chunkSize) { std::vector>> chunks; auto offset (objects.begin()); auto end (objects.end()); @@ -308,7 +308,7 @@ std::vector>> RedisWriter::ChunkObjects( return std::move(chunks); } -void RedisWriter::DeleteKeys(const std::vector& keys) { +void IcingaDB::DeleteKeys(const std::vector& keys) { std::vector query = {"DEL"}; for (auto& key : keys) { query.emplace_back(key); @@ -317,7 +317,7 @@ void RedisWriter::DeleteKeys(const std::vector& keys) { m_Rcon->FireAndForgetQuery(std::move(query)); } -std::vector RedisWriter::GetTypeObjectKeys(const String& type) +std::vector IcingaDB::GetTypeObjectKeys(const String& type) { std::vector keys = { m_PrefixConfigObject + type, @@ -361,7 +361,7 @@ static ConfigObject::Ptr GetObjectByName(const String& name) return ConfigObject::GetObject(name); } -void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map>& hMSets, +void IcingaDB::InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map>& hMSets, std::map>& publishes, bool runtimeUpdate) { String objectKey = GetObjectIdentifier(object); @@ -780,7 +780,7 @@ void RedisWriter::InsertObjectDependencies(const ConfigObject::Ptr& object, cons } } -void RedisWriter::UpdateState(const Checkable::Ptr& checkable) +void IcingaDB::UpdateState(const Checkable::Ptr& checkable) { Dictionary::Ptr stateAttrs = SerializeState(checkable); @@ -788,7 +788,7 @@ void RedisWriter::UpdateState(const Checkable::Ptr& checkable) } // Used to update a single object, used for runtime updates -void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate) +void IcingaDB::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate) { if (!m_Rcon || !m_Rcon->IsConnected()) return; @@ -836,7 +836,7 @@ void RedisWriter::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtime // Takes object and collects IcingaDB relevant attributes and computes checksums. Returns whether the object is relevant // for IcingaDB. -bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& attributes, Dictionary::Ptr& checksums) +bool IcingaDB::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& attributes, Dictionary::Ptr& checksums) { attributes->Set("name_checksum", CalculateCheckSumString(object->GetName())); attributes->Set("environment_id", CalculateCheckSumString(GetEnvironment())); @@ -1093,7 +1093,7 @@ bool RedisWriter::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr * icinga:config:object:downtime) need to be prepended. There is nothing to indicate success or failure. */ void -RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, const String typeName, std::map>& hMSets, +IcingaDB::CreateConfigUpdate(const ConfigObject::Ptr& object, const String typeName, std::map>& hMSets, std::map>& publishes, bool runtimeUpdate) { /* TODO: This isn't essentially correct as we don't keep track of config objects ourselves. This would avoid duplicated config updates at startup. @@ -1128,7 +1128,7 @@ RedisWriter::CreateConfigUpdate(const ConfigObject::Ptr& object, const String ty } } -void RedisWriter::SendConfigDelete(const ConfigObject::Ptr& object) +void IcingaDB::SendConfigDelete(const ConfigObject::Ptr& object) { String typeName = object->GetReflectionType()->GetName().ToLower(); String objectKey = GetObjectIdentifier(object); @@ -1152,7 +1152,7 @@ unsigned short GetPreviousHardState(const Checkable::Ptr& checkable, const Servi } } -void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type) +void IcingaDB::SendStatusUpdate(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type) { if (!m_Rcon || !m_Rcon->IsConnected()) return; @@ -1228,7 +1228,7 @@ void RedisWriter::SendStatusUpdate(const ConfigObject::Ptr& object, const CheckR m_Rcon->FireAndForgetQuery(std::move(xAdd)); } -void RedisWriter::SendSentNotification( +void IcingaDB::SendSentNotification( const Notification::Ptr& notification, const Checkable::Ptr& checkable, size_t users, NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text ) @@ -1276,7 +1276,7 @@ void RedisWriter::SendSentNotification( m_Rcon->FireAndForgetQuery(std::move(xAdd)); } -void RedisWriter::SendAddedDowntime(const Downtime::Ptr& downtime) +void IcingaDB::SendAddedDowntime(const Downtime::Ptr& downtime) { if (!m_Rcon || !m_Rcon->IsConnected()) return; @@ -1343,7 +1343,7 @@ void RedisWriter::SendAddedDowntime(const Downtime::Ptr& downtime) m_Rcon->FireAndForgetQuery(std::move(xAdd)); } -void RedisWriter::SendStartedDowntime(const Downtime::Ptr& downtime) +void IcingaDB::SendStartedDowntime(const Downtime::Ptr& downtime) { if (!m_Rcon || !m_Rcon->IsConnected()) return; @@ -1412,7 +1412,7 @@ void RedisWriter::SendStartedDowntime(const Downtime::Ptr& downtime) m_Rcon->FireAndForgetQuery(std::move(xAdd)); } -void RedisWriter::SendRemovedDowntime(const Downtime::Ptr& downtime) +void IcingaDB::SendRemovedDowntime(const Downtime::Ptr& downtime) { if (!m_Rcon || !m_Rcon->IsConnected()) return; @@ -1480,7 +1480,7 @@ void RedisWriter::SendRemovedDowntime(const Downtime::Ptr& downtime) m_Rcon->FireAndForgetQuery(std::move(xAdd)); } -void RedisWriter::SendAddedComment(const Comment::Ptr& comment) +void IcingaDB::SendAddedComment(const Comment::Ptr& comment) { if (!m_Rcon || !m_Rcon->IsConnected()) return; @@ -1524,7 +1524,7 @@ void RedisWriter::SendAddedComment(const Comment::Ptr& comment) m_Rcon->FireAndForgetQuery(std::move(xAdd)); } -void RedisWriter::SendRemovedComment(const Comment::Ptr& comment) +void IcingaDB::SendRemovedComment(const Comment::Ptr& comment) { if (!m_Rcon || !m_Rcon->IsConnected()) return; @@ -1569,7 +1569,7 @@ void RedisWriter::SendRemovedComment(const Comment::Ptr& comment) m_Rcon->FireAndForgetQuery(std::move(xAdd)); } -void RedisWriter::SendFlappingChanged(const Checkable::Ptr& checkable, const Value& value) +void IcingaDB::SendFlappingChanged(const Checkable::Ptr& checkable, const Value& value) { if (!m_Rcon || !m_Rcon->IsConnected()) return; @@ -1611,7 +1611,7 @@ void RedisWriter::SendFlappingChanged(const Checkable::Ptr& checkable, const Val m_Rcon->FireAndForgetQuery(std::move(xAdd)); } -Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) +Dictionary::Ptr IcingaDB::SerializeState(const Checkable::Ptr& checkable) { Dictionary::Ptr attrs = new Dictionary(); @@ -1709,7 +1709,7 @@ Dictionary::Ptr RedisWriter::SerializeState(const Checkable::Ptr& checkable) } std::vector -RedisWriter::UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, +IcingaDB::UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, const String& typeNameOverride) { Type::Ptr type = object->GetReflectionType(); @@ -1751,73 +1751,73 @@ RedisWriter::UpdateObjectAttrs(const ConfigObject::Ptr& object, int fieldType, //m_Rcon->FireAndForgetQuery({"HSET", keyPrefix + typeName, GetObjectIdentifier(object), JsonEncode(attrs)}); } -void RedisWriter::StateChangeHandler(const ConfigObject::Ptr& object) +void IcingaDB::StateChangeHandler(const ConfigObject::Ptr& object) { auto checkable (dynamic_pointer_cast(object)); if (checkable) { - RedisWriter::StateChangeHandler(object, checkable->GetLastCheckResult(), checkable->GetStateType()); + IcingaDB::StateChangeHandler(object, checkable->GetLastCheckResult(), checkable->GetStateType()); } } -void RedisWriter::StateChangeHandler(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type) +void IcingaDB::StateChangeHandler(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type) { - for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { + for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType()) { rw->m_WorkQueue.Enqueue([rw, object, cr, type]() { rw->SendStatusUpdate(object, cr, type); }); } } -void RedisWriter::VersionChangedHandler(const ConfigObject::Ptr& object) +void IcingaDB::VersionChangedHandler(const ConfigObject::Ptr& object) { Type::Ptr type = object->GetReflectionType(); if (object->IsActive()) { // Create or update the object config - for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { + for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType()) { if (rw) rw->m_WorkQueue.Enqueue([rw, object]() { rw->SendConfigUpdate(object, true); }); } } else if (!object->IsActive() && object->GetExtension("ConfigObjectDeleted")) { // same as in apilistener-configsync.cpp // Delete object config - for (const RedisWriter::Ptr& rw : ConfigType::GetObjectsByType()) { + for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType()) { if (rw) rw->m_WorkQueue.Enqueue([rw, object]() { rw->SendConfigDelete(object); }); } } } -void RedisWriter::DowntimeAddedHandler(const Downtime::Ptr& downtime) +void IcingaDB::DowntimeAddedHandler(const Downtime::Ptr& downtime) { - for (auto& rw : ConfigType::GetObjectsByType()) { + for (auto& rw : ConfigType::GetObjectsByType()) { rw->m_WorkQueue.Enqueue([rw, downtime]() { rw->SendAddedDowntime(downtime); }); } } -void RedisWriter::DowntimeStartedHandler(const Downtime::Ptr& downtime) +void IcingaDB::DowntimeStartedHandler(const Downtime::Ptr& downtime) { StateChangeHandler(downtime->GetCheckable()); - for (auto& rw : ConfigType::GetObjectsByType()) { + for (auto& rw : ConfigType::GetObjectsByType()) { rw->m_WorkQueue.Enqueue([rw, downtime]() { rw->SendStartedDowntime(downtime); }); } } -void RedisWriter::DowntimeRemovedHandler(const Downtime::Ptr& downtime) +void IcingaDB::DowntimeRemovedHandler(const Downtime::Ptr& downtime) { StateChangeHandler(downtime->GetCheckable()); - for (auto& rw : ConfigType::GetObjectsByType()) { + for (auto& rw : ConfigType::GetObjectsByType()) { rw->m_WorkQueue.Enqueue([rw, downtime]() { rw->SendRemovedDowntime(downtime); }); } } -void RedisWriter::NotificationSentToAllUsersHandler( +void IcingaDB::NotificationSentToAllUsersHandler( const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set& users, NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text ) { - auto rws (ConfigType::GetObjectsByType()); + auto rws (ConfigType::GetObjectsByType()); if (!rws.empty()) { auto usersAmount (users.size()); @@ -1831,23 +1831,23 @@ void RedisWriter::NotificationSentToAllUsersHandler( } } -void RedisWriter::CommentAddedHandler(const Comment::Ptr& comment) +void IcingaDB::CommentAddedHandler(const Comment::Ptr& comment) { - for (auto& rw : ConfigType::GetObjectsByType()) { + for (auto& rw : ConfigType::GetObjectsByType()) { rw->m_WorkQueue.Enqueue([rw, comment]() { rw->SendAddedComment(comment); }); } } -void RedisWriter::CommentRemovedHandler(const Comment::Ptr& comment) +void IcingaDB::CommentRemovedHandler(const Comment::Ptr& comment) { - for (auto& rw : ConfigType::GetObjectsByType()) { + for (auto& rw : ConfigType::GetObjectsByType()) { rw->m_WorkQueue.Enqueue([rw, comment]() { rw->SendRemovedComment(comment); }); } } -void RedisWriter::FlappingChangedHandler(const Checkable::Ptr& checkable, const Value& value) +void IcingaDB::FlappingChangedHandler(const Checkable::Ptr& checkable, const Value& value) { - for (auto& rw : ConfigType::GetObjectsByType()) { + for (auto& rw : ConfigType::GetObjectsByType()) { rw->m_WorkQueue.Enqueue([rw, checkable, value]() { rw->SendFlappingChanged(checkable, value); }); } } diff --git a/lib/redis/rediswriter-stats.cpp b/lib/redis/rediswriter-stats.cpp index 8ed024cf2..d007aeb66 100644 --- a/lib/redis/rediswriter-stats.cpp +++ b/lib/redis/rediswriter-stats.cpp @@ -26,7 +26,7 @@ using namespace icinga; -Dictionary::Ptr RedisWriter::GetStats() +Dictionary::Ptr IcingaDB::GetStats() { Dictionary::Ptr stats = new Dictionary(); diff --git a/lib/redis/rediswriter-utility.cpp b/lib/redis/rediswriter-utility.cpp index f2a4a5281..0a687f772 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/redis/rediswriter-utility.cpp @@ -41,7 +41,7 @@ using namespace icinga; -String RedisWriter::FormatCheckSumBinary(const String& str) +String IcingaDB::FormatCheckSumBinary(const String& str) { char output[20*2+1]; for (int i = 0; i < 20; i++) @@ -50,7 +50,7 @@ String RedisWriter::FormatCheckSumBinary(const String& str) return output; } -String RedisWriter::FormatCommandLine(const Value& commandLine) +String IcingaDB::FormatCommandLine(const Value& commandLine) { String result; if (commandLine.IsObjectType()) { @@ -77,12 +77,12 @@ String RedisWriter::FormatCommandLine(const Value& commandLine) return result; } -String RedisWriter::GetEnvironment() +String IcingaDB::GetEnvironment() { return ConfigType::GetObjectsByType()[0]->GetEnvironment(); } -String RedisWriter::GetObjectIdentifier(const ConfigObject::Ptr& object) +String IcingaDB::GetObjectIdentifier(const ConfigObject::Ptr& object) { Type::Ptr type = object->GetReflectionType(); @@ -92,12 +92,12 @@ String RedisWriter::GetObjectIdentifier(const ConfigObject::Ptr& object) return HashValue((Array::Ptr)new Array({GetEnvironment(), object->GetName()})); } -String RedisWriter::CalculateCheckSumString(const String& str) +String IcingaDB::CalculateCheckSumString(const String& str) { return SHA1(str); } -String RedisWriter::CalculateCheckSumArray(const Array::Ptr& arr) +String IcingaDB::CalculateCheckSumArray(const Array::Ptr& arr) { /* Ensure that checksums happen in a defined order. */ Array::Ptr tmpArr = arr->ShallowClone(); @@ -107,7 +107,7 @@ String RedisWriter::CalculateCheckSumArray(const Array::Ptr& arr) return SHA1(PackObject(tmpArr)); } -String RedisWriter::CalculateCheckSumProperties(const ConfigObject::Ptr& object, const std::set& propertiesBlacklist) +String IcingaDB::CalculateCheckSumProperties(const ConfigObject::Ptr& object, const std::set& propertiesBlacklist) { //TODO: consider precision of 6 for double values; use specific config fields for hashing? return HashValue(object, propertiesBlacklist); @@ -115,12 +115,12 @@ String RedisWriter::CalculateCheckSumProperties(const ConfigObject::Ptr& object, static const std::set metadataWhitelist ({"package", "source_location", "templates"}); -String RedisWriter::CalculateCheckSumMetadata(const ConfigObject::Ptr& object) +String IcingaDB::CalculateCheckSumMetadata(const ConfigObject::Ptr& object) { return HashValue(object, metadataWhitelist, true); } -String RedisWriter::CalculateCheckSumVars(const CustomVarObject::Ptr& object) +String IcingaDB::CalculateCheckSumVars(const CustomVarObject::Ptr& object) { Dictionary::Ptr vars = object->GetVars(); @@ -169,7 +169,7 @@ String RedisWriter::CalculateCheckSumVars(const CustomVarObject::Ptr& object) * * @return JSON-like data structure for Redis */ -Dictionary::Ptr RedisWriter::SerializeVars(const CustomVarObject::Ptr& object) +Dictionary::Ptr IcingaDB::SerializeVars(const CustomVarObject::Ptr& object) { Dictionary::Ptr vars = object->GetVars(); @@ -199,12 +199,12 @@ Dictionary::Ptr RedisWriter::SerializeVars(const CustomVarObject::Ptr& object) static const std::set propertiesBlacklistEmpty; -String RedisWriter::HashValue(const Value& value) +String IcingaDB::HashValue(const Value& value) { return HashValue(value, propertiesBlacklistEmpty); } -String RedisWriter::HashValue(const Value& value, const std::set& propertiesBlacklist, bool propertiesWhitelist) +String IcingaDB::HashValue(const Value& value, const std::set& propertiesBlacklist, bool propertiesWhitelist) { Value temp; bool mutabl; @@ -252,11 +252,11 @@ String RedisWriter::HashValue(const Value& value, const std::set& proper return SHA1(PackObject(temp)); } -String RedisWriter::GetLowerCaseTypeNameDB(const ConfigObject::Ptr& obj) +String IcingaDB::GetLowerCaseTypeNameDB(const ConfigObject::Ptr& obj) { return obj->GetReflectionType()->GetName().ToLower(); } -long long RedisWriter::TimestampToMilliseconds(double timestamp) { +long long IcingaDB::TimestampToMilliseconds(double timestamp) { return static_cast(timestamp * 1000); } diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp index eaa2609a9..6b3677cdd 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/redis/rediswriter.cpp @@ -35,14 +35,14 @@ using namespace icinga; //TODO Make configurable and figure out a sane default #define MAX_EVENTS_DEFAULT 5000 -REGISTER_TYPE(RedisWriter); +REGISTER_TYPE(IcingaDB); -RedisWriter::RedisWriter() +IcingaDB::IcingaDB() : m_Rcon(nullptr) { m_Rcon = nullptr; - m_WorkQueue.SetName("RedisWriter"); + m_WorkQueue.SetName("IcingaDB"); m_PrefixConfigObject = "icinga:config:"; m_PrefixConfigCheckSum = "icinga:checksum:"; @@ -52,11 +52,11 @@ RedisWriter::RedisWriter() /** * Starts the component. */ -void RedisWriter::Start(bool runtimeCreated) +void IcingaDB::Start(bool runtimeCreated) { - ObjectImpl::Start(runtimeCreated); + ObjectImpl::Start(runtimeCreated); - Log(LogInformation, "RedisWriter") + Log(LogInformation, "IcingaDB") << "'" << GetName() << "' started."; m_ConfigDumpInProgress = false; @@ -83,27 +83,27 @@ void RedisWriter::Start(bool runtimeCreated) m_StatsTimer->OnTimerExpired.connect([this](const Timer * const&) { PublishStatsTimerHandler(); }); m_StatsTimer->Start(); - m_WorkQueue.SetName("RedisWriter"); + m_WorkQueue.SetName("IcingaDB"); - boost::thread thread(&RedisWriter::HandleEvents, this); + boost::thread thread(&IcingaDB::HandleEvents, this); thread.detach(); } -void RedisWriter::ExceptionHandler(boost::exception_ptr exp) +void IcingaDB::ExceptionHandler(boost::exception_ptr exp) { - Log(LogCritical, "RedisWriter", "Exception during redis query. Verify that Redis is operational."); + Log(LogCritical, "IcingaDB", "Exception during redis query. Verify that Redis is operational."); - Log(LogDebug, "RedisWriter") + Log(LogDebug, "IcingaDB") << "Exception during redis operation: " << DiagnosticInformation(exp); } -void RedisWriter::ReconnectTimerHandler() +void IcingaDB::ReconnectTimerHandler() { m_WorkQueue.Enqueue([this]() { TryToReconnect(); }); } -void RedisWriter::TryToReconnect() +void IcingaDB::TryToReconnect() { AssertOnWorkQueue(); @@ -131,16 +131,16 @@ void RedisWriter::TryToReconnect() m_ConfigDumpInProgress = false; } -void RedisWriter::UpdateSubscriptionsTimerHandler() +void IcingaDB::UpdateSubscriptionsTimerHandler() { m_WorkQueue.Enqueue([this]() { UpdateSubscriptions(); }); } -void RedisWriter::UpdateSubscriptions() +void IcingaDB::UpdateSubscriptions() { AssertOnWorkQueue(); - Log(LogInformation, "RedisWriter", "Updating Redis subscriptions"); + Log(LogInformation, "IcingaDB", "Updating Redis subscriptions"); /* TODO: * Silently return in this case. Usually the RedisConnection checks for connectivity and logs in failure case. @@ -168,8 +168,8 @@ void RedisWriter::UpdateSubscriptions() RedisSubscriptionInfo rsi; - if (!RedisWriter::GetSubscriptionTypes(key, rsi)) { - Log(LogInformation, "RedisWriter") + if (!IcingaDB::GetSubscriptionTypes(key, rsi)) { + Log(LogInformation, "IcingaDB") << "Subscription \"" << key << "\" has no types listed."; } else { m_Subscriptions[key.SubStr(keyPrefix.GetLength())] = rsi; @@ -177,11 +177,11 @@ void RedisWriter::UpdateSubscriptions() } } while (cursor != "0"); - Log(LogInformation, "RedisWriter") + Log(LogInformation, "IcingaDB") << "Current Redis event subscriptions: " << m_Subscriptions.size(); } -bool RedisWriter::GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi) +bool IcingaDB::GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi) { try { Array::Ptr redisReply = m_Rcon->GetResultOfQuery({ "SMEMBERS", key }); @@ -197,11 +197,11 @@ bool RedisWriter::GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi) } } - Log(LogInformation, "RedisWriter") + Log(LogInformation, "IcingaDB") << "Subscriber Info - Key: " << key << " Value: " << Value(Array::FromSet(rsi.EventTypes)); } catch (const std::exception& ex) { - Log(LogWarning, "RedisWriter") + Log(LogWarning, "IcingaDB") << "Invalid Redis subscriber info for subscriber '" << key << "': " << DiagnosticInformation(ex); return false; @@ -210,12 +210,12 @@ bool RedisWriter::GetSubscriptionTypes(String key, RedisSubscriptionInfo& rsi) return true; } -void RedisWriter::PublishStatsTimerHandler(void) +void IcingaDB::PublishStatsTimerHandler(void) { m_WorkQueue.Enqueue([this]() { PublishStats(); }); } -void RedisWriter::PublishStats() +void IcingaDB::PublishStats() { AssertOnWorkQueue(); @@ -229,7 +229,7 @@ void RedisWriter::PublishStats() m_Rcon->FireAndForgetQuery({ "PUBLISH", "icinga:stats", jsonStats }); } -void RedisWriter::HandleEvents() +void IcingaDB::HandleEvents() { String queueName = Utility::NewUniqueID(); EventQueue::Ptr queue = new EventQueue(queueName); @@ -265,7 +265,7 @@ void RedisWriter::HandleEvents() EventQueue::UnregisterIfUnused(queueName, queue); } -void RedisWriter::HandleEvent(const Dictionary::Ptr& event) +void IcingaDB::HandleEvent(const Dictionary::Ptr& event) { AssertOnWorkQueue(); @@ -284,7 +284,7 @@ void RedisWriter::HandleEvent(const Dictionary::Ptr& event) if (maxExists != 0) { String redisReply = m_Rcon->GetResultOfQuery({ "GET", "icinga:subscription:" + name + ":limit"}); - Log(LogInformation, "RedisWriter") + Log(LogInformation, "IcingaDB") << "Got limit " << redisReply << " for " << name; maxEvents = Convert::ToLong(redisReply); @@ -298,7 +298,7 @@ void RedisWriter::HandleEvent(const Dictionary::Ptr& event) } } -void RedisWriter::SendEvent(const Dictionary::Ptr& event) +void IcingaDB::SendEvent(const Dictionary::Ptr& event) { AssertOnWorkQueue(); @@ -347,7 +347,7 @@ void RedisWriter::SendEvent(const Dictionary::Ptr& event) String body = JsonEncode(event); -// Log(LogInformation, "RedisWriter") +// Log(LogInformation, "IcingaDB") // << "Sending event \"" << body << "\""; m_Rcon->FireAndForgetQueries({ @@ -355,15 +355,15 @@ void RedisWriter::SendEvent(const Dictionary::Ptr& event) { "PUBLISH", "icinga:event:" + event->Get("type"), body }}); } -void RedisWriter::Stop(bool runtimeRemoved) +void IcingaDB::Stop(bool runtimeRemoved) { - Log(LogInformation, "RedisWriter") + Log(LogInformation, "IcingaDB") << "'" << GetName() << "' stopped."; - ObjectImpl::Stop(runtimeRemoved); + ObjectImpl::Stop(runtimeRemoved); } -void RedisWriter::AssertOnWorkQueue() +void IcingaDB::AssertOnWorkQueue() { ASSERT(m_WorkQueue.IsWorkerThread()); } diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp index 666f9b1b7..597d2fcc6 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/redis/rediswriter.hpp @@ -42,13 +42,13 @@ struct RedisSubscriptionInfo /** * @ingroup redis */ -class RedisWriter : public ObjectImpl +class IcingaDB : public ObjectImpl { public: - DECLARE_OBJECT(RedisWriter); - DECLARE_OBJECTNAME(RedisWriter); + DECLARE_OBJECT(IcingaDB); + DECLARE_OBJECTNAME(IcingaDB); - RedisWriter(); + IcingaDB(); static void ConfigStaticInitialize(); diff --git a/lib/redis/rediswriter.ti b/lib/redis/rediswriter.ti index db3df0ab5..07572eba2 100644 --- a/lib/redis/rediswriter.ti +++ b/lib/redis/rediswriter.ti @@ -24,7 +24,7 @@ library redis; namespace icinga { -class RedisWriter : ConfigObject +class IcingaDB : ConfigObject { [config] String host { default {{{ return "127.0.0.1"; }}} From 13ab7eb60922da8f7a9c5286352fa38a1607aa01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Aleksandrovi=C4=8D=20Klimov?= Date: Tue, 29 Oct 2019 18:36:16 +0100 Subject: [PATCH 198/219] Rename redis to icingadb --- etc/icinga2/features-available/icingadb.conf | 5 +++++ etc/icinga2/features-available/redis.conf | 14 -------------- icinga-app/CMakeLists.txt | 2 +- lib/CMakeLists.txt | 2 +- lib/{redis => icingadb}/CMakeLists.txt | 16 ++++++++-------- .../icingadb-objects.cpp} | 4 ++-- .../icingadb-stats.cpp} | 2 +- .../icingadb-utility.cpp} | 2 +- .../rediswriter.cpp => icingadb/icingadb.cpp} | 6 +++--- .../rediswriter.hpp => icingadb/icingadb.hpp} | 6 +++--- .../rediswriter.ti => icingadb/icingadb.ti} | 0 lib/{redis => icingadb}/redisconnection.cpp | 2 +- lib/{redis => icingadb}/redisconnection.hpp | 2 +- 13 files changed, 27 insertions(+), 36 deletions(-) create mode 100644 etc/icinga2/features-available/icingadb.conf delete mode 100644 etc/icinga2/features-available/redis.conf rename lib/{redis => icingadb}/CMakeLists.txt (71%) rename lib/{redis/rediswriter-objects.cpp => icingadb/icingadb-objects.cpp} (99%) rename lib/{redis/rediswriter-stats.cpp => icingadb/icingadb-stats.cpp} (98%) rename lib/{redis/rediswriter-utility.cpp => icingadb/icingadb-utility.cpp} (99%) rename lib/{redis/rediswriter.cpp => icingadb/icingadb.cpp} (98%) rename lib/{redis/rediswriter.hpp => icingadb/icingadb.hpp} (98%) rename lib/{redis/rediswriter.ti => icingadb/icingadb.ti} (100%) rename lib/{redis => icingadb}/redisconnection.cpp (99%) rename lib/{redis => icingadb}/redisconnection.hpp (99%) diff --git a/etc/icinga2/features-available/icingadb.conf b/etc/icinga2/features-available/icingadb.conf new file mode 100644 index 000000000..9fe55ec8e --- /dev/null +++ b/etc/icinga2/features-available/icingadb.conf @@ -0,0 +1,5 @@ +object IcingaDB "icingadb" { + //host = "127.0.0.1" + //port = 6379 + //password = "xxx" +} diff --git a/etc/icinga2/features-available/redis.conf b/etc/icinga2/features-available/redis.conf deleted file mode 100644 index 4be873ef3..000000000 --- a/etc/icinga2/features-available/redis.conf +++ /dev/null @@ -1,14 +0,0 @@ -/** - * The redis library implements functionality for putting Icinga - * event data into a redis database. - * - * NOTE: This is experimental and may change without further notice. - */ - -library "redis" - -object IcingaDB "redis" { - //host = "127.0.0.1" - //port = 6379 - //password = "xxx" -} diff --git a/icinga-app/CMakeLists.txt b/icinga-app/CMakeLists.txt index 0ba780316..db8daae30 100644 --- a/icinga-app/CMakeLists.txt +++ b/icinga-app/CMakeLists.txt @@ -54,7 +54,7 @@ if(ICINGA2_WITH_PERFDATA) endif() if(ICINGA2_WITH_REDIS) - list(APPEND icinga_app_SOURCES $) + list(APPEND icinga_app_SOURCES $) endif() add_executable(icinga-app diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index a00cf4db8..c3baa2755 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -54,7 +54,7 @@ if(ICINGA2_WITH_PERFDATA) endif() if(ICINGA2_WITH_REDIS) - add_subdirectory(redis) + add_subdirectory(icingadb) endif() set(CPACK_NSIS_EXTRA_INSTALL_COMMANDS "${CPACK_NSIS_EXTRA_INSTALL_COMMANDS}" PARENT_SCOPE) diff --git a/lib/redis/CMakeLists.txt b/lib/icingadb/CMakeLists.txt similarity index 71% rename from lib/redis/CMakeLists.txt rename to lib/icingadb/CMakeLists.txt index bb00ca343..a2ece0054 100644 --- a/lib/redis/CMakeLists.txt +++ b/lib/icingadb/CMakeLists.txt @@ -15,29 +15,29 @@ # along with this program; if not, write to the Free Software Foundation # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. -mkclass_target(rediswriter.ti rediswriter-ti.cpp rediswriter-ti.hpp) +mkclass_target(icingadb.ti icingadb-ti.cpp icingadb-ti.hpp) -set(redis_SOURCES - rediswriter.cpp rediswriter-objects.cpp rediswriter-stats.cpp rediswriter-utility.cpp redisconnection.cpp rediswriter-ti.hpp +set(icingadb_SOURCES + icingadb.cpp icingadb-objects.cpp icingadb-stats.cpp icingadb-utility.cpp redisconnection.cpp icingadb-ti.hpp ) if(ICINGA2_UNITY_BUILD) - mkunity_target(redis redis redis_SOURCES) + mkunity_target(icingadb icingadb icingadb_SOURCES) endif() -add_library(redis OBJECT ${redis_SOURCES}) +add_library(icingadb OBJECT ${icingadb_SOURCES}) include_directories(${icinga2_SOURCE_DIR}/third-party) -add_dependencies(redis base config icinga remote) +add_dependencies(icingadb base config icinga remote) set_target_properties ( - redis PROPERTIES + icingadb PROPERTIES FOLDER Components ) install_if_not_exists( - ${PROJECT_SOURCE_DIR}/etc/icinga2/features-available/redis.conf + ${PROJECT_SOURCE_DIR}/etc/icinga2/features-available/icingadb.conf ${CMAKE_INSTALL_SYSCONFDIR}/icinga2/features-available ) diff --git a/lib/redis/rediswriter-objects.cpp b/lib/icingadb/icingadb-objects.cpp similarity index 99% rename from lib/redis/rediswriter-objects.cpp rename to lib/icingadb/icingadb-objects.cpp index b6f9424f4..e7ddc90da 100644 --- a/lib/redis/rediswriter-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -17,8 +17,8 @@ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * ******************************************************************************/ -#include "redis/rediswriter.hpp" -#include "redis/redisconnection.hpp" +#include "icingadb/icingadb.hpp" +#include "icingadb/redisconnection.hpp" #include "icinga/command.hpp" #include "icinga/compatutility.hpp" #include "base/configtype.hpp" diff --git a/lib/redis/rediswriter-stats.cpp b/lib/icingadb/icingadb-stats.cpp similarity index 98% rename from lib/redis/rediswriter-stats.cpp rename to lib/icingadb/icingadb-stats.cpp index d007aeb66..ea1dcecf0 100644 --- a/lib/redis/rediswriter-stats.cpp +++ b/lib/icingadb/icingadb-stats.cpp @@ -17,7 +17,7 @@ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * ******************************************************************************/ -#include "redis/rediswriter.hpp" +#include "icingadb/icingadb.hpp" #include "base/json.hpp" #include "base/logger.hpp" #include "base/serializer.hpp" diff --git a/lib/redis/rediswriter-utility.cpp b/lib/icingadb/icingadb-utility.cpp similarity index 99% rename from lib/redis/rediswriter-utility.cpp rename to lib/icingadb/icingadb-utility.cpp index 0a687f772..dbf3f7a6e 100644 --- a/lib/redis/rediswriter-utility.cpp +++ b/lib/icingadb/icingadb-utility.cpp @@ -17,7 +17,7 @@ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * ******************************************************************************/ -#include "redis/rediswriter.hpp" +#include "icingadb/icingadb.hpp" #include "icinga/customvarobject.hpp" #include "icinga/checkcommand.hpp" #include "icinga/notificationcommand.hpp" diff --git a/lib/redis/rediswriter.cpp b/lib/icingadb/icingadb.cpp similarity index 98% rename from lib/redis/rediswriter.cpp rename to lib/icingadb/icingadb.cpp index 6b3677cdd..020788682 100644 --- a/lib/redis/rediswriter.cpp +++ b/lib/icingadb/icingadb.cpp @@ -17,9 +17,9 @@ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * ******************************************************************************/ -#include "redis/rediswriter.hpp" -#include "redis/rediswriter-ti.cpp" -#include "redis/redisconnection.hpp" +#include "icingadb/icingadb.hpp" +#include "icingadb/icingadb-ti.cpp" +#include "icingadb/redisconnection.hpp" #include "remote/eventqueue.hpp" #include "base/json.hpp" #include "icinga/checkable.hpp" diff --git a/lib/redis/rediswriter.hpp b/lib/icingadb/icingadb.hpp similarity index 98% rename from lib/redis/rediswriter.hpp rename to lib/icingadb/icingadb.hpp index 597d2fcc6..5437422c5 100644 --- a/lib/redis/rediswriter.hpp +++ b/lib/icingadb/icingadb.hpp @@ -20,12 +20,12 @@ #ifndef REDISWRITER_H #define REDISWRITER_H -#include "redis/rediswriter-ti.hpp" +#include "icingadb/icingadb-ti.hpp" #include "icinga/customvarobject.hpp" #include "remote/messageorigin.hpp" #include "base/timer.hpp" #include "base/workqueue.hpp" -#include "redis/redisconnection.hpp" +#include "icingadb/redisconnection.hpp" #include "icinga/checkable.hpp" #include "icinga/service.hpp" #include "icinga/downtime.hpp" @@ -40,7 +40,7 @@ struct RedisSubscriptionInfo }; /** - * @ingroup redis + * @ingroup icingadb */ class IcingaDB : public ObjectImpl { diff --git a/lib/redis/rediswriter.ti b/lib/icingadb/icingadb.ti similarity index 100% rename from lib/redis/rediswriter.ti rename to lib/icingadb/icingadb.ti diff --git a/lib/redis/redisconnection.cpp b/lib/icingadb/redisconnection.cpp similarity index 99% rename from lib/redis/redisconnection.cpp rename to lib/icingadb/redisconnection.cpp index 7445a2a8b..c6d94f535 100644 --- a/lib/redis/redisconnection.cpp +++ b/lib/icingadb/redisconnection.cpp @@ -17,7 +17,7 @@ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * ******************************************************************************/ -#include "redis/redisconnection.hpp" +#include "icingadb/redisconnection.hpp" #include "base/array.hpp" #include "base/convert.hpp" #include "base/defer.hpp" diff --git a/lib/redis/redisconnection.hpp b/lib/icingadb/redisconnection.hpp similarity index 99% rename from lib/redis/redisconnection.hpp rename to lib/icingadb/redisconnection.hpp index 9ef6b5566..9d2684ed1 100644 --- a/lib/redis/redisconnection.hpp +++ b/lib/icingadb/redisconnection.hpp @@ -55,7 +55,7 @@ namespace icinga /** * An Async Redis connection. * - * @ingroup redis + * @ingroup icingadb */ class RedisConnection final : public Object { From 04dfbb07ce79c5ae30fdd094c473df8aae9ee51e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Aleksandrovi=C4=8D=20Klimov?= Date: Wed, 30 Oct 2019 10:10:21 +0100 Subject: [PATCH 199/219] Rename ICINGA2_WITH_REDIS to ICINGA2_WITH_ICINGADB --- CMakeLists.txt | 2 +- icinga-app/CMakeLists.txt | 2 +- lib/CMakeLists.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 59e0533d5..d2d77926a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ option(ICINGA2_WITH_COMPAT "Build the compat module" ON) option(ICINGA2_WITH_LIVESTATUS "Build the Livestatus module" ON) option(ICINGA2_WITH_NOTIFICATION "Build the notification module" ON) option(ICINGA2_WITH_PERFDATA "Build the perfdata module" ON) -option(ICINGA2_WITH_REDIS "Build the redis module" ON) +option(ICINGA2_WITH_ICINGADB "Build the IcingaDB module" ON) option(ICINGA2_WITH_TESTS "Run unit tests" ON) option (USE_SYSTEMD diff --git a/icinga-app/CMakeLists.txt b/icinga-app/CMakeLists.txt index db8daae30..ef71ad999 100644 --- a/icinga-app/CMakeLists.txt +++ b/icinga-app/CMakeLists.txt @@ -53,7 +53,7 @@ if(ICINGA2_WITH_PERFDATA) list(APPEND icinga_app_SOURCES $) endif() -if(ICINGA2_WITH_REDIS) +if(ICINGA2_WITH_ICINGADB) list(APPEND icinga_app_SOURCES $) endif() diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index c3baa2755..aadbb39ad 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -53,7 +53,7 @@ if(ICINGA2_WITH_PERFDATA) add_subdirectory(perfdata) endif() -if(ICINGA2_WITH_REDIS) +if(ICINGA2_WITH_ICINGADB) add_subdirectory(icingadb) endif() From 5eb1c42a2876a5866ffc423b4df55d03e73ce27c Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Thu, 31 Oct 2019 09:53:06 +0100 Subject: [PATCH 200/219] History: Use same names for event_time and event_type --- lib/icingadb/icingadb-objects.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index e7ddc90da..ff20c2107 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1189,7 +1189,6 @@ void IcingaDB::SendStatusUpdate(const ConfigObject::Ptr& object, const CheckResu "XADD", "icinga:history:stream:state", "*", "id", Utility::NewUniqueID(), "environment_id", SHA1(GetEnvironment()), - "change_time", Convert::ToString(TimestampToMilliseconds(cr ? cr->GetExecutionEnd() : Utility::GetTime())), "state_type", Convert::ToString(type), "soft_state", Convert::ToString(cr ? cr->GetState() : 99), "hard_state", Convert::ToString(service ? service->GetLastHardState() : host->GetLastHardState()), @@ -1202,6 +1201,7 @@ void IcingaDB::SendStatusUpdate(const ConfigObject::Ptr& object, const CheckResu "long_output", Utility::ValidateUTF8(std::move(output.second)), "check_source", cr->GetCheckSource(), "max_check_attempts", Convert::ToString(checkable->GetMaxCheckAttempts()), + "event_time", Convert::ToString(TimestampToMilliseconds(cr ? cr->GetExecutionEnd() : Utility::GetTime())), "event_id", Utility::NewUniqueID(), "event_type", "state" }); @@ -1244,12 +1244,12 @@ void IcingaDB::SendSentNotification( "environment_id", SHA1(GetEnvironment()), "notification_id", GetObjectIdentifier(notification), "type", Convert::ToString(type), - "send_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), "state", Convert::ToString(cr->GetState()), "previous_hard_state", Convert::ToString(GetPreviousHardState(checkable, service)), "author", Utility::ValidateUTF8(author), "text", Utility::ValidateUTF8(text), "users_notified", Convert::ToString(users), + "event_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), "event_id", Utility::NewUniqueID(), "event_type", "notification" }); @@ -1580,11 +1580,10 @@ void IcingaDB::SendFlappingChanged(const Checkable::Ptr& checkable, const Value& "XADD", "icinga:history:stream:flapping", "*", "id", Utility::NewUniqueID(), "environment_id", SHA1(GetEnvironment()), - "change_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), - "change_type", value.ToBool() ? "start" : "end", "percent_state_change", Convert::ToString(checkable->GetFlappingCurrent()), "flapping_threshold_low", Convert::ToString(checkable->GetFlappingThresholdLow()), "flapping_threshold_high", Convert::ToString(checkable->GetFlappingThresholdHigh()), + "event_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), "event_id", Utility::NewUniqueID(), "event_type", value.ToBool() ? "flapping_start" : "flapping_end" }); From 4a43dd1ce666054a0a3cf9da149947e6af66498f Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 30 Oct 2019 17:09:07 +0100 Subject: [PATCH 201/219] Rename event_type state to state_change --- lib/icingadb/icingadb-objects.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index ff20c2107..498dd1e56 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1203,7 +1203,7 @@ void IcingaDB::SendStatusUpdate(const ConfigObject::Ptr& object, const CheckResu "max_check_attempts", Convert::ToString(checkable->GetMaxCheckAttempts()), "event_time", Convert::ToString(TimestampToMilliseconds(cr ? cr->GetExecutionEnd() : Utility::GetTime())), "event_id", Utility::NewUniqueID(), - "event_type", "state" + "event_type", "state_change" }); if (service) { From 0aa885573d9e4775f7f97c5273d614c184ea4684 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Thu, 31 Oct 2019 13:32:06 +0100 Subject: [PATCH 202/219] Fix notification history not setting text if notification is triggered by checkresult --- lib/icingadb/icingadb-objects.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 498dd1e56..73fb00094 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1238,6 +1238,11 @@ void IcingaDB::SendSentNotification( auto service (dynamic_pointer_cast(checkable)); + auto finalText = text; + if (finalText == "" && cr) { + finalText = cr->GetOutput(); + } + std::vector xAdd ({ "XADD", "icinga:history:stream:notification", "*", "id", Utility::NewUniqueID(), @@ -1247,7 +1252,7 @@ void IcingaDB::SendSentNotification( "state", Convert::ToString(cr->GetState()), "previous_hard_state", Convert::ToString(GetPreviousHardState(checkable, service)), "author", Utility::ValidateUTF8(author), - "text", Utility::ValidateUTF8(text), + "text", Utility::ValidateUTF8(finalText), "users_notified", Convert::ToString(users), "event_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), "event_id", Utility::NewUniqueID(), From 6a7e83a5e6f70fc088aaf76db86d8235c8bd071d Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Thu, 31 Oct 2019 11:18:54 +0100 Subject: [PATCH 203/219] Improve Downtime & DowntimeHistory --- lib/icingadb/icingadb-objects.cpp | 117 ++++++------------------------ lib/icingadb/icingadb.hpp | 2 - 2 files changed, 24 insertions(+), 95 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 73fb00094..0ff9ff1e2 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -87,8 +87,6 @@ void IcingaDB::ConfigStaticInitialize() IcingaDB::VersionChangedHandler(object); }); - /* fixed/flexible downtime add */ - Downtime::OnDowntimeAdded.connect(&IcingaDB::DowntimeAddedHandler); /* fixed downtime start */ Downtime::OnDowntimeStarted.connect(&IcingaDB::DowntimeStartedHandler); /* flexible downtime start */ @@ -1031,8 +1029,10 @@ bool IcingaDB::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& a attributes->Set("flexible_duration", TimestampToMilliseconds(downtime->GetDuration())); attributes->Set("is_flexible", !downtime->GetFixed()); attributes->Set("is_in_effect", downtime->IsInEffect()); - if (downtime->IsInEffect()) - attributes->Set("actual_start_time", TimestampToMilliseconds(downtime->GetTriggerTime())); + if (downtime->IsInEffect()) { + attributes->Set("start_time", TimestampToMilliseconds(downtime->GetTriggerTime())); + attributes->Set("end_time", TimestampToMilliseconds(downtime->GetFixed() ? downtime->GetEndTime() : (downtime->GetTriggerTime() + downtime->GetDuration()))); + } Host::Ptr host; Service::Ptr service; @@ -1048,6 +1048,11 @@ bool IcingaDB::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& a attributes->Set("service_id", "00000000000000000000000000000000"); } + auto triggeredBy (Downtime::GetByName(downtime->GetTriggeredBy())); + if (triggeredBy) { + attributes->Set("triggered_by_id", GetObjectIdentifier(triggeredBy)); + } + return true; } @@ -1281,73 +1286,6 @@ void IcingaDB::SendSentNotification( m_Rcon->FireAndForgetQuery(std::move(xAdd)); } -void IcingaDB::SendAddedDowntime(const Downtime::Ptr& downtime) -{ - if (!m_Rcon || !m_Rcon->IsConnected()) - return; - - auto checkable (downtime->GetCheckable()); - auto service (dynamic_pointer_cast(checkable)); - auto triggeredBy (Downtime::GetByName(downtime->GetTriggeredBy())); - - std::vector xAdd ({ - "XADD", "icinga:history:stream:downtime", "*", - "downtime_id", GetObjectIdentifier(downtime), - "environment_id", SHA1(GetEnvironment()), - "entry_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEntryTime())), - "author", Utility::ValidateUTF8(downtime->GetAuthor()), - "comment", Utility::ValidateUTF8(downtime->GetComment()), - "is_flexible", Convert::ToString((unsigned short)!downtime->GetFixed()), - "flexible_duration", Convert::ToString(downtime->GetDuration()), - "scheduled_start_time", Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime())), - "scheduled_end_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())), - "was_started", "0", - "was_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()), - "is_in_effect", Convert::ToString((unsigned short)downtime->IsInEffect()), - "trigger_time", Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime())), - "event_id", Utility::NewUniqueID(), - "event_type", "downtime_schedule" - }); - - if (service) { - xAdd.emplace_back("object_type"); - xAdd.emplace_back("service"); - xAdd.emplace_back("service_id"); - xAdd.emplace_back(GetObjectIdentifier(checkable)); - } else { - xAdd.emplace_back("object_type"); - xAdd.emplace_back("host"); - xAdd.emplace_back("host_id"); - xAdd.emplace_back(GetObjectIdentifier(checkable)); - } - - if (triggeredBy) { - xAdd.emplace_back("triggered_by_id"); - xAdd.emplace_back(GetObjectIdentifier(triggeredBy)); - } - - if (downtime->GetFixed()) { - xAdd.emplace_back("actual_start_time"); - xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime()))); - xAdd.emplace_back("actual_end_time"); - xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime()))); - } else { - xAdd.emplace_back("actual_start_time"); - xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime()))); - xAdd.emplace_back("actual_end_time"); - xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime() + downtime->GetDuration()))); - } - - auto endpoint (Endpoint::GetLocalEndpoint()); - - if (endpoint) { - xAdd.emplace_back("endpoint_id"); - xAdd.emplace_back(GetObjectIdentifier(endpoint)); - } - - m_Rcon->FireAndForgetQuery(std::move(xAdd)); -} - void IcingaDB::SendStartedDowntime(const Downtime::Ptr& downtime) { if (!m_Rcon || !m_Rcon->IsConnected()) @@ -1370,9 +1308,7 @@ void IcingaDB::SendStartedDowntime(const Downtime::Ptr& downtime) "flexible_duration", Convert::ToString(downtime->GetDuration()), "scheduled_start_time", Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime())), "scheduled_end_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())), - "was_started", "1", - "was_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()), - "is_in_effect", Convert::ToString((unsigned short)downtime->IsInEffect()), + "has_been_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()), "trigger_time", Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime())), "event_id", Utility::NewUniqueID(), "event_type", "downtime_start" @@ -1396,14 +1332,14 @@ void IcingaDB::SendStartedDowntime(const Downtime::Ptr& downtime) } if (downtime->GetFixed()) { - xAdd.emplace_back("actual_start_time"); + xAdd.emplace_back("start_time"); xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime()))); - xAdd.emplace_back("actual_end_time"); + xAdd.emplace_back("end_time"); xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime()))); } else { - xAdd.emplace_back("actual_start_time"); + xAdd.emplace_back("start_time"); xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime()))); - xAdd.emplace_back("actual_end_time"); + xAdd.emplace_back("end_time"); xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime() + downtime->GetDuration()))); } @@ -1426,6 +1362,10 @@ void IcingaDB::SendRemovedDowntime(const Downtime::Ptr& downtime) auto service (dynamic_pointer_cast(checkable)); auto triggeredBy (Downtime::GetByName(downtime->GetTriggeredBy())); + // Downtime never got triggered (didn't send "downtime_start") so we don't want to send "downtime_end" + if (downtime->GetTriggerTime() == 0) + return; + std::vector xAdd ({ "XADD", "icinga:history:stream:downtime", "*", "downtime_id", GetObjectIdentifier(downtime), @@ -1437,11 +1377,9 @@ void IcingaDB::SendRemovedDowntime(const Downtime::Ptr& downtime) "flexible_duration", Convert::ToString(downtime->GetDuration()), "scheduled_start_time", Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime())), "scheduled_end_time", Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime())), - "was_started", "1", - "was_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()), - "is_in_effect", Convert::ToString((unsigned short)downtime->IsInEffect()), + "has_been_cancelled", Convert::ToString((unsigned short)downtime->GetWasCancelled()), "trigger_time", Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime())), - "deletion_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), + "cancel_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), "event_id", Utility::NewUniqueID(), "event_type", "downtime_end" }); @@ -1464,14 +1402,14 @@ void IcingaDB::SendRemovedDowntime(const Downtime::Ptr& downtime) } if (downtime->GetFixed()) { - xAdd.emplace_back("actual_start_time"); + xAdd.emplace_back("start_time"); xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetStartTime()))); - xAdd.emplace_back("actual_end_time"); + xAdd.emplace_back("end_time"); xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetEndTime()))); } else { - xAdd.emplace_back("actual_start_time"); + xAdd.emplace_back("start_time"); xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime()))); - xAdd.emplace_back("actual_end_time"); + xAdd.emplace_back("end_time"); xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(downtime->GetTriggerTime() + downtime->GetDuration()))); } @@ -1791,13 +1729,6 @@ void IcingaDB::VersionChangedHandler(const ConfigObject::Ptr& object) } } -void IcingaDB::DowntimeAddedHandler(const Downtime::Ptr& downtime) -{ - for (auto& rw : ConfigType::GetObjectsByType()) { - rw->m_WorkQueue.Enqueue([rw, downtime]() { rw->SendAddedDowntime(downtime); }); - } -} - void IcingaDB::DowntimeStartedHandler(const Downtime::Ptr& downtime) { StateChangeHandler(downtime->GetCheckable()); diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index 5437422c5..aeb955d4a 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -87,7 +87,6 @@ private: NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text ); - void SendAddedDowntime(const Downtime::Ptr& downtime); void SendStartedDowntime(const Downtime::Ptr& downtime); void SendRemovedDowntime(const Downtime::Ptr& downtime); void SendAddedComment(const Comment::Ptr& comment); @@ -123,7 +122,6 @@ private: static void StateChangeHandler(const ConfigObject::Ptr& object); static void StateChangeHandler(const ConfigObject::Ptr& object, const CheckResult::Ptr& cr, StateType type); static void VersionChangedHandler(const ConfigObject::Ptr& object); - static void DowntimeAddedHandler(const Downtime::Ptr& downtime); static void DowntimeStartedHandler(const Downtime::Ptr& downtime); static void DowntimeRemovedHandler(const Downtime::Ptr& downtime); From 4e43c766cae0d8844c6edb76260f8ea40731a7e3 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Wed, 30 Oct 2019 19:16:52 +0100 Subject: [PATCH 204/219] Improve CommentHistory --- lib/icingadb/icingadb-objects.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 0ff9ff1e2..e98713e8c 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1485,7 +1485,6 @@ void IcingaDB::SendRemovedComment(const Comment::Ptr& comment) "entry_type", Convert::ToString(comment->GetEntryType()), "is_persistent", Convert::ToString((unsigned short)comment->GetPersistent()), "expire_time", Convert::ToString(TimestampToMilliseconds(comment->GetExpireTime())), - "deletion_time", Convert::ToString(TimestampToMilliseconds(Utility::GetTime())), "event_id", Utility::NewUniqueID(), "event_type", "comment_remove" }); @@ -1549,6 +1548,16 @@ void IcingaDB::SendFlappingChanged(const Checkable::Ptr& checkable, const Value& xAdd.emplace_back("endpoint_id"); xAdd.emplace_back(GetObjectIdentifier(endpoint)); } + + if (comment->GetExpireTime() < Utility::GetTime()) { + xAdd.emplace_back("remove_time"); + xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(Utility::GetTime()))); + xAdd.emplace_back("has_been_removed"); + xAdd.emplace_back("1"); + } else { + xAdd.emplace_back("has_been_removed"); + xAdd.emplace_back("0"); + } m_Rcon->FireAndForgetQuery(std::move(xAdd)); } From 91ecfc35cf4d25602b9d91f7a9928c6a00806db0 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Thu, 31 Oct 2019 13:59:16 +0100 Subject: [PATCH 205/219] Ensure that execution_time never goes below 0 --- lib/icingadb/icingadb-objects.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index e98713e8c..7d63bcc56 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1616,7 +1616,7 @@ Dictionary::Ptr IcingaDB::SerializeState(const Checkable::Ptr& checkable) if (!cr->GetCommand().IsEmpty()) attrs->Set("commandline", FormatCommandLine(cr->GetCommand())); - attrs->Set("execution_time", TimestampToMilliseconds(cr->CalculateExecutionTime())); + attrs->Set("execution_time", TimestampToMilliseconds(fmax(0.0, cr->CalculateExecutionTime()))); attrs->Set("latency", TimestampToMilliseconds(cr->CalculateLatency())); attrs->Set("check_source", cr->GetCheckSource()); } From 67909210a61ab9707c905939485acbc1e92de956 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 31 Oct 2019 14:28:27 +0100 Subject: [PATCH 206/219] RedisConnection: introduce high-priority queue refs #57 --- lib/icingadb/redisconnection.cpp | 244 ++++++++++++++++--------------- lib/icingadb/redisconnection.hpp | 27 ++-- 2 files changed, 144 insertions(+), 127 deletions(-) diff --git a/lib/icingadb/redisconnection.cpp b/lib/icingadb/redisconnection.cpp index c6d94f535..79cf240d2 100644 --- a/lib/icingadb/redisconnection.cpp +++ b/lib/icingadb/redisconnection.cpp @@ -85,7 +85,7 @@ void LogQuery(RedisConnection::Query& query, Log& msg) } } -void RedisConnection::FireAndForgetQuery(RedisConnection::Query query) +void RedisConnection::FireAndForgetQuery(RedisConnection::Query query, bool highPrio) { { Log msg (LogNotice, "IcingaDB", "Firing and forgetting query:"); @@ -94,13 +94,13 @@ void RedisConnection::FireAndForgetQuery(RedisConnection::Query query) auto item (std::make_shared(std::move(query))); - asio::post(m_Strand, [this, item]() { - m_Queues.Writes.emplace(WriteQueueItem{item, nullptr, nullptr, nullptr}); + asio::post(m_Strand, [this, item, highPrio]() { + (highPrio ? &m_Queues.HighPrioWrites : &m_Queues.Writes)->emplace(WriteQueueItem{item, nullptr, nullptr, nullptr}); m_QueuedWrites.Set(); }); } -void RedisConnection::FireAndForgetQueries(RedisConnection::Queries queries) +void RedisConnection::FireAndForgetQueries(RedisConnection::Queries queries, bool highPrio) { for (auto& query : queries) { Log msg (LogNotice, "IcingaDB", "Firing and forgetting query:"); @@ -109,13 +109,13 @@ void RedisConnection::FireAndForgetQueries(RedisConnection::Queries queries) auto item (std::make_shared(std::move(queries))); - asio::post(m_Strand, [this, item]() { - m_Queues.Writes.emplace(WriteQueueItem{nullptr, item, nullptr, nullptr}); + asio::post(m_Strand, [this, item, highPrio]() { + (highPrio ? &m_Queues.HighPrioWrites : &m_Queues.Writes)->emplace(WriteQueueItem{nullptr, item, nullptr, nullptr}); m_QueuedWrites.Set(); }); } -RedisConnection::Reply RedisConnection::GetResultOfQuery(RedisConnection::Query query) +RedisConnection::Reply RedisConnection::GetResultOfQuery(RedisConnection::Query query, bool highPrio) { { Log msg (LogNotice, "IcingaDB", "Executing query:"); @@ -126,8 +126,8 @@ RedisConnection::Reply RedisConnection::GetResultOfQuery(RedisConnection::Query auto future (promise.get_future()); auto item (std::make_shared(std::move(query), std::move(promise))); - asio::post(m_Strand, [this, item]() { - m_Queues.Writes.emplace(WriteQueueItem{nullptr, nullptr, item, nullptr}); + asio::post(m_Strand, [this, item, highPrio]() { + (highPrio ? &m_Queues.HighPrioWrites : &m_Queues.Writes)->emplace(WriteQueueItem{nullptr, nullptr, item, nullptr}); m_QueuedWrites.Set(); }); @@ -136,7 +136,7 @@ RedisConnection::Reply RedisConnection::GetResultOfQuery(RedisConnection::Query return future.get(); } -RedisConnection::Replies RedisConnection::GetResultsOfQueries(RedisConnection::Queries queries) +RedisConnection::Replies RedisConnection::GetResultsOfQueries(RedisConnection::Queries queries, bool highPrio) { for (auto& query : queries) { Log msg (LogNotice, "IcingaDB", "Executing query:"); @@ -147,8 +147,8 @@ RedisConnection::Replies RedisConnection::GetResultsOfQueries(RedisConnection::Q auto future (promise.get_future()); auto item (std::make_shared(std::move(queries), std::move(promise))); - asio::post(m_Strand, [this, item]() { - m_Queues.Writes.emplace(WriteQueueItem{nullptr, nullptr, nullptr, item}); + asio::post(m_Strand, [this, item, highPrio]() { + (highPrio ? &m_Queues.HighPrioWrites : &m_Queues.Writes)->emplace(WriteQueueItem{nullptr, nullptr, nullptr, item}); m_QueuedWrites.Set(); }); @@ -264,111 +264,21 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) for (;;) { m_QueuedWrites.Wait(yc); - while (!m_Queues.Writes.empty()) { - auto next (std::move(m_Queues.Writes.front())); - m_Queues.Writes.pop(); - - if (next.FireAndForgetQuery) { - auto& item (*next.FireAndForgetQuery); - - try { - WriteOne(item, yc); - } catch (const boost::coroutines::detail::forced_unwind&) { - throw; - } catch (const std::exception& ex) { - Log msg (LogCritical, "IcingaDB", "Error during sending query"); - LogQuery(item, msg); - msg << " which has been fired and forgotten: " << ex.what(); - continue; - } catch (...) { - Log msg (LogCritical, "IcingaDB", "Error during sending query"); - LogQuery(item, msg); - msg << " which has been fired and forgotten"; - continue; - } - - if (m_Queues.FutureResponseActions.empty() || m_Queues.FutureResponseActions.back().Action != ResponseAction::Ignore) { - m_Queues.FutureResponseActions.emplace(FutureResponseAction{1, ResponseAction::Ignore}); + for (;;) { + if (m_Queues.HighPrioWrites.empty()) { + if (m_Queues.Writes.empty()) { + break; } else { - ++m_Queues.FutureResponseActions.back().Amount; + auto next (std::move(m_Queues.Writes.front())); + m_Queues.Writes.pop(); + + WriteItem(yc, std::move(next)); } + } else { + auto next (std::move(m_Queues.HighPrioWrites.front())); + m_Queues.HighPrioWrites.pop(); - m_QueuedReads.Set(); - } - - if (next.FireAndForgetQueries) { - auto& item (*next.FireAndForgetQueries); - size_t i = 0; - - try { - for (auto& query : item) { - WriteOne(query, yc); - ++i; - } - } catch (const boost::coroutines::detail::forced_unwind&) { - throw; - } catch (const std::exception& ex) { - Log msg (LogCritical, "IcingaDB", "Error during sending query"); - LogQuery(item[i], msg); - msg << " which has been fired and forgotten: " << ex.what(); - continue; - } catch (...) { - Log msg (LogCritical, "IcingaDB", "Error during sending query"); - LogQuery(item[i], msg); - msg << " which has been fired and forgotten"; - continue; - } - - if (m_Queues.FutureResponseActions.empty() || m_Queues.FutureResponseActions.back().Action != ResponseAction::Ignore) { - m_Queues.FutureResponseActions.emplace(FutureResponseAction{item.size(), ResponseAction::Ignore}); - } else { - m_Queues.FutureResponseActions.back().Amount += item.size(); - } - - m_QueuedReads.Set(); - } - - if (next.GetResultOfQuery) { - auto& item (*next.GetResultOfQuery); - - try { - WriteOne(item.first, yc); - } catch (const boost::coroutines::detail::forced_unwind&) { - throw; - } catch (...) { - item.second.set_exception(std::current_exception()); - continue; - } - - m_Queues.ReplyPromises.emplace(std::move(item.second)); - - if (m_Queues.FutureResponseActions.empty() || m_Queues.FutureResponseActions.back().Action != ResponseAction::Deliver) { - m_Queues.FutureResponseActions.emplace(FutureResponseAction{1, ResponseAction::Deliver}); - } else { - ++m_Queues.FutureResponseActions.back().Amount; - } - - m_QueuedReads.Set(); - } - - if (next.GetResultsOfQueries) { - auto& item (*next.GetResultsOfQueries); - - try { - for (auto& query : item.first) { - WriteOne(query, yc); - } - } catch (const boost::coroutines::detail::forced_unwind&) { - throw; - } catch (...) { - item.second.set_exception(std::current_exception()); - continue; - } - - m_Queues.RepliesPromises.emplace(std::move(item.second)); - m_Queues.FutureResponseActions.emplace(FutureResponseAction{item.first.size(), ResponseAction::DeliverBulk}); - - m_QueuedReads.Set(); + WriteItem(yc, std::move(next)); } } @@ -376,6 +286,112 @@ void RedisConnection::WriteLoop(asio::yield_context& yc) } } +void RedisConnection::WriteItem(boost::asio::yield_context& yc, RedisConnection::WriteQueueItem next) +{ + if (next.FireAndForgetQuery) { + auto& item (*next.FireAndForgetQuery); + + try { + WriteOne(item, yc); + } catch (const boost::coroutines::detail::forced_unwind&) { + throw; + } catch (const std::exception& ex) { + Log msg (LogCritical, "IcingaDB", "Error during sending query"); + LogQuery(item, msg); + msg << " which has been fired and forgotten: " << ex.what(); + return; + } catch (...) { + Log msg (LogCritical, "IcingaDB", "Error during sending query"); + LogQuery(item, msg); + msg << " which has been fired and forgotten"; + return; + } + + if (m_Queues.FutureResponseActions.empty() || m_Queues.FutureResponseActions.back().Action != ResponseAction::Ignore) { + m_Queues.FutureResponseActions.emplace(FutureResponseAction{1, ResponseAction::Ignore}); + } else { + ++m_Queues.FutureResponseActions.back().Amount; + } + + m_QueuedReads.Set(); + } + + if (next.FireAndForgetQueries) { + auto& item (*next.FireAndForgetQueries); + size_t i = 0; + + try { + for (auto& query : item) { + WriteOne(query, yc); + ++i; + } + } catch (const boost::coroutines::detail::forced_unwind&) { + throw; + } catch (const std::exception& ex) { + Log msg (LogCritical, "IcingaDB", "Error during sending query"); + LogQuery(item[i], msg); + msg << " which has been fired and forgotten: " << ex.what(); + return; + } catch (...) { + Log msg (LogCritical, "IcingaDB", "Error during sending query"); + LogQuery(item[i], msg); + msg << " which has been fired and forgotten"; + return; + } + + if (m_Queues.FutureResponseActions.empty() || m_Queues.FutureResponseActions.back().Action != ResponseAction::Ignore) { + m_Queues.FutureResponseActions.emplace(FutureResponseAction{item.size(), ResponseAction::Ignore}); + } else { + m_Queues.FutureResponseActions.back().Amount += item.size(); + } + + m_QueuedReads.Set(); + } + + if (next.GetResultOfQuery) { + auto& item (*next.GetResultOfQuery); + + try { + WriteOne(item.first, yc); + } catch (const boost::coroutines::detail::forced_unwind&) { + throw; + } catch (...) { + item.second.set_exception(std::current_exception()); + return; + } + + m_Queues.ReplyPromises.emplace(std::move(item.second)); + + if (m_Queues.FutureResponseActions.empty() || m_Queues.FutureResponseActions.back().Action != ResponseAction::Deliver) { + m_Queues.FutureResponseActions.emplace(FutureResponseAction{1, ResponseAction::Deliver}); + } else { + ++m_Queues.FutureResponseActions.back().Amount; + } + + m_QueuedReads.Set(); + } + + if (next.GetResultsOfQueries) { + auto& item (*next.GetResultsOfQueries); + + try { + for (auto& query : item.first) { + WriteOne(query, yc); + } + } catch (const boost::coroutines::detail::forced_unwind&) { + throw; + } catch (...) { + item.second.set_exception(std::current_exception()); + return; + } + + m_Queues.RepliesPromises.emplace(std::move(item.second)); + m_Queues.FutureResponseActions.emplace(FutureResponseAction{item.first.size(), ResponseAction::DeliverBulk}); + + m_QueuedReads.Set(); + } +} + RedisConnection::Reply RedisConnection::ReadOne(boost::asio::yield_context& yc) { if (m_Path.IsEmpty()) { diff --git a/lib/icingadb/redisconnection.hpp b/lib/icingadb/redisconnection.hpp index 9d2684ed1..abfdbea85 100644 --- a/lib/icingadb/redisconnection.hpp +++ b/lib/icingadb/redisconnection.hpp @@ -73,11 +73,11 @@ namespace icinga bool IsConnected(); - void FireAndForgetQuery(Query query); - void FireAndForgetQueries(Queries queries); + void FireAndForgetQuery(Query query, bool highPrio = false); + void FireAndForgetQueries(Queries queries, bool highPrio = false); - Reply GetResultOfQuery(Query query); - Replies GetResultsOfQueries(Queries queries); + Reply GetResultOfQuery(Query query, bool highPrio = false); + Replies GetResultsOfQueries(Queries queries, bool highPrio = false); private: enum class ResponseAction : unsigned char @@ -91,6 +91,14 @@ namespace icinga ResponseAction Action; }; + struct WriteQueueItem + { + std::shared_ptr FireAndForgetQuery; + std::shared_ptr FireAndForgetQueries; + std::shared_ptr>> GetResultOfQuery; + std::shared_ptr>> GetResultsOfQueries; + }; + typedef boost::asio::ip::tcp Tcp; typedef boost::asio::local::stream_protocol Unix; @@ -114,6 +122,7 @@ namespace icinga void Connect(boost::asio::yield_context& yc); void ReadLoop(boost::asio::yield_context& yc); void WriteLoop(boost::asio::yield_context& yc); + void WriteItem(boost::asio::yield_context& yc, WriteQueueItem item); Reply ReadOne(boost::asio::yield_context& yc); void WriteOne(Query& query, boost::asio::yield_context& yc); @@ -134,16 +143,8 @@ namespace icinga std::shared_ptr m_UnixConn; Atomic m_Connecting, m_Connected, m_Started; - struct WriteQueueItem - { - std::shared_ptr FireAndForgetQuery; - std::shared_ptr FireAndForgetQueries; - std::shared_ptr>> GetResultOfQuery; - std::shared_ptr>> GetResultsOfQueries; - }; - struct { - std::queue Writes; + std::queue Writes, HighPrioWrites; std::queue> ReplyPromises; std::queue> RepliesPromises; std::queue FutureResponseActions; From 071a1489aa7335b437f6305eff69b8190f74e638 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Thu, 31 Oct 2019 14:52:46 +0100 Subject: [PATCH 207/219] PUBLISH to icinga:stats with high priority refs #57 --- lib/icingadb/icingadb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/icingadb/icingadb.cpp b/lib/icingadb/icingadb.cpp index 020788682..27ba58b6f 100644 --- a/lib/icingadb/icingadb.cpp +++ b/lib/icingadb/icingadb.cpp @@ -226,7 +226,7 @@ void IcingaDB::PublishStats() status->Set("config_dump_in_progress", m_ConfigDumpInProgress); String jsonStats = JsonEncode(status); - m_Rcon->FireAndForgetQuery({ "PUBLISH", "icinga:stats", jsonStats }); + m_Rcon->FireAndForgetQuery({ "PUBLISH", "icinga:stats", jsonStats }, true); } void IcingaDB::HandleEvents() From b65e5f5547f765b765bd514d8d8fea17fcff7989 Mon Sep 17 00:00:00 2001 From: Noah Hilverling Date: Thu, 31 Oct 2019 17:14:06 +0100 Subject: [PATCH 208/219] IcingaDB/Comment: Add is_sticky --- lib/icingadb/icingadb-objects.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 7d63bcc56..2376fe800 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1000,6 +1000,7 @@ bool IcingaDB::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& a attributes->Set("entry_type", comment->GetEntryType()); attributes->Set("entry_time", TimestampToMilliseconds(comment->GetEntryTime())); attributes->Set("is_persistent", comment->GetPersistent()); + attributes->Set("is_sticky", comment->GetEntryType() == CommentAcknowledgement && comment->GetCheckable()->GetAcknowledgement() == AcknowledgementSticky); attributes->Set("expire_time", TimestampToMilliseconds(comment->GetExpireTime())); Host::Ptr host; @@ -1440,6 +1441,7 @@ void IcingaDB::SendAddedComment(const Comment::Ptr& comment) "comment", Utility::ValidateUTF8(comment->GetText()), "entry_type", Convert::ToString(comment->GetEntryType()), "is_persistent", Convert::ToString((unsigned short)comment->GetPersistent()), + "is_sticky", Convert::ToString((unsigned short)(comment->GetEntryType() == CommentAcknowledgement && comment->GetCheckable()->GetAcknowledgement() == AcknowledgementSticky)), "expire_time", Convert::ToString(TimestampToMilliseconds(comment->GetExpireTime())), "event_id", Utility::NewUniqueID(), "event_type", "comment_add" @@ -1484,6 +1486,7 @@ void IcingaDB::SendRemovedComment(const Comment::Ptr& comment) "comment", Utility::ValidateUTF8(comment->GetText()), "entry_type", Convert::ToString(comment->GetEntryType()), "is_persistent", Convert::ToString((unsigned short)comment->GetPersistent()), + "is_sticky", Convert::ToString((unsigned short)(comment->GetEntryType() == CommentAcknowledgement && comment->GetCheckable()->GetAcknowledgement() == AcknowledgementSticky)), "expire_time", Convert::ToString(TimestampToMilliseconds(comment->GetExpireTime())), "event_id", Utility::NewUniqueID(), "event_type", "comment_remove" From f9b3e88bbb8f57c837ad5f93a483935bc29b1564 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Sat, 2 Nov 2019 11:59:58 +0100 Subject: [PATCH 209/219] Fix non-unity builds --- lib/icingadb/icingadb-utility.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/icingadb/icingadb-utility.cpp b/lib/icingadb/icingadb-utility.cpp index dbf3f7a6e..0d2441f56 100644 --- a/lib/icingadb/icingadb-utility.cpp +++ b/lib/icingadb/icingadb-utility.cpp @@ -33,6 +33,7 @@ #include "base/array.hpp" #include "base/scriptglobal.hpp" #include "base/convert.hpp" +#include "base/json.hpp" #include #include #include From e89fea12d535450a3dd87dceaa6cf0f8abdd9616 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Sat, 2 Nov 2019 12:02:46 +0100 Subject: [PATCH 210/219] Disable IcingaDB on Windows Rationale: Masters running with IcingaDB are supported, agents are not. --- CMakeLists.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d2d77926a..1eefc9938 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,9 +20,15 @@ option(ICINGA2_WITH_COMPAT "Build the compat module" ON) option(ICINGA2_WITH_LIVESTATUS "Build the Livestatus module" ON) option(ICINGA2_WITH_NOTIFICATION "Build the notification module" ON) option(ICINGA2_WITH_PERFDATA "Build the perfdata module" ON) -option(ICINGA2_WITH_ICINGADB "Build the IcingaDB module" ON) option(ICINGA2_WITH_TESTS "Run unit tests" ON) +# IcingaDB only is supported on modern Linux/Unix master systems +if(NOT WIN32) + option(ICINGA2_WITH_ICINGADB "Build the IcingaDB module" ON) +else() + option(ICINGA2_WITH_ICINGADB "Build the IcingaDB module" OFF) +endif() + option (USE_SYSTEMD "Configure icinga as native systemd service instead of a SysV initscript" OFF) From af20b32dafa062527eeb7e20e601d592f20f4a1a Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Sat, 2 Nov 2019 12:08:55 +0100 Subject: [PATCH 211/219] Purge obsolete docs --- doc/99-redis.md | 54 ------------------------------------------------- 1 file changed, 54 deletions(-) delete mode 100644 doc/99-redis.md diff --git a/doc/99-redis.md b/doc/99-redis.md deleted file mode 100644 index eb6b32f8e..000000000 --- a/doc/99-redis.md +++ /dev/null @@ -1,54 +0,0 @@ -# Icinga2 Redis -## Subscriptions and events - -Using the redis feature allows you to add subscriptions for events which will then be served by Icinga 2 over Redis. - -Possible event types: - -* CheckResult -* StateChange -* Notification -* AcknowledgementSet -* AcknowledgementCleared -* CommentAdded -* CommentRemoved -* DowntimeAdded -* DowntimeRemoved -* DowntimeStarted -* DowntimeTriggered - -### Creating a subscription -A subscription is created by creating a new key `icinga:subscription:` in redis with a set of event types. All -events matching the the type of those listed will then be added to a list at `icinga:event:`. The events are -rotated on a first-in-first-out basis, the default limit is (TODO) and can be overwritten by setting -`icinga:subscription::limit` to the desired ammount. - -The events are saved as json encoded strings, similar to the API. - -It is recommended to use `LPOP` to read from the list and discard read events at the same time. - -Example: - -``` -$ redis-cli -127.0.0.1:6379> SADD icinga:subscription:noma-2 "Notification" "CheckResult" -(integer) 2 -127.0.0.1:6379> SET icinga:subscription:noma-2:limit 500 -OK -... -127.0.0.1:6379> LRANGE icinga:event:noma-2 0 1 -1) "{\"check_result\":{\"active\":true,\"check_source\":\"icinga-1\",\"command\":[\"/usr/lib/nagios/plugins/check_ping\" ... ]}}" -1) "{\"check_result\":{\"active\":true,\"check_source\":\"icinga-2\",\"command\":[\"/usr/lib/nagios/plugins/check_ping\" ... ]}}" -``` - - -All Keys are prefixed with "icinga:" - -Key | Type | Description | Dev notes ------------------------|--------|-----------------|----------- -status:{feature} | Hash | Feature status | Currently the hash only contains one key (name) which returns a json string -{type}state.{SHA1} | String | Host state | json string of the current state of the object SHA1 is of the object name -config:{type} | Hash | Config | Has all the config with name as key and json string of config as value -config:{type}:checksum | Hash | Checksums | Key is name, returns a json encoded string of checksums -subscription:{name} | String | sub description | json string describing the subsciption -event:{name} | List | sub output | Publish endpoint for subscription of the same name From 61d9130dd0ceb64bcbd94e55c0aed74311ec0f69 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Sat, 2 Nov 2019 14:00:06 +0100 Subject: [PATCH 212/219] Fix Copyright headers --- lib/icingadb/CMakeLists.txt | 17 +---------------- lib/icingadb/icingadb-objects.cpp | 19 +------------------ lib/icingadb/icingadb-stats.cpp | 19 +------------------ lib/icingadb/icingadb-utility.cpp | 19 +------------------ lib/icingadb/icingadb.cpp | 19 +------------------ lib/icingadb/icingadb.hpp | 19 +------------------ lib/icingadb/icingadb.ti | 19 +------------------ lib/icingadb/redisconnection.cpp | 19 +------------------ lib/icingadb/redisconnection.hpp | 19 +------------------ 9 files changed, 9 insertions(+), 160 deletions(-) diff --git a/lib/icingadb/CMakeLists.txt b/lib/icingadb/CMakeLists.txt index a2ece0054..71a7c67f2 100644 --- a/lib/icingadb/CMakeLists.txt +++ b/lib/icingadb/CMakeLists.txt @@ -1,19 +1,4 @@ -# Icinga 2 -# Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation -# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. +# Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ mkclass_target(icingadb.ti icingadb-ti.cpp icingadb-ti.hpp) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 2376fe800..877c99b9d 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1,21 +1,4 @@ -/****************************************************************************** - * Icinga 2 * - * Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) * - * * - * This program is free software; you can redistribute it and/or * - * modify it under the terms of the GNU General Public License * - * as published by the Free Software Foundation; either version 2 * - * of the License, or (at your option) any later version. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU General Public License for more details. * - * * - * You should have received a copy of the GNU General Public License * - * along with this program; if not, write to the Free Software Foundation * - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * - ******************************************************************************/ +/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */ #include "icingadb/icingadb.hpp" #include "icingadb/redisconnection.hpp" diff --git a/lib/icingadb/icingadb-stats.cpp b/lib/icingadb/icingadb-stats.cpp index ea1dcecf0..125b8fd44 100644 --- a/lib/icingadb/icingadb-stats.cpp +++ b/lib/icingadb/icingadb-stats.cpp @@ -1,21 +1,4 @@ -/****************************************************************************** - * Icinga 2 * - * Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) * - * * - * This program is free software; you can redistribute it and/or * - * modify it under the terms of the GNU General Public License * - * as published by the Free Software Foundation; either version 2 * - * of the License, or (at your option) any later version. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU General Public License for more details. * - * * - * You should have received a copy of the GNU General Public License * - * along with this program; if not, write to the Free Software Foundation * - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * - ******************************************************************************/ +/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */ #include "icingadb/icingadb.hpp" #include "base/json.hpp" diff --git a/lib/icingadb/icingadb-utility.cpp b/lib/icingadb/icingadb-utility.cpp index 0d2441f56..fd0270d8a 100644 --- a/lib/icingadb/icingadb-utility.cpp +++ b/lib/icingadb/icingadb-utility.cpp @@ -1,21 +1,4 @@ -/****************************************************************************** - * Icinga 2 * - * Copyright (C) 2012-2017 Icinga Development Team (https://www.icinga.com/) * - * * - * This program is free software; you can redistribute it and/or * - * modify it under the terms of the GNU General Public License * - * as published by the Free Software Foundation; either version 2 * - * of the License, or (at your option) any later version. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU General Public License for more details. * - * * - * You should have received a copy of the GNU General Public License * - * along with this program; if not, write to the Free Software Foundation * - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * - ******************************************************************************/ +/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */ #include "icingadb/icingadb.hpp" #include "icinga/customvarobject.hpp" diff --git a/lib/icingadb/icingadb.cpp b/lib/icingadb/icingadb.cpp index 27ba58b6f..4ec1c4df5 100644 --- a/lib/icingadb/icingadb.cpp +++ b/lib/icingadb/icingadb.cpp @@ -1,21 +1,4 @@ -/****************************************************************************** - * Icinga 2 * - * Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) * - * * - * This program is free software; you can redistribute it and/or * - * modify it under the terms of the GNU General Public License * - * as published by the Free Software Foundation; either version 2 * - * of the License, or (at your option) any later version. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU General Public License for more details. * - * * - * You should have received a copy of the GNU General Public License * - * along with this program; if not, write to the Free Software Foundation * - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * - ******************************************************************************/ +/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */ #include "icingadb/icingadb.hpp" #include "icingadb/icingadb-ti.cpp" diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index aeb955d4a..4de7e6038 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -1,21 +1,4 @@ -/****************************************************************************** - * Icinga 2 * - * Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) * - * * - * This program is free software; you can redistribute it and/or * - * modify it under the terms of the GNU General Public License * - * as published by the Free Software Foundation; either version 2 * - * of the License, or (at your option) any later version. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU General Public License for more details. * - * * - * You should have received a copy of the GNU General Public License * - * along with this program; if not, write to the Free Software Foundation * - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * - ******************************************************************************/ +/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */ #ifndef REDISWRITER_H #define REDISWRITER_H diff --git a/lib/icingadb/icingadb.ti b/lib/icingadb/icingadb.ti index 07572eba2..e5857341d 100644 --- a/lib/icingadb/icingadb.ti +++ b/lib/icingadb/icingadb.ti @@ -1,21 +1,4 @@ -/****************************************************************************** - * Icinga 2 * - * Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) * - * * - * This program is free software; you can redistribute it and/or * - * modify it under the terms of the GNU General Public License * - * as published by the Free Software Foundation; either version 2 * - * of the License, or (at your option) any later version. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU General Public License for more details. * - * * - * You should have received a copy of the GNU General Public License * - * along with this program; if not, write to the Free Software Foundation * - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * - ******************************************************************************/ +/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */ #include "base/configobject.hpp" diff --git a/lib/icingadb/redisconnection.cpp b/lib/icingadb/redisconnection.cpp index 79cf240d2..9a0078aef 100644 --- a/lib/icingadb/redisconnection.cpp +++ b/lib/icingadb/redisconnection.cpp @@ -1,21 +1,4 @@ -/****************************************************************************** - * Icinga 2 * - * Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) * - * * - * This program is free software; you can redistribute it and/or * - * modify it under the terms of the GNU General Public License * - * as published by the Free Software Foundation; either version 2 * - * of the License, or (at your option) any later version. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU General Public License for more details. * - * * - * You should have received a copy of the GNU General Public License * - * along with this program; if not, write to the Free Software Foundation * - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * - ******************************************************************************/ +/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */ #include "icingadb/redisconnection.hpp" #include "base/array.hpp" diff --git a/lib/icingadb/redisconnection.hpp b/lib/icingadb/redisconnection.hpp index abfdbea85..02c9236f3 100644 --- a/lib/icingadb/redisconnection.hpp +++ b/lib/icingadb/redisconnection.hpp @@ -1,21 +1,4 @@ -/****************************************************************************** - * Icinga 2 * - * Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) * - * * - * This program is free software; you can redistribute it and/or * - * modify it under the terms of the GNU General Public License * - * as published by the Free Software Foundation; either version 2 * - * of the License, or (at your option) any later version. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU General Public License for more details. * - * * - * You should have received a copy of the GNU General Public License * - * along with this program; if not, write to the Free Software Foundation * - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * - ******************************************************************************/ +/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */ #ifndef REDISCONNECTION_H #define REDISCONNECTION_H From 923524698517b95d9cc7a7a1d4f92ca58519da07 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Sat, 2 Nov 2019 14:18:20 +0100 Subject: [PATCH 213/219] Fix rebase against master branch --- lib/icingadb/icingadb-objects.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 877c99b9d..d92eb8fd8 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1494,6 +1494,16 @@ void IcingaDB::SendRemovedComment(const Comment::Ptr& comment) xAdd.emplace_back(GetObjectIdentifier(endpoint)); } + if (comment->GetExpireTime() < Utility::GetTime()) { + xAdd.emplace_back("remove_time"); + xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(Utility::GetTime()))); + xAdd.emplace_back("has_been_removed"); + xAdd.emplace_back("1"); + } else { + xAdd.emplace_back("has_been_removed"); + xAdd.emplace_back("0"); + } + m_Rcon->FireAndForgetQuery(std::move(xAdd)); } @@ -1534,16 +1544,6 @@ void IcingaDB::SendFlappingChanged(const Checkable::Ptr& checkable, const Value& xAdd.emplace_back("endpoint_id"); xAdd.emplace_back(GetObjectIdentifier(endpoint)); } - - if (comment->GetExpireTime() < Utility::GetTime()) { - xAdd.emplace_back("remove_time"); - xAdd.emplace_back(Convert::ToString(TimestampToMilliseconds(Utility::GetTime()))); - xAdd.emplace_back("has_been_removed"); - xAdd.emplace_back("1"); - } else { - xAdd.emplace_back("has_been_removed"); - xAdd.emplace_back("0"); - } m_Rcon->FireAndForgetQuery(std::move(xAdd)); } From 2dbf5bd0d0f354ebecff75e957b2a6339fe43ec4 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Sat, 2 Nov 2019 15:34:10 +0100 Subject: [PATCH 214/219] Docs: Add IcingaDB chapter, needs details --- doc/14-features.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/doc/14-features.md b/doc/14-features.md index c119abd91..4b56737bf 100644 --- a/doc/14-features.md +++ b/doc/14-features.md @@ -44,6 +44,30 @@ By default, log files will be rotated daily. The REST API is documented [here](12-icinga2-api.md#icinga2-api) as a core feature. +### Icinga DB + +Icinga DB provides a new core backend and aims to replace the IDO backend +output. It consists of different components: + +* Icinga 2 provides the `icingadb` feature which stores monitoring data in a memory database +* The [IcingaDB service](https://github.com/icinga/icingadb) collects and synchronizes monitoring data into its backend +* Icinga Web reads monitoring data from the new IcingaDB backend + +Requirements: + +* Local Redis instance +* MySQL/MariaDB server with `icingadb` database, user and schema imports +* Icinga 2's `icingadb` feature enabled +* IcingaDB service requires Redis and MySQL/MariaDB server +* Icinga Web module + +> TODO: Detailed instructions. + +``` +icinga2 feature enable icingadb +``` + + ### IDO Database (DB IDO) The IDO (Icinga Data Output) feature for Icinga 2 takes care of exporting all From 36d18ed23a5fea7bf55aa57f392e3158cd01b5a9 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Sat, 2 Nov 2019 15:43:58 +0100 Subject: [PATCH 215/219] IcingaDB: Log where we are connecting to --- lib/icingadb/redisconnection.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/icingadb/redisconnection.cpp b/lib/icingadb/redisconnection.cpp index 9a0078aef..c1d04b774 100644 --- a/lib/icingadb/redisconnection.cpp +++ b/lib/icingadb/redisconnection.cpp @@ -144,14 +144,18 @@ void RedisConnection::Connect(asio::yield_context& yc) { Defer notConnecting ([this]() { m_Connecting.store(m_Connected.load()); }); - Log(LogInformation, "IcingaDB", "Trying to connect to Redis server (async)"); - try { if (m_Path.IsEmpty()) { + Log(LogInformation, "IcingaDB") + << "Trying to connect to Redis server (async) on host '" << m_Host << ":" << m_Port << "'"; + decltype(m_TcpConn) conn (new TcpConn(m_Strand.context())); icinga::Connect(conn->next_layer(), m_Host, Convert::ToString(m_Port), yc); m_TcpConn = std::move(conn); } else { + Log(LogInformation, "IcingaDB") + << "Trying to connect to Redis server (async) on unix socket path '" << m_Path << "'"; + decltype(m_UnixConn) conn (new UnixConn(m_Strand.context())); conn->next_layer().async_connect(Unix::endpoint(m_Path.CStr()), yc); m_UnixConn = std::move(conn); From 904f2ce7d4fcfde56dfa30e4f2e6aafe0575cb79 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Sat, 2 Nov 2019 15:47:51 +0100 Subject: [PATCH 216/219] IcingaDB: Silence some developer logging --- lib/icingadb/icingadb.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/icingadb/icingadb.cpp b/lib/icingadb/icingadb.cpp index 4ec1c4df5..abb25cc54 100644 --- a/lib/icingadb/icingadb.cpp +++ b/lib/icingadb/icingadb.cpp @@ -123,7 +123,7 @@ void IcingaDB::UpdateSubscriptions() { AssertOnWorkQueue(); - Log(LogInformation, "IcingaDB", "Updating Redis subscriptions"); + Log(LogNotice, "IcingaDB", "Updating Redis subscriptions"); /* TODO: * Silently return in this case. Usually the RedisConnection checks for connectivity and logs in failure case. @@ -160,7 +160,7 @@ void IcingaDB::UpdateSubscriptions() } } while (cursor != "0"); - Log(LogInformation, "IcingaDB") + Log(LogNotice, "IcingaDB") << "Current Redis event subscriptions: " << m_Subscriptions.size(); } From 631a4ee9faba548db30ffff75aedf7ef49dd3939 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Sat, 2 Nov 2019 17:24:34 +0100 Subject: [PATCH 217/219] Fix header guards --- lib/icingadb/icingadb.hpp | 6 +++--- lib/icingadb/icingadb.ti | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index 4de7e6038..4a86e1cca 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -1,7 +1,7 @@ /* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */ -#ifndef REDISWRITER_H -#define REDISWRITER_H +#ifndef ICINGADB_H +#define ICINGADB_H #include "icingadb/icingadb-ti.hpp" #include "icinga/customvarobject.hpp" @@ -138,4 +138,4 @@ private: }; } -#endif /* REDISWRITER_H */ +#endif /* ICINGADB_H */ diff --git a/lib/icingadb/icingadb.ti b/lib/icingadb/icingadb.ti index e5857341d..7ffc80b09 100644 --- a/lib/icingadb/icingadb.ti +++ b/lib/icingadb/icingadb.ti @@ -2,7 +2,7 @@ #include "base/configobject.hpp" -library redis; +library icingadb; namespace icinga { From c3f6fa2e418cb4cbb370e3b0a427e49b4ccf4039 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Sat, 2 Nov 2019 17:34:00 +0100 Subject: [PATCH 218/219] Docs: Add object type IcingaDB --- doc/09-object-types.md | 24 ++++++++++++++++++++++++ doc/15-troubleshooting.md | 4 ++-- doc/16-upgrading-icinga-2.md | 2 +- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/doc/09-object-types.md b/doc/09-object-types.md index 40f161bc4..efe796e5c 100644 --- a/doc/09-object-types.md +++ b/doc/09-object-types.md @@ -1379,6 +1379,30 @@ Configuration Attributes: vars | Dictionary | **Optional.** A dictionary containing custom variables that are available globally. environment | String | **Optional.** Specify the Icinga environment. This overrides the `Environment` constant specified in the configuration or on the CLI with `--define`. Defaults to empty. + +### IcingaDB + +The IcingaDB object implements the [icingadb feauture](14-features.md#core-backends-icingadb). + +Example: + +``` +object IcingaDB "icingadb" { + //host = "127.0.0.1" + //port = 6379 + //password = "xxx" +} +``` + +Configuration Attributes: + + Name | Type | Description + --------------------------|-----------------------|---------------------------------- + host | String | **Optional.** Redis host for IcingaDB. Defaults to `127.0.0.1`. + port | Number | **Optional.** Redis port for IcingaDB. Defaults to `6379`. + path | String | **Optional.** Redix unix socket path. Can be used instead of `host` and `port` attributes. + password | String | **Optional.** Redis auth password for IcingaDB. + ### IdoMySqlConnection IDO database adapter for MySQL. diff --git a/doc/15-troubleshooting.md b/doc/15-troubleshooting.md index 5e7cc43ae..3ae53ce4f 100644 --- a/doc/15-troubleshooting.md +++ b/doc/15-troubleshooting.md @@ -1626,9 +1626,9 @@ it is valid to just sync their zones via the config sync. The following restores the Zone/Endpoint objects as config objects outside of `zones.d` in your master/satellite's zones.conf with rendering them as external objects in the Director. -[Example](06-distributed-monitoring.md#three-levels-with-masters-satellites-and-agents) +[Example](06-distributed-monitoring.md#distributed-monitoring-scenarios-master-satellite-agents) for a 3 level setup with the masters and satellites knowing about the zone hierarchy -outside defined in [zones.conf](#zones-conf): +outside defined in [zones.conf](04-configuration.md#zones-conf): ``` object Endpoint "icinga-master1.localdomain" { diff --git a/doc/16-upgrading-icinga-2.md b/doc/16-upgrading-icinga-2.md index 7dacfb764..95e08a2d7 100644 --- a/doc/16-upgrading-icinga-2.md +++ b/doc/16-upgrading-icinga-2.md @@ -188,7 +188,7 @@ being set but no `zone` defined. The most convenient way with e.g. managing the objects in `conf.d` is to move them into the `master` zone. Please continue in the -[troubleshooting docs](#troubleshooting-cluster-command-endpoint-errors-agent-hosts-command-endpoint-zone) +[troubleshooting docs](15-troubleshooting.md#troubleshooting-cluster-command-endpoint-errors-agent-hosts-command-endpoint-zone) for further instructions. #### Config Sync From 9d9804d50a08012237fe70e48379b73037cfb3f8 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Sat, 2 Nov 2019 18:01:31 +0100 Subject: [PATCH 219/219] Styleguide for IcingaDB --- lib/icingadb/icingadb-objects.cpp | 22 +++++++++++----------- lib/icingadb/icingadb-utility.cpp | 13 ++++++------- lib/icingadb/icingadb.cpp | 14 ++++++-------- lib/icingadb/icingadb.hpp | 6 +++--- lib/icingadb/redisconnection.cpp | 14 +++++++++++++- lib/icingadb/redisconnection.hpp | 19 ++++++++++++------- 6 files changed, 51 insertions(+), 37 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index d92eb8fd8..0869aeb97 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -2,10 +2,19 @@ #include "icingadb/icingadb.hpp" #include "icingadb/redisconnection.hpp" -#include "icinga/command.hpp" -#include "icinga/compatutility.hpp" #include "base/configtype.hpp" #include "base/configobject.hpp" +#include "base/json.hpp" +#include "base/logger.hpp" +#include "base/serializer.hpp" +#include "base/tlsutility.hpp" +#include "base/initialize.hpp" +#include "base/convert.hpp" +#include "base/array.hpp" +#include "base/exception.hpp" +#include "base/utility.hpp" +#include "icinga/command.hpp" +#include "icinga/compatutility.hpp" #include "icinga/customvarobject.hpp" #include "icinga/host.hpp" #include "icinga/service.hpp" @@ -18,15 +27,6 @@ #include "icinga/timeperiod.hpp" #include "icinga/pluginutility.hpp" #include "remote/zone.hpp" -#include "base/json.hpp" -#include "base/logger.hpp" -#include "base/serializer.hpp" -#include "base/tlsutility.hpp" -#include "base/initialize.hpp" -#include "base/convert.hpp" -#include "base/array.hpp" -#include "base/exception.hpp" -#include "base/utility.hpp" #include #include #include diff --git a/lib/icingadb/icingadb-utility.cpp b/lib/icingadb/icingadb-utility.cpp index fd0270d8a..7ac1a1fe9 100644 --- a/lib/icingadb/icingadb-utility.cpp +++ b/lib/icingadb/icingadb-utility.cpp @@ -1,11 +1,6 @@ /* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */ #include "icingadb/icingadb.hpp" -#include "icinga/customvarobject.hpp" -#include "icinga/checkcommand.hpp" -#include "icinga/notificationcommand.hpp" -#include "icinga/eventcommand.hpp" -#include "icinga/host.hpp" #include "base/configtype.hpp" #include "base/object-packer.hpp" #include "base/logger.hpp" @@ -17,11 +12,15 @@ #include "base/scriptglobal.hpp" #include "base/convert.hpp" #include "base/json.hpp" +#include "icinga/customvarobject.hpp" +#include "icinga/checkcommand.hpp" +#include "icinga/notificationcommand.hpp" +#include "icinga/eventcommand.hpp" +#include "icinga/host.hpp" +#include #include #include #include -#include - using namespace icinga; diff --git a/lib/icingadb/icingadb.cpp b/lib/icingadb/icingadb.cpp index abb25cc54..7964b414d 100644 --- a/lib/icingadb/icingadb.cpp +++ b/lib/icingadb/icingadb.cpp @@ -7,21 +7,18 @@ #include "base/json.hpp" #include "icinga/checkable.hpp" #include "icinga/host.hpp" - -#include #include +#include #include - using namespace icinga; -//TODO Make configurable and figure out a sane default #define MAX_EVENTS_DEFAULT 5000 REGISTER_TYPE(IcingaDB); IcingaDB::IcingaDB() -: m_Rcon(nullptr) + : m_Rcon(nullptr) { m_Rcon = nullptr; @@ -292,11 +289,13 @@ void IcingaDB::SendEvent(const Dictionary::Ptr& event) if (type == "CheckResult") { Checkable::Ptr checkable; + if (event->Contains("service")) { checkable = Service::GetByNamePair(event->Get("host"), event->Get("service")); } else { checkable = Host::GetByName(event->Get("host")); } + // Update State for icingaweb m_WorkQueue.Enqueue([this, checkable]() { UpdateState(checkable); }); } @@ -315,6 +314,7 @@ void IcingaDB::SendEvent(const Dictionary::Ptr& event) if (type == "AcknowledgementSet") { Timestamp entry = 0; Comment::Ptr AckComment; + for (const Comment::Ptr& c : checkable->GetComments()) { if (c->GetEntryType() == CommentAcknowledgement) { if (c->GetEntryTime() > entry) { @@ -324,15 +324,13 @@ void IcingaDB::SendEvent(const Dictionary::Ptr& event) } } } + event->Set("comment_id", GetObjectIdentifier(AckComment)); } } String body = JsonEncode(event); -// Log(LogInformation, "IcingaDB") -// << "Sending event \"" << body << "\""; - m_Rcon->FireAndForgetQueries({ { "PUBLISH", "icinga:event:all", body }, { "PUBLISH", "icinga:event:" + event->Get("type"), body }}); diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index 4a86e1cca..4ebe26409 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -4,14 +4,14 @@ #define ICINGADB_H #include "icingadb/icingadb-ti.hpp" -#include "icinga/customvarobject.hpp" -#include "remote/messageorigin.hpp" +#include "icingadb/redisconnection.hpp" #include "base/timer.hpp" #include "base/workqueue.hpp" -#include "icingadb/redisconnection.hpp" +#include "icinga/customvarobject.hpp" #include "icinga/checkable.hpp" #include "icinga/service.hpp" #include "icinga/downtime.hpp" +#include "remote/messageorigin.hpp" #include namespace icinga diff --git a/lib/icingadb/redisconnection.cpp b/lib/icingadb/redisconnection.cpp index c1d04b774..243644bf4 100644 --- a/lib/icingadb/redisconnection.cpp +++ b/lib/icingadb/redisconnection.cpp @@ -22,7 +22,7 @@ using namespace icinga; namespace asio = boost::asio; -RedisConnection::RedisConnection(const String host, const int port, const String path, const String password, const int db) : +RedisConnection::RedisConnection(const String& host, const int port, const String& path, const String& password, const int db) : RedisConnection(IoEngine::Get().GetIoContext(), host, port, path, password, db) { } @@ -192,12 +192,15 @@ void RedisConnection::ReadLoop(asio::yield_context& yc) } catch (const std::exception& ex) { Log(LogCritical, "IcingaDB") << "Error during receiving the response to a query which has been fired and forgotten: " << ex.what(); + continue; } catch (...) { Log(LogCritical, "IcingaDB") << "Error during receiving the response to a query which has been fired and forgotten"; + continue; } + break; case ResponseAction::Deliver: for (auto i (item.Amount); i; --i) { @@ -212,11 +215,13 @@ void RedisConnection::ReadLoop(asio::yield_context& yc) throw; } catch (...) { promise.set_exception(std::current_exception()); + continue; } promise.set_value(std::move(reply)); } + break; case ResponseAction::DeliverBulk: { @@ -233,6 +238,7 @@ void RedisConnection::ReadLoop(asio::yield_context& yc) throw; } catch (...) { promise.set_exception(std::current_exception()); + continue; } } @@ -286,11 +292,13 @@ void RedisConnection::WriteItem(boost::asio::yield_context& yc, RedisConnection: Log msg (LogCritical, "IcingaDB", "Error during sending query"); LogQuery(item, msg); msg << " which has been fired and forgotten: " << ex.what(); + return; } catch (...) { Log msg (LogCritical, "IcingaDB", "Error during sending query"); LogQuery(item, msg); msg << " which has been fired and forgotten"; + return; } @@ -318,11 +326,13 @@ void RedisConnection::WriteItem(boost::asio::yield_context& yc, RedisConnection: Log msg (LogCritical, "IcingaDB", "Error during sending query"); LogQuery(item[i], msg); msg << " which has been fired and forgotten: " << ex.what(); + return; } catch (...) { Log msg (LogCritical, "IcingaDB", "Error during sending query"); LogQuery(item[i], msg); msg << " which has been fired and forgotten"; + return; } @@ -344,6 +354,7 @@ void RedisConnection::WriteItem(boost::asio::yield_context& yc, RedisConnection: throw; } catch (...) { item.second.set_exception(std::current_exception()); + return; } @@ -369,6 +380,7 @@ void RedisConnection::WriteItem(boost::asio::yield_context& yc, RedisConnection: throw; } catch (...) { item.second.set_exception(std::current_exception()); + return; } diff --git a/lib/icingadb/redisconnection.hpp b/lib/icingadb/redisconnection.hpp index 02c9236f3..9f277f591 100644 --- a/lib/icingadb/redisconnection.hpp +++ b/lib/icingadb/redisconnection.hpp @@ -3,6 +3,12 @@ #ifndef REDISCONNECTION_H #define REDISCONNECTION_H +#include "base/array.hpp" +#include "base/atomic.hpp" +#include "base/io-engine.hpp" +#include "base/object.hpp" +#include "base/string.hpp" +#include "base/value.hpp" #include #include #include @@ -26,12 +32,6 @@ #include #include #include -#include "base/array.hpp" -#include "base/atomic.hpp" -#include "base/io-engine.hpp" -#include "base/object.hpp" -#include "base/string.hpp" -#include "base/value.hpp" namespace icinga { @@ -50,7 +50,8 @@ namespace icinga typedef Value Reply; typedef std::vector Replies; - RedisConnection(const String host, const int port, const String path, const String password = "", const int db = 0); + RedisConnection(const String& host, const int port, const String& path, + const String& password = "", const int db = 0); void Start(); @@ -221,6 +222,7 @@ RedisConnection::Reply RedisConnection::ReadOne(StreamPtr& stream, boost::asio:: m_Connected.store(false); stream = nullptr; } + throw; } } @@ -244,6 +246,7 @@ void RedisConnection::WriteOne(StreamPtr& stream, RedisConnection::Query& query, m_Connected.store(false); stream = nullptr; } + throw; } } @@ -370,7 +373,9 @@ void RedisConnection::WriteRESP(AsyncWriteStream& stream, const Query& query, bo for (auto& arg : query) { asio::async_write(stream, asio::const_buffer("$", 1), yc); + WriteInt(stream, arg.GetLength(), yc); + asio::async_write(stream, asio::const_buffer("\r\n", 2), yc); asio::async_write(stream, asio::const_buffer(arg.CStr(), arg.GetLength()), yc); asio::async_write(stream, asio::const_buffer("\r\n", 2), yc);