From ef93f945a241a32960525e51e9085d5079eeac77 Mon Sep 17 00:00:00 2001 From: Alvar Penning Date: Mon, 21 Oct 2024 16:00:39 +0200 Subject: [PATCH 01/40] IcingaDB: Start keeping track of Host/Service to Dependency relationship This does not work in this state! Trying to refresh Dependency if a Host or Service being member of this Dependency has a state change. --- lib/icingadb/icingadb-objects.cpp | 200 ++++++++++++++++++++++++++++++ lib/icingadb/icingadb.cpp | 2 + lib/icingadb/icingadb.hpp | 3 + 3 files changed, 205 insertions(+) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 920251969f2..e46012ba1e9 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -19,6 +19,7 @@ #include "icinga/command.hpp" #include "icinga/compatutility.hpp" #include "icinga/customvarobject.hpp" +#include "icinga/dependency.hpp" #include "icinga/host.hpp" #include "icinga/service.hpp" #include "icinga/hostgroup.hpp" @@ -61,6 +62,7 @@ std::vector IcingaDB::GetTypes() // Then sync them for similar reasons. Downtime::TypeInstance, Comment::TypeInstance, + Dependency::TypeInstance, HostGroup::TypeInstance, ServiceGroup::TypeInstance, @@ -791,6 +793,137 @@ void IcingaDB::InsertObjectDependencies(const ConfigObject::Ptr& object, const S return; } + if (type == Dependency::TypeInstance) { + auto& dependencyNodes (hMSets[m_PrefixConfigObject + "dependency:node"]); + auto& dependencyEdges (hMSets[m_PrefixConfigObject + "dependency:edge"]); + auto& redundancyGroups (hMSets[m_PrefixConfigObject + "redundancygroup"]); + + Dependency::Ptr dependency = static_pointer_cast(object); + + Host::Ptr parentHost, childHost; + Service::Ptr parentService, childService; + tie(parentHost, parentService) = GetHostService(dependency->GetParent()); + tie(childHost, childService) = GetHostService(dependency->GetChild()); + String redundancyGroup = dependency->GetRedundancyGroup(); + + String redundancyGroupId, dependencyNodeParentId, dependencyNodeChildId, dependencyNodeReduId; + + Dictionary::Ptr parentNodeData, childNodeData; + + if (parentService) { + dependencyNodeParentId = HashValue(new Array({ + m_EnvironmentId, + GetObjectIdentifier(parentHost), + GetObjectIdentifier(parentService)})); + parentNodeData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"host_id", GetObjectIdentifier(parentHost)}, + {"service_id", GetObjectIdentifier(parentService)}}); + + m_CheckablesToDependencies->Set(GetObjectIdentifier(parentService), dependency); + } else { + dependencyNodeParentId = HashValue(new Array({ + m_EnvironmentId, + GetObjectIdentifier(parentHost)})); + parentNodeData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"host_id", GetObjectIdentifier(parentHost)}}); + + m_CheckablesToDependencies->Set(GetObjectIdentifier(parentHost), dependency); + } + + if (childService) { + dependencyNodeChildId = HashValue(new Array({ + m_EnvironmentId, + GetObjectIdentifier(childHost), + GetObjectIdentifier(childService)})); + childNodeData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"host_id", GetObjectIdentifier(childHost)}, + {"service_id", GetObjectIdentifier(childService)}}); + + m_CheckablesToDependencies->Set(GetObjectIdentifier(childService), dependency); + } else { + dependencyNodeChildId = HashValue(new Array({ + m_EnvironmentId, + GetObjectIdentifier(childHost)})); + childNodeData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"host_id", GetObjectIdentifier(childHost)}}); + + m_CheckablesToDependencies->Set(GetObjectIdentifier(childHost), dependency); + } + + dependencyNodes.emplace_back(dependencyNodeParentId); + dependencyNodes.emplace_back(JsonEncode(parentNodeData)); + dependencyNodes.emplace_back(dependencyNodeChildId); + dependencyNodes.emplace_back(JsonEncode(childNodeData)); + + if (runtimeUpdate) { + AddObjectDataToRuntimeUpdates(runtimeUpdates, dependencyNodeParentId, m_PrefixConfigObject + "dependency:node", parentNodeData); + AddObjectDataToRuntimeUpdates(runtimeUpdates, dependencyNodeChildId, m_PrefixConfigObject + "dependency:node", childNodeData); + } + + if (!redundancyGroup.IsEmpty()) { + /* TODO: name should be suffixed with names of all children. + * however, at this point I don't have this information, + * only the direct neighbors. + */ + redundancyGroupId = HashValue(new Array({m_EnvironmentId, redundancyGroup, dependencyNodeChildId})); + dependencyNodeReduId = redundancyGroupId; + + redundancyGroups.emplace_back(redundancyGroupId); + Dictionary::Ptr groupData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"name", redundancyGroupId}, + {"display_name", redundancyGroup}}); + redundancyGroups.emplace_back(JsonEncode(groupData)); + + dependencyNodes.emplace_back(dependencyNodeReduId); + Dictionary::Ptr reduNodeData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"redundancy_group_id", redundancyGroupId}}); + dependencyNodes.emplace_back(JsonEncode(reduNodeData)); + + String edgeInId = HashValue(new Array({m_EnvironmentId, dependencyNodeChildId, dependencyNodeReduId})); + dependencyEdges.emplace_back(edgeInId); + Dictionary::Ptr edgeInData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"from_node_id", dependencyNodeChildId}, + {"to_node_id", dependencyNodeReduId}}); + dependencyEdges.emplace_back(JsonEncode(edgeInData)); + + String edgeOutId = HashValue(new Array({m_EnvironmentId, dependencyNodeReduId, dependencyNodeParentId})); + dependencyEdges.emplace_back(edgeOutId); + Dictionary::Ptr edgeOutData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"from_node_id", dependencyNodeReduId}, + {"to_node_id", dependencyNodeParentId}, + {"dependency_id", GetObjectIdentifier(dependency)}}); + dependencyEdges.emplace_back(JsonEncode(edgeOutData)); + + if (runtimeUpdate) { + AddObjectDataToRuntimeUpdates(runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "redundancygroup", groupData); + AddObjectDataToRuntimeUpdates(runtimeUpdates, dependencyNodeReduId, m_PrefixConfigObject + "dependency:node", reduNodeData); + AddObjectDataToRuntimeUpdates(runtimeUpdates, edgeInId, m_PrefixConfigObject + "dependency:edge", edgeInData); + AddObjectDataToRuntimeUpdates(runtimeUpdates, edgeOutId, m_PrefixConfigObject + "dependency:edge", edgeOutData); + } + } else { + String edgeId = HashValue(new Array({m_EnvironmentId, dependencyNodeChildId, dependencyNodeParentId})); + dependencyEdges.emplace_back(edgeId); + Dictionary::Ptr edgeData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"from_node_id", dependencyNodeChildId}, + {"to_node_id", dependencyNodeParentId}, + {"dependency_id", GetObjectIdentifier(dependency)}}); + dependencyEdges.emplace_back(JsonEncode(edgeData)); + + if (runtimeUpdate) { + AddObjectDataToRuntimeUpdates(runtimeUpdates, edgeId, m_PrefixConfigObject + "dependency:edge", edgeData); + } + } + } + if (type == TimePeriod::TypeInstance) { TimePeriod::Ptr timeperiod = static_pointer_cast(object); @@ -1121,6 +1254,47 @@ void IcingaDB::InsertObjectDependencies(const ConfigObject::Ptr& object, const S } } +void IcingaDB::UpdateDependencyState(const Dependency::Ptr& dependency) +{ + if (!m_Rcon || !m_Rcon->IsConnected()) { + return; + } + + auto& redundancyGroupStates (hMSets[m_PrefixConfigObject + "redundancygroup:state"]); + + String redundancyGroup = dependency->GetRedundancyGroup(); + + if (!redundancyGroup.IsEmpty()) { + Host::Ptr childHost; + Service::Ptr childService; + tie(childHost, childService) = GetHostService(dependency->GetChild()); + + String dependencyNodeChildId = HashValue( + (childService) + ? new Array({ m_EnvironmentId, GetObjectIdentifier(childHost), GetObjectIdentifier(childService) }) + : new Array({ m_EnvironmentId, GetObjectIdentifier(childHost) })); + String redundancyGroupId = HashValue(new Array({ + m_EnvironmentId, + redundancyGroup, + dependencyNodeChildId})); + + redundancyGroupStates.emplace_back(redundancyGroupId); + Dictionary::Ptr groupStateData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"redundancy_group_id", redundancyGroupId}, + {"failed", !((childService) ? childService->IsReachable() : childHost->IsReachable())}, + {"last_state_change", TimestampToMilliseconds(Utility::GetTime())}}); + redundancyGroupStates.emplace_back(JsonEncode(groupStateData)); + + // TODO + // AddObjectDataToRuntimeUpdates(runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "redundancygroup:state", groupStateData); + // dataClone->Set("id", objectKey); // redundancyGroupId + // dataClone->Set("redis_key", redisKey); // m_PrefixConfigObject + "redundancygroup:state" + // dataClone->Set("runtime_type", "upsert"); + // runtimeUpdates.emplace_back(dataClone); + } +} + /** * Update the state information of a checkable in Redis. * @@ -1450,6 +1624,32 @@ bool IcingaDB::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& a return true; } + if (type == Dependency::TypeInstance) { + Dependency::Ptr dependency = static_pointer_cast(object); + String redundancyGroup = dependency->GetRedundancyGroup(); + + attributes->Set("name", GetObjectIdentifier(dependency)); + + if (!redundancyGroup.IsEmpty()) { + Host::Ptr childHost; + Service::Ptr childService; + tie(childHost, childService) = GetHostService(dependency->GetChild()); + + String dependencyNodeChildId = HashValue( + (childService) + ? new Array({ m_EnvironmentId, GetObjectIdentifier(childHost), GetObjectIdentifier(childService) }) + : new Array({ m_EnvironmentId, GetObjectIdentifier(childHost) })); + String redundancyGroupId = HashValue(new Array({ + m_EnvironmentId, + redundancyGroup, + dependencyNodeChildId})); + + attributes->Set("redundancy_group_id", redundancyGroupId); + } + + return true; + } + if (type == Downtime::TypeInstance) { Downtime::Ptr downtime = static_pointer_cast(object); diff --git a/lib/icingadb/icingadb.cpp b/lib/icingadb/icingadb.cpp index 8d3b9099bd7..3c623259b58 100644 --- a/lib/icingadb/icingadb.cpp +++ b/lib/icingadb/icingadb.cpp @@ -38,6 +38,8 @@ IcingaDB::IcingaDB() m_PrefixConfigObject = "icinga:"; m_PrefixConfigCheckSum = "icinga:checksum:"; + + m_CheckablesToDependencies = new Dictionary(); } void IcingaDB::Validate(int types, const ValidationUtils& utils) diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index b943d7f4b1f..c5b49931848 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -103,6 +103,7 @@ class IcingaDB : public ObjectImpl std::vector GetTypeDumpSignalKeys(const Type::Ptr& type); void InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map>& hMSets, std::vector& runtimeUpdates, bool runtimeUpdate); + void UpdateDependencyState(const Dependency::Ptr& dependency); void UpdateState(const Checkable::Ptr& checkable, StateUpdate mode); void SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate); void CreateConfigUpdate(const ConfigObject::Ptr& object, const String type, std::map>& hMSets, @@ -224,6 +225,8 @@ class IcingaDB : public ObjectImpl std::unordered_map m_Rcons; std::atomic_size_t m_PendingRcons; + Dictionary::Ptr m_CheckablesToDependencies; + struct { DumpedGlobals CustomVar, ActionUrl, NotesUrl, IconImage; } m_DumpedGlobals; From d6b289e1cde1740db7a04810bdb38e81c4e88b76 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 2 Dec 2024 14:15:00 +0100 Subject: [PATCH 02/40] Checkable: Introduce `GetAllChildrenCount()` method The previous limit (32) doesn't seem to make sense, and appears to be some random number. So, this limit is set to 256 to match the limit in IsReachable(). --- lib/icinga/checkable-dependency.cpp | 55 +++++++++++++++++++++++------ lib/icinga/checkable.hpp | 1 + 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index 58d6b578bb8..a58b55c05a5 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -7,6 +7,14 @@ using namespace icinga; +/** + * The maximum number of dependency recursion levels allowed. + * + * This is a subjective limit how deep the dependency tree should be allowed to go, as anything beyond this level + * is just madness and will likely result in a stack overflow or other undefined behavior. + */ +static constexpr int l_MaxDependencyRecursionLevel(256); + void Checkable::AddDependency(const Dependency::Ptr& dep) { std::unique_lock lock(m_DependencyMutex); @@ -45,12 +53,9 @@ std::vector Checkable::GetReverseDependencies() const bool Checkable::IsReachable(DependencyType dt, Dependency::Ptr *failedDependency, int rstack) const { - /* Anything greater than 256 causes recursion bus errors. */ - int limit = 256; - - if (rstack > limit) { + if (rstack > l_MaxDependencyRecursionLevel) { Log(LogWarning, "Checkable") - << "Too many nested dependencies (>" << limit << ") for checkable '" << GetName() << "': Dependency failed."; + << "Too many nested dependencies (>" << l_MaxDependencyRecursionLevel << ") for checkable '" << GetName() << "': Dependency failed."; return false; } @@ -145,6 +150,22 @@ std::set Checkable::GetChildren() const return parents; } +/** + * Retrieve the total number of all the children of the current Checkable. + * + * Note, due to the max recursion limit of 256, the returned number may not reflect + * the actual total number of children involved in the dependency chain. + * + * @return int - Returns the total number of all the children of the current Checkable. + */ +size_t Checkable::GetAllChildrenCount() const +{ + // Are you thinking in making this more efficient? Please, don't. + // In order not to count the same child multiple times, we need to maintain a separate set of visited children, + // which is basically the same as what GetAllChildren() does. So, we're using it here! + return GetAllChildren().size(); +} + std::set Checkable::GetAllChildren() const { std::set children = GetChildren(); @@ -154,22 +175,34 @@ std::set Checkable::GetAllChildren() const return children; } +/** + * Retrieve all direct and indirect children of the current Checkable. + * + * Note, this function performs a recursive call chain traversing all the children of the current Checkable + * up to a certain limit (256). When that limit is reached, it will log a warning message and abort the operation. + * + * @param children - The set of children to be filled with all the children of the current Checkable. + * @param level - The current level of recursion. + */ void Checkable::GetAllChildrenInternal(std::set& children, int level) const { - if (level > 32) - return; + if (level > l_MaxDependencyRecursionLevel) { + Log(LogWarning, "Checkable") + << "Too many nested dependencies (>" << l_MaxDependencyRecursionLevel << ") for checkable '" << GetName() << "': aborting traversal."; + return ; + } std::set localChildren; for (const Checkable::Ptr& checkable : children) { - std::set cChildren = checkable->GetChildren(); - - if (!cChildren.empty()) { + if (auto cChildren(checkable->GetChildren()); !cChildren.empty()) { GetAllChildrenInternal(cChildren, level + 1); localChildren.insert(cChildren.begin(), cChildren.end()); } - localChildren.insert(checkable); + if (level != 0) { // Recursion level 0 is the initiator, so checkable is already in the set. + localChildren.insert(checkable); + } } children.insert(localChildren.begin(), localChildren.end()); diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index fcfbca9b281..c9e54b0f5a4 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -77,6 +77,7 @@ class Checkable : public ObjectImpl std::set GetParents() const; std::set GetChildren() const; std::set GetAllChildren() const; + size_t GetAllChildrenCount() const; void AddGroup(const String& name); From 297b62d841797c8d635a37834a2e81f1f8da2926 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 2 Dec 2024 14:19:55 +0100 Subject: [PATCH 03/40] IcingaDB: Add `affected_children` to `Host/Service` Redis updates --- lib/icingadb/icingadb-objects.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index e46012ba1e9..ac111c745a0 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1482,6 +1482,11 @@ bool IcingaDB::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& a attributes->Set("notes", checkable->GetNotes()); attributes->Set("icon_image_alt", checkable->GetIconImageAlt()); + if (size_t totalChildren (checkable->GetAllChildrenCount()); totalChildren > 0) { + // Only set the Redis key if the Checkable has actually some child dependencies. + attributes->Set("total_children", totalChildren); + } + attributes->Set("checkcommand_id", GetObjectIdentifier(checkable->GetCheckCommand())); Endpoint::Ptr commandEndpoint = checkable->GetCommandEndpoint(); From 632160667171041177e7a16320b7ffbdb667a3f7 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 2 Dec 2024 14:37:09 +0100 Subject: [PATCH 04/40] IcingaDB: Sync `affects_children` as part of runtime state updates --- lib/icinga/checkable-dependency.cpp | 29 +++++++++++++++++++++++++++++ lib/icinga/checkable.hpp | 1 + lib/icingadb/icingadb-objects.cpp | 1 + 3 files changed, 31 insertions(+) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index a58b55c05a5..d10d6f3cefb 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -122,6 +122,35 @@ bool Checkable::IsReachable(DependencyType dt, Dependency::Ptr *failedDependency return true; } +/** + * Checks whether the last check result of this Checkable affects its child dependencies. + * + * A Checkable affects its child dependencies if it runs into a non-OK state and results in any of its child + * Checkables to become unreachable. Though, that unavailable dependency may not necessarily cause the child + * Checkable to be in unreachable state as it might have some other dependencies that are still reachable, instead + * it just indicates whether the edge/connection between this and the child Checkable is broken or not. + * + * @return bool - Returns true if the Checkable affects its child dependencies, otherwise false. + */ +bool Checkable::AffectsChildren() const +{ + if (!GetLastCheckResult() || !IsReachable()) { + // If there is no check result, or the Checkable is not reachable, we can't safely determine whether + // the Checkable affects its child dependencies. + return false; + } + + for (auto& dep : GetReverseDependencies()) { + if (!dep->IsAvailable(DependencyState)) { + // If one of the child dependency is not available, then it's definitely due to the + // current Checkable state, so we don't need to verify the remaining ones. + return true; + } + } + + return false; +} + std::set Checkable::GetParents() const { std::set parents; diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index c9e54b0f5a4..c6413fa1b09 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -82,6 +82,7 @@ class Checkable : public ObjectImpl void AddGroup(const String& name); bool IsReachable(DependencyType dt = DependencyState, intrusive_ptr *failedDependency = nullptr, int rstack = 0) const; + bool AffectsChildren() const; AcknowledgementType GetAcknowledgement(); diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index ac111c745a0..a7618e75aa3 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -2828,6 +2828,7 @@ Dictionary::Ptr IcingaDB::SerializeState(const Checkable::Ptr& checkable) attrs->Set("check_attempt", checkable->GetCheckAttempt()); attrs->Set("is_active", checkable->IsActive()); + attrs->Set("affects_children", checkable->AffectsChildren()); CheckResult::Ptr cr = checkable->GetLastCheckResult(); From c64ae1af0fcde711ded92366fd82de9cddf9bbf7 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Thu, 5 Dec 2024 09:09:31 +0100 Subject: [PATCH 05/40] Dependency: Don't allow to change `redundancy_group` at runtime Otherwise, it would require too much code changes to properly handle redundancy group runtime modification in Icinga DB for no real benefit. --- lib/icinga/dependency.ti | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/icinga/dependency.ti b/lib/icinga/dependency.ti index 41de7ba23cf..38f5859aeaa 100644 --- a/lib/icinga/dependency.ti +++ b/lib/icinga/dependency.ti @@ -77,7 +77,7 @@ class Dependency : CustomVarObject < DependencyNameComposer }}} }; - [config] String redundancy_group; + [config, no_user_modify] String redundancy_group; [config, navigation] name(TimePeriod) period (PeriodRaw) { navigate {{{ From 772420a438537556e9c6b769ea49a281688e7fd7 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Thu, 6 Feb 2025 08:58:03 +0100 Subject: [PATCH 06/40] Checkable: Don't always trigger reachablity changed signal But only when the current check result being processed affects the child Checkables in any way. --- lib/icinga/checkable-check.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/icinga/checkable-check.cpp b/lib/icinga/checkable-check.cpp index 6e3b8764b83..31993fc871a 100644 --- a/lib/icinga/checkable-check.cpp +++ b/lib/icinga/checkable-check.cpp @@ -154,6 +154,10 @@ Checkable::ProcessingResult Checkable::ProcessCheckResult(const CheckResult::Ptr bool reachable = IsReachable(); bool notification_reachable = IsReachable(DependencyNotification); + // Cache whether the previous state of this Checkable affects its children before overwriting the last check result. + // This will be used to determine whether the on reachability changed event should be triggered. + bool affectsPreviousStateChildren(reachable && AffectsChildren()); + ObjectLock olock(this); CheckResult::Ptr old_cr = GetLastCheckResult(); @@ -533,7 +537,7 @@ Checkable::ProcessingResult Checkable::ProcessCheckResult(const CheckResult::Ptr } /* update reachability for child objects */ - if ((stateChange || hardChange) && !children.empty()) + if ((stateChange || hardChange) && !children.empty() && (affectsPreviousStateChildren || AffectsChildren())) OnReachabilityChanged(this, cr, children, origin); return Result::Ok; From c02b9d74a9e42a168dc9e5509d55d9766ecff5ac Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Wed, 15 Jan 2025 17:27:28 +0100 Subject: [PATCH 07/40] IcingaDB: Send reachablity state updates for all children --- lib/icingadb/icingadb-objects.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index a7618e75aa3..cbdc4802653 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -96,8 +96,11 @@ void IcingaDB::ConfigStaticInitialize() AcknowledgementClearedHandler(checkable, removedBy, changeTime); }); - Checkable::OnReachabilityChanged.connect([](const Checkable::Ptr&, const CheckResult::Ptr&, std::set children, const MessageOrigin::Ptr&) { - IcingaDB::ReachabilityChangeHandler(children); + Checkable::OnReachabilityChanged.connect([](const Checkable::Ptr& parent, const CheckResult::Ptr&, std::set, const MessageOrigin::Ptr&) { + // Icinga DB Web needs to know about the reachability of all children, not just the direct ones. + // These might get updated with their next check result anyway, but we can't rely on that, since + // they might not be actively checked or have a very high check interval. + IcingaDB::ReachabilityChangeHandler(parent->GetAllChildren()); }); /* triggered on create, update and delete objects */ From e0ce0ccff6ecb489794bd9739be8249d8b6b1f01 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Fri, 10 Jan 2025 13:24:27 +0100 Subject: [PATCH 08/40] Activate `Dependency` objects before their parent objects --- lib/icinga/dependency.ti | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/icinga/dependency.ti b/lib/icinga/dependency.ti index 38f5859aeaa..b5887710418 100644 --- a/lib/icinga/dependency.ti +++ b/lib/icinga/dependency.ti @@ -20,6 +20,8 @@ public: class Dependency : CustomVarObject < DependencyNameComposer { + activation_priority -10; + load_after Host; load_after Service; From c465f45200698b3c81d632fa5cfdfc1107507332 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Fri, 31 Jan 2025 16:30:13 +0100 Subject: [PATCH 09/40] Rewrite `Checkable::GetAllChildrenInternal()` method The previous wasn't per-se wrong, but it was way too inefficient. With this commit each and every Checkable is going to be visited only once, and we won't traverse the same Checkable's children multiple times somewhere in the dependency chain. --- lib/icinga/checkable-dependency.cpp | 26 +++++++++----------------- lib/icinga/checkable.hpp | 2 +- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index d10d6f3cefb..40d66f59f90 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -8,7 +8,7 @@ using namespace icinga; /** - * The maximum number of dependency recursion levels allowed. + * The maximum number of allowed dependency recursion levels. * * This is a subjective limit how deep the dependency tree should be allowed to go, as anything beyond this level * is just madness and will likely result in a stack overflow or other undefined behavior. @@ -197,7 +197,7 @@ size_t Checkable::GetAllChildrenCount() const std::set Checkable::GetAllChildren() const { - std::set children = GetChildren(); + std::set children; GetAllChildrenInternal(children, 0); @@ -210,29 +210,21 @@ std::set Checkable::GetAllChildren() const * Note, this function performs a recursive call chain traversing all the children of the current Checkable * up to a certain limit (256). When that limit is reached, it will log a warning message and abort the operation. * - * @param children - The set of children to be filled with all the children of the current Checkable. + * @param seenChildren - A container to store all the traversed children into. * @param level - The current level of recursion. */ -void Checkable::GetAllChildrenInternal(std::set& children, int level) const +void Checkable::GetAllChildrenInternal(std::set& seenChildren, int level) const { if (level > l_MaxDependencyRecursionLevel) { Log(LogWarning, "Checkable") << "Too many nested dependencies (>" << l_MaxDependencyRecursionLevel << ") for checkable '" << GetName() << "': aborting traversal."; - return ; + return; } - std::set localChildren; - - for (const Checkable::Ptr& checkable : children) { - if (auto cChildren(checkable->GetChildren()); !cChildren.empty()) { - GetAllChildrenInternal(cChildren, level + 1); - localChildren.insert(cChildren.begin(), cChildren.end()); - } - - if (level != 0) { // Recursion level 0 is the initiator, so checkable is already in the set. - localChildren.insert(checkable); + for (const Checkable::Ptr& checkable : GetChildren()) { + if (auto [_, inserted] = seenChildren.insert(checkable); inserted) { + seenChildren.emplace(checkable); + checkable->GetAllChildrenInternal(seenChildren, level + 1); } } - - children.insert(localChildren.begin(), localChildren.end()); } diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index c6413fa1b09..12e620ed592 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -249,7 +249,7 @@ class Checkable : public ObjectImpl std::set > m_Dependencies; std::set > m_ReverseDependencies; - void GetAllChildrenInternal(std::set& children, int level = 0) const; + void GetAllChildrenInternal(std::set& seenChildren, int level = 0) const; /* Flapping */ static const std::map m_FlappingStateFilterMap; From 67664ad7b7a5526606e0d1059e0be9be0940fa11 Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Thu, 6 Feb 2025 15:46:24 +0100 Subject: [PATCH 10/40] Checkable::GetAllChildrenInternal: remove redundant emplace call `checkable` is already added to the set by the insert call above, so calling emplace for the same checkable doesn't do anything useful and can be removed. --- lib/icinga/checkable-dependency.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index 40d66f59f90..a302f3aec62 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -223,7 +223,6 @@ void Checkable::GetAllChildrenInternal(std::set& seenChildren, i for (const Checkable::Ptr& checkable : GetChildren()) { if (auto [_, inserted] = seenChildren.insert(checkable); inserted) { - seenChildren.emplace(checkable); checkable->GetAllChildrenInternal(seenChildren, level + 1); } } From 93d9fad5654dfa3fabe085f16b0e20ca224fb4a5 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Thu, 12 Dec 2024 18:31:05 +0100 Subject: [PATCH 11/40] Checkable: Drop unused `failedDependency` argument from `IsReachable()` --- lib/icinga/checkable-dependency.cpp | 16 ++-------------- lib/icinga/checkable.hpp | 2 +- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index a302f3aec62..f8a84a56b50 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -51,7 +51,7 @@ std::vector Checkable::GetReverseDependencies() const return std::vector(m_ReverseDependencies.begin(), m_ReverseDependencies.end()); } -bool Checkable::IsReachable(DependencyType dt, Dependency::Ptr *failedDependency, int rstack) const +bool Checkable::IsReachable(DependencyType dt, int rstack) const { if (rstack > l_MaxDependencyRecursionLevel) { Log(LogWarning, "Checkable") @@ -61,7 +61,7 @@ bool Checkable::IsReachable(DependencyType dt, Dependency::Ptr *failedDependency } for (const Checkable::Ptr& checkable : GetParents()) { - if (!checkable->IsReachable(dt, failedDependency, rstack + 1)) + if (!checkable->IsReachable(dt, rstack + 1)) return false; } @@ -71,9 +71,6 @@ bool Checkable::IsReachable(DependencyType dt, Dependency::Ptr *failedDependency Host::Ptr host = service->GetHost(); if (host && host->GetState() != HostUp && host->GetStateType() == StateTypeHard) { - if (failedDependency) - *failedDependency = nullptr; - return false; } } @@ -90,9 +87,6 @@ bool Checkable::IsReachable(DependencyType dt, Dependency::Ptr *failedDependency Log(LogDebug, "Checkable") << "Non-redundant dependency '" << dep->GetName() << "' failed for checkable '" << GetName() << "': Marking as unreachable."; - if (failedDependency) - *failedDependency = dep; - return false; } @@ -110,15 +104,9 @@ bool Checkable::IsReachable(DependencyType dt, Dependency::Ptr *failedDependency Log(LogDebug, "Checkable") << "All dependencies in redundancy group '" << violator->first << "' have failed for checkable '" << GetName() << "': Marking as unreachable."; - if (failedDependency) - *failedDependency = violator->second; - return false; } - if (failedDependency) - *failedDependency = nullptr; - return true; } diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index 12e620ed592..e34b34ef0bc 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -81,7 +81,7 @@ class Checkable : public ObjectImpl void AddGroup(const String& name); - bool IsReachable(DependencyType dt = DependencyState, intrusive_ptr *failedDependency = nullptr, int rstack = 0) const; + bool IsReachable(DependencyType dt = DependencyState, int rstack = 0) const; bool AffectsChildren() const; AcknowledgementType GetAcknowledgement(); From d7c9e6687e8aefc0f84fcea448b4053531403530 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Thu, 5 Dec 2024 10:42:21 +0100 Subject: [PATCH 12/40] Introduce `DependencyGroup` helper class --- lib/icinga/CMakeLists.txt | 2 +- lib/icinga/checkable-dependency.cpp | 12 + lib/icinga/checkable.hpp | 4 + lib/icinga/dependency-group.cpp | 340 ++++++++++++++++++++++++++++ lib/icinga/dependency.hpp | 143 +++++++++++- 5 files changed, 499 insertions(+), 2 deletions(-) create mode 100644 lib/icinga/dependency-group.cpp diff --git a/lib/icinga/CMakeLists.txt b/lib/icinga/CMakeLists.txt index 62077bce74d..8187d48e884 100644 --- a/lib/icinga/CMakeLists.txt +++ b/lib/icinga/CMakeLists.txt @@ -39,7 +39,7 @@ set(icinga_SOURCES comment.cpp comment.hpp comment-ti.hpp compatutility.cpp compatutility.hpp customvarobject.cpp customvarobject.hpp customvarobject-ti.hpp - dependency.cpp dependency.hpp dependency-ti.hpp dependency-apply.cpp + dependency.cpp dependency-group.cpp dependency.hpp dependency-ti.hpp dependency-apply.cpp downtime.cpp downtime.hpp downtime-ti.hpp envresolver.cpp envresolver.hpp eventcommand.cpp eventcommand.hpp eventcommand-ti.hpp diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index f8a84a56b50..8a838a7d5fb 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -15,6 +15,18 @@ using namespace icinga; */ static constexpr int l_MaxDependencyRecursionLevel(256); +void Checkable::AddDependencyGroup(const DependencyGroup::Ptr& dependencyGroup) +{ + std::unique_lock lock(m_DependencyMutex); + m_DependencyGroups.insert(dependencyGroup); +} + +void Checkable::RemoveDependencyGroup(const DependencyGroup::Ptr& dependencyGroup) +{ + std::unique_lock lock(m_DependencyMutex); + m_DependencyGroups.erase(dependencyGroup); +} + void Checkable::AddDependency(const Dependency::Ptr& dep) { std::unique_lock lock(m_DependencyMutex); diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index e34b34ef0bc..39cc8f57059 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -57,6 +57,7 @@ enum FlappingStateFilter class CheckCommand; class EventCommand; class Dependency; +class DependencyGroup; /** * An Icinga service. @@ -184,6 +185,8 @@ class Checkable : public ObjectImpl bool IsFlapping() const; /* Dependencies */ + void AddDependencyGroup(const intrusive_ptr& dependencyGroup); + void RemoveDependencyGroup(const intrusive_ptr& dependencyGroup); void AddDependency(const intrusive_ptr& dep); void RemoveDependency(const intrusive_ptr& dep); std::vector > GetDependencies() const; @@ -246,6 +249,7 @@ class Checkable : public ObjectImpl /* Dependencies */ mutable std::mutex m_DependencyMutex; + std::set> m_DependencyGroups; std::set > m_Dependencies; std::set > m_ReverseDependencies; diff --git a/lib/icinga/dependency-group.cpp b/lib/icinga/dependency-group.cpp new file mode 100644 index 00000000000..7d686fc0fcc --- /dev/null +++ b/lib/icinga/dependency-group.cpp @@ -0,0 +1,340 @@ +/* Icinga 2 | (c) 2024 Icinga GmbH | GPLv2+ */ + +#include "icinga/dependency.hpp" +#include "base/object-packer.hpp" + +using namespace icinga; + +std::mutex DependencyGroup::m_RegistryMutex; +DependencyGroup::RegistryType DependencyGroup::m_Registry; + +/** + * Refresh the global registry of dependency groups. + * + * Registers the provided dependency object to an existing dependency group with the same redundancy + * group name (if any), or creates a new one and registers it to the child Checkable and the registry. + * + * Note: This is a helper function intended for internal use only, and you should acquire the global registry mutex + * before calling this function. + * + * @param dependency The dependency object to refresh the registry for. + * @param unregister A flag indicating whether the provided dependency object should be unregistered from the registry. + */ +void DependencyGroup::RefreshRegistry(const Dependency::Ptr& dependency, bool unregister) +{ + auto registerRedundancyGroup = [](const DependencyGroup::Ptr& dependencyGroup) { + if (auto [it, inserted](m_Registry.insert(dependencyGroup.get())); !inserted) { + DependencyGroup::Ptr existingGroup(*it); + dependencyGroup->CopyDependenciesTo(existingGroup); + } + }; + + // Retrieve all the dependency groups with the same redundancy group name of the provided dependency object. + // This allows us to shorten the lookup for the _one_ optimal group to (un)register the dependency from/to. + auto [begin, end] = m_Registry.get<1>().equal_range(dependency->GetRedundancyGroup()); + for (auto it(begin); it != end; ++it) { + DependencyGroup::Ptr existingGroup(*it); + auto child(dependency->GetChild()); + if (auto dependencies(existingGroup->GetDependenciesForChild(child.get())); !dependencies.empty()) { + m_Registry.erase(existingGroup->GetCompositeKey()); // Will be re-registered when needed down below. + if (unregister) { + existingGroup->RemoveDependency(dependency); + // Remove the connection between the child Checkable and the dependency group if it has no members + // left or the above removed member was the only member of the group that the child depended on. + if (existingGroup->IsEmpty() || dependencies.size() == 1) { + child->RemoveDependencyGroup(existingGroup); + } + } + + size_t totalDependencies(existingGroup->GetDependenciesCount()); + // If the existing dependency group has an identical member already, or the child Checkable of the + // dependency object is the only member of it (totalDependencies == dependencies.size()), we can simply + // add the dependency object to the existing group. + if (!unregister && (existingGroup->HasParentWithConfig(dependency) || totalDependencies == dependencies.size())) { + existingGroup->AddDependency(dependency); + } else if (!unregister || (dependencies.size() > 1 && totalDependencies >= dependencies.size())) { + // The child Checkable is going to have a new dependency group, so we must detach the existing one. + child->RemoveDependencyGroup(existingGroup); + + Ptr replacementGroup(unregister ? nullptr : new DependencyGroup(existingGroup->GetRedundancyGroupName(), dependency)); + for (auto& existingDependency : dependencies) { + if (existingDependency != dependency) { + existingGroup->RemoveDependency(existingDependency); + if (replacementGroup) { + replacementGroup->AddDependency(existingDependency); + } else { + replacementGroup = new DependencyGroup(existingGroup->GetRedundancyGroupName(), existingDependency); + } + } + } + + child->AddDependencyGroup(replacementGroup); + registerRedundancyGroup(replacementGroup); + } + + if (!existingGroup->IsEmpty()) { + registerRedundancyGroup(existingGroup); + } + return; + } + } + + if (!unregister) { + // We couldn't find any existing dependency group to register the dependency to, so we must + // initiate a new one and attach it to the child Checkable and register to the global registry. + DependencyGroup::Ptr newGroup(new DependencyGroup(dependency->GetRedundancyGroup())); + newGroup->AddDependency(dependency); + dependency->GetChild()->AddDependencyGroup(newGroup); + registerRedundancyGroup(newGroup); + } +} + +/** + * Register the provided dependency to the global dependency group registry. + * + * @param dependency The dependency to register. + */ +void DependencyGroup::Register(const Dependency::Ptr& dependency) +{ + std::lock_guard lock(m_RegistryMutex); + RefreshRegistry(dependency, false); +} + +/** + * Unregister the provided dependency from the dependency group it was member of. + * + * @param dependency The dependency to unregister. + */ +void DependencyGroup::Unregister(const Dependency::Ptr& dependency) +{ + std::lock_guard lock(m_RegistryMutex); + RefreshRegistry(dependency, true); +} + +/** + * Retrieve the size of the global dependency group registry. + * + * @return size_t - Returns the size of the global dependency groups registry. + */ +size_t DependencyGroup::GetRegistrySize() +{ + std::lock_guard lock(m_RegistryMutex); + return m_Registry.size(); +} + +DependencyGroup::DependencyGroup(String name): m_RedundancyGroupName(std::move(name)) +{ +} + +/** + * Create a composite key for the provided dependency. + * + * The composite key consists of all the properties of the provided dependency object that influence its availability. + * + * @param dependency The dependency object to create a composite key for. + * + * @return - Returns the composite key for the provided dependency. + */ +DependencyGroup::CompositeKeyType DependencyGroup::MakeCompositeKeyFor(const Dependency::Ptr& dependency) +{ + return std::make_tuple( + dependency->GetParent().get(), + dependency->GetPeriod().get(), + dependency->GetStateFilter(), + dependency->GetIgnoreSoftStates() + ); +} + +/** + * Check if the current dependency group is empty. + * + * @return bool - Returns true if the current dependency group has no members, otherwise false. + */ +bool DependencyGroup::IsEmpty() const +{ + std::lock_guard lock(m_Mutex); + return m_Members.empty(); +} + +/** + * Retrieve all dependency objects of the current dependency group the provided child Checkable depend on. + * + * @param child The child Checkable to get the dependencies for. + * + * @return - Returns all the dependencies of the provided child Checkable in the current dependency group. + */ +std::vector DependencyGroup::GetDependenciesForChild(const Checkable* child) const +{ + std::lock_guard lock(m_Mutex); + std::vector dependencies; + for (auto& [_, children] : m_Members) { + auto [begin, end] = children.equal_range(child); + std::transform(begin, end, std::back_inserter(dependencies), [](const auto& pair) { + return pair.second; + }); + } + return dependencies; +} + +/** + * Retrieve the number of dependency objects in the current dependency group. + * + * This function mainly exists for optimization purposes, i.e. instead of getting a copy of the members and + * counting them, we can directly query the number of dependencies in the group. + * + * @return size_t + */ +size_t DependencyGroup::GetDependenciesCount() const +{ + std::lock_guard lock(m_Mutex); + size_t count(0); + for (auto& [_, dependencies] : m_Members) { + count += dependencies.size(); + } + return count; +} + +/** + * Add a dependency object to the current dependency group. + * + * @param dependency The dependency to add to the dependency group. + */ +void DependencyGroup::AddDependency(const Dependency::Ptr& dependency) +{ + std::lock_guard lock(m_Mutex); + auto compositeKey(MakeCompositeKeyFor(dependency)); + if (auto it(m_Members.find(compositeKey)); it != m_Members.end()) { + it->second.emplace(dependency->GetChild().get(), dependency.get()); + } else { + m_Members.emplace(compositeKey, MemberValueType{{dependency->GetChild().get(), dependency.get()}}); + } +} + +/** + * Remove a dependency object from the current dependency group. + * + * @param dependency The dependency to remove from the dependency group. + */ +void DependencyGroup::RemoveDependency(const Dependency::Ptr& dependency) +{ + std::lock_guard lock(m_Mutex); + if (auto it(m_Members.find(MakeCompositeKeyFor(dependency))); it != m_Members.end()) { + auto [begin, end] = it->second.equal_range(dependency->GetChild().get()); + for (auto childrenIt(begin); childrenIt != end; ++childrenIt) { + if (childrenIt->second == dependency) { + // This will also remove the child Checkable from the multimap container + // entirely if this was the last child of it. + it->second.erase(childrenIt); + // If the composite key has no more children left, we can remove it entirely as well. + if (it->second.empty()) { + m_Members.erase(it); + } + return; + } + } + } +} + +/** + * Copy the dependency objects of the current dependency group to the provided dependency group (destination). + * + * @param dest The dependency group to move the dependencies to. + */ +void DependencyGroup::CopyDependenciesTo(const DependencyGroup::Ptr& dest) +{ + VERIFY(this != dest); // Prevent from doing something stupid, i.e. deadlocking ourselves. + + std::lock_guard lock(m_Mutex); + DependencyGroup::Ptr thisPtr(this); // Just in case the Checkable below was our last reference. + for (auto& [_, children] : m_Members) { + Checkable::Ptr previousChild; + for (auto& [checkable, dependency] : children) { + dest->AddDependency(dependency); + if (!previousChild || previousChild != checkable) { + previousChild = dependency->GetChild(); + previousChild->RemoveDependencyGroup(thisPtr); + previousChild->AddDependencyGroup(dest); + } + } + } +} + +/** + * Set the Icinga DB identifier for the current dependency group. + * + * The only usage of this function is the Icinga DB feature used to cache the unique hash of this dependency groups. + * + * @param identifier The Icinga DB identifier to set. + */ +void DependencyGroup::SetIcingaDBIdentifier(const String& identifier) +{ + std::lock_guard lock(m_Mutex); + m_IcingaDBIdentifier = identifier; +} + +/** + * Retrieve the Icinga DB identifier for the current dependency group. + * + * When the identifier is not already set by Icinga DB via the SetIcingaDBIdentifier method, + * this will just return an empty string. + * + * @return - Returns the Icinga DB identifier for the current dependency group. + */ +String DependencyGroup::GetIcingaDBIdentifier() const +{ + std::lock_guard lock(m_Mutex); + return m_IcingaDBIdentifier; +} + +/** + * Retrieve the redundancy group name of the current dependency group. + * + * If the current dependency group doesn't represent a redundancy group, this will return an empty string. + * + * @return - Returns the name of the current dependency group. + */ +const String& DependencyGroup::GetRedundancyGroupName() const +{ + // We don't need to lock the mutex here, as the name is set once during + // the object construction and never changed afterwards. + return m_RedundancyGroupName; +} + +/** + * Retrieve the unique composite key of the current dependency group. + * + * The composite key consists of some unique data of the group members, and should be used to generate + * a unique deterministic hash for the dependency group. Additionally, for explicitly configured redundancy + * groups, the non-unique dependency group name is also included on top of the composite keys. + * + * @return - Returns the composite key of the current dependency group. + */ +String DependencyGroup::GetCompositeKey() +{ + // This a copy of the CompositeKeyType definition but with the String type instead of Checkable* and TimePeriod*. + // This is because we need to produce a deterministic value from the composite key after each restart and that's + // not achievable using pointers. + using StringTuple = std::tuple; + std::vector compositeKeys; + { + std::lock_guard lock(m_Mutex); + for (auto& [compositeKey, _] : m_Members) { + auto [parent, tp, stateFilter, ignoreSoftStates] = compositeKey; + compositeKeys.emplace_back(parent->GetName(), tp ? tp->GetName() : "", stateFilter, ignoreSoftStates); + } + } + + // IMPORTANT: The order of the composite keys must be sorted to ensure the deterministic hash value. + std::sort(compositeKeys.begin(), compositeKeys.end()); + + Array::Ptr data(new Array{GetRedundancyGroupName()}); + for (auto& compositeKey : compositeKeys) { + auto [parent, tp, stateFilter, ignoreSoftStates] = compositeKey; + data->Add(std::move(parent)); + data->Add(std::move(tp)); + data->Add(stateFilter); + data->Add(ignoreSoftStates); + } + + return PackObject(data); +} diff --git a/lib/icinga/dependency.hpp b/lib/icinga/dependency.hpp index 32bd8e70dc2..afc5cab4956 100644 --- a/lib/icinga/dependency.hpp +++ b/lib/icinga/dependency.hpp @@ -3,9 +3,17 @@ #ifndef DEPENDENCY_H #define DEPENDENCY_H +#include "base/shared-object.hpp" +#include "config/configitem.hpp" #include "icinga/i2-icinga.hpp" #include "icinga/dependency-ti.hpp" -#include "config/configitem.hpp" +#include "icinga/timeperiod.hpp" +#include +#include +#include +#include +#include +#include namespace icinga { @@ -60,6 +68,139 @@ class Dependency final : public ObjectImpl static void BeforeOnAllConfigLoadedHandler(const ConfigItems& items); }; +/** + * A DependencyGroup represents a set of dependencies that are somehow related to each other. + * + * Specifically, a DependencyGroup is a container for Dependency objects of different Checkables that share the same + * child -> parent relationship config, thus forming a group of dependencies. All dependencies of a Checkable that + * have the same "redundancy_group" attribute value set are guaranteed to be part of the same DependencyGroup object, + * and another Checkable will join that group if and only if it has identical set of dependencies, that is, the same + * parent(s), same redundancy group name and all other dependency attributes required to form a composite key. + * + * More specifically, let's say we have a dependency graph like this: + * @verbatim + * PP1 PP2 + * /\ /\ + * || || + * ––––||–––––––––––––––||––––– + * P1 - ( "RG1" ) - P2 + * –––––––––––––––––––––––––––– + * /\ /\ + * || || + * C1 C2 + * @endverbatim + * The arrows represent a dependency relationship from bottom to top, i.e. both "C1" and "C2" depend on + * their "RG1" redundancy group, and "P1" and "P2" depend each on their respective parents (PP1, PP2 - no group). + * Now, as one can see, both "C1" and "C2" have identical dependencies, that is, they both depend on the same + * redundancy group "RG1" (these could e.g. be constructed through some Apply Rules). + * + * So, instead of having to maintain two separate copies of that graph, we can bring that imaginary redundancy group + * into reality by putting both "P1" and "P2" into an actual DependencyGroup object. However, we don't really put "P1" + * and "P2" objects into that group, but rather the actual Dependency objects of both child Checkables. Therefore, the + * group wouldn't just contain 2 dependencies, but 4 in total, i.e. 2 for each child Checkable (C1 -> {P1, P2} and + * C2 -> {P1, P2}). This way, both child Checkables can just refer to that very same DependencyGroup object. + * + * However, since not all dependencies are part of a redundancy group, we also have to consider the case where + * a Checkable has dependencies that are not part of any redundancy group, like P1 -> PP1. In such situations, + * each of the child Checkables (e.g. P1, P2) will have their own (sharable) DependencyGroup object just like for RGs. + * This allows us to keep the implementation simple and treat redundant and non-redundant dependencies in the same + * way, without having to introduce any special cases everywhere. So, in the end, we'd have 3 dependency groups in + * total, i.e. one for the redundancy group "RG1" (shared by C1 and C2), and two distinct groups for P1 and P2. + * + * @ingroup icinga + */ +class DependencyGroup final : public SharedObject +{ +public: + DECLARE_PTR_TYPEDEFS(DependencyGroup); + + /** + * Defines the key type of each dependency group members. + * + * This tuple consists of the dependency parent Checkable, the dependency time period (nullptr if not configured), + * the state filter, and the ignore soft states flag. Each of these values influences the availability of the + * dependency object, and thus used to group similar dependencies from different Checkables together. + */ + using CompositeKeyType = std::tuple; + + /** + * Represents the value type of each dependency group members. + * + * It stores the dependency objects of any given Checkable that produce the same composite key (CompositeKeyType). + * In other words, when looking at the dependency graph from the class description, the two dependency objects + * {C1, C2} -> P1 produce the same composite key, thus they are mapped to the same MemberValueType container with + * "C1" and "C2" as their keys respectively. Since Icinga 2 allows to construct different identical dependencies + * (duplicates), we're using a multimap instead of a simple map here. + */ + using MemberValueType = std::unordered_multimap; + using MembersMap = std::map; + + explicit DependencyGroup(String name); + + static void Register(const Dependency::Ptr& dependency); + static void Unregister(const Dependency::Ptr& dependency); + static size_t GetRegistrySize(); + + static CompositeKeyType MakeCompositeKeyFor(const Dependency::Ptr& dependency); + + /** + * Check whether the current dependency group represents an explicitly configured redundancy group. + * + * @return bool - Returns true if it's a redundancy group, false otherwise. + */ + inline bool IsRedundancyGroup() const + { + return !m_RedundancyGroupName.IsEmpty(); + } + + bool IsEmpty() const; + std::vector GetDependenciesForChild(const Checkable* child) const; + size_t GetDependenciesCount() const; + + void SetIcingaDBIdentifier(const String& identifier); + String GetIcingaDBIdentifier() const; + + const String& GetRedundancyGroupName() const; + String GetCompositeKey(); + +protected: + void AddDependency(const Dependency::Ptr& dependency); + void RemoveDependency(const Dependency::Ptr& dependency); + void CopyDependenciesTo(const DependencyGroup::Ptr& dest); + + static void RefreshRegistry(const Dependency::Ptr& dependency, bool unregister); + +private: + mutable std::mutex m_Mutex; + String m_IcingaDBIdentifier; + String m_RedundancyGroupName; + MembersMap m_Members; + + using RegistryType = boost::multi_index_container< + DependencyGroup*, // The type of the elements stored in the container. + boost::multi_index::indexed_by< + // This unique index allows to search/erase dependency groups by their composite key in an efficient manner. + boost::multi_index::hashed_unique< + boost::multi_index::mem_fun, + std::hash + >, + // This non-unique index allows to search for dependency groups by their name, and reduces the overall + // runtime complexity. Without this index, we would have to iterate over all elements to find the one + // with the desired members and since containers don't allow erasing elements while iterating, we would + // have to copy each of them to a temporary container, and then erase and reinsert them back to the original + // container. This produces way too much overhead, and slows down the startup time of Icinga 2 significantly. + boost::multi_index::hashed_non_unique< + boost::multi_index::const_mem_fun, + std::hash + > + > + >; + + // The global registry of dependency groups. + static std::mutex m_RegistryMutex; + static RegistryType m_Registry; +}; + } #endif /* DEPENDENCY_H */ From 1820955993980feab04cf0e1aaafec00aead15cb Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Thu, 5 Dec 2024 11:21:01 +0100 Subject: [PATCH 13/40] Add `DependencyGroup::GetState()` helper method --- lib/icinga/dependency-group.cpp | 43 +++++++++++++++++++++++++++++++++ lib/icinga/dependency.hpp | 8 ++++++ 2 files changed, 51 insertions(+) diff --git a/lib/icinga/dependency-group.cpp b/lib/icinga/dependency-group.cpp index 7d686fc0fcc..8b5aac92dbf 100644 --- a/lib/icinga/dependency-group.cpp +++ b/lib/icinga/dependency-group.cpp @@ -338,3 +338,46 @@ String DependencyGroup::GetCompositeKey() return PackObject(data); } + +/** + * Retrieve the state of the current dependency group. + * + * The state of the dependency group is determined based on the state of the parent Checkables and dependency objects + * of the group. A dependency group is considered unreachable when none of the parent Checkables is reachable. However, + * a dependency group may still be marked as failed even when it has reachable parent Checkables, but an unreachable + * group has always a failed state. + * + * @return - Returns the state of the current dependency group. + */ +DependencyGroup::State DependencyGroup::GetState(DependencyType dt, int rstack) const +{ + MembersMap members; + { + // We don't want to hold the mutex lock for the entire evaluation, thus we just need to operate on a copy. + std::lock_guard lock(m_Mutex); + members = m_Members; + } + + State state{false /* Reachable */, false /* OK */}; + for (auto& [_, children] : members) { + for (auto& [checkable, dependency] : children) { + state.Reachable = dependency->GetParent()->IsReachable(dt, rstack); + if (!state.Reachable && !IsRedundancyGroup()) { + return state; + } + + if (state.Reachable) { + state.OK = dependency->IsAvailable(dt); + // If this is a redundancy group, and we have found one functional path, that's enough and we can return. + // Likewise, if this is a non-redundant dependency group, and we have found one non-functional path, + // we have to mark the group as failed and return. + if (state.OK == IsRedundancyGroup()) { // OK && IsRedundancyGroup() || !OK && !IsRedundancyGroup() + return state; + } + } + break; // Move on to the next batch of group members (next composite key). + } + } + + return state; +} diff --git a/lib/icinga/dependency.hpp b/lib/icinga/dependency.hpp index afc5cab4956..34667eec120 100644 --- a/lib/icinga/dependency.hpp +++ b/lib/icinga/dependency.hpp @@ -163,6 +163,14 @@ class DependencyGroup final : public SharedObject const String& GetRedundancyGroupName() const; String GetCompositeKey(); + struct State + { + bool Reachable; // Whether the dependency group is reachable. + bool OK; // Whether the dependency group is reachable and OK. + }; + + State GetState(DependencyType dt = DependencyState, int rstack = 0) const; + protected: void AddDependency(const Dependency::Ptr& dependency); void RemoveDependency(const Dependency::Ptr& dependency); From ff0dabe287978f187eb982f4cffdbdf0028734ac Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Wed, 4 Dec 2024 10:55:16 +0100 Subject: [PATCH 14/40] Checkable: Store dependencies grouped by their redundancy group --- lib/icinga/checkable-dependency.cpp | 19 +++++++++---------- lib/icinga/checkable.hpp | 4 +--- lib/icinga/dependency.cpp | 4 ++-- test/icinga-dependencies.cpp | 4 ++-- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index 8a838a7d5fb..a176a10add0 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -27,22 +27,21 @@ void Checkable::RemoveDependencyGroup(const DependencyGroup::Ptr& dependencyGrou m_DependencyGroups.erase(dependencyGroup); } -void Checkable::AddDependency(const Dependency::Ptr& dep) +std::vector Checkable::GetDependencyGroups() const { - std::unique_lock lock(m_DependencyMutex); - m_Dependencies.insert(dep); -} - -void Checkable::RemoveDependency(const Dependency::Ptr& dep) -{ - std::unique_lock lock(m_DependencyMutex); - m_Dependencies.erase(dep); + std::lock_guard lock(m_DependencyMutex); + return {m_DependencyGroups.begin(), m_DependencyGroups.end()}; } std::vector Checkable::GetDependencies() const { std::unique_lock lock(m_DependencyMutex); - return std::vector(m_Dependencies.begin(), m_Dependencies.end()); + std::vector dependencies; + for (const auto& dependencyGroup : m_DependencyGroups) { + auto tmpDependencies(dependencyGroup->GetDependenciesForChild(this)); + dependencies.insert(dependencies.end(), tmpDependencies.begin(), tmpDependencies.end()); + } + return dependencies; } void Checkable::AddReverseDependency(const Dependency::Ptr& dep) diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index 39cc8f57059..04c12d4f94d 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -187,8 +187,7 @@ class Checkable : public ObjectImpl /* Dependencies */ void AddDependencyGroup(const intrusive_ptr& dependencyGroup); void RemoveDependencyGroup(const intrusive_ptr& dependencyGroup); - void AddDependency(const intrusive_ptr& dep); - void RemoveDependency(const intrusive_ptr& dep); + std::vector> GetDependencyGroups() const; std::vector > GetDependencies() const; void AddReverseDependency(const intrusive_ptr& dep); @@ -250,7 +249,6 @@ class Checkable : public ObjectImpl /* Dependencies */ mutable std::mutex m_DependencyMutex; std::set> m_DependencyGroups; - std::set > m_Dependencies; std::set > m_ReverseDependencies; void GetAllChildrenInternal(std::set& seenChildren, int level = 0) const; diff --git a/lib/icinga/dependency.cpp b/lib/icinga/dependency.cpp index a9a7bf3725d..2f2482136de 100644 --- a/lib/icinga/dependency.cpp +++ b/lib/icinga/dependency.cpp @@ -251,7 +251,7 @@ void Dependency::OnAllConfigLoaded() // InitChildParentReferences() has to be called before. VERIFY(m_Child && m_Parent); - m_Child->AddDependency(this); + DependencyGroup::Register(this); m_Parent->AddReverseDependency(this); } @@ -259,7 +259,7 @@ void Dependency::Stop(bool runtimeRemoved) { ObjectImpl::Stop(runtimeRemoved); - GetChild()->RemoveDependency(this); + DependencyGroup::Unregister(this); GetParent()->RemoveReverseDependency(this); } diff --git a/test/icinga-dependencies.cpp b/test/icinga-dependencies.cpp index 929b6ca0de5..86735cdb90d 100644 --- a/test/icinga-dependencies.cpp +++ b/test/icinga-dependencies.cpp @@ -54,7 +54,7 @@ BOOST_AUTO_TEST_CASE(multi_parent) dep1->SetStateFilter(StateFilterUp); // Reverse dependencies - childHost->AddDependency(dep1); + DependencyGroup::Register(dep1); parentHost1->AddReverseDependency(dep1); Dependency::Ptr dep2 = new Dependency(); @@ -64,7 +64,7 @@ BOOST_AUTO_TEST_CASE(multi_parent) dep2->SetStateFilter(StateFilterUp); // Reverse dependencies - childHost->AddDependency(dep2); + DependencyGroup::Register(dep2); parentHost2->AddReverseDependency(dep2); From 27f11a09559c9e1565be39f717ee3fd207194133 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Fri, 7 Feb 2025 11:09:34 +0100 Subject: [PATCH 15/40] Checkable: Introduce `HasAnyDependencies()` method --- lib/icinga/checkable-dependency.cpp | 6 ++++++ lib/icinga/checkable.hpp | 1 + 2 files changed, 7 insertions(+) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index a176a10add0..afa1aca6d9e 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -44,6 +44,12 @@ std::vector Checkable::GetDependencies() const return dependencies; } +bool Checkable::HasAnyDependencies() const +{ + std::unique_lock lock(m_DependencyMutex); + return !m_DependencyGroups.empty() || !m_ReverseDependencies.empty(); +} + void Checkable::AddReverseDependency(const Dependency::Ptr& dep) { std::unique_lock lock(m_DependencyMutex); diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index 04c12d4f94d..2b901414355 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -189,6 +189,7 @@ class Checkable : public ObjectImpl void RemoveDependencyGroup(const intrusive_ptr& dependencyGroup); std::vector> GetDependencyGroups() const; std::vector > GetDependencies() const; + bool HasAnyDependencies() const; void AddReverseDependency(const intrusive_ptr& dep); void RemoveReverseDependency(const intrusive_ptr& dep); From d094581b4b2105970f878dab35c4ef8d2646088d Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 16 Dec 2024 09:22:21 +0100 Subject: [PATCH 16/40] Checkable: Use redundancy groups state in `IsReachable` --- lib/icinga/checkable-dependency.cpp | 39 +++++------------------------ 1 file changed, 6 insertions(+), 33 deletions(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index afa1aca6d9e..0904c09cf49 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -77,11 +77,6 @@ bool Checkable::IsReachable(DependencyType dt, int rstack) const return false; } - for (const Checkable::Ptr& checkable : GetParents()) { - if (!checkable->IsReachable(dt, rstack + 1)) - return false; - } - /* implicit dependency on host if this is a service */ const auto *service = dynamic_cast(this); if (service && (dt == DependencyState || dt == DependencyNotification)) { @@ -92,38 +87,16 @@ bool Checkable::IsReachable(DependencyType dt, int rstack) const } } - auto deps = GetDependencies(); - - std::unordered_map violated; // key: redundancy group, value: nullptr if satisfied, violating dependency otherwise - - for (const Dependency::Ptr& dep : deps) { - std::string redundancy_group = dep->GetRedundancyGroup(); - - if (!dep->IsAvailable(dt)) { - if (redundancy_group.empty()) { - Log(LogDebug, "Checkable") - << "Non-redundant dependency '" << dep->GetName() << "' failed for checkable '" << GetName() << "': Marking as unreachable."; + for (auto& dependencyGroup : GetDependencyGroups()) { + if (auto state(dependencyGroup->GetState(dt, rstack + 1)); !state.Reachable || !state.OK) { + Log(LogDebug, "Checkable") + << "Dependency group '" << dependencyGroup->GetRedundancyGroupName() << "' have failed for checkable '" + << GetName() << "': Marking as unreachable."; - return false; - } - - // tentatively mark this dependency group as failed unless it is already marked; - // so it either passed before (don't overwrite) or already failed (so don't care) - // note that std::unordered_map::insert() will not overwrite an existing entry - violated.insert(std::make_pair(redundancy_group, dep)); - } else if (!redundancy_group.empty()) { - violated[redundancy_group] = nullptr; + return false; } } - auto violator = std::find_if(violated.begin(), violated.end(), [](auto& v) { return v.second != nullptr; }); - if (violator != violated.end()) { - Log(LogDebug, "Checkable") - << "All dependencies in redundancy group '" << violator->first << "' have failed for checkable '" << GetName() << "': Marking as unreachable."; - - return false; - } - return true; } From 2616c99891b4dc7f6a5b0747daf03f8bf1a82f97 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 16 Dec 2024 09:22:52 +0100 Subject: [PATCH 17/40] tests: Add unittests for the redundancy groups registry --- test/CMakeLists.txt | 3 + test/icinga-dependencies.cpp | 287 ++++++++++++++++++++++++++++++----- 2 files changed, 256 insertions(+), 34 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c4b1041ddb5..7857c126104 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -232,6 +232,9 @@ add_boost_test(base icinga_checkresult/service_flapping_notification icinga_checkresult/suppressed_notification icinga_dependencies/multi_parent + icinga_dependencies/default_redundancy_group_registration_unregistration + icinga_dependencies/simple_redundancy_group_registration_unregistration + icinga_dependencies/mixed_redundancy_group_registration_unregsitration icinga_notification/strings icinga_notification/state_filter icinga_notification/type_filter diff --git a/test/icinga-dependencies.cpp b/test/icinga-dependencies.cpp index 86735cdb90d..e02385938ac 100644 --- a/test/icinga-dependencies.cpp +++ b/test/icinga-dependencies.cpp @@ -9,6 +9,52 @@ using namespace icinga; BOOST_AUTO_TEST_SUITE(icinga_dependencies) +static Host::Ptr CreateHost(const std::string& name) +{ + Host::Ptr host = new Host(); + host->SetName(name); + return host; +} + +static Dependency::Ptr CreateDependency(Checkable::Ptr parent, Checkable::Ptr child, const std::string& name) +{ + Dependency::Ptr dep = new Dependency(); + dep->SetParent(parent); + dep->SetChild(child); + dep->SetName(name + "!" + child->GetName()); + return dep; +} + +static void RegisterDependency(Dependency::Ptr dep, const std::string& redundancyGroup) +{ + dep->SetRedundancyGroup(redundancyGroup); + DependencyGroup::Register(dep); + dep->GetParent()->AddReverseDependency(dep); +} + +static void AssertCheckableRedundancyGroup(Checkable::Ptr checkable, int dependencyCount, int groupCount, int totalDependenciesCount) +{ + BOOST_CHECK_MESSAGE( + dependencyCount == checkable->GetDependencies().size(), + "Dependency count mismatch for '" << checkable->GetName() << "' - expected=" << dependencyCount << "; got=" + << checkable->GetDependencies().size() + ); + auto dependencyGroups(checkable->GetDependencyGroups()); + BOOST_CHECK_MESSAGE( + groupCount == dependencyGroups.size(), + "Dependency group count mismatch for '" << checkable->GetName() << "'" << " - expected=" << groupCount + << "; got=" << dependencyGroups.size() + ); + if (groupCount > 0) { + BOOST_REQUIRE_MESSAGE(1 <= dependencyGroups.size(), "Checkable '" << checkable->GetName() << "' should have at least one dependency group."); + BOOST_CHECK_MESSAGE( + totalDependenciesCount == dependencyGroups.begin()->get()->GetDependenciesCount(), + "Member count mismatch for '" << checkable->GetName() << "'" << " - expected=" << totalDependenciesCount + << "; got=" << dependencyGroups.begin()->get()->GetDependenciesCount() + ); + } +} + BOOST_AUTO_TEST_CASE(multi_parent) { /* One child host, two parent hosts. Simulate multi-parent dependencies. */ @@ -20,53 +66,28 @@ BOOST_AUTO_TEST_CASE(multi_parent) * - Parent objects need a CheckResult object * - Dependencies need a StateFilter */ - Host::Ptr parentHost1 = new Host(); - parentHost1->SetActive(true); - parentHost1->SetMaxCheckAttempts(1); - parentHost1->Activate(); - parentHost1->SetAuthority(true); + Host::Ptr parentHost1 = CreateHost("parentHost1"); parentHost1->SetStateRaw(ServiceCritical); parentHost1->SetStateType(StateTypeHard); parentHost1->SetLastCheckResult(new CheckResult()); - Host::Ptr parentHost2 = new Host(); - parentHost2->SetActive(true); - parentHost2->SetMaxCheckAttempts(1); - parentHost2->Activate(); - parentHost2->SetAuthority(true); + Host::Ptr parentHost2 = CreateHost("parentHost2"); parentHost2->SetStateRaw(ServiceOK); parentHost2->SetStateType(StateTypeHard); parentHost2->SetLastCheckResult(new CheckResult()); - Host::Ptr childHost = new Host(); - childHost->SetActive(true); - childHost->SetMaxCheckAttempts(1); - childHost->Activate(); - childHost->SetAuthority(true); + Host::Ptr childHost = CreateHost("childHost"); childHost->SetStateRaw(ServiceOK); childHost->SetStateType(StateTypeHard); /* Build the dependency tree. */ - Dependency::Ptr dep1 = new Dependency(); - - dep1->SetParent(parentHost1); - dep1->SetChild(childHost); + Dependency::Ptr dep1 (CreateDependency(parentHost1, childHost, "dep1")); dep1->SetStateFilter(StateFilterUp); + RegisterDependency(dep1, ""); - // Reverse dependencies - DependencyGroup::Register(dep1); - parentHost1->AddReverseDependency(dep1); - - Dependency::Ptr dep2 = new Dependency(); - - dep2->SetParent(parentHost2); - dep2->SetChild(childHost); + Dependency::Ptr dep2 (CreateDependency(parentHost2, childHost, "dep2")); dep2->SetStateFilter(StateFilterUp); - - // Reverse dependencies - DependencyGroup::Register(dep2); - parentHost2->AddReverseDependency(dep2); - + RegisterDependency(dep2, ""); /* Test the reachability from this point. * parentHost1 is DOWN, parentHost2 is UP. @@ -77,18 +98,42 @@ BOOST_AUTO_TEST_CASE(multi_parent) BOOST_CHECK(childHost->IsReachable() == false); + Dependency::Ptr duplicateDep (CreateDependency(parentHost1, childHost, "dep4")); + duplicateDep->SetIgnoreSoftStates(false, true); + RegisterDependency(duplicateDep, ""); + parentHost1->SetStateType(StateTypeSoft); + + // It should still be unreachable, due to the duplicated dependency object above with ignore_soft_states set to false. + BOOST_CHECK(childHost->IsReachable() == false); + parentHost1->SetStateType(StateTypeHard); + DependencyGroup::Unregister(duplicateDep); + /* The only DNS server is DOWN. * Expected result: childHost is unreachable. */ - dep1->SetRedundancyGroup("DNS"); + DependencyGroup::Unregister(dep1); // Remove the dep and re-add it with a configured redundancy group. + RegisterDependency(dep1, "DNS"); BOOST_CHECK(childHost->IsReachable() == false); /* 1/2 DNS servers is DOWN. * Expected result: childHost is reachable. */ - dep2->SetRedundancyGroup("DNS"); + DependencyGroup::Unregister(dep2); + RegisterDependency(dep2, "DNS"); BOOST_CHECK(childHost->IsReachable() == true); + auto grandParentHost(CreateHost("GrandParentHost")); + grandParentHost->SetLastCheckResult(new CheckResult()); + grandParentHost->SetStateRaw(ServiceCritical); + grandParentHost->SetStateType(StateTypeHard); + + Dependency::Ptr dep3 (CreateDependency(grandParentHost, parentHost1, "dep3")); + dep3->SetStateFilter(StateFilterUp); + RegisterDependency(dep3, ""); + // The grandparent is DOWN but the DNS redundancy group has to be still reachable. + BOOST_CHECK_EQUAL(true, childHost->IsReachable()); + DependencyGroup::Unregister(dep3); + /* Both DNS servers are DOWN. * Expected result: childHost is unreachable. */ @@ -98,4 +143,178 @@ BOOST_AUTO_TEST_CASE(multi_parent) BOOST_CHECK(childHost->IsReachable() == false); } +BOOST_AUTO_TEST_CASE(default_redundancy_group_registration_unregistration) +{ + Checkable::Ptr childHostC(CreateHost("C")); + Dependency::Ptr depCA(CreateDependency(CreateHost("A"), childHostC, "depCA")); + RegisterDependency(depCA, ""); + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depCB(CreateDependency(CreateHost("B"), childHostC, "depCB")); + RegisterDependency(depCB, ""); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 2); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Checkable::Ptr childHostD(CreateHost("D")); + Dependency::Ptr depDA(CreateDependency(depCA->GetParent(), childHostD, "depDA")); + RegisterDependency(depDA, ""); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depDB(CreateDependency(depCB->GetParent(), childHostD, "depDB")); + RegisterDependency(depDB, ""); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depCA); + DependencyGroup::Unregister(depDA); + AssertCheckableRedundancyGroup(childHostC, 1, 1, 2); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depCB); + DependencyGroup::Unregister(depDB); + AssertCheckableRedundancyGroup(childHostC, 0, 0, 0); + AssertCheckableRedundancyGroup(childHostD, 0, 0, 0); + BOOST_CHECK_EQUAL(0, DependencyGroup::GetRegistrySize()); +} + +BOOST_AUTO_TEST_CASE(simple_redundancy_group_registration_unregistration) +{ + Checkable::Ptr childHostC(CreateHost("childC")); + + Dependency::Ptr depCA(CreateDependency(CreateHost("A"), childHostC, "depCA")); + RegisterDependency(depCA, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depCB(CreateDependency(CreateHost("B"), childHostC, "depCB")); + RegisterDependency(depCB, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 2); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Checkable::Ptr childHostD(CreateHost("childD")); + Dependency::Ptr depDA (CreateDependency(depCA->GetParent(), childHostD, "depDA")); + RegisterDependency(depDA, "redundant"); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depDB(CreateDependency(depCB->GetParent(), childHostD, "depDB")); + RegisterDependency(depDB, "redundant"); + // Still 1 redundancy group, but there should be 4 dependencies now, i.e. 2 for each child Checkable. + AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depCA); + // After unregistering depCA, childHostC should have a new redundancy group with only depCB as dependency, and... + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + // ...childHostD should still have the same redundancy group as before but also with only two dependencies. + AssertCheckableRedundancyGroup(childHostD, 2, 1, 2); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depDA); + // Nothing should have changed for childHostC, but childHostD should now have a fewer group dependency, i.e. + // both child hosts should have the same redundancy group with only depCB and depDB as dependency. + AssertCheckableRedundancyGroup(childHostC, 1, 1, 2); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Register(depDA); + DependencyGroup::Unregister(depDB); + // Nothing should have changed for childHostC, but both should now have a separate group with only depCB and depDA as dependency. + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depCB); + DependencyGroup::Unregister(depDA); + AssertCheckableRedundancyGroup(childHostC, 0, 0, 0); + AssertCheckableRedundancyGroup(childHostD, 0, 0, 0); + BOOST_CHECK_EQUAL(0, DependencyGroup::GetRegistrySize()); +} + +BOOST_AUTO_TEST_CASE(mixed_redundancy_group_registration_unregsitration) +{ + Checkable::Ptr childHostC(CreateHost("childC")); + Dependency::Ptr depCA(CreateDependency(CreateHost("A"), childHostC, "depCA")); + RegisterDependency(depCA, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Checkable::Ptr childHostD(CreateHost("childD")); + Dependency::Ptr depDA(CreateDependency(depCA->GetParent(), childHostD, "depDA")); + RegisterDependency(depDA, "redundant"); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depCB(CreateDependency(CreateHost("B"), childHostC, "depCB")); + RegisterDependency(depCB, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 2); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depDB(CreateDependency(depCB->GetParent(), childHostD, "depDB")); + RegisterDependency(depDB, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Checkable::Ptr childHostE(CreateHost("childE")); + Dependency::Ptr depEA(CreateDependency(depCA->GetParent(), childHostE, "depEA")); + RegisterDependency(depEA, "redundant"); + AssertCheckableRedundancyGroup(childHostE, 1, 1, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depEB(CreateDependency(depCB->GetParent(), childHostE, "depEB")); + RegisterDependency(depEB, "redundant"); + // All 3 hosts share the same group, and each host has 2 dependencies, thus 6 dependencies in total. + AssertCheckableRedundancyGroup(childHostC, 2, 1, 6); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 6); + AssertCheckableRedundancyGroup(childHostE, 2, 1, 6); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depEZ(CreateDependency(CreateHost("Z"), childHostE, "depEZ")); + RegisterDependency(depEZ, "redundant"); + // Child host E should have a new redundancy group with 3 dependencies and the other two should still share the same group. + AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostE, 3, 1, 3); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depEA); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostE, 2, 1, 2); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Register(depEA); // Re-register depEA and instead... + DependencyGroup::Unregister(depEZ); // ...unregister depEZ and check if all the hosts share the same group again. + // All 3 hosts share the same group again, and each host has 2 dependencies, thus 6 dependencies in total. + AssertCheckableRedundancyGroup(childHostC, 2, 1, 6); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 6); + AssertCheckableRedundancyGroup(childHostE, 2, 1, 6); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depCA); + DependencyGroup::Unregister(depDB); + DependencyGroup::Unregister(depEB); + // Child host C has now a separate group with only depCB as dependency, and child hosts D and E share the same group. + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); + AssertCheckableRedundancyGroup(childHostE, 1, 1, 2); + // Child host C has now a separate group with only depCB as member, and child hosts D and E share the same group. + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depCB); + DependencyGroup::Unregister(depDA); + DependencyGroup::Unregister(depEA); + AssertCheckableRedundancyGroup(childHostC, 0, 0, 0); + AssertCheckableRedundancyGroup(childHostD, 0, 0, 0); + AssertCheckableRedundancyGroup(childHostE, 0, 0, 0); + BOOST_CHECK_EQUAL(0, DependencyGroup::GetRegistrySize()); +} + BOOST_AUTO_TEST_SUITE_END() From 4bfaefadfabf00eaa9ea3b9786aadbdb4c6840a8 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 16 Dec 2024 09:23:47 +0100 Subject: [PATCH 18/40] IcingaDB: Bump expected redis version to `6` --- lib/icingadb/icingadb-objects.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index cbdc4802653..e087f429a56 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -179,7 +179,7 @@ void IcingaDB::ConfigStaticInitialize() void IcingaDB::UpdateAllConfigObjects() { m_Rcon->Sync(); - m_Rcon->FireAndForgetQuery({"XADD", "icinga:schema", "MAXLEN", "1", "*", "version", "5"}, Prio::Heartbeat); + m_Rcon->FireAndForgetQuery({"XADD", "icinga:schema", "MAXLEN", "1", "*", "version", "6"}, Prio::Heartbeat); Log(LogInformation, "IcingaDB") << "Starting initial config/status dump"; double startTime = Utility::GetTime(); From c6466ee0eaea6e76c9a1800aae6719a8d091d167 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Wed, 4 Dec 2024 11:00:26 +0100 Subject: [PATCH 19/40] IcingaDB: Dump checkables dependencies config to redis correctly --- lib/icingadb/icingadb-objects.cpp | 376 +++++++++++++++--------------- lib/icingadb/icingadb.cpp | 2 - lib/icingadb/icingadb.hpp | 17 +- 3 files changed, 199 insertions(+), 196 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index e087f429a56..33cf973ce0a 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -62,7 +62,6 @@ std::vector IcingaDB::GetTypes() // Then sync them for similar reasons. Downtime::TypeInstance, Comment::TypeInstance, - Dependency::TypeInstance, HostGroup::TypeInstance, ServiceGroup::TypeInstance, @@ -208,10 +207,17 @@ void IcingaDB::UpdateAllConfigObjects() m_Rcon->FireAndForgetQuery({"XADD", "icinga:dump", "MAXLEN", "1", "*", "key", "*", "state", "wip"}, Prio::Config); const std::vector globalKeys = { - m_PrefixConfigObject + "customvar", - m_PrefixConfigObject + "action:url", - m_PrefixConfigObject + "notes:url", - m_PrefixConfigObject + "icon:image", + m_PrefixConfigObject + "customvar", + m_PrefixConfigObject + "action:url", + m_PrefixConfigObject + "notes:url", + m_PrefixConfigObject + "icon:image", + + // These keys aren't tied to a specific Checkable type but apply to both "Host" and "Service" types, + // and as such we've to make sure to clear them before we actually start dumping the actual objects. + // This allows us to wait on both types to be dumped before we send a config dump done signal for those keys. + m_PrefixConfigObject + "dependency:node", + m_PrefixConfigObject + "dependency:edge", + m_PrefixConfigObject + "redundancygroup", }; DeleteKeys(m_Rcon, globalKeys, Prio::Config); DeleteKeys(m_Rcon, {"icinga:nextupdate:host", "icinga:nextupdate:service"}, Prio::Config); @@ -222,6 +228,7 @@ void IcingaDB::UpdateAllConfigObjects() m_DumpedGlobals.ActionUrl.Reset(); m_DumpedGlobals.NotesUrl.Reset(); m_DumpedGlobals.IconImage.Reset(); + m_DumpedGlobals.DependencyGroup.Reset(); }); upq.ParallelFor(types, false, [this](const Type::Ptr& type) { @@ -793,138 +800,9 @@ void IcingaDB::InsertObjectDependencies(const ConfigObject::Ptr& object, const S } } - return; - } - - if (type == Dependency::TypeInstance) { - auto& dependencyNodes (hMSets[m_PrefixConfigObject + "dependency:node"]); - auto& dependencyEdges (hMSets[m_PrefixConfigObject + "dependency:edge"]); - auto& redundancyGroups (hMSets[m_PrefixConfigObject + "redundancygroup"]); - - Dependency::Ptr dependency = static_pointer_cast(object); - - Host::Ptr parentHost, childHost; - Service::Ptr parentService, childService; - tie(parentHost, parentService) = GetHostService(dependency->GetParent()); - tie(childHost, childService) = GetHostService(dependency->GetChild()); - String redundancyGroup = dependency->GetRedundancyGroup(); - - String redundancyGroupId, dependencyNodeParentId, dependencyNodeChildId, dependencyNodeReduId; - - Dictionary::Ptr parentNodeData, childNodeData; - - if (parentService) { - dependencyNodeParentId = HashValue(new Array({ - m_EnvironmentId, - GetObjectIdentifier(parentHost), - GetObjectIdentifier(parentService)})); - parentNodeData = new Dictionary({ - {"environment_id", m_EnvironmentId}, - {"host_id", GetObjectIdentifier(parentHost)}, - {"service_id", GetObjectIdentifier(parentService)}}); - - m_CheckablesToDependencies->Set(GetObjectIdentifier(parentService), dependency); - } else { - dependencyNodeParentId = HashValue(new Array({ - m_EnvironmentId, - GetObjectIdentifier(parentHost)})); - parentNodeData = new Dictionary({ - {"environment_id", m_EnvironmentId}, - {"host_id", GetObjectIdentifier(parentHost)}}); - - m_CheckablesToDependencies->Set(GetObjectIdentifier(parentHost), dependency); - } - - if (childService) { - dependencyNodeChildId = HashValue(new Array({ - m_EnvironmentId, - GetObjectIdentifier(childHost), - GetObjectIdentifier(childService)})); - childNodeData = new Dictionary({ - {"environment_id", m_EnvironmentId}, - {"host_id", GetObjectIdentifier(childHost)}, - {"service_id", GetObjectIdentifier(childService)}}); - - m_CheckablesToDependencies->Set(GetObjectIdentifier(childService), dependency); - } else { - dependencyNodeChildId = HashValue(new Array({ - m_EnvironmentId, - GetObjectIdentifier(childHost)})); - childNodeData = new Dictionary({ - {"environment_id", m_EnvironmentId}, - {"host_id", GetObjectIdentifier(childHost)}}); - - m_CheckablesToDependencies->Set(GetObjectIdentifier(childHost), dependency); - } - - dependencyNodes.emplace_back(dependencyNodeParentId); - dependencyNodes.emplace_back(JsonEncode(parentNodeData)); - dependencyNodes.emplace_back(dependencyNodeChildId); - dependencyNodes.emplace_back(JsonEncode(childNodeData)); - - if (runtimeUpdate) { - AddObjectDataToRuntimeUpdates(runtimeUpdates, dependencyNodeParentId, m_PrefixConfigObject + "dependency:node", parentNodeData); - AddObjectDataToRuntimeUpdates(runtimeUpdates, dependencyNodeChildId, m_PrefixConfigObject + "dependency:node", childNodeData); - } - - if (!redundancyGroup.IsEmpty()) { - /* TODO: name should be suffixed with names of all children. - * however, at this point I don't have this information, - * only the direct neighbors. - */ - redundancyGroupId = HashValue(new Array({m_EnvironmentId, redundancyGroup, dependencyNodeChildId})); - dependencyNodeReduId = redundancyGroupId; - - redundancyGroups.emplace_back(redundancyGroupId); - Dictionary::Ptr groupData = new Dictionary({ - {"environment_id", m_EnvironmentId}, - {"name", redundancyGroupId}, - {"display_name", redundancyGroup}}); - redundancyGroups.emplace_back(JsonEncode(groupData)); - - dependencyNodes.emplace_back(dependencyNodeReduId); - Dictionary::Ptr reduNodeData = new Dictionary({ - {"environment_id", m_EnvironmentId}, - {"redundancy_group_id", redundancyGroupId}}); - dependencyNodes.emplace_back(JsonEncode(reduNodeData)); + InsertCheckableDependencies(checkable, hMSets, runtimeUpdate ? &runtimeUpdates : nullptr); - String edgeInId = HashValue(new Array({m_EnvironmentId, dependencyNodeChildId, dependencyNodeReduId})); - dependencyEdges.emplace_back(edgeInId); - Dictionary::Ptr edgeInData = new Dictionary({ - {"environment_id", m_EnvironmentId}, - {"from_node_id", dependencyNodeChildId}, - {"to_node_id", dependencyNodeReduId}}); - dependencyEdges.emplace_back(JsonEncode(edgeInData)); - - String edgeOutId = HashValue(new Array({m_EnvironmentId, dependencyNodeReduId, dependencyNodeParentId})); - dependencyEdges.emplace_back(edgeOutId); - Dictionary::Ptr edgeOutData = new Dictionary({ - {"environment_id", m_EnvironmentId}, - {"from_node_id", dependencyNodeReduId}, - {"to_node_id", dependencyNodeParentId}, - {"dependency_id", GetObjectIdentifier(dependency)}}); - dependencyEdges.emplace_back(JsonEncode(edgeOutData)); - - if (runtimeUpdate) { - AddObjectDataToRuntimeUpdates(runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "redundancygroup", groupData); - AddObjectDataToRuntimeUpdates(runtimeUpdates, dependencyNodeReduId, m_PrefixConfigObject + "dependency:node", reduNodeData); - AddObjectDataToRuntimeUpdates(runtimeUpdates, edgeInId, m_PrefixConfigObject + "dependency:edge", edgeInData); - AddObjectDataToRuntimeUpdates(runtimeUpdates, edgeOutId, m_PrefixConfigObject + "dependency:edge", edgeOutData); - } - } else { - String edgeId = HashValue(new Array({m_EnvironmentId, dependencyNodeChildId, dependencyNodeParentId})); - dependencyEdges.emplace_back(edgeId); - Dictionary::Ptr edgeData = new Dictionary({ - {"environment_id", m_EnvironmentId}, - {"from_node_id", dependencyNodeChildId}, - {"to_node_id", dependencyNodeParentId}, - {"dependency_id", GetObjectIdentifier(dependency)}}); - dependencyEdges.emplace_back(JsonEncode(edgeData)); - - if (runtimeUpdate) { - AddObjectDataToRuntimeUpdates(runtimeUpdates, edgeId, m_PrefixConfigObject + "dependency:edge", edgeData); - } - } + return; } if (type == TimePeriod::TypeInstance) { @@ -1257,44 +1135,156 @@ void IcingaDB::InsertObjectDependencies(const ConfigObject::Ptr& object, const S } } -void IcingaDB::UpdateDependencyState(const Dependency::Ptr& dependency) +/** + * Inserts the dependency data for a Checkable object into the given Redis HMSETs and runtime updates. + * + * This function is responsible for serializing the in memory representation Checkable dependencies into + * Redis HMSETs and runtime updates (if any) according to the Icinga DB schema. The serialized data consists + * of the following Redis HMSETs: + * - RedisKey::DependencyNode: Contains dependency node data representing each host, service, and redundancy group + * in the dependency graph. + * - RedisKey::DependencyEdge: Dependency edge information representing all connections between the nodes. + * - RedisKey::RedundancyGroup: Redundancy group data representing all redundancy groups in the graph. + * + * For initial dumps, it shouldn't be necessary to set the `runtimeUpdates` parameter. + * + * @param checkable The checkable object to extract dependencies from. + * @param hMSets The map of Redis HMSETs to insert the dependency data into. + * @param runtimeUpdates If set, runtime updates are additionally added to this vector. + */ +void IcingaDB::InsertCheckableDependencies( + const Checkable::Ptr& checkable, + std::map& hMSets, + std::vector* runtimeUpdates +) { - if (!m_Rcon || !m_Rcon->IsConnected()) { + // Only generate a dependency node event if the Checkable is actually part of some dependency graph. + // That's, it either depends on other Checkables or others depend on it, and in both cases, we have + // to at least generate a dependency node entry for it. + if (!checkable->HasAnyDependencies()) { return; } - auto& redundancyGroupStates (hMSets[m_PrefixConfigObject + "redundancygroup:state"]); + // First and foremost, generate a dependency node entry for the provided Checkable object and insert it into + // the HMSETs map and if set, the `runtimeUpdates` vector. + auto [host, service] = GetHostService(checkable); + auto checkableId(GetObjectIdentifier(checkable)); + { + Dictionary::Ptr data(new Dictionary{{"environment_id", m_EnvironmentId}, {"host_id", GetObjectIdentifier(host)}}); + if (service) { + data->Set("service_id", checkableId); + } + + AddDataToHmSets(hMSets, RedisKey::DependencyNode, checkableId, data); + if (runtimeUpdates) { + AddObjectDataToRuntimeUpdates(*runtimeUpdates, checkableId, m_PrefixConfigObject + "dependency:node", data); + } + } + + for (auto& dependencyGroup : checkable->GetDependencyGroups()) { + String edgeFromNodeId(checkableId); - String redundancyGroup = dependency->GetRedundancyGroup(); + if (dependencyGroup->IsRedundancyGroup()) { + auto redundancyGroupId(HashValue(new Array{m_EnvironmentId, dependencyGroup->GetCompositeKey()})); + dependencyGroup->SetIcingaDBIdentifier(redundancyGroupId); - if (!redundancyGroup.IsEmpty()) { - Host::Ptr childHost; - Service::Ptr childService; - tie(childHost, childService) = GetHostService(dependency->GetChild()); + edgeFromNodeId = redundancyGroupId; - String dependencyNodeChildId = HashValue( - (childService) - ? new Array({ m_EnvironmentId, GetObjectIdentifier(childHost), GetObjectIdentifier(childService) }) - : new Array({ m_EnvironmentId, GetObjectIdentifier(childHost) })); - String redundancyGroupId = HashValue(new Array({ - m_EnvironmentId, - redundancyGroup, - dependencyNodeChildId})); + // During the initial config sync, multiple children can depend on the same redundancy group, sync it only + // the first time it is encountered. Though, if this is a runtime update, we have to re-serialize and sync + // the redundancy group unconditionally, as we don't know whether it was already synced or the context that + // triggered this update. + if (runtimeUpdates || m_DumpedGlobals.DependencyGroup.IsNew(redundancyGroupId)) { + Dictionary::Ptr groupData(new Dictionary{ + {"environment_id", m_EnvironmentId}, + {"display_name", dependencyGroup->GetRedundancyGroupName()}, + }); + // Set/refresh the redundancy group data in the Redis HMSETs (redundancy_group database table). + AddDataToHmSets(hMSets, RedisKey::RedundancyGroup, redundancyGroupId, groupData); - redundancyGroupStates.emplace_back(redundancyGroupId); - Dictionary::Ptr groupStateData = new Dictionary({ + Dictionary::Ptr nodeData(new Dictionary{ + {"environment_id", m_EnvironmentId}, + {"redundancy_group_id", redundancyGroupId}, + }); + // Obviously, the redundancy group is part of some dependency chain, thus we have to generate + // dependency node entry for it as well. + AddDataToHmSets(hMSets, RedisKey::DependencyNode, redundancyGroupId, nodeData); + + if (runtimeUpdates) { + // Send the same data sent to the Redis HMSETs to the runtime updates stream as well. + AddObjectDataToRuntimeUpdates(*runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "redundancygroup", groupData); + AddObjectDataToRuntimeUpdates(*runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "dependency:node", nodeData); + } + } + + Dictionary::Ptr data(new Dictionary{ {"environment_id", m_EnvironmentId}, - {"redundancy_group_id", redundancyGroupId}, - {"failed", !((childService) ? childService->IsReachable() : childHost->IsReachable())}, - {"last_state_change", TimestampToMilliseconds(Utility::GetTime())}}); - redundancyGroupStates.emplace_back(JsonEncode(groupStateData)); + {"from_node_id", checkableId}, + {"to_node_id", redundancyGroupId}, + // All redundancy group members share the same state, thus use the group ID as a reference. + {"dependency_edge_state_id", redundancyGroupId}, + {"display_name", dependencyGroup->GetRedundancyGroupName()}, + }); + + // Generate a dependency edge entry representing the connection between the Checkable and the redundancy + // group. This Checkable dependes on the redundancy group (is a child of it), thus the "dependency_edge_state_id" + // is set to the redundancy group ID. Note that if this group has multiple children, they all will have the + // same "dependency_edge_state_id" value. + auto edgeId(HashValue(new Array{checkableId, redundancyGroupId})); + AddDataToHmSets(hMSets, RedisKey::DependencyEdge, edgeId, data); + + if (runtimeUpdates) { + AddObjectDataToRuntimeUpdates(*runtimeUpdates, edgeId, m_PrefixConfigObject + "dependency:edge", data); + } + } + + auto dependencies(dependencyGroup->GetDependenciesForChild(checkable.get())); + // Sort the dependencies by their parent Checkable object to ensure that all dependency objects that share the + // same parent Checkable are placed next to each other in the container. See the while loop below for more info! + std::sort(dependencies.begin(), dependencies.end(), [](const Dependency::Ptr& lhs, const Dependency::Ptr& rhs) { + return lhs->GetParent() < rhs->GetParent(); + }); + + // Traverse through each dependency objects within the current dependency group the provided Checkable depend + // on and generate a dependency edge entry. The generated dependency edge "from_node_id" may vary depending on + // whether the dependency group is a redundancy group or not. If it's a redundancy group, the "from_node_id" + // will be the redundancy group ID; otherwise, it will be the current Checkable ID. However, the "to_node_id" + // value will always be the parent Checkable ID of the dependency object. + for (auto it(dependencies.begin()); it != dependencies.end(); /* no increment */) { + auto dependency(*it); + auto parent(dependency->GetParent()); + auto displayName(dependency->GetShortName()); + + // In case there are multiple Dependency objects with the same parent, these are merged into a single edge + // to prevent duplicate edges in the resulting graph. All objects with the same parent were placed next to + // each other by the sort function above. + // + // Additionally, the iterator for the surrounding loop is incremented by this loop: after it has finished, + // "it" will either point to the next dependency with a different parent or to the end of the container. + while (++it != dependencies.end() && (*it)->GetParent() == parent) { + displayName += ", " + (*it)->GetShortName(); + } - // TODO - // AddObjectDataToRuntimeUpdates(runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "redundancygroup:state", groupStateData); - // dataClone->Set("id", objectKey); // redundancyGroupId - // dataClone->Set("redis_key", redisKey); // m_PrefixConfigObject + "redundancygroup:state" - // dataClone->Set("runtime_type", "upsert"); - // runtimeUpdates.emplace_back(dataClone); + Dictionary::Ptr data(new Dictionary{ + {"environment_id", m_EnvironmentId}, + {"from_node_id", edgeFromNodeId}, + {"to_node_id", GetObjectIdentifier(parent)}, + {"dependency_edge_state_id", HashValue(new Array{ + dependencyGroup->IsRedundancyGroup() + ? dependencyGroup->GetIcingaDBIdentifier() + : dependencyGroup->GetCompositeKey(), + GetObjectIdentifier(dependency->GetParent()), + })}, + {"display_name", std::move(displayName)}, + }); + + auto edgeId(HashValue(new Array{data->Get("from_node_id"), data->Get("to_node_id")})); + AddDataToHmSets(hMSets, RedisKey::DependencyEdge, edgeId, data); + + if (runtimeUpdates) { + AddObjectDataToRuntimeUpdates(*runtimeUpdates, edgeId, m_PrefixConfigObject + "dependency:edge", data); + } + } } } @@ -1632,32 +1622,6 @@ bool IcingaDB::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& a return true; } - if (type == Dependency::TypeInstance) { - Dependency::Ptr dependency = static_pointer_cast(object); - String redundancyGroup = dependency->GetRedundancyGroup(); - - attributes->Set("name", GetObjectIdentifier(dependency)); - - if (!redundancyGroup.IsEmpty()) { - Host::Ptr childHost; - Service::Ptr childService; - tie(childHost, childService) = GetHostService(dependency->GetChild()); - - String dependencyNodeChildId = HashValue( - (childService) - ? new Array({ m_EnvironmentId, GetObjectIdentifier(childHost), GetObjectIdentifier(childService) }) - : new Array({ m_EnvironmentId, GetObjectIdentifier(childHost) })); - String redundancyGroupId = HashValue(new Array({ - m_EnvironmentId, - redundancyGroup, - dependencyNodeChildId})); - - attributes->Set("redundancy_group_id", redundancyGroupId); - } - - return true; - } - if (type == Downtime::TypeInstance) { Downtime::Ptr downtime = static_pointer_cast(object); @@ -3182,3 +3146,35 @@ void IcingaDB::DeleteRelationship(const String& id, const String& redisKeyWithou m_Rcon->FireAndForgetQueries(queries, Prio::Config); } + +/** + * Add the provided data to the Redis HMSETs map. + * + * Adds the provided data to the Redis HMSETs map for the provided Redis key. The actual Redis key is determined by + * the provided RedisKey enum. The data will be json encoded before being added to the Redis HMSETs map. + * + * @param hMSets The map of RedisConnection::Query you want to add the data to. + * @param redisKey The key of the Redis object you want to add the data to. + * @param id Unique Redis identifier for the provided data. + * @param data The actual data you want to add the Redis HMSETs map. + */ +void IcingaDB::AddDataToHmSets(std::map& hMSets, RedisKey redisKey, const String& id, const Dictionary::Ptr& data) const +{ + RedisConnection::Query* query; + switch (redisKey) { + case RedisKey::RedundancyGroup: + query = &hMSets[m_PrefixConfigObject + "redundancygroup"]; + break; + case RedisKey::DependencyNode: + query = &hMSets[m_PrefixConfigObject + "dependency:node"]; + break; + case RedisKey::DependencyEdge: + query = &hMSets[m_PrefixConfigObject + "dependency:edge"]; + break; + default: + BOOST_THROW_EXCEPTION(std::invalid_argument("Invalid RedisKey provided")); + } + + query->emplace_back(id); + query->emplace_back(JsonEncode(data)); +} diff --git a/lib/icingadb/icingadb.cpp b/lib/icingadb/icingadb.cpp index 3c623259b58..8d3b9099bd7 100644 --- a/lib/icingadb/icingadb.cpp +++ b/lib/icingadb/icingadb.cpp @@ -38,8 +38,6 @@ IcingaDB::IcingaDB() m_PrefixConfigObject = "icinga:"; m_PrefixConfigCheckSum = "icinga:checksum:"; - - m_CheckablesToDependencies = new Dictionary(); } void IcingaDB::Validate(int types, const ValidationUtils& utils) diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index c5b49931848..f24dcdc444c 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -90,6 +90,15 @@ class IcingaDB : public ObjectImpl Full = Volatile | RuntimeOnly, }; + enum class RedisKey : uint8_t + { + RedundancyGroup, + DependencyNode, + DependencyEdge, + RedundancyGroupState, + DependencyEdgeState, + }; + void OnConnectedHandler(); void PublishStatsTimerHandler(); @@ -101,9 +110,10 @@ class IcingaDB : public ObjectImpl void DeleteKeys(const RedisConnection::Ptr& conn, const std::vector& keys, RedisConnection::QueryPriority priority); std::vector GetTypeOverwriteKeys(const String& type); std::vector GetTypeDumpSignalKeys(const Type::Ptr& type); + void InsertCheckableDependencies(const Checkable::Ptr& checkable, std::map& hMSets, + std::vector* runtimeUpdates); void InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map>& hMSets, std::vector& runtimeUpdates, bool runtimeUpdate); - void UpdateDependencyState(const Dependency::Ptr& dependency); void UpdateState(const Checkable::Ptr& checkable, StateUpdate mode); void SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate); void CreateConfigUpdate(const ConfigObject::Ptr& object, const String type, std::map>& hMSets, @@ -113,6 +123,7 @@ class IcingaDB : public ObjectImpl void AddObjectDataToRuntimeUpdates(std::vector& runtimeUpdates, const String& objectKey, const String& redisKey, const Dictionary::Ptr& data); void DeleteRelationship(const String& id, const String& redisKeyWithoutPrefix, bool hasChecksum = false); + void AddDataToHmSets(std::map& hMSets, RedisKey redisKey, const String& id, const Dictionary::Ptr& data) const; void SendSentNotification( const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set& users, @@ -225,10 +236,8 @@ class IcingaDB : public ObjectImpl std::unordered_map m_Rcons; std::atomic_size_t m_PendingRcons; - Dictionary::Ptr m_CheckablesToDependencies; - struct { - DumpedGlobals CustomVar, ActionUrl, NotesUrl, IconImage; + DumpedGlobals CustomVar, ActionUrl, NotesUrl, IconImage, DependencyGroup; } m_DumpedGlobals; // m_EnvironmentId is shared across all IcingaDB objects (typically there is at most one, but it is perfectly fine From f502993eb4034262471744f04ef04f01d24e2b28 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Wed, 4 Dec 2024 16:47:32 +0100 Subject: [PATCH 20/40] IcingaDB: Sync dependencies states to Redis --- lib/icinga/dependency.hpp | 9 ++++ lib/icingadb/icingadb-objects.cpp | 87 +++++++++++++++++++++++++++++++ lib/icingadb/icingadb-utility.cpp | 59 +++++++++++++++++++++ lib/icingadb/icingadb.hpp | 3 ++ 4 files changed, 158 insertions(+) diff --git a/lib/icinga/dependency.hpp b/lib/icinga/dependency.hpp index 34667eec120..63bab92ad51 100644 --- a/lib/icinga/dependency.hpp +++ b/lib/icinga/dependency.hpp @@ -180,6 +180,15 @@ class DependencyGroup final : public SharedObject private: mutable std::mutex m_Mutex; + /** + * This identifier is used by Icinga DB to cache the unique hash of this dependency group. + * + * For redundancy groups, once Icinga DB sets this identifier, it will never change again for the lifetime + * of the object. For non-redundant dependency groups, this identifier is (mis)used to cache the shared edge + * state ID of the group. Specifically, non-redundant dependency groups are irrelevant for Icinga DB, so since + * this field isn't going to be used for anything else, we use it to cache the computed shared edge state ID. + * Likewise, if that gets set, it will never change again for the lifetime of the object as well. + */ String m_IcingaDBIdentifier; String m_RedundancyGroupName; MembersMap m_Members; diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 33cf973ce0a..e61aee3ce5e 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -217,7 +217,9 @@ void IcingaDB::UpdateAllConfigObjects() // This allows us to wait on both types to be dumped before we send a config dump done signal for those keys. m_PrefixConfigObject + "dependency:node", m_PrefixConfigObject + "dependency:edge", + m_PrefixConfigObject + "dependency:edge:state", m_PrefixConfigObject + "redundancygroup", + m_PrefixConfigObject + "redundancygroup:state", }; DeleteKeys(m_Rcon, globalKeys, Prio::Config); DeleteKeys(m_Rcon, {"icinga:nextupdate:host", "icinga:nextupdate:service"}, Prio::Config); @@ -1344,6 +1346,90 @@ void IcingaDB::UpdateState(const Checkable::Ptr& checkable, StateUpdate mode) } } +/** + * Send dependencies state information of the given Checkable to Redis. + * + * If the dependencyGroup parameter is set, only the dependencies state of that group are sent. Otherwise, all + * dependency groups of the provided Checkable are processed. + * + * @param checkable The Checkable you want to send the dependencies state update for + * @param onlyDependencyGroup If set, send state updates only for this dependency group and its dependencies. + */ +void IcingaDB::UpdateDependenciesState(const Checkable::Ptr& checkable, const DependencyGroup::Ptr& onlyDependencyGroup) const +{ + if (!m_Rcon || !m_Rcon->IsConnected()) { + return; + } + + std::vector dependencyGroups{onlyDependencyGroup}; + if (!onlyDependencyGroup) { + dependencyGroups = checkable->GetDependencyGroups(); + if (dependencyGroups.empty()) { + return; + } + } + + RedisConnection::Queries streamStates; + auto addDependencyStateToStream([this, &streamStates](const String& redisKey, const Dictionary::Ptr& stateAttrs) { + RedisConnection::Query xAdd{ + "XADD", "icinga:runtime:state", "MAXLEN", "~", "1000000", "*", "runtime_type", "upsert", + "redis_key", redisKey + }; + ObjectLock olock(stateAttrs); + for (auto& [key, value] : stateAttrs) { + xAdd.emplace_back(key); + xAdd.emplace_back(IcingaToStreamValue(value)); + } + streamStates.emplace_back(std::move(xAdd)); + }); + + for (auto& dependencyGroup : dependencyGroups) { + bool isRedundancyGroup(dependencyGroup->IsRedundancyGroup()); + if (isRedundancyGroup && dependencyGroup->GetIcingaDBIdentifier().IsEmpty()) { + // Way too soon! The Icinga DB hash will be set during the initial config dump, but this state + // update seems to occur way too early. So, we've to skip it for now and wait for the next one. + // The m_ConfigDumpInProgress flag is probably still set to true at this point! + continue; + } + + auto dependencies(dependencyGroup->GetDependenciesForChild(checkable.get())); + std::sort(dependencies.begin(), dependencies.end(), [](const Dependency::Ptr& lhs, const Dependency::Ptr& rhs) { + return lhs->GetParent() < rhs->GetParent(); + }); + for (auto it(dependencies.begin()); it != dependencies.end(); /* no increment */) { + const auto& dependency(*it); + + Dictionary::Ptr stateAttrs; + // Note: The following loop is intended to cover some possible special cases but may not occur in practice + // that often. That is, having two or more dependency objects that point to the same parent Checkable. + // So, traverse all those duplicates and merge their relevant state information into a single edge. + for (; it != dependencies.end() && (*it)->GetParent() == dependency->GetParent(); ++it) { + if (!stateAttrs || stateAttrs->Get("failed") == false) { + stateAttrs = SerializeDependencyEdgeState(dependencyGroup, *it); + } + } + + addDependencyStateToStream(m_PrefixConfigObject + "dependency:edge:state", stateAttrs); + } + + if (isRedundancyGroup) { + Dictionary::Ptr stateAttrs(SerializeRedundancyGroupState(dependencyGroup)); + + Dictionary::Ptr sharedGroupState(stateAttrs->ShallowClone()); + sharedGroupState->Remove("redundancy_group_id"); + sharedGroupState->Remove("is_reachable"); + sharedGroupState->Remove("last_state_change"); + + addDependencyStateToStream(m_PrefixConfigObject + "redundancygroup:state", stateAttrs); + addDependencyStateToStream(m_PrefixConfigObject + "dependency:edge:state", sharedGroupState); + } + } + + if (!streamStates.empty()) { + m_Rcon->FireAndForgetQueries(std::move(streamStates), Prio::RuntimeStateStream, {0, 1}); + } +} + // Used to update a single object, used for runtime updates void IcingaDB::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate) { @@ -2933,6 +3019,7 @@ void IcingaDB::ReachabilityChangeHandler(const std::set& childre for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType()) { for (auto& checkable : children) { rw->UpdateState(checkable, StateUpdate::Full); + rw->UpdateDependenciesState(checkable); } } } diff --git a/lib/icingadb/icingadb-utility.cpp b/lib/icingadb/icingadb-utility.cpp index 35f503ab53d..f53055a50a2 100644 --- a/lib/icingadb/icingadb-utility.cpp +++ b/lib/icingadb/icingadb-utility.cpp @@ -159,6 +159,65 @@ Dictionary::Ptr IcingaDB::SerializeVars(const Dictionary::Ptr& vars) return res; } +/** + * Serialize a dependency edge state for Icinga DB + * + * @param dependencyGroup The state of the group the dependency is part of. + * @param dep The dependency object to serialize. + * + * @return A dictionary with the serialized state. + */ +Dictionary::Ptr IcingaDB::SerializeDependencyEdgeState(const DependencyGroup::Ptr& dependencyGroup, const Dependency::Ptr& dep) +{ + String edgeStateId; + // The edge state ID is computed a bit differently depending on whether this is for a redundancy group or not. + // For redundancy groups, the state ID is supposed to represent the connection state between the redundancy group + // and the parent Checkable of the given dependency. Hence, the outcome will always be different for each parent + // Checkable of the redundancy group. + if (dependencyGroup->IsRedundancyGroup()) { + edgeStateId = HashValue(new Array{ + dependencyGroup->GetIcingaDBIdentifier(), + GetObjectIdentifier(dep->GetParent()), + }); + } else if (dependencyGroup->GetIcingaDBIdentifier().IsEmpty()) { + // For non-redundant dependency groups, on the other hand, all dependency objects within that group will + // always have the same parent Checkable. Likewise, the state ID will be always the same as well it doesn't + // matter which dependency object is used to compute it. Therefore, it's sufficient to compute it only once + // and all the other dependency objects can reuse the cached state ID. + edgeStateId = HashValue(new Array{dependencyGroup->GetCompositeKey(), GetObjectIdentifier(dep->GetParent())}); + dependencyGroup->SetIcingaDBIdentifier(edgeStateId); + } else { + // Use the already computed state ID for the dependency group. + edgeStateId = dependencyGroup->GetIcingaDBIdentifier(); + } + + return new Dictionary{ + {"id", std::move(edgeStateId)}, + {"environment_id", m_EnvironmentId}, + {"failed", !dep->IsAvailable(DependencyState) || !dep->GetParent()->IsReachable()} + }; +} + +/** + * Serialize the provided redundancy group state attributes. + * + * @param redundancyGroup The redundancy group object to serialize the state for. + * + * @return A dictionary with the serialized redundancy group state. + */ +Dictionary::Ptr IcingaDB::SerializeRedundancyGroupState(const DependencyGroup::Ptr& redundancyGroup) +{ + auto state(redundancyGroup->GetState()); + return new Dictionary{ + {"id", redundancyGroup->GetIcingaDBIdentifier()}, + {"environment_id", m_EnvironmentId}, + {"redundancy_group_id", redundancyGroup->GetIcingaDBIdentifier()}, + {"failed", !state.Reachable || !state.OK}, + {"is_reachable", state.Reachable}, + {"last_state_change", TimestampToMilliseconds(Utility::GetTime())}, + }; +} + const char* IcingaDB::GetNotificationTypeByEnum(NotificationType type) { switch (type) { diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index f24dcdc444c..73ee4e8ae67 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -114,6 +114,7 @@ class IcingaDB : public ObjectImpl std::vector* runtimeUpdates); void InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map>& hMSets, std::vector& runtimeUpdates, bool runtimeUpdate); + void UpdateDependenciesState(const Checkable::Ptr& checkable, const DependencyGroup::Ptr& onlyDependencyGroup = nullptr) const; void UpdateState(const Checkable::Ptr& checkable, StateUpdate mode); void SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate); void CreateConfigUpdate(const ConfigObject::Ptr& object, const String type, std::map>& hMSets, @@ -169,6 +170,8 @@ class IcingaDB : public ObjectImpl static String CalcEventID(const char* eventType, const ConfigObject::Ptr& object, double eventTime = 0, NotificationType nt = NotificationType(0)); static const char* GetNotificationTypeByEnum(NotificationType type); static Dictionary::Ptr SerializeVars(const Dictionary::Ptr& vars); + static Dictionary::Ptr SerializeDependencyEdgeState(const DependencyGroup::Ptr& dependencyGroup, const Dependency::Ptr& dep); + static Dictionary::Ptr SerializeRedundancyGroupState(const DependencyGroup::Ptr& redundancyGroup); static String HashValue(const Value& value); static String HashValue(const Value& value, const std::set& propertiesBlacklist, bool propertiesWhitelist = false); From db3f8dec275e135a43f7ce01bdcced5cd297f94b Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Thu, 5 Dec 2024 16:21:32 +0100 Subject: [PATCH 21/40] IcingaDB: Sync dependencies initial states on config dump --- lib/icingadb/icingadb-objects.cpp | 45 ++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index e61aee3ce5e..ecae7c201dc 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1147,6 +1147,9 @@ void IcingaDB::InsertObjectDependencies(const ConfigObject::Ptr& object, const S * in the dependency graph. * - RedisKey::DependencyEdge: Dependency edge information representing all connections between the nodes. * - RedisKey::RedundancyGroup: Redundancy group data representing all redundancy groups in the graph. + * - RedisKey::RedundancyGroupState: State information for redundancy groups. + * - RedisKey::DependencyEdgeState: State information for (each) dependency edge. Multiple edges may share the + * same state. * * For initial dumps, it shouldn't be necessary to set the `runtimeUpdates` parameter. * @@ -1185,8 +1188,14 @@ void IcingaDB::InsertCheckableDependencies( for (auto& dependencyGroup : checkable->GetDependencyGroups()) { String edgeFromNodeId(checkableId); + bool syncSharedEdgeState(false); - if (dependencyGroup->IsRedundancyGroup()) { + if (!dependencyGroup->IsRedundancyGroup()) { + // Non-redundant dependency groups are just placeholders and never get synced to Redis, thus just figure + // out whether we have to sync the shared edge state. For runtime updates the states are sent via the + // UpdateDependenciesState() method, thus we don't have to sync them here. + syncSharedEdgeState = !runtimeUpdates && m_DumpedGlobals.DependencyGroup.IsNew(dependencyGroup->GetCompositeKey()); + } else { auto redundancyGroupId(HashValue(new Array{m_EnvironmentId, dependencyGroup->GetCompositeKey()})); dependencyGroup->SetIcingaDBIdentifier(redundancyGroupId); @@ -1216,6 +1225,20 @@ void IcingaDB::InsertCheckableDependencies( // Send the same data sent to the Redis HMSETs to the runtime updates stream as well. AddObjectDataToRuntimeUpdates(*runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "redundancygroup", groupData); AddObjectDataToRuntimeUpdates(*runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "dependency:node", nodeData); + } else { + syncSharedEdgeState = true; + + // Serialize and sync the redundancy group state information a) to the RedundancyGroupState and b) + // to the DependencyEdgeState HMSETs. The latter is shared by all child Checkables of the current + // redundancy group, and since they all depend on the redundancy group, the state of that group is + // basically the state of the dependency edges between the children and the redundancy group. + auto stateAttrs(SerializeRedundancyGroupState(dependencyGroup)); + AddDataToHmSets(hMSets, RedisKey::RedundancyGroupState, redundancyGroupId, stateAttrs); + AddDataToHmSets(hMSets, RedisKey::DependencyEdgeState, redundancyGroupId, Dictionary::Ptr(new Dictionary{ + {"id", redundancyGroupId}, + {"environment_id", m_EnvironmentId}, + {"failed", stateAttrs->Get("failed")}, + })); } } @@ -1257,6 +1280,8 @@ void IcingaDB::InsertCheckableDependencies( auto parent(dependency->GetParent()); auto displayName(dependency->GetShortName()); + Dictionary::Ptr edgeStateAttrs(SerializeDependencyEdgeState(dependencyGroup, dependency)); + // In case there are multiple Dependency objects with the same parent, these are merged into a single edge // to prevent duplicate edges in the resulting graph. All objects with the same parent were placed next to // each other by the sort function above. @@ -1265,18 +1290,16 @@ void IcingaDB::InsertCheckableDependencies( // "it" will either point to the next dependency with a different parent or to the end of the container. while (++it != dependencies.end() && (*it)->GetParent() == parent) { displayName += ", " + (*it)->GetShortName(); + if (syncSharedEdgeState && edgeStateAttrs->Get("failed") == false) { + edgeStateAttrs = SerializeDependencyEdgeState(dependencyGroup, *it); + } } Dictionary::Ptr data(new Dictionary{ {"environment_id", m_EnvironmentId}, {"from_node_id", edgeFromNodeId}, {"to_node_id", GetObjectIdentifier(parent)}, - {"dependency_edge_state_id", HashValue(new Array{ - dependencyGroup->IsRedundancyGroup() - ? dependencyGroup->GetIcingaDBIdentifier() - : dependencyGroup->GetCompositeKey(), - GetObjectIdentifier(dependency->GetParent()), - })}, + {"dependency_edge_state_id", edgeStateAttrs->Get("id")}, {"display_name", std::move(displayName)}, }); @@ -1285,6 +1308,8 @@ void IcingaDB::InsertCheckableDependencies( if (runtimeUpdates) { AddObjectDataToRuntimeUpdates(*runtimeUpdates, edgeId, m_PrefixConfigObject + "dependency:edge", data); + } else if (syncSharedEdgeState) { + AddDataToHmSets(hMSets, RedisKey::DependencyEdgeState, edgeStateAttrs->Get("id"), edgeStateAttrs); } } } @@ -3258,6 +3283,12 @@ void IcingaDB::AddDataToHmSets(std::map& hMSets, case RedisKey::DependencyEdge: query = &hMSets[m_PrefixConfigObject + "dependency:edge"]; break; + case RedisKey::RedundancyGroupState: + query = &hMSets[m_PrefixConfigObject + "redundancygroup:state"]; + break; + case RedisKey::DependencyEdgeState: + query = &hMSets[m_PrefixConfigObject + "dependency:edge:state"]; + break; default: BOOST_THROW_EXCEPTION(std::invalid_argument("Invalid RedisKey provided")); } From aed1bb6294a07d088d051339edebdb44619e2ec7 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 24 Feb 2025 13:11:12 +0100 Subject: [PATCH 22/40] IcingaDB: Introduce `ExecuteRedisTransaction()` helper method --- lib/icingadb/icingadb-objects.cpp | 122 +++++++++++++----------------- lib/icingadb/icingadb.hpp | 3 + 2 files changed, 57 insertions(+), 68 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index ecae7c201dc..e407373fa55 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -273,11 +273,6 @@ void IcingaDB::UpdateAllConfigObjects() upqObjectType.ParallelFor(objectChunks, [&](decltype(objectChunks)::const_reference chunk) { std::map> hMSets; - // Two values are appended per object: Object ID (Hash encoded) and Object State (IcingaDB::SerializeState() -> JSON encoded) - std::vector states = {"HMSET", m_PrefixConfigObject + lcType + ":state"}; - // Two values are appended per object: Object ID (Hash encoded) and State Checksum ({ "checksum": checksum } -> JSON encoded) - std::vector statesChksms = {"HMSET", m_PrefixConfigCheckSum + lcType + ":state"}; - std::vector > transaction = {{"MULTI"}}; std::vector hostZAdds = {"ZADD", "icinga:nextupdate:host"}, serviceZAdds = {"ZADD", "icinga:nextupdate:service"}; auto skimObjects ([&]() { @@ -317,9 +312,11 @@ void IcingaDB::UpdateAllConfigObjects() String objectKey = GetObjectIdentifier(object); Dictionary::Ptr state = SerializeState(dynamic_pointer_cast(object)); + auto& states = hMSets[m_PrefixConfigObject + lcType + ":state"]; states.emplace_back(objectKey); states.emplace_back(JsonEncode(state)); + auto& statesChksms = hMSets[m_PrefixConfigCheckSum + lcType + ":state"]; statesChksms.emplace_back(objectKey); statesChksms.emplace_back(JsonEncode(new Dictionary({{"checksum", HashValue(state)}}))); } @@ -328,27 +325,9 @@ void IcingaDB::UpdateAllConfigObjects() if (!(bulkCounter % 100)) { skimObjects(); - for (auto& kv : hMSets) { - if (!kv.second.empty()) { - kv.second.insert(kv.second.begin(), {"HMSET", kv.first}); - transaction.emplace_back(std::move(kv.second)); - } - } - - if (states.size() > 2) { - transaction.emplace_back(std::move(states)); - transaction.emplace_back(std::move(statesChksms)); - states = {"HMSET", m_PrefixConfigObject + lcType + ":state"}; - statesChksms = {"HMSET", m_PrefixConfigCheckSum + lcType + ":state"}; - } + ExecuteRedisTransaction(rcon, hMSets, {}); hMSets = decltype(hMSets)(); - - if (transaction.size() > 1) { - transaction.push_back({"EXEC"}); - rcon->FireAndForgetQueries(std::move(transaction), Prio::Config); - transaction = {{"MULTI"}}; - } } auto checkable (dynamic_pointer_cast(object)); @@ -371,22 +350,7 @@ void IcingaDB::UpdateAllConfigObjects() skimObjects(); - for (auto& kv : hMSets) { - if (!kv.second.empty()) { - kv.second.insert(kv.second.begin(), {"HMSET", kv.first}); - transaction.emplace_back(std::move(kv.second)); - } - } - - if (states.size() > 2) { - transaction.emplace_back(std::move(states)); - transaction.emplace_back(std::move(statesChksms)); - } - - if (transaction.size() > 1) { - transaction.push_back({"EXEC"}); - rcon->FireAndForgetQueries(std::move(transaction), Prio::Config); - } + ExecuteRedisTransaction(rcon, hMSets, {}); for (auto zAdds : {&hostZAdds, &serviceZAdds}) { if (zAdds->size() > 2u) { @@ -1472,34 +1436,7 @@ void IcingaDB::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpd UpdateState(checkable, runtimeUpdate ? StateUpdate::Full : StateUpdate::Volatile); } - std::vector > transaction = {{"MULTI"}}; - - for (auto& kv : hMSets) { - if (!kv.second.empty()) { - kv.second.insert(kv.second.begin(), {"HMSET", kv.first}); - transaction.emplace_back(std::move(kv.second)); - } - } - - for (auto& objectAttributes : runtimeUpdates) { - std::vector xAdd({"XADD", "icinga:runtime", "MAXLEN", "~", "1000000", "*"}); - ObjectLock olock(objectAttributes); - - for (const Dictionary::Pair& kv : objectAttributes) { - String value = IcingaToStreamValue(kv.second); - if (!value.IsEmpty()) { - xAdd.emplace_back(kv.first); - xAdd.emplace_back(value); - } - } - - transaction.emplace_back(std::move(xAdd)); - } - - if (transaction.size() > 1) { - transaction.push_back({"EXEC"}); - m_Rcon->FireAndForgetQueries(std::move(transaction), Prio::Config, {1}); - } + ExecuteRedisTransaction(m_Rcon, hMSets, runtimeUpdates); if (checkable) { SendNextUpdate(checkable); @@ -3296,3 +3233,52 @@ void IcingaDB::AddDataToHmSets(std::map& hMSets, query->emplace_back(id); query->emplace_back(JsonEncode(data)); } + +/** + * Execute the provided HMSET values and runtime updates in a single Redis transaction on the provided Redis connection. + * + * The HMSETs should just contain the necessary key value pairs to be set in Redis, i.e, without the HMSET command + * itself. This function will then go through each of the map keys and prepend the HMSET command when transforming the + * map into valid Redis queries. Likewise, the runtime updates should just contain the key value pairs to be streamed + * to the icinga:runtime pipeline, and this function will generate a XADD query for each one of the vector elements. + * + * @param rcon The Redis connection to execute the transaction on. + * @param hMSets A map of Redis keys and their respective HMSET values. + * @param runtimeUpdates A list of dictionaries to be sent to the icinga:runtime stream. + */ +void IcingaDB::ExecuteRedisTransaction(const RedisConnection::Ptr& rcon, std::map& hMSets, + const std::vector& runtimeUpdates) +{ + RedisConnection::Queries transaction{{"MULTI"}}; + for (auto& [redisKey, query] : hMSets) { + if (!query.empty()) { + query.insert(query.begin(), {"HSET", redisKey}); + transaction.emplace_back(std::move(query)); + } + } + + for (auto& attrs : runtimeUpdates) { + RedisConnection::Query xAdd{"XADD", "icinga:runtime", "MAXLEN", "~", "1000000", "*"}; + + ObjectLock olock(attrs); + for (auto& [key, value] : attrs) { + if (auto streamVal(IcingaToStreamValue(value)); !streamVal.IsEmpty()) { + xAdd.emplace_back(key); + xAdd.emplace_back(std::move(streamVal)); + } + } + + transaction.emplace_back(std::move(xAdd)); + } + + if (transaction.size() > 1) { + transaction.emplace_back(RedisConnection::Query{"EXEC"}); + if (!runtimeUpdates.empty()) { + rcon->FireAndForgetQueries(std::move(transaction), Prio::Config, {1}); + } else { + // This is likely triggered by the initial Redis config dump, so a) we don't need to record the number of + // affected objects and b) we don't really know how many objects are going to be affected by this tx. + rcon->FireAndForgetQueries(std::move(transaction), Prio::Config); + } + } +} diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index 73ee4e8ae67..d8f7843a510 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -210,6 +210,9 @@ class IcingaDB : public ObjectImpl static void CommandArgumentsChangedHandler(const ConfigObject::Ptr& command, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues); static void CustomVarsChangedHandler(const ConfigObject::Ptr& object, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues); + static void ExecuteRedisTransaction(const RedisConnection::Ptr& rcon, std::map& hMSets, + const std::vector& runtimeUpdates); + void AssertOnWorkQueue(); void ExceptionHandler(boost::exception_ptr exp); From 26f46fe021f8c993e9d9e1cf278180e6a0afef0c Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Fri, 7 Feb 2025 11:52:15 +0100 Subject: [PATCH 23/40] Simplify dependency group registration Co-Authored-By: Yonas Habteab --- lib/icinga/checkable-dependency.cpp | 80 ++++++++++++++-- lib/icinga/checkable.hpp | 7 +- lib/icinga/dependency-group.cpp | 140 +++++++++------------------- lib/icinga/dependency.cpp | 4 +- lib/icinga/dependency.hpp | 52 ++++++----- 5 files changed, 147 insertions(+), 136 deletions(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index 0904c09cf49..21cbcd71f9b 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -15,29 +15,91 @@ using namespace icinga; */ static constexpr int l_MaxDependencyRecursionLevel(256); -void Checkable::AddDependencyGroup(const DependencyGroup::Ptr& dependencyGroup) +std::vector Checkable::GetDependencyGroups() const { - std::unique_lock lock(m_DependencyMutex); - m_DependencyGroups.insert(dependencyGroup); + std::lock_guard lock(m_DependencyMutex); + std::vector dependencyGroups; + for (const auto& [_, dependencyGroup] : m_DependencyGroups) { + dependencyGroups.emplace_back(dependencyGroup); + } + return dependencyGroups; } -void Checkable::RemoveDependencyGroup(const DependencyGroup::Ptr& dependencyGroup) +/** + * Get the key for the provided dependency group. + * + * The key is either the parent Checkable object or the redundancy group name of the dependency object. + * This is used to uniquely identify the dependency group within a given Checkable object. + * + * @param dependency The dependency to get the key for. + * + * @return - Returns the key for the provided dependency group. + */ +static std::variant GetDependencyGroupKey(const Dependency::Ptr& dependency) { - std::unique_lock lock(m_DependencyMutex); - m_DependencyGroups.erase(dependencyGroup); + if (auto redundancyGroup(dependency->GetRedundancyGroup()); !redundancyGroup.IsEmpty()) { + return std::move(redundancyGroup); + } + + return dependency->GetParent().get(); } -std::vector Checkable::GetDependencyGroups() const +/** + * Add the provided dependency to the current Checkable list of dependencies. + * + * @param dependency The dependency to add. + */ +void Checkable::AddDependency(const Dependency::Ptr& dependency) { std::lock_guard lock(m_DependencyMutex); - return {m_DependencyGroups.begin(), m_DependencyGroups.end()}; + + auto dependencyGroupKey(GetDependencyGroupKey(dependency)); + std::set dependencies; + if (auto it(m_DependencyGroups.find(dependencyGroupKey)); it != m_DependencyGroups.end()) { + dependencies = DependencyGroup::Unregister(it->second, this); + m_DependencyGroups.erase(it); + } + + dependencies.emplace(dependency); + + m_DependencyGroups.emplace( + dependencyGroupKey, + DependencyGroup::Register(new DependencyGroup(dependency->GetRedundancyGroup(), dependencies)) + ); +} + +/** + * Remove the provided dependency from the current Checkable list of dependencies. + * + * @param dependency The dependency to remove. + */ +void Checkable::RemoveDependency(const Dependency::Ptr& dependency) +{ + std::lock_guard lock(m_DependencyMutex); + + auto dependencyGroupKey(GetDependencyGroupKey(dependency)); + auto it = m_DependencyGroups.find(dependencyGroupKey); + if (it == m_DependencyGroups.end()) { + return; + } + + std::set dependencies(DependencyGroup::Unregister(it->second, this)); + m_DependencyGroups.erase(it); + dependencies.erase(dependency); + + if (!dependencies.empty()) { + m_DependencyGroups.emplace( + dependencyGroupKey, + DependencyGroup::Register(new DependencyGroup(dependency->GetRedundancyGroup(), dependencies)) + ); + } } std::vector Checkable::GetDependencies() const { std::unique_lock lock(m_DependencyMutex); std::vector dependencies; - for (const auto& dependencyGroup : m_DependencyGroups) { + for (const auto& [_, dependencyGroup] : m_DependencyGroups) { auto tmpDependencies(dependencyGroup->GetDependenciesForChild(this)); dependencies.insert(dependencies.end(), tmpDependencies.begin(), tmpDependencies.end()); } diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index 2b901414355..2d29bbec066 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -18,6 +18,7 @@ #include #include #include +#include namespace icinga { @@ -185,9 +186,9 @@ class Checkable : public ObjectImpl bool IsFlapping() const; /* Dependencies */ - void AddDependencyGroup(const intrusive_ptr& dependencyGroup); - void RemoveDependencyGroup(const intrusive_ptr& dependencyGroup); std::vector> GetDependencyGroups() const; + void AddDependency(const intrusive_ptr& dependency); + void RemoveDependency(const intrusive_ptr& dependency); std::vector > GetDependencies() const; bool HasAnyDependencies() const; @@ -249,7 +250,7 @@ class Checkable : public ObjectImpl /* Dependencies */ mutable std::mutex m_DependencyMutex; - std::set> m_DependencyGroups; + std::map, intrusive_ptr> m_DependencyGroups; std::set > m_ReverseDependencies; void GetAllChildrenInternal(std::set& seenChildren, int level = 0) const; diff --git a/lib/icinga/dependency-group.cpp b/lib/icinga/dependency-group.cpp index 8b5aac92dbf..eabe69a0be6 100644 --- a/lib/icinga/dependency-group.cpp +++ b/lib/icinga/dependency-group.cpp @@ -9,106 +9,52 @@ std::mutex DependencyGroup::m_RegistryMutex; DependencyGroup::RegistryType DependencyGroup::m_Registry; /** - * Refresh the global registry of dependency groups. + * Register the provided dependency group to the global dependency group registry. * - * Registers the provided dependency object to an existing dependency group with the same redundancy - * group name (if any), or creates a new one and registers it to the child Checkable and the registry. + * In case there is already an identical dependency group in the registry, the provided dependency group is merged + * with the existing one, and that group is returned. Otherwise, the provided dependency group is registered as is, + * and it's returned. * - * Note: This is a helper function intended for internal use only, and you should acquire the global registry mutex - * before calling this function. - * - * @param dependency The dependency object to refresh the registry for. - * @param unregister A flag indicating whether the provided dependency object should be unregistered from the registry. + * @param dependencyGroup The dependency group to register. */ -void DependencyGroup::RefreshRegistry(const Dependency::Ptr& dependency, bool unregister) +DependencyGroup::Ptr DependencyGroup::Register(const DependencyGroup::Ptr& dependencyGroup) { - auto registerRedundancyGroup = [](const DependencyGroup::Ptr& dependencyGroup) { - if (auto [it, inserted](m_Registry.insert(dependencyGroup.get())); !inserted) { - DependencyGroup::Ptr existingGroup(*it); - dependencyGroup->CopyDependenciesTo(existingGroup); - } - }; - - // Retrieve all the dependency groups with the same redundancy group name of the provided dependency object. - // This allows us to shorten the lookup for the _one_ optimal group to (un)register the dependency from/to. - auto [begin, end] = m_Registry.get<1>().equal_range(dependency->GetRedundancyGroup()); - for (auto it(begin); it != end; ++it) { - DependencyGroup::Ptr existingGroup(*it); - auto child(dependency->GetChild()); - if (auto dependencies(existingGroup->GetDependenciesForChild(child.get())); !dependencies.empty()) { - m_Registry.erase(existingGroup->GetCompositeKey()); // Will be re-registered when needed down below. - if (unregister) { - existingGroup->RemoveDependency(dependency); - // Remove the connection between the child Checkable and the dependency group if it has no members - // left or the above removed member was the only member of the group that the child depended on. - if (existingGroup->IsEmpty() || dependencies.size() == 1) { - child->RemoveDependencyGroup(existingGroup); - } - } - - size_t totalDependencies(existingGroup->GetDependenciesCount()); - // If the existing dependency group has an identical member already, or the child Checkable of the - // dependency object is the only member of it (totalDependencies == dependencies.size()), we can simply - // add the dependency object to the existing group. - if (!unregister && (existingGroup->HasParentWithConfig(dependency) || totalDependencies == dependencies.size())) { - existingGroup->AddDependency(dependency); - } else if (!unregister || (dependencies.size() > 1 && totalDependencies >= dependencies.size())) { - // The child Checkable is going to have a new dependency group, so we must detach the existing one. - child->RemoveDependencyGroup(existingGroup); - - Ptr replacementGroup(unregister ? nullptr : new DependencyGroup(existingGroup->GetRedundancyGroupName(), dependency)); - for (auto& existingDependency : dependencies) { - if (existingDependency != dependency) { - existingGroup->RemoveDependency(existingDependency); - if (replacementGroup) { - replacementGroup->AddDependency(existingDependency); - } else { - replacementGroup = new DependencyGroup(existingGroup->GetRedundancyGroupName(), existingDependency); - } - } - } - - child->AddDependencyGroup(replacementGroup); - registerRedundancyGroup(replacementGroup); - } - - if (!existingGroup->IsEmpty()) { - registerRedundancyGroup(existingGroup); - } - return; - } - } - - if (!unregister) { - // We couldn't find any existing dependency group to register the dependency to, so we must - // initiate a new one and attach it to the child Checkable and register to the global registry. - DependencyGroup::Ptr newGroup(new DependencyGroup(dependency->GetRedundancyGroup())); - newGroup->AddDependency(dependency); - dependency->GetChild()->AddDependencyGroup(newGroup); - registerRedundancyGroup(newGroup); + std::lock_guard lock(m_RegistryMutex); + if (auto [it, inserted] = m_Registry.insert(dependencyGroup); !inserted) { + dependencyGroup->CopyDependenciesTo(*it); + return *it; } + return dependencyGroup; } /** - * Register the provided dependency to the global dependency group registry. + * Detach the provided child Checkable from the specified dependency group. * - * @param dependency The dependency to register. - */ -void DependencyGroup::Register(const Dependency::Ptr& dependency) -{ - std::lock_guard lock(m_RegistryMutex); - RefreshRegistry(dependency, false); -} - -/** - * Unregister the provided dependency from the dependency group it was member of. + * Unregisters all the dependency objects the child Checkable depends on from the provided dependency group and + * removes the dependency group from the global registry if it becomes empty afterward. * - * @param dependency The dependency to unregister. + * @param dependencyGroup The dependency group to unregister the child Checkable from. + * @param child The child Checkable to detach from the dependency group. + * + * @return - Returns the dependency objects of the child Checkable that were member of the provided dependency group. */ -void DependencyGroup::Unregister(const Dependency::Ptr& dependency) +std::set DependencyGroup::Unregister(const DependencyGroup::Ptr& dependencyGroup, const Checkable::Ptr& child) { std::lock_guard lock(m_RegistryMutex); - RefreshRegistry(dependency, true); + std::vector dependencies; + if (auto it(m_Registry.find(dependencyGroup)); it != m_Registry.end()) { + const auto& existingGroup(*it); + dependencies = existingGroup->GetDependenciesForChild(child.get()); + + for (const auto& dependency : dependencies) { + existingGroup->RemoveDependency(dependency); + } + + if (existingGroup->IsEmpty()) { + m_Registry.erase(it); + } + } + return {dependencies.begin(), dependencies.end()}; } /** @@ -126,6 +72,13 @@ DependencyGroup::DependencyGroup(String name): m_RedundancyGroupName(std::move(n { } +DependencyGroup::DependencyGroup(String name, const std::set& dependencies): m_RedundancyGroupName(std::move(name)) +{ + for (const auto& dependency : dependencies) { + AddDependency(dependency); + } +} + /** * Create a composite key for the provided dependency. * @@ -245,17 +198,10 @@ void DependencyGroup::CopyDependenciesTo(const DependencyGroup::Ptr& dest) VERIFY(this != dest); // Prevent from doing something stupid, i.e. deadlocking ourselves. std::lock_guard lock(m_Mutex); - DependencyGroup::Ptr thisPtr(this); // Just in case the Checkable below was our last reference. for (auto& [_, children] : m_Members) { - Checkable::Ptr previousChild; - for (auto& [checkable, dependency] : children) { - dest->AddDependency(dependency); - if (!previousChild || previousChild != checkable) { - previousChild = dependency->GetChild(); - previousChild->RemoveDependencyGroup(thisPtr); - previousChild->AddDependencyGroup(dest); - } - } + std::for_each(children.begin(), children.end(), [&dest](const auto& pair) { + dest->AddDependency(pair.second); + }); } } diff --git a/lib/icinga/dependency.cpp b/lib/icinga/dependency.cpp index 2f2482136de..a9a7bf3725d 100644 --- a/lib/icinga/dependency.cpp +++ b/lib/icinga/dependency.cpp @@ -251,7 +251,7 @@ void Dependency::OnAllConfigLoaded() // InitChildParentReferences() has to be called before. VERIFY(m_Child && m_Parent); - DependencyGroup::Register(this); + m_Child->AddDependency(this); m_Parent->AddReverseDependency(this); } @@ -259,7 +259,7 @@ void Dependency::Stop(bool runtimeRemoved) { ObjectImpl::Stop(runtimeRemoved); - DependencyGroup::Unregister(this); + GetChild()->RemoveDependency(this); GetParent()->RemoveReverseDependency(this); } diff --git a/lib/icinga/dependency.hpp b/lib/icinga/dependency.hpp index 63bab92ad51..0698a3e0ae8 100644 --- a/lib/icinga/dependency.hpp +++ b/lib/icinga/dependency.hpp @@ -8,9 +8,6 @@ #include "icinga/i2-icinga.hpp" #include "icinga/dependency-ti.hpp" #include "icinga/timeperiod.hpp" -#include -#include -#include #include #include #include @@ -136,9 +133,10 @@ class DependencyGroup final : public SharedObject using MembersMap = std::map; explicit DependencyGroup(String name); + DependencyGroup(String name, const std::set& dependencies); - static void Register(const Dependency::Ptr& dependency); - static void Unregister(const Dependency::Ptr& dependency); + static DependencyGroup::Ptr Register(const DependencyGroup::Ptr& dependencyGroup); + static std::set Unregister(const DependencyGroup::Ptr& dependencyGroup, const Checkable::Ptr& child); static size_t GetRegistrySize(); static CompositeKeyType MakeCompositeKeyFor(const Dependency::Ptr& dependency); @@ -176,7 +174,29 @@ class DependencyGroup final : public SharedObject void RemoveDependency(const Dependency::Ptr& dependency); void CopyDependenciesTo(const DependencyGroup::Ptr& dest); - static void RefreshRegistry(const Dependency::Ptr& dependency, bool unregister); + struct Hash + { + size_t operator()(const DependencyGroup::Ptr& dependencyGroup) const + { + size_t hash = 0; + for (const auto& [key, group] : dependencyGroup->m_Members) { + boost::hash_combine(hash, key); + } + return hash; + } + }; + + struct Equal + { + bool operator()(const DependencyGroup::Ptr& lhs, const DependencyGroup::Ptr& rhs) const + { + return std::equal( + lhs->m_Members.begin(), lhs->m_Members.end(), + rhs->m_Members.begin(), rhs->m_Members.end(), + [](const auto& l, const auto& r) { return l.first == r.first; } + ); + } + }; private: mutable std::mutex m_Mutex; @@ -193,25 +213,7 @@ class DependencyGroup final : public SharedObject String m_RedundancyGroupName; MembersMap m_Members; - using RegistryType = boost::multi_index_container< - DependencyGroup*, // The type of the elements stored in the container. - boost::multi_index::indexed_by< - // This unique index allows to search/erase dependency groups by their composite key in an efficient manner. - boost::multi_index::hashed_unique< - boost::multi_index::mem_fun, - std::hash - >, - // This non-unique index allows to search for dependency groups by their name, and reduces the overall - // runtime complexity. Without this index, we would have to iterate over all elements to find the one - // with the desired members and since containers don't allow erasing elements while iterating, we would - // have to copy each of them to a temporary container, and then erase and reinsert them back to the original - // container. This produces way too much overhead, and slows down the startup time of Icinga 2 significantly. - boost::multi_index::hashed_non_unique< - boost::multi_index::const_mem_fun, - std::hash - > - > - >; + using RegistryType = std::unordered_set; // The global registry of dependency groups. static std::mutex m_RegistryMutex; From 67a4889945de730965c391cf1136703dcc901c2e Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Fri, 7 Feb 2025 12:22:51 +0100 Subject: [PATCH 24/40] Checkable: Delay dependency group global registration on startup --- lib/icinga/checkable-dependency.cpp | 35 ++++++++++++++++++++++++++++- lib/icinga/checkable.cpp | 2 ++ lib/icinga/checkable.hpp | 2 ++ lib/icinga/dependency.hpp | 6 ++--- 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index 21cbcd71f9b..47147bd3651 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -3,7 +3,6 @@ #include "icinga/service.hpp" #include "icinga/dependency.hpp" #include "base/logger.hpp" -#include using namespace icinga; @@ -15,9 +14,34 @@ using namespace icinga; */ static constexpr int l_MaxDependencyRecursionLevel(256); +/** + * Register all the dependency groups of the current Checkable to the global dependency group registry. + * + * Initially, each Checkable object tracks locally its own dependency groups on Icinga 2 startup, and once the start + * signal of that Checkable is emitted, it pushes all the local tracked dependency groups to the global registry. + * Once the global registry is populated with all the local dependency groups, this Checkable may not necessarily + * contain the exact same dependency groups as it did before, as identical groups are merged together in the registry, + * but it's guaranteed to have the same *number* of dependency groups as before. + */ +void Checkable::PushDependencyGroupsToRegistry() +{ + std::lock_guard lock(m_DependencyMutex); + if (!m_DependencyGroupsPushedToRegistry) { + m_DependencyGroupsPushedToRegistry = true; + + decltype(m_DependencyGroups) dependencyGroups; + m_DependencyGroups.swap(dependencyGroups); + + for (auto& [dependencyGroupKey, dependencyGroup] : dependencyGroups) { + m_DependencyGroups.emplace(dependencyGroupKey, DependencyGroup::Register(dependencyGroup)); + } + } +} + std::vector Checkable::GetDependencyGroups() const { std::lock_guard lock(m_DependencyMutex); + std::vector dependencyGroups; for (const auto& [_, dependencyGroup] : m_DependencyGroups) { dependencyGroups.emplace_back(dependencyGroup); @@ -54,6 +78,15 @@ void Checkable::AddDependency(const Dependency::Ptr& dependency) std::lock_guard lock(m_DependencyMutex); auto dependencyGroupKey(GetDependencyGroupKey(dependency)); + if (!m_DependencyGroupsPushedToRegistry) { + auto& dependencyGroup = m_DependencyGroups[dependencyGroupKey]; + if (!dependencyGroup) { + dependencyGroup = new DependencyGroup(dependency->GetRedundancyGroup()); + } + dependencyGroup->AddDependency(dependency); + return; + } + std::set dependencies; if (auto it(m_DependencyGroups.find(dependencyGroupKey)); it != m_DependencyGroups.end()) { dependencies = DependencyGroup::Unregister(it->second, this); diff --git a/lib/icinga/checkable.cpp b/lib/icinga/checkable.cpp index ddf84cd1fd2..13fd778a303 100644 --- a/lib/icinga/checkable.cpp +++ b/lib/icinga/checkable.cpp @@ -80,6 +80,8 @@ void Checkable::OnAllConfigLoaded() void Checkable::Start(bool runtimeCreated) { + PushDependencyGroupsToRegistry(); + double now = Utility::GetTime(); { diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index 2d29bbec066..c3ac2961299 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -186,6 +186,7 @@ class Checkable : public ObjectImpl bool IsFlapping() const; /* Dependencies */ + void PushDependencyGroupsToRegistry(); std::vector> GetDependencyGroups() const; void AddDependency(const intrusive_ptr& dependency); void RemoveDependency(const intrusive_ptr& dependency); @@ -250,6 +251,7 @@ class Checkable : public ObjectImpl /* Dependencies */ mutable std::mutex m_DependencyMutex; + bool m_DependencyGroupsPushedToRegistry{false}; std::map, intrusive_ptr> m_DependencyGroups; std::set > m_ReverseDependencies; diff --git a/lib/icinga/dependency.hpp b/lib/icinga/dependency.hpp index 0698a3e0ae8..0f3c257eec5 100644 --- a/lib/icinga/dependency.hpp +++ b/lib/icinga/dependency.hpp @@ -152,6 +152,8 @@ class DependencyGroup final : public SharedObject } bool IsEmpty() const; + void AddDependency(const Dependency::Ptr& dependency); + void RemoveDependency(const Dependency::Ptr& dependency); std::vector GetDependenciesForChild(const Checkable* child) const; size_t GetDependenciesCount() const; @@ -169,9 +171,7 @@ class DependencyGroup final : public SharedObject State GetState(DependencyType dt = DependencyState, int rstack = 0) const; -protected: - void AddDependency(const Dependency::Ptr& dependency); - void RemoveDependency(const Dependency::Ptr& dependency); +private: void CopyDependenciesTo(const DependencyGroup::Ptr& dest); struct Hash From 6a0ec7013174f56c29742277c04e47e441d512ec Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Tue, 28 Jan 2025 16:14:55 +0100 Subject: [PATCH 25/40] Fix & adjust dependencies unittests --- test/icinga-dependencies.cpp | 143 +++++++++++++++++++++++++---------- 1 file changed, 104 insertions(+), 39 deletions(-) diff --git a/test/icinga-dependencies.cpp b/test/icinga-dependencies.cpp index e02385938ac..d9e7f651efc 100644 --- a/test/icinga-dependencies.cpp +++ b/test/icinga-dependencies.cpp @@ -16,7 +16,7 @@ static Host::Ptr CreateHost(const std::string& name) return host; } -static Dependency::Ptr CreateDependency(Checkable::Ptr parent, Checkable::Ptr child, const std::string& name) +static Dependency::Ptr CreateDependency(Checkable::Ptr parent, Checkable::Ptr child, const String& name) { Dependency::Ptr dep = new Dependency(); dep->SetParent(parent); @@ -25,10 +25,10 @@ static Dependency::Ptr CreateDependency(Checkable::Ptr parent, Checkable::Ptr ch return dep; } -static void RegisterDependency(Dependency::Ptr dep, const std::string& redundancyGroup) +static void RegisterDependency(Dependency::Ptr dep, const String& redundancyGroup) { dep->SetRedundancyGroup(redundancyGroup); - DependencyGroup::Register(dep); + dep->GetChild()->AddDependency(dep); dep->GetParent()->AddReverseDependency(dep); } @@ -42,17 +42,29 @@ static void AssertCheckableRedundancyGroup(Checkable::Ptr checkable, int depende auto dependencyGroups(checkable->GetDependencyGroups()); BOOST_CHECK_MESSAGE( groupCount == dependencyGroups.size(), - "Dependency group count mismatch for '" << checkable->GetName() << "'" << " - expected=" << groupCount + "Dependency group count mismatch for '" << checkable->GetName() << "' - expected=" << groupCount << "; got=" << dependencyGroups.size() ); - if (groupCount > 0) { - BOOST_REQUIRE_MESSAGE(1 <= dependencyGroups.size(), "Checkable '" << checkable->GetName() << "' should have at least one dependency group."); + + for (auto& dependencyGroup : dependencyGroups) { BOOST_CHECK_MESSAGE( - totalDependenciesCount == dependencyGroups.begin()->get()->GetDependenciesCount(), - "Member count mismatch for '" << checkable->GetName() << "'" << " - expected=" << totalDependenciesCount - << "; got=" << dependencyGroups.begin()->get()->GetDependenciesCount() + totalDependenciesCount == dependencyGroup->GetDependenciesCount(), + "Dependency group '" << dependencyGroup->GetRedundancyGroupName() << "' and Checkable '" << checkable->GetName() + << "' total dependencies count mismatch - expected=" << totalDependenciesCount << "; got=" + << dependencyGroup->GetDependenciesCount() ); } + + if (groupCount > 0) { + BOOST_REQUIRE_MESSAGE(!dependencyGroups.empty(), "Checkable '" << checkable->GetName() << "' should have at least one dependency group."); + } +} + +static std::vector ExtractGroups(const Checkable::Ptr& checkable) +{ + auto dependencyGroups(checkable->GetDependencyGroups()); + std::sort(dependencyGroups.begin(), dependencyGroups.end()); + return dependencyGroups; } BOOST_AUTO_TEST_CASE(multi_parent) @@ -106,19 +118,19 @@ BOOST_AUTO_TEST_CASE(multi_parent) // It should still be unreachable, due to the duplicated dependency object above with ignore_soft_states set to false. BOOST_CHECK(childHost->IsReachable() == false); parentHost1->SetStateType(StateTypeHard); - DependencyGroup::Unregister(duplicateDep); + childHost->RemoveDependency(duplicateDep); /* The only DNS server is DOWN. * Expected result: childHost is unreachable. */ - DependencyGroup::Unregister(dep1); // Remove the dep and re-add it with a configured redundancy group. + childHost->RemoveDependency(dep1); // Remove the dep and re-add it with a configured redundancy group. RegisterDependency(dep1, "DNS"); BOOST_CHECK(childHost->IsReachable() == false); /* 1/2 DNS servers is DOWN. * Expected result: childHost is reachable. */ - DependencyGroup::Unregister(dep2); + childHost->RemoveDependency(dep2); RegisterDependency(dep2, "DNS"); BOOST_CHECK(childHost->IsReachable() == true); @@ -132,7 +144,7 @@ BOOST_AUTO_TEST_CASE(multi_parent) RegisterDependency(dep3, ""); // The grandparent is DOWN but the DNS redundancy group has to be still reachable. BOOST_CHECK_EQUAL(true, childHost->IsReachable()); - DependencyGroup::Unregister(dep3); + childHost->RemoveDependency(dep3); /* Both DNS servers are DOWN. * Expected result: childHost is unreachable. @@ -153,29 +165,72 @@ BOOST_AUTO_TEST_CASE(default_redundancy_group_registration_unregistration) Dependency::Ptr depCB(CreateDependency(CreateHost("B"), childHostC, "depCB")); RegisterDependency(depCB, ""); - AssertCheckableRedundancyGroup(childHostC, 2, 1, 2); - BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + AssertCheckableRedundancyGroup(childHostC, 2, 2, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); Checkable::Ptr childHostD(CreateHost("D")); Dependency::Ptr depDA(CreateDependency(depCA->GetParent(), childHostD, "depDA")); RegisterDependency(depDA, ""); - AssertCheckableRedundancyGroup(childHostD, 1, 1, 1); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); Dependency::Ptr depDB(CreateDependency(depCB->GetParent(), childHostD, "depDB")); RegisterDependency(depDB, ""); - AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); - AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); - BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + AssertCheckableRedundancyGroup(childHostD, 2, 2, 2); + AssertCheckableRedundancyGroup(childHostC, 2, 2, 2); + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + // This is an exact duplicate of depCA, but with a different dependency name. + Dependency::Ptr depCA2(CreateDependency(depCA->GetParent(), childHostC, "depCA2")); + // This is a duplicate of depCA, but with a different state filter. + Dependency::Ptr depCA3(CreateDependency(depCA->GetParent(), childHostC, "depCA3")); + depCA3->SetStateFilter(StateFilterUp, true); + // This is a duplicate of depCA, but with a different ignore_soft_states flag. + Dependency::Ptr depCA4(CreateDependency(depCA->GetParent(), childHostC, "depCA4")); + depCA4->SetIgnoreSoftStates(false, true); + + for (auto& dependency : {depCA2, depCA3, depCA4}) { + bool isAnExactDuplicate = dependency == depCA2; + RegisterDependency(dependency, ""); + + if (isAnExactDuplicate) { + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); + } + + for (auto& dependencyGroup : childHostD->GetDependencyGroups()) { + if (dependency->GetParent() == dependencyGroup->GetDependenciesForChild(childHostD.get()).front()->GetParent()) { + BOOST_CHECK_EQUAL(isAnExactDuplicate ? 3 : 1, dependencyGroup->GetDependenciesCount()); + } else { + BOOST_CHECK_EQUAL(2, dependencyGroup->GetDependenciesCount()); + } + BOOST_CHECK_EQUAL(2, childHostD->GetDependencies().size()); + } + + for (auto& dependencyGroup : childHostC->GetDependencyGroups()) { + if (dependency->GetParent() == dependencyGroup->GetDependenciesForChild(childHostC.get()).front()->GetParent()) { + // If depCA2 is currently being processed, then the group should have 3 dependencies, that's because + // depCA2 is an exact duplicate of depCA, and depCA shares the same group with depDA. + BOOST_CHECK_EQUAL(isAnExactDuplicate ? 3 : 2, dependencyGroup->GetDependenciesCount()); + } else { + BOOST_CHECK_EQUAL(2, dependencyGroup->GetDependenciesCount()); + } + // The 3 dependencies are depCA, depCB, and the current one from the loop. + BOOST_CHECK_EQUAL(3, childHostC->GetDependencies().size()); + } + BOOST_CHECK_EQUAL(isAnExactDuplicate ? 2 : 3, DependencyGroup::GetRegistrySize()); + childHostC->RemoveDependency(dependency); + } - DependencyGroup::Unregister(depCA); - DependencyGroup::Unregister(depDA); + childHostC->RemoveDependency(depCA); + childHostD->RemoveDependency(depDA); AssertCheckableRedundancyGroup(childHostC, 1, 1, 2); AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); - DependencyGroup::Unregister(depCB); - DependencyGroup::Unregister(depDB); + childHostC->RemoveDependency(depCB); + childHostD->RemoveDependency(depDB); AssertCheckableRedundancyGroup(childHostC, 0, 0, 0); AssertCheckableRedundancyGroup(childHostD, 0, 0, 0); BOOST_CHECK_EQUAL(0, DependencyGroup::GetRegistrySize()); @@ -206,31 +261,33 @@ BOOST_AUTO_TEST_CASE(simple_redundancy_group_registration_unregistration) // Still 1 redundancy group, but there should be 4 dependencies now, i.e. 2 for each child Checkable. AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); - DependencyGroup::Unregister(depCA); + childHostC->RemoveDependency(depCA); // After unregistering depCA, childHostC should have a new redundancy group with only depCB as dependency, and... AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); // ...childHostD should still have the same redundancy group as before but also with only two dependencies. AssertCheckableRedundancyGroup(childHostD, 2, 1, 2); BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); - DependencyGroup::Unregister(depDA); + childHostD->RemoveDependency(depDA); // Nothing should have changed for childHostC, but childHostD should now have a fewer group dependency, i.e. // both child hosts should have the same redundancy group with only depCB and depDB as dependency. AssertCheckableRedundancyGroup(childHostC, 1, 1, 2); AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); - DependencyGroup::Register(depDA); - DependencyGroup::Unregister(depDB); + RegisterDependency(depDA, depDA->GetRedundancyGroup()); + childHostD->RemoveDependency(depDB); // Nothing should have changed for childHostC, but both should now have a separate group with only depCB and depDA as dependency. AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); AssertCheckableRedundancyGroup(childHostD, 1, 1, 1); BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); - DependencyGroup::Unregister(depCB); - DependencyGroup::Unregister(depDA); + childHostC->RemoveDependency(depCB); + childHostD->RemoveDependency(depDA); AssertCheckableRedundancyGroup(childHostC, 0, 0, 0); AssertCheckableRedundancyGroup(childHostD, 0, 0, 0); BOOST_CHECK_EQUAL(0, DependencyGroup::GetRegistrySize()); @@ -248,6 +305,7 @@ BOOST_AUTO_TEST_CASE(mixed_redundancy_group_registration_unregsitration) Dependency::Ptr depDA(CreateDependency(depCA->GetParent(), childHostD, "depDA")); RegisterDependency(depDA, "redundant"); AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); Dependency::Ptr depCB(CreateDependency(CreateHost("B"), childHostC, "depCB")); @@ -260,6 +318,7 @@ BOOST_AUTO_TEST_CASE(mixed_redundancy_group_registration_unregsitration) RegisterDependency(depDB, "redundant"); AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); Checkable::Ptr childHostE(CreateHost("childE")); @@ -274,6 +333,8 @@ BOOST_AUTO_TEST_CASE(mixed_redundancy_group_registration_unregsitration) AssertCheckableRedundancyGroup(childHostC, 2, 1, 6); AssertCheckableRedundancyGroup(childHostD, 2, 1, 6); AssertCheckableRedundancyGroup(childHostE, 2, 1, 6); + auto childHostCGroups(ExtractGroups(childHostC)); + BOOST_TEST((childHostCGroups == ExtractGroups(childHostD) && childHostCGroups == ExtractGroups(childHostE))); BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); Dependency::Ptr depEZ(CreateDependency(CreateHost("Z"), childHostE, "depEZ")); @@ -281,36 +342,40 @@ BOOST_AUTO_TEST_CASE(mixed_redundancy_group_registration_unregsitration) // Child host E should have a new redundancy group with 3 dependencies and the other two should still share the same group. AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); AssertCheckableRedundancyGroup(childHostE, 3, 1, 3); BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); - DependencyGroup::Unregister(depEA); + childHostE->RemoveDependency(depEA); AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); AssertCheckableRedundancyGroup(childHostE, 2, 1, 2); BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); - DependencyGroup::Register(depEA); // Re-register depEA and instead... - DependencyGroup::Unregister(depEZ); // ...unregister depEZ and check if all the hosts share the same group again. + RegisterDependency(depEA, depEA->GetRedundancyGroup()); // Re-register depEA and instead... + childHostE->RemoveDependency(depEZ); // ...unregister depEZ and check if all the hosts share the same group again. // All 3 hosts share the same group again, and each host has 2 dependencies, thus 6 dependencies in total. AssertCheckableRedundancyGroup(childHostC, 2, 1, 6); AssertCheckableRedundancyGroup(childHostD, 2, 1, 6); AssertCheckableRedundancyGroup(childHostE, 2, 1, 6); + childHostCGroups = ExtractGroups(childHostC); + BOOST_TEST((childHostCGroups == ExtractGroups(childHostD) && childHostCGroups == ExtractGroups(childHostE))); BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); - DependencyGroup::Unregister(depCA); - DependencyGroup::Unregister(depDB); - DependencyGroup::Unregister(depEB); + childHostC->RemoveDependency(depCA); + childHostD->RemoveDependency(depDB); + childHostE->RemoveDependency(depEB); // Child host C has now a separate group with only depCB as dependency, and child hosts D and E share the same group. AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); AssertCheckableRedundancyGroup(childHostE, 1, 1, 2); - // Child host C has now a separate group with only depCB as member, and child hosts D and E share the same group. + BOOST_TEST(ExtractGroups(childHostD) == ExtractGroups(childHostE), boost::test_tools::per_element()); BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); - DependencyGroup::Unregister(depCB); - DependencyGroup::Unregister(depDA); - DependencyGroup::Unregister(depEA); + childHostC->RemoveDependency(depCB); + childHostD->RemoveDependency(depDA); + childHostE->RemoveDependency(depEA); AssertCheckableRedundancyGroup(childHostC, 0, 0, 0); AssertCheckableRedundancyGroup(childHostD, 0, 0, 0); AssertCheckableRedundancyGroup(childHostE, 0, 0, 0); From b462028b4f809427a8878f8b8b643e19b1f3b1e4 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Tue, 28 Jan 2025 16:30:00 +0100 Subject: [PATCH 26/40] Add basic unittests for bulk group registration --- test/CMakeLists.txt | 1 + test/icinga-dependencies.cpp | 52 +++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7857c126104..810b35befd7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -232,6 +232,7 @@ add_boost_test(base icinga_checkresult/service_flapping_notification icinga_checkresult/suppressed_notification icinga_dependencies/multi_parent + icinga_dependencies/push_dependency_groups_to_registry icinga_dependencies/default_redundancy_group_registration_unregistration icinga_dependencies/simple_redundancy_group_registration_unregistration icinga_dependencies/mixed_redundancy_group_registration_unregsitration diff --git a/test/icinga-dependencies.cpp b/test/icinga-dependencies.cpp index d9e7f651efc..b1fbf7b4c1c 100644 --- a/test/icinga-dependencies.cpp +++ b/test/icinga-dependencies.cpp @@ -9,10 +9,13 @@ using namespace icinga; BOOST_AUTO_TEST_SUITE(icinga_dependencies) -static Host::Ptr CreateHost(const std::string& name) +static Host::Ptr CreateHost(const std::string& name, bool pushDependencyGroupsToRegistry = true) { Host::Ptr host = new Host(); host->SetName(name); + if (pushDependencyGroupsToRegistry) { + host->PushDependencyGroupsToRegistry(); + } return host; } @@ -155,6 +158,53 @@ BOOST_AUTO_TEST_CASE(multi_parent) BOOST_CHECK(childHost->IsReachable() == false); } +BOOST_AUTO_TEST_CASE(push_dependency_groups_to_registry) +{ + Checkable::Ptr childHostC(CreateHost("C", false)); + Checkable::Ptr childHostD(CreateHost("D", false)); + std::set dependencies; // Keep track of all dependencies to avoid unexpected deletions. + for (auto& parent : {String("A"), String("B"), String("E")}) { + Dependency::Ptr depC(CreateDependency(CreateHost(parent), childHostC, "depC" + parent)); + Dependency::Ptr depD(CreateDependency(depC->GetParent(), childHostD, "depD" + parent)); + if (parent == "A") { + Dependency::Ptr depCA2(CreateDependency(depC->GetParent(), childHostC, "depCA2")); + childHostC->AddDependency(depCA2); + dependencies.emplace(depCA2); + } else { + depC->SetRedundancyGroup("redundant", true); + depD->SetRedundancyGroup("redundant", true); + + if (parent == "B") { // Create an exact duplicate of depC, but with a different name. + Dependency::Ptr depCB2(CreateDependency(depC->GetParent(), childHostC, "depCB2")); + depCB2->SetRedundancyGroup("redundant", true); + childHostC->AddDependency(depCB2); + dependencies.emplace(depCB2); + } + } + childHostC->AddDependency(depC); + childHostD->AddDependency(depD); + dependencies.insert({depC, depD}); + } + + childHostC->PushDependencyGroupsToRegistry(); + childHostD->PushDependencyGroupsToRegistry(); + + BOOST_TEST(ExtractGroups(childHostC) == ExtractGroups(childHostD), boost::test_tools::per_element()); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + for (auto& checkable : {childHostC, childHostD}) { + BOOST_CHECK_EQUAL(2, checkable->GetDependencyGroups().size()); + for (auto& dependencyGroup : checkable->GetDependencyGroups()) { + if (dependencyGroup->IsRedundancyGroup()) { + BOOST_CHECK_EQUAL(5, dependencyGroup->GetDependenciesCount()); + BOOST_CHECK_EQUAL(checkable == childHostC ? 5 : 3, checkable->GetDependencies().size()); + } else { + BOOST_CHECK_EQUAL(3, dependencyGroup->GetDependenciesCount()); + BOOST_CHECK_EQUAL(checkable == childHostC ? 5 : 3, checkable->GetDependencies().size()); + } + } + } +} + BOOST_AUTO_TEST_CASE(default_redundancy_group_registration_unregistration) { Checkable::Ptr childHostC(CreateHost("C")); From 806fff950c290af375931abb74ee7a76d345a837 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Fri, 7 Feb 2025 16:51:00 +0100 Subject: [PATCH 27/40] Checkable: Emit boost signals when changing dependency groups at runtime --- lib/icinga/checkable-dependency.cpp | 49 +++++++++++++++++++++-------- lib/icinga/checkable.hpp | 2 +- lib/icinga/dependency-group.cpp | 16 ++++++---- lib/icinga/dependency.cpp | 2 +- lib/icinga/dependency.hpp | 5 ++- 5 files changed, 52 insertions(+), 22 deletions(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index 47147bd3651..51e054e6047 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -75,7 +75,7 @@ static std::variant GetDependencyGroupKey(const Dependency:: */ void Checkable::AddDependency(const Dependency::Ptr& dependency) { - std::lock_guard lock(m_DependencyMutex); + std::unique_lock lock(m_DependencyMutex); auto dependencyGroupKey(GetDependencyGroupKey(dependency)); if (!m_DependencyGroupsPushedToRegistry) { @@ -88,27 +88,38 @@ void Checkable::AddDependency(const Dependency::Ptr& dependency) } std::set dependencies; + bool removeGroup(false); + + DependencyGroup::Ptr existingGroup; if (auto it(m_DependencyGroups.find(dependencyGroupKey)); it != m_DependencyGroups.end()) { - dependencies = DependencyGroup::Unregister(it->second, this); + existingGroup = it->second; + std::tie(dependencies, removeGroup) = DependencyGroup::Unregister(existingGroup, this); m_DependencyGroups.erase(it); } dependencies.emplace(dependency); - m_DependencyGroups.emplace( - dependencyGroupKey, - DependencyGroup::Register(new DependencyGroup(dependency->GetRedundancyGroup(), dependencies)) - ); + auto dependencyGroup(DependencyGroup::Register(new DependencyGroup(dependency->GetRedundancyGroup(), dependencies))); + m_DependencyGroups.emplace(dependencyGroupKey, dependencyGroup); + + lock.unlock(); + + if (existingGroup) { + dependencies.erase(dependency); + DependencyGroup::OnChildRemoved(existingGroup, {dependencies.begin(), dependencies.end()}, removeGroup); + } + DependencyGroup::OnChildRegistered(this, dependencyGroup); } /** * Remove the provided dependency from the current Checkable list of dependencies. * * @param dependency The dependency to remove. + * @param runtimeRemoved Whether the given dependency object is being removed at runtime. */ -void Checkable::RemoveDependency(const Dependency::Ptr& dependency) +void Checkable::RemoveDependency(const Dependency::Ptr& dependency, bool runtimeRemoved) { - std::lock_guard lock(m_DependencyMutex); + std::unique_lock lock(m_DependencyMutex); auto dependencyGroupKey(GetDependencyGroupKey(dependency)); auto it = m_DependencyGroups.find(dependencyGroupKey); @@ -116,15 +127,27 @@ void Checkable::RemoveDependency(const Dependency::Ptr& dependency) return; } - std::set dependencies(DependencyGroup::Unregister(it->second, this)); + DependencyGroup::Ptr existingGroup(it->second); + auto [dependencies, removeGroup] = DependencyGroup::Unregister(existingGroup, this); + m_DependencyGroups.erase(it); dependencies.erase(dependency); + DependencyGroup::Ptr newDependencyGroup; if (!dependencies.empty()) { - m_DependencyGroups.emplace( - dependencyGroupKey, - DependencyGroup::Register(new DependencyGroup(dependency->GetRedundancyGroup(), dependencies)) - ); + newDependencyGroup = DependencyGroup::Register(new DependencyGroup(dependency->GetRedundancyGroup(), dependencies)); + m_DependencyGroups.emplace(dependencyGroupKey, newDependencyGroup); + } + + lock.unlock(); + + if (runtimeRemoved) { + dependencies.emplace(dependency); + DependencyGroup::OnChildRemoved(existingGroup, {dependencies.begin(), dependencies.end()}, removeGroup); + + if (newDependencyGroup) { + DependencyGroup::OnChildRegistered(this, newDependencyGroup); + } } } diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index c3ac2961299..53db79d721f 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -189,7 +189,7 @@ class Checkable : public ObjectImpl void PushDependencyGroupsToRegistry(); std::vector> GetDependencyGroups() const; void AddDependency(const intrusive_ptr& dependency); - void RemoveDependency(const intrusive_ptr& dependency); + void RemoveDependency(const intrusive_ptr& dependency, bool runtimeRemoved = false); std::vector > GetDependencies() const; bool HasAnyDependencies() const; diff --git a/lib/icinga/dependency-group.cpp b/lib/icinga/dependency-group.cpp index eabe69a0be6..47e43316f30 100644 --- a/lib/icinga/dependency-group.cpp +++ b/lib/icinga/dependency-group.cpp @@ -5,6 +5,9 @@ using namespace icinga; +boost::signals2::signal DependencyGroup::OnChildRegistered; +boost::signals2::signal&, bool)> DependencyGroup::OnChildRemoved; + std::mutex DependencyGroup::m_RegistryMutex; DependencyGroup::RegistryType DependencyGroup::m_Registry; @@ -36,15 +39,15 @@ DependencyGroup::Ptr DependencyGroup::Register(const DependencyGroup::Ptr& depen * @param dependencyGroup The dependency group to unregister the child Checkable from. * @param child The child Checkable to detach from the dependency group. * - * @return - Returns the dependency objects of the child Checkable that were member of the provided dependency group. + * @return - Returns the dependency objects of the child Checkable that were member of the provided dependency group + * and a boolean indicating whether the dependency group has been erased from the global registry. */ -std::set DependencyGroup::Unregister(const DependencyGroup::Ptr& dependencyGroup, const Checkable::Ptr& child) +std::pair, bool> DependencyGroup::Unregister(const DependencyGroup::Ptr& dependencyGroup, const Checkable::Ptr& child) { std::lock_guard lock(m_RegistryMutex); - std::vector dependencies; if (auto it(m_Registry.find(dependencyGroup)); it != m_Registry.end()) { - const auto& existingGroup(*it); - dependencies = existingGroup->GetDependenciesForChild(child.get()); + auto existingGroup(*it); + auto dependencies(existingGroup->GetDependenciesForChild(child.get())); for (const auto& dependency : dependencies) { existingGroup->RemoveDependency(dependency); @@ -53,8 +56,9 @@ std::set DependencyGroup::Unregister(const DependencyGroup::Ptr if (existingGroup->IsEmpty()) { m_Registry.erase(it); } + return {{dependencies.begin(), dependencies.end()}, existingGroup->IsEmpty()}; } - return {dependencies.begin(), dependencies.end()}; + return {{}, false}; } /** diff --git a/lib/icinga/dependency.cpp b/lib/icinga/dependency.cpp index a9a7bf3725d..c45015e8f89 100644 --- a/lib/icinga/dependency.cpp +++ b/lib/icinga/dependency.cpp @@ -259,7 +259,7 @@ void Dependency::Stop(bool runtimeRemoved) { ObjectImpl::Stop(runtimeRemoved); - GetChild()->RemoveDependency(this); + GetChild()->RemoveDependency(this, runtimeRemoved); GetParent()->RemoveReverseDependency(this); } diff --git a/lib/icinga/dependency.hpp b/lib/icinga/dependency.hpp index 0f3c257eec5..a46b785c47e 100644 --- a/lib/icinga/dependency.hpp +++ b/lib/icinga/dependency.hpp @@ -136,7 +136,7 @@ class DependencyGroup final : public SharedObject DependencyGroup(String name, const std::set& dependencies); static DependencyGroup::Ptr Register(const DependencyGroup::Ptr& dependencyGroup); - static std::set Unregister(const DependencyGroup::Ptr& dependencyGroup, const Checkable::Ptr& child); + static std::pair, bool> Unregister(const DependencyGroup::Ptr& dependencyGroup, const Checkable::Ptr& child); static size_t GetRegistrySize(); static CompositeKeyType MakeCompositeKeyFor(const Dependency::Ptr& dependency); @@ -171,6 +171,9 @@ class DependencyGroup final : public SharedObject State GetState(DependencyType dt = DependencyState, int rstack = 0) const; + static boost::signals2::signal OnChildRegistered; + static boost::signals2::signal&, bool)> OnChildRemoved; + private: void CopyDependenciesTo(const DependencyGroup::Ptr& dest); From 8640a3f84e86dd6da1a8ca2ece0ac0e43ee2ad79 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 10 Feb 2025 08:30:32 +0100 Subject: [PATCH 28/40] Checkable: Extract parents directly from dependency groups --- lib/icinga/checkable-dependency.cpp | 8 ++------ lib/icinga/dependency-group.cpp | 14 ++++++++++++++ lib/icinga/dependency.hpp | 1 + 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index 51e054e6047..b82aebc88aa 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -250,12 +250,8 @@ bool Checkable::AffectsChildren() const std::set Checkable::GetParents() const { std::set parents; - - for (const Dependency::Ptr& dep : GetDependencies()) { - Checkable::Ptr parent = dep->GetParent(); - - if (parent && parent.get() != this) - parents.insert(parent); + for (auto& dependencyGroup : GetDependencyGroups()) { + dependencyGroup->LoadParents(parents); } return parents; diff --git a/lib/icinga/dependency-group.cpp b/lib/icinga/dependency-group.cpp index 47e43316f30..01520467e9a 100644 --- a/lib/icinga/dependency-group.cpp +++ b/lib/icinga/dependency-group.cpp @@ -133,6 +133,20 @@ std::vector DependencyGroup::GetDependenciesForChild(const Chec return dependencies; } +/** + * Load all parent Checkables of the current dependency group. + * + * @param parents The set to load the parent Checkables into. + */ +void DependencyGroup::LoadParents(std::set& parents) const +{ + std::lock_guard lock(m_Mutex); + for (auto& [compositeKey, children] : m_Members) { + ASSERT(!children.empty()); // We should never have an empty map for any given key at any given time. + parents.insert(std::get<0>(compositeKey)); + } +} + /** * Retrieve the number of dependency objects in the current dependency group. * diff --git a/lib/icinga/dependency.hpp b/lib/icinga/dependency.hpp index a46b785c47e..fe005dbb2d4 100644 --- a/lib/icinga/dependency.hpp +++ b/lib/icinga/dependency.hpp @@ -155,6 +155,7 @@ class DependencyGroup final : public SharedObject void AddDependency(const Dependency::Ptr& dependency); void RemoveDependency(const Dependency::Ptr& dependency); std::vector GetDependenciesForChild(const Checkable* child) const; + void LoadParents(std::set& parents) const; size_t GetDependenciesCount() const; void SetIcingaDBIdentifier(const String& identifier); From 915ea6427e5ffb6826942ca00e1dab1afb177a23 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 10 Feb 2025 08:33:29 +0100 Subject: [PATCH 29/40] Use `GetParents()` in `FireSppressedNotifications()` It's way efficient than accessing them through the dependency objects, plus we won't have any duplicates. --- lib/icinga/checkable-notification.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/icinga/checkable-notification.cpp b/lib/icinga/checkable-notification.cpp index 2a115055672..282b95d3212 100644 --- a/lib/icinga/checkable-notification.cpp +++ b/lib/icinga/checkable-notification.cpp @@ -167,8 +167,7 @@ void Checkable::FireSuppressedNotifications() } } - for (auto& dep : GetDependencies()) { - auto parent (dep->GetParent()); + for (auto& parent : GetParents()) { ObjectLock oLock (parent); if (!parent->GetProblem() && parent->GetLastStateChange() >= threshold) { From 0ab50fd82d5cdf4e9068f8d4e466f42f970d2a7f Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 10 Feb 2025 08:40:29 +0100 Subject: [PATCH 30/40] IcingaDB: Process dependencies runtime updates --- lib/icingadb/icingadb-objects.cpp | 173 +++++++++++++++++++++++++++++- lib/icingadb/icingadb.hpp | 8 +- 2 files changed, 177 insertions(+), 4 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index e407373fa55..0a5e5eb0fcd 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -110,6 +110,9 @@ void IcingaDB::ConfigStaticInitialize() IcingaDB::VersionChangedHandler(object); }); + DependencyGroup::OnChildRegistered.connect(&IcingaDB::DependencyGroupChildRegisteredHandler); + DependencyGroup::OnChildRemoved.connect(&IcingaDB::DependencyGroupChildRemovedHandler); + /* downtime start */ Downtime::OnDowntimeTriggered.connect(&IcingaDB::DowntimeStartedHandler); /* fixed/flexible downtime end or remove */ @@ -1115,16 +1118,20 @@ void IcingaDB::InsertObjectDependencies(const ConfigObject::Ptr& object, const S * - RedisKey::DependencyEdgeState: State information for (each) dependency edge. Multiple edges may share the * same state. * - * For initial dumps, it shouldn't be necessary to set the `runtimeUpdates` parameter. + * If the `onlyDependencyGroup` parameter is set, only dependencies from this group are processed. This is useful + * when only a specific dependency group should be processed, e.g. during runtime updates. For initial config dumps, + * it shouldn't be necessary to set the `runtimeUpdates` and `onlyDependencyGroup` parameters. * * @param checkable The checkable object to extract dependencies from. * @param hMSets The map of Redis HMSETs to insert the dependency data into. * @param runtimeUpdates If set, runtime updates are additionally added to this vector. + * @param onlyDependencyGroup If set, only process dependency objects from this group. */ void IcingaDB::InsertCheckableDependencies( const Checkable::Ptr& checkable, std::map& hMSets, - std::vector* runtimeUpdates + std::vector* runtimeUpdates, + const DependencyGroup::Ptr& onlyDependencyGroup ) { // Only generate a dependency node event if the Checkable is actually part of some dependency graph. @@ -1150,7 +1157,14 @@ void IcingaDB::InsertCheckableDependencies( } } - for (auto& dependencyGroup : checkable->GetDependencyGroups()) { + // If `onlyDependencyGroup` is provided, process the dependencies only from that group; otherwise, + // retrieve all the dependency groups that the Checkable object is part of. + std::vector dependencyGroups{onlyDependencyGroup}; + if (!onlyDependencyGroup) { + dependencyGroups = checkable->GetDependencyGroups(); + } + + for (auto& dependencyGroup : dependencyGroups) { String edgeFromNodeId(checkableId); bool syncSharedEdgeState(false); @@ -2800,6 +2814,98 @@ void IcingaDB::SendCustomVarsChanged(const ConfigObject::Ptr& object, const Dict } } +void IcingaDB::SendDependencyGroupChildRegistered(const Checkable::Ptr& child, const DependencyGroup::Ptr& dependencyGroup) +{ + if (!m_Rcon || !m_Rcon->IsConnected()) { + return; + } + + std::vector runtimeUpdates; + std::map hMSets; + InsertCheckableDependencies(child, hMSets, &runtimeUpdates, dependencyGroup); + ExecuteRedisTransaction(m_Rcon, hMSets, runtimeUpdates); + + UpdateState(child, StateUpdate::Full); + UpdateDependenciesState(child, dependencyGroup); + + std::set parents; + dependencyGroup->LoadParents(parents); + for (const auto& parent : parents) { + // The total_children and affects_children columns might now have different outcome, so update the parent + // Checkable as well. The grandparent Checkable may still have wrong numbers of total children, though it's not + // worth traversing the whole tree way up and sending config updates for each one of them, as the next Redis + // config dump is going to fix it anyway. + SendConfigUpdate(parent, true); + } +} + +void IcingaDB::SendDependencyGroupChildRemoved( + const DependencyGroup::Ptr& dependencyGroup, + const std::vector& dependencies, + bool removeGroup +) +{ + if (!m_Rcon || !m_Rcon->IsConnected() || dependencies.empty()) { + return; + } + + Checkable::Ptr child; + std::set detachedParents; + for (const auto& dependency : dependencies) { + child = dependency->GetChild(); // All dependencies have the same child. + const auto& parent(dependency->GetParent()); + if (auto [_, inserted] = detachedParents.insert(dependency->GetParent().get()); inserted) { + String edgeId; + if (dependencyGroup->IsRedundancyGroup()) { + // If the redundancy group has no members left, it's going to be removed as well, so we need to + // delete dependency edges from that group to the parent Checkables. + if (removeGroup) { + auto id(HashValue(new Array{dependencyGroup->GetIcingaDBIdentifier(), GetObjectIdentifier(parent)})); + DeleteRelationship(id, RedisKey::DependencyEdge); + DeleteState(id, RedisKey::DependencyEdgeState); + } + + // Remove the connection from the child Checkable to the redundancy group. + edgeId = HashValue(new Array{GetObjectIdentifier(child), dependencyGroup->GetIcingaDBIdentifier()}); + } else { + // Remove the edge between the parent and child Checkable linked through the removed dependency. + edgeId = HashValue(new Array{GetObjectIdentifier(child), GetObjectIdentifier(parent)}); + } + + DeleteRelationship(edgeId, RedisKey::DependencyEdge); + + // The total_children and affects_children columns might now have different outcome, so update the parent + // Checkable as well. The grandparent Checkable may still have wrong numbers of total children, though it's + // not worth traversing the whole tree way up and sending config updates for each one of them, as the next + // Redis config dump is going to fix it anyway. + SendConfigUpdate(parent, true); + + if (!parent->HasAnyDependencies()) { + // If the parent Checkable isn't part of any other dependency chain anymore, drop its dependency node entry. + DeleteRelationship(GetObjectIdentifier(parent), RedisKey::DependencyNode); + } + } + } + + if (removeGroup && dependencyGroup->IsRedundancyGroup()) { + String redundancyGroupId(dependencyGroup->GetIcingaDBIdentifier()); + DeleteRelationship(redundancyGroupId, RedisKey::DependencyNode); + DeleteRelationship(redundancyGroupId, RedisKey::RedundancyGroup); + + DeleteState(redundancyGroupId, RedisKey::RedundancyGroupState); + DeleteState(redundancyGroupId, RedisKey::DependencyEdgeState); + } else if (removeGroup) { + // Note: The Icinga DB identifier of a non-redundant dependency group is used as the edge state ID + // and shared by all of its dependency objects. See also SerializeDependencyEdgeState() for details. + DeleteState(dependencyGroup->GetIcingaDBIdentifier(), RedisKey::DependencyEdgeState); + } + + if (!child->HasAnyDependencies()) { + // If the child Checkable has no parent and reverse dependencies, we can safely remove the dependency node. + DeleteRelationship(GetObjectIdentifier(child), RedisKey::DependencyNode); + } +} + Dictionary::Ptr IcingaDB::SerializeState(const Checkable::Ptr& checkable) { Dictionary::Ptr attrs = new Dictionary(); @@ -3078,6 +3184,20 @@ void IcingaDB::NextCheckUpdatedHandler(const Checkable::Ptr& checkable) } } +void IcingaDB::DependencyGroupChildRegisteredHandler(const Checkable::Ptr& child, const DependencyGroup::Ptr& dependencyGroup) +{ + for (const auto& rw : ConfigType::GetObjectsByType()) { + rw->SendDependencyGroupChildRegistered(child, dependencyGroup); + } +} + +void IcingaDB::DependencyGroupChildRemovedHandler(const DependencyGroup::Ptr& dependencyGroup, const std::vector& dependencies, bool removeGroup) +{ + for (const auto& rw : ConfigType::GetObjectsByType()) { + rw->SendDependencyGroupChildRemoved(dependencyGroup, dependencies, removeGroup); + } +} + void IcingaDB::HostProblemChangedHandler(const Service::Ptr& service) { for (auto& rw : ConfigType::GetObjectsByType()) { /* Host state changes affect is_handled and severity of services. */ @@ -3196,6 +3316,53 @@ void IcingaDB::DeleteRelationship(const String& id, const String& redisKeyWithou m_Rcon->FireAndForgetQueries(queries, Prio::Config); } +void IcingaDB::DeleteRelationship(const String& id, RedisKey redisKey, bool hasChecksum) +{ + switch (redisKey) { + case RedisKey::RedundancyGroup: + DeleteRelationship(id, "redundancygroup", hasChecksum); + break; + case RedisKey::DependencyNode: + DeleteRelationship(id, "dependency:node", hasChecksum); + break; + case RedisKey::DependencyEdge: + DeleteRelationship(id, "dependency:edge", hasChecksum); + break; + default: + BOOST_THROW_EXCEPTION(std::invalid_argument("Invalid RedisKey provided")); + } +} + +void IcingaDB::DeleteState(const String& id, RedisKey redisKey, bool hasChecksum) const +{ + String redisKeyWithoutPrefix; + switch (redisKey) { + case RedisKey::RedundancyGroupState: + redisKeyWithoutPrefix = "redundancygroup:state"; + break; + case RedisKey::DependencyEdgeState: + redisKeyWithoutPrefix = "dependency:edge:state"; + break; + default: + BOOST_THROW_EXCEPTION(std::invalid_argument("Invalid state RedisKey provided")); + } + + Log(LogNotice, "IcingaDB") + << "Deleting state " << std::quoted(redisKeyWithoutPrefix.CStr()) << " -> " << std::quoted(id.CStr()); + + RedisConnection::Queries hdels; + if (hasChecksum) { + hdels.emplace_back(RedisConnection::Query{"HDEL", m_PrefixConfigCheckSum + redisKeyWithoutPrefix, id}); + } + hdels.emplace_back(RedisConnection::Query{"HDEL", m_PrefixConfigObject + redisKeyWithoutPrefix, id}); + + m_Rcon->FireAndForgetQueries(std::move(hdels), Prio::RuntimeStateSync); + m_Rcon->FireAndForgetQueries({{ + "XADD", "icinga:runtime:state", "MAXLEN", "~", "1000000", "*", + "redis_key", m_PrefixConfigObject + redisKeyWithoutPrefix, "id", id, "runtime_type", "delete" + }}, Prio::RuntimeStateStream, {0, 1}); +} + /** * Add the provided data to the Redis HMSETs map. * diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index d8f7843a510..df454e31a0c 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -111,7 +111,7 @@ class IcingaDB : public ObjectImpl std::vector GetTypeOverwriteKeys(const String& type); std::vector GetTypeDumpSignalKeys(const Type::Ptr& type); void InsertCheckableDependencies(const Checkable::Ptr& checkable, std::map& hMSets, - std::vector* runtimeUpdates); + std::vector* runtimeUpdates, const DependencyGroup::Ptr& onlyDependencyGroup = nullptr); void InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map>& hMSets, std::vector& runtimeUpdates, bool runtimeUpdate); void UpdateDependenciesState(const Checkable::Ptr& checkable, const DependencyGroup::Ptr& onlyDependencyGroup = nullptr) const; @@ -124,6 +124,8 @@ class IcingaDB : public ObjectImpl void AddObjectDataToRuntimeUpdates(std::vector& runtimeUpdates, const String& objectKey, const String& redisKey, const Dictionary::Ptr& data); void DeleteRelationship(const String& id, const String& redisKeyWithoutPrefix, bool hasChecksum = false); + void DeleteRelationship(const String& id, RedisKey redisKey, bool hasChecksum = false); + void DeleteState(const String& id, RedisKey redisKey, bool hasChecksum = false) const; void AddDataToHmSets(std::map& hMSets, RedisKey redisKey, const String& id, const Dictionary::Ptr& data) const; void SendSentNotification( @@ -149,6 +151,8 @@ class IcingaDB : public ObjectImpl void SendCommandEnvChanged(const ConfigObject::Ptr& command, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues); void SendCommandArgumentsChanged(const ConfigObject::Ptr& command, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues); void SendCustomVarsChanged(const ConfigObject::Ptr& object, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues); + void SendDependencyGroupChildRegistered(const Checkable::Ptr& child, const DependencyGroup::Ptr& dependencyGroup); + void SendDependencyGroupChildRemoved(const DependencyGroup::Ptr& dependencyGroup, const std::vector& dependencies, bool removeGroup); void ForwardHistoryEntries(); @@ -195,6 +199,8 @@ class IcingaDB : public ObjectImpl static void FlappingChangeHandler(const Checkable::Ptr& checkable, double changeTime); static void NewCheckResultHandler(const Checkable::Ptr& checkable); static void NextCheckUpdatedHandler(const Checkable::Ptr& checkable); + static void DependencyGroupChildRegisteredHandler(const Checkable::Ptr& child, const DependencyGroup::Ptr& dependencyGroup); + static void DependencyGroupChildRemovedHandler(const DependencyGroup::Ptr& dependencyGroup, const std::vector& dependencies, bool removeGroup); static void HostProblemChangedHandler(const Service::Ptr& service); static void AcknowledgementSetHandler(const Checkable::Ptr& checkable, const String& author, const String& comment, AcknowledgementType type, bool persistent, double changeTime, double expiry); static void AcknowledgementClearedHandler(const Checkable::Ptr& checkable, const String& removedBy, double changeTime); From ce1ed8556c8a7c8ad693c197c3665cc5a2fbc99b Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Wed, 12 Feb 2025 11:18:12 +0100 Subject: [PATCH 31/40] Simplify DependencyGroup::GetState() implementation The new implementation just counts reachable and available parents and determines the overall result by comparing numbers, see inline comments for more information. This also fixes an issue in the previous implementation: if it didn't return early from the loop, it would just return the state of the last parent considered which may not actually represent the group state accurately. --- lib/icinga/dependency-group.cpp | 42 ++++++++++++++++++++------------- lib/icinga/dependency.hpp | 6 ++++- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/lib/icinga/dependency-group.cpp b/lib/icinga/dependency-group.cpp index 01520467e9a..043e8f51473 100644 --- a/lib/icinga/dependency-group.cpp +++ b/lib/icinga/dependency-group.cpp @@ -322,26 +322,34 @@ DependencyGroup::State DependencyGroup::GetState(DependencyType dt, int rstack) members = m_Members; } - State state{false /* Reachable */, false /* OK */}; - for (auto& [_, children] : members) { - for (auto& [checkable, dependency] : children) { - state.Reachable = dependency->GetParent()->IsReachable(dt, rstack); - if (!state.Reachable && !IsRedundancyGroup()) { - return state; - } + size_t reachable = 0, available = 0; - if (state.Reachable) { - state.OK = dependency->IsAvailable(dt); - // If this is a redundancy group, and we have found one functional path, that's enough and we can return. - // Likewise, if this is a non-redundant dependency group, and we have found one non-functional path, - // we have to mark the group as failed and return. - if (state.OK == IsRedundancyGroup()) { // OK && IsRedundancyGroup() || !OK && !IsRedundancyGroup() - return state; - } + for (auto& [_, dependencies] : members) { + ASSERT(!dependencies.empty()); + + // Dependencies are grouped by parent and all config attributes affecting their availability. Hence, only + // one dependency from the map entry has to be considered, all others will share the same state. + if (auto& [_, dependency] = *dependencies.begin(); dependency->GetParent()->IsReachable(dt, rstack)) { + reachable++; + + // Only reachable parents are considered for availability. If they are unreachable and checks are disabled, + // they could be incorrectly treated as available otherwise. + if (dependency->IsAvailable(dt)) { + available++; } - break; // Move on to the next batch of group members (next composite key). } } - return state; + if (IsRedundancyGroup()) { + // The state of a redundancy group is determined by the best state of any parent. If any parent ist reachable, + // the redundancy group is reachable, analogously for availability. Note that an unreachable group cannot be + // available as reachable = 0 implies available = 0. + return {reachable > 0, available > 0}; + } else { + // For dependencies without a redundancy group, members.size() will be 1 in almost all cases. It will only + // contain more elements if there are duplicate dependency config objects between two checkables. In this case, + // all of them have to be reachable or available as they don't provide redundancy. Note that unreachable implies + // unavailable here as well as only reachable parents count towards the number of available parents. + return {reachable >= members.size(), available == members.size()}; + } } diff --git a/lib/icinga/dependency.hpp b/lib/icinga/dependency.hpp index fe005dbb2d4..49fb93989a5 100644 --- a/lib/icinga/dependency.hpp +++ b/lib/icinga/dependency.hpp @@ -182,7 +182,7 @@ class DependencyGroup final : public SharedObject { size_t operator()(const DependencyGroup::Ptr& dependencyGroup) const { - size_t hash = 0; + size_t hash = std::hash{}(dependencyGroup->GetRedundancyGroupName()); for (const auto& [key, group] : dependencyGroup->m_Members) { boost::hash_combine(hash, key); } @@ -194,6 +194,10 @@ class DependencyGroup final : public SharedObject { bool operator()(const DependencyGroup::Ptr& lhs, const DependencyGroup::Ptr& rhs) const { + if (lhs->GetRedundancyGroupName() != rhs->GetRedundancyGroupName()) { + return false; + } + return std::equal( lhs->m_Members.begin(), lhs->m_Members.end(), rhs->m_Members.begin(), rhs->m_Members.end(), From 7fbb8f74527e2f11c1843064362a4ba1ee46723d Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Fri, 14 Feb 2025 13:11:24 +0100 Subject: [PATCH 32/40] Evaluate dependency group state only for a specific child Previously the dependency state was evaluated by picking the first dependency object from the batched members. However, since the dependency `disable_{checks,notifications` attributes aren't taken into account when batching the members, the evaluated state may yield a wrong result for some Checkables due to some random dependency from other Checkable of that group that has the `disable_{checks,notifications` attrs set. This commit forces the callers to always provide the child Checkable the state is evaluated for and picks only the dependency objects of that child Checkable. --- lib/icinga/checkable-dependency.cpp | 2 +- lib/icinga/dependency-group.cpp | 30 ++++++++++++----------------- lib/icinga/dependency.hpp | 2 +- lib/icingadb/icingadb-objects.cpp | 4 ++-- lib/icingadb/icingadb-utility.cpp | 5 +++-- lib/icingadb/icingadb.hpp | 2 +- 6 files changed, 20 insertions(+), 25 deletions(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index b82aebc88aa..921b36a7e35 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -206,7 +206,7 @@ bool Checkable::IsReachable(DependencyType dt, int rstack) const } for (auto& dependencyGroup : GetDependencyGroups()) { - if (auto state(dependencyGroup->GetState(dt, rstack + 1)); !state.Reachable || !state.OK) { + if (auto state(dependencyGroup->GetState(this, dt, rstack + 1)); !state.Reachable || !state.OK) { Log(LogDebug, "Checkable") << "Dependency group '" << dependencyGroup->GetRedundancyGroupName() << "' have failed for checkable '" << GetName() << "': Marking as unreachable."; diff --git a/lib/icinga/dependency-group.cpp b/lib/icinga/dependency-group.cpp index 043e8f51473..fdf58b5e2d0 100644 --- a/lib/icinga/dependency-group.cpp +++ b/lib/icinga/dependency-group.cpp @@ -311,29 +311,23 @@ String DependencyGroup::GetCompositeKey() * a dependency group may still be marked as failed even when it has reachable parent Checkables, but an unreachable * group has always a failed state. * + * @param child The child Checkable to evaluate the state for. + * @param dt The dependency type to evaluate the state for, defaults to DependencyState. + * @param rstack The recursion stack level to prevent infinite recursion, defaults to 0. + * * @return - Returns the state of the current dependency group. */ -DependencyGroup::State DependencyGroup::GetState(DependencyType dt, int rstack) const +DependencyGroup::State DependencyGroup::GetState(const Checkable* child, DependencyType dt, int rstack) const { - MembersMap members; - { - // We don't want to hold the mutex lock for the entire evaluation, thus we just need to operate on a copy. - std::lock_guard lock(m_Mutex); - members = m_Members; - } - + auto dependencies(GetDependenciesForChild(child)); size_t reachable = 0, available = 0; - for (auto& [_, dependencies] : members) { - ASSERT(!dependencies.empty()); - - // Dependencies are grouped by parent and all config attributes affecting their availability. Hence, only - // one dependency from the map entry has to be considered, all others will share the same state. - if (auto& [_, dependency] = *dependencies.begin(); dependency->GetParent()->IsReachable(dt, rstack)) { + for (const auto& dependency : dependencies) { + if (dependency->GetParent()->IsReachable(dt, rstack)) { reachable++; - // Only reachable parents are considered for availability. If they are unreachable and checks are disabled, - // they could be incorrectly treated as available otherwise. + // Only reachable parents are considered for availability. If they are unreachable and checks are + // disabled, they could be incorrectly treated as available otherwise. if (dependency->IsAvailable(dt)) { available++; } @@ -346,10 +340,10 @@ DependencyGroup::State DependencyGroup::GetState(DependencyType dt, int rstack) // available as reachable = 0 implies available = 0. return {reachable > 0, available > 0}; } else { - // For dependencies without a redundancy group, members.size() will be 1 in almost all cases. It will only + // For dependencies without a redundancy group, dependencies.size() will be 1 in almost all cases. It will only // contain more elements if there are duplicate dependency config objects between two checkables. In this case, // all of them have to be reachable or available as they don't provide redundancy. Note that unreachable implies // unavailable here as well as only reachable parents count towards the number of available parents. - return {reachable >= members.size(), available == members.size()}; + return {reachable == dependencies.size(), available == dependencies.size()}; } } diff --git a/lib/icinga/dependency.hpp b/lib/icinga/dependency.hpp index 49fb93989a5..dab2bc4b8db 100644 --- a/lib/icinga/dependency.hpp +++ b/lib/icinga/dependency.hpp @@ -170,7 +170,7 @@ class DependencyGroup final : public SharedObject bool OK; // Whether the dependency group is reachable and OK. }; - State GetState(DependencyType dt = DependencyState, int rstack = 0) const; + State GetState(const Checkable* child, DependencyType dt = DependencyState, int rstack = 0) const; static boost::signals2::signal OnChildRegistered; static boost::signals2::signal&, bool)> OnChildRemoved; diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 0a5e5eb0fcd..283770490a2 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1210,7 +1210,7 @@ void IcingaDB::InsertCheckableDependencies( // to the DependencyEdgeState HMSETs. The latter is shared by all child Checkables of the current // redundancy group, and since they all depend on the redundancy group, the state of that group is // basically the state of the dependency edges between the children and the redundancy group. - auto stateAttrs(SerializeRedundancyGroupState(dependencyGroup)); + auto stateAttrs(SerializeRedundancyGroupState(checkable, dependencyGroup)); AddDataToHmSets(hMSets, RedisKey::RedundancyGroupState, redundancyGroupId, stateAttrs); AddDataToHmSets(hMSets, RedisKey::DependencyEdgeState, redundancyGroupId, Dictionary::Ptr(new Dictionary{ {"id", redundancyGroupId}, @@ -1416,7 +1416,7 @@ void IcingaDB::UpdateDependenciesState(const Checkable::Ptr& checkable, const De } if (isRedundancyGroup) { - Dictionary::Ptr stateAttrs(SerializeRedundancyGroupState(dependencyGroup)); + Dictionary::Ptr stateAttrs(SerializeRedundancyGroupState(checkable, dependencyGroup)); Dictionary::Ptr sharedGroupState(stateAttrs->ShallowClone()); sharedGroupState->Remove("redundancy_group_id"); diff --git a/lib/icingadb/icingadb-utility.cpp b/lib/icingadb/icingadb-utility.cpp index f53055a50a2..b93aff67a24 100644 --- a/lib/icingadb/icingadb-utility.cpp +++ b/lib/icingadb/icingadb-utility.cpp @@ -201,13 +201,14 @@ Dictionary::Ptr IcingaDB::SerializeDependencyEdgeState(const DependencyGroup::Pt /** * Serialize the provided redundancy group state attributes. * + * @param child The child checkable object to serialize the state for. * @param redundancyGroup The redundancy group object to serialize the state for. * * @return A dictionary with the serialized redundancy group state. */ -Dictionary::Ptr IcingaDB::SerializeRedundancyGroupState(const DependencyGroup::Ptr& redundancyGroup) +Dictionary::Ptr IcingaDB::SerializeRedundancyGroupState(const Checkable::Ptr& child, const DependencyGroup::Ptr& redundancyGroup) { - auto state(redundancyGroup->GetState()); + auto state(redundancyGroup->GetState(child.get())); return new Dictionary{ {"id", redundancyGroup->GetIcingaDBIdentifier()}, {"environment_id", m_EnvironmentId}, diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index df454e31a0c..39a899efcba 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -175,7 +175,7 @@ class IcingaDB : public ObjectImpl static const char* GetNotificationTypeByEnum(NotificationType type); static Dictionary::Ptr SerializeVars(const Dictionary::Ptr& vars); static Dictionary::Ptr SerializeDependencyEdgeState(const DependencyGroup::Ptr& dependencyGroup, const Dependency::Ptr& dep); - static Dictionary::Ptr SerializeRedundancyGroupState(const DependencyGroup::Ptr& redundancyGroup); + static Dictionary::Ptr SerializeRedundancyGroupState(const Checkable::Ptr& child, const DependencyGroup::Ptr& redundancyGroup); static String HashValue(const Value& value); static String HashValue(const Value& value, const std::set& propertiesBlacklist, bool propertiesWhitelist = false); From a9bb11b16d4b3939cf18534b3bf32c2dff230ab5 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Fri, 14 Feb 2025 16:53:47 +0100 Subject: [PATCH 33/40] (Un)register dependencies from parent prior to child Checkable --- lib/icinga/dependency.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/icinga/dependency.cpp b/lib/icinga/dependency.cpp index c45015e8f89..52456ff4bd5 100644 --- a/lib/icinga/dependency.cpp +++ b/lib/icinga/dependency.cpp @@ -251,16 +251,24 @@ void Dependency::OnAllConfigLoaded() // InitChildParentReferences() has to be called before. VERIFY(m_Child && m_Parent); - m_Child->AddDependency(this); + // Icinga DB will implicitly send config updates for the parent Checkable to refresh its affects_children and + // affected_children columns when registering the dependency from the child Checkable. So, we need to register + // the dependency from the parent Checkable first, otherwise the config update of the parent Checkable will change + // nothing at all. m_Parent->AddReverseDependency(this); + m_Child->AddDependency(this); } void Dependency::Stop(bool runtimeRemoved) { ObjectImpl::Stop(runtimeRemoved); - GetChild()->RemoveDependency(this, runtimeRemoved); + // Icinga DB will implicitly send config updates for the parent Checkable to refresh its affects_children and + // affected_children columns when removing the dependency from the child Checkable. So, we need to remove the + // dependency from the parent Checkable first, otherwise the config update of the parent Checkable will change + // nothing at all. GetParent()->RemoveReverseDependency(this); + GetChild()->RemoveDependency(this, runtimeRemoved); } bool Dependency::IsAvailable(DependencyType dt) const From 21cd5e00fa53d52923af6ddc15b24e06fa84cc8a Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 17 Feb 2025 13:49:39 +0100 Subject: [PATCH 34/40] Dependency: Don't allow to update `{period,states,ignore_soft_states}` at runtime --- lib/icinga/dependency.ti | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/icinga/dependency.ti b/lib/icinga/dependency.ti index b5887710418..a033420ea8d 100644 --- a/lib/icinga/dependency.ti +++ b/lib/icinga/dependency.ti @@ -81,16 +81,16 @@ class Dependency : CustomVarObject < DependencyNameComposer [config, no_user_modify] String redundancy_group; - [config, navigation] name(TimePeriod) period (PeriodRaw) { + [config, no_user_modify, navigation] name(TimePeriod) period (PeriodRaw) { navigate {{{ return TimePeriod::GetByName(GetPeriodRaw()); }}} }; - [config] array(Value) states; + [config, no_user_modify] array(Value) states; [no_user_view, no_user_modify] int state_filter_real (StateFilter); - [config] bool ignore_soft_states { + [config, no_user_modify] bool ignore_soft_states { default {{{ return true; }}} }; From da637c3741b0482de241b66908b7a4a8ce57d345 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 17 Feb 2025 17:47:17 +0100 Subject: [PATCH 35/40] IcingaDB: Always send dependencies state HSET updates to Redis --- lib/icingadb/icingadb-objects.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 283770490a2..04043db53ba 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1386,6 +1386,7 @@ void IcingaDB::UpdateDependenciesState(const Checkable::Ptr& checkable, const De streamStates.emplace_back(std::move(xAdd)); }); + std::map hMSets; for (auto& dependencyGroup : dependencyGroups) { bool isRedundancyGroup(dependencyGroup->IsRedundancyGroup()); if (isRedundancyGroup && dependencyGroup->GetIcingaDBIdentifier().IsEmpty()) { @@ -1413,6 +1414,7 @@ void IcingaDB::UpdateDependenciesState(const Checkable::Ptr& checkable, const De } addDependencyStateToStream(m_PrefixConfigObject + "dependency:edge:state", stateAttrs); + AddDataToHmSets(hMSets, RedisKey::DependencyEdgeState, stateAttrs->Get("id"), stateAttrs); } if (isRedundancyGroup) { @@ -1425,10 +1427,19 @@ void IcingaDB::UpdateDependenciesState(const Checkable::Ptr& checkable, const De addDependencyStateToStream(m_PrefixConfigObject + "redundancygroup:state", stateAttrs); addDependencyStateToStream(m_PrefixConfigObject + "dependency:edge:state", sharedGroupState); + AddDataToHmSets(hMSets, RedisKey::RedundancyGroupState, dependencyGroup->GetIcingaDBIdentifier(), stateAttrs); + AddDataToHmSets(hMSets, RedisKey::DependencyEdgeState, dependencyGroup->GetIcingaDBIdentifier(), sharedGroupState); } } if (!streamStates.empty()) { + RedisConnection::Queries queries; + for (auto& [redisKey, query] : hMSets) { + query.insert(query.begin(), {"HSET", redisKey}); + queries.emplace_back(std::move(query)); + } + + m_Rcon->FireAndForgetQueries(std::move(queries), Prio::RuntimeStateSync); m_Rcon->FireAndForgetQueries(std::move(streamStates), Prio::RuntimeStateStream, {0, 1}); } } From 945a79e37fd22d2108c378d6e4097b6d38786fc3 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Fri, 28 Feb 2025 17:20:03 +0100 Subject: [PATCH 36/40] IcingaDB: Don't send useless dependencies state updates --- lib/icingadb/icingadb-objects.cpp | 19 +++++++++++++++++-- lib/icingadb/icingadb.hpp | 3 ++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 04043db53ba..bd8e7432248 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1357,8 +1357,10 @@ void IcingaDB::UpdateState(const Checkable::Ptr& checkable, StateUpdate mode) * * @param checkable The Checkable you want to send the dependencies state update for * @param onlyDependencyGroup If set, send state updates only for this dependency group and its dependencies. + * @param seenGroups A container to track already processed DependencyGroups to avoid duplicate state updates. */ -void IcingaDB::UpdateDependenciesState(const Checkable::Ptr& checkable, const DependencyGroup::Ptr& onlyDependencyGroup) const +void IcingaDB::UpdateDependenciesState(const Checkable::Ptr& checkable, const DependencyGroup::Ptr& onlyDependencyGroup, + std::set* seenGroups) const { if (!m_Rcon || !m_Rcon->IsConnected()) { return; @@ -1396,6 +1398,18 @@ void IcingaDB::UpdateDependenciesState(const Checkable::Ptr& checkable, const De continue; } + if (seenGroups && !seenGroups->insert(dependencyGroup.get()).second) { + // Usually, if the seenGroups set is provided, IcingaDB is triggering a runtime state update for ALL + // children of a given initiator Checkable (parent). In such cases, we may end up with lots of useless + // state updates as all the children of a non-redundant group a) share the same entry in the database b) + // it doesn't matter which child triggers the state update first all the subsequent updates are just useless. + // + // Likewise, for redundancy groups, all children of a redundancy group share the same set of parents + // and thus the resulting state information would be the same from each child Checkable perspective. + // So, serializing the redundancy group state information only once is sufficient. + continue; + } + auto dependencies(dependencyGroup->GetDependenciesForChild(checkable.get())); std::sort(dependencies.begin(), dependencies.end(), [](const Dependency::Ptr& lhs, const Dependency::Ptr& rhs) { return lhs->GetParent() < rhs->GetParent(); @@ -3096,9 +3110,10 @@ void IcingaDB::StateChangeHandler(const ConfigObject::Ptr& object, const CheckRe void IcingaDB::ReachabilityChangeHandler(const std::set& children) { for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType()) { + std::set seenGroups; for (auto& checkable : children) { rw->UpdateState(checkable, StateUpdate::Full); - rw->UpdateDependenciesState(checkable); + rw->UpdateDependenciesState(checkable, nullptr, &seenGroups); } } } diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index 39a899efcba..af58a977db7 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -114,7 +114,8 @@ class IcingaDB : public ObjectImpl std::vector* runtimeUpdates, const DependencyGroup::Ptr& onlyDependencyGroup = nullptr); void InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map>& hMSets, std::vector& runtimeUpdates, bool runtimeUpdate); - void UpdateDependenciesState(const Checkable::Ptr& checkable, const DependencyGroup::Ptr& onlyDependencyGroup = nullptr) const; + void UpdateDependenciesState(const Checkable::Ptr& checkable, const DependencyGroup::Ptr& onlyDependencyGroup = nullptr, + std::set* seenGroups = nullptr) const; void UpdateState(const Checkable::Ptr& checkable, StateUpdate mode); void SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate); void CreateConfigUpdate(const ConfigObject::Ptr& object, const String type, std::map>& hMSets, From 693d094ebc48606675c38c6a18636e597098e817 Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Thu, 13 Mar 2025 13:13:06 +0100 Subject: [PATCH 37/40] DependencyGroup: don't change the keys of m_Members after construction This prevents the use of DependencyGroup for storing the dependencies during the early registration (m_DependencyGroupsPushedToRegistry = false), m_PendingDependencies is introduced as a replacement to store the dependencies at that time. --- lib/icinga/checkable-dependency.cpp | 30 +++++++++++----------- lib/icinga/checkable.hpp | 14 ++++++++-- lib/icinga/dependency-group.cpp | 40 ++++++++++++----------------- lib/icinga/dependency.cpp | 2 +- lib/icinga/dependency.hpp | 3 +-- 5 files changed, 46 insertions(+), 43 deletions(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index 921b36a7e35..27ea5c0ae98 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -26,15 +26,12 @@ static constexpr int l_MaxDependencyRecursionLevel(256); void Checkable::PushDependencyGroupsToRegistry() { std::lock_guard lock(m_DependencyMutex); - if (!m_DependencyGroupsPushedToRegistry) { - m_DependencyGroupsPushedToRegistry = true; - - decltype(m_DependencyGroups) dependencyGroups; - m_DependencyGroups.swap(dependencyGroups); - - for (auto& [dependencyGroupKey, dependencyGroup] : dependencyGroups) { - m_DependencyGroups.emplace(dependencyGroupKey, DependencyGroup::Register(dependencyGroup)); + if (m_PendingDependencies != nullptr) { + for (const auto& [key, dependencies] : *m_PendingDependencies) { + String redundancyGroup = std::holds_alternative(key) ? std::get(key) : ""; + m_DependencyGroups.emplace(key, DependencyGroup::Register(new DependencyGroup(redundancyGroup, dependencies))); } + m_PendingDependencies.reset(); } } @@ -78,12 +75,8 @@ void Checkable::AddDependency(const Dependency::Ptr& dependency) std::unique_lock lock(m_DependencyMutex); auto dependencyGroupKey(GetDependencyGroupKey(dependency)); - if (!m_DependencyGroupsPushedToRegistry) { - auto& dependencyGroup = m_DependencyGroups[dependencyGroupKey]; - if (!dependencyGroup) { - dependencyGroup = new DependencyGroup(dependency->GetRedundancyGroup()); - } - dependencyGroup->AddDependency(dependency); + if (m_PendingDependencies != nullptr) { + (*m_PendingDependencies)[dependencyGroupKey].emplace(dependency); return; } @@ -151,10 +144,17 @@ void Checkable::RemoveDependency(const Dependency::Ptr& dependency, bool runtime } } -std::vector Checkable::GetDependencies() const +std::vector Checkable::GetDependencies(bool includePending) const { std::unique_lock lock(m_DependencyMutex); std::vector dependencies; + + if (includePending && m_PendingDependencies != nullptr) { + for (const auto& [group, groupDeps] : *m_PendingDependencies) { + dependencies.insert(dependencies.end(), groupDeps.begin(), groupDeps.end()); + } + } + for (const auto& [_, dependencyGroup] : m_DependencyGroups) { auto tmpDependencies(dependencyGroup->GetDependenciesForChild(this)); dependencies.insert(dependencies.end(), tmpDependencies.begin(), tmpDependencies.end()); diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index 53db79d721f..98c015ed60f 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -190,7 +190,7 @@ class Checkable : public ObjectImpl std::vector> GetDependencyGroups() const; void AddDependency(const intrusive_ptr& dependency); void RemoveDependency(const intrusive_ptr& dependency, bool runtimeRemoved = false); - std::vector > GetDependencies() const; + std::vector > GetDependencies(bool includePending = false) const; bool HasAnyDependencies() const; void AddReverseDependency(const intrusive_ptr& dep); @@ -251,9 +251,19 @@ class Checkable : public ObjectImpl /* Dependencies */ mutable std::mutex m_DependencyMutex; - bool m_DependencyGroupsPushedToRegistry{false}; std::map, intrusive_ptr> m_DependencyGroups; std::set > m_ReverseDependencies; + /** + * Registering a checkable to its parent DependencyGroups is delayed during config loading until all dependencies + * were registered on the checkable. m_PendingDependencies is used to temporarily store the dependencies until then. + * It is a pointer type for two reasons: + * 1. The field is no longer needed after the DependencyGroups were registered, having it as a pointer reduces the + * overhead from sizeof(std::map<>) to sizeof(std::map<>*). + * 2. It allows the field to also be used as a flag: the delayed group registration is only done until it is reset + * to nullptr. + */ + std::unique_ptr, std::set>>> + m_PendingDependencies {std::make_unique()}; void GetAllChildrenInternal(std::set& seenChildren, int level = 0) const; diff --git a/lib/icinga/dependency-group.cpp b/lib/icinga/dependency-group.cpp index fdf58b5e2d0..29d628832fb 100644 --- a/lib/icinga/dependency-group.cpp +++ b/lib/icinga/dependency-group.cpp @@ -46,17 +46,18 @@ std::pair, bool> DependencyGroup::Unregister(const Dep { std::lock_guard lock(m_RegistryMutex); if (auto it(m_Registry.find(dependencyGroup)); it != m_Registry.end()) { - auto existingGroup(*it); + auto& existingGroup(*it); auto dependencies(existingGroup->GetDependenciesForChild(child.get())); for (const auto& dependency : dependencies) { existingGroup->RemoveDependency(dependency); } - if (existingGroup->IsEmpty()) { + bool remove = !existingGroup->HasChildren(); + if (remove) { m_Registry.erase(it); } - return {{dependencies.begin(), dependencies.end()}, existingGroup->IsEmpty()}; + return {{dependencies.begin(), dependencies.end()}, remove}; } return {{}, false}; } @@ -72,14 +73,11 @@ size_t DependencyGroup::GetRegistrySize() return m_Registry.size(); } -DependencyGroup::DependencyGroup(String name): m_RedundancyGroupName(std::move(name)) -{ -} - -DependencyGroup::DependencyGroup(String name, const std::set& dependencies): m_RedundancyGroupName(std::move(name)) +DependencyGroup::DependencyGroup(String name, const std::set& dependencies) + : m_RedundancyGroupName(std::move(name)) { for (const auto& dependency : dependencies) { - AddDependency(dependency); + m_Members[MakeCompositeKeyFor(dependency)].emplace(dependency->GetChild().get(), dependency.get()); } } @@ -103,14 +101,14 @@ DependencyGroup::CompositeKeyType DependencyGroup::MakeCompositeKeyFor(const Dep } /** - * Check if the current dependency group is empty. + * Check if the current dependency has any children. * - * @return bool - Returns true if the current dependency group has no members, otherwise false. + * @return bool - Returns true if the current dependency group has children, otherwise false. */ -bool DependencyGroup::IsEmpty() const +bool DependencyGroup::HasChildren() const { std::lock_guard lock(m_Mutex); - return m_Members.empty(); + return std::any_of(m_Members.begin(), m_Members.end(), [](const auto& pair) { return !pair.second.empty(); }); } /** @@ -142,7 +140,6 @@ void DependencyGroup::LoadParents(std::set& parents) const { std::lock_guard lock(m_Mutex); for (auto& [compositeKey, children] : m_Members) { - ASSERT(!children.empty()); // We should never have an empty map for any given key at any given time. parents.insert(std::get<0>(compositeKey)); } } @@ -174,11 +171,12 @@ void DependencyGroup::AddDependency(const Dependency::Ptr& dependency) { std::lock_guard lock(m_Mutex); auto compositeKey(MakeCompositeKeyFor(dependency)); - if (auto it(m_Members.find(compositeKey)); it != m_Members.end()) { - it->second.emplace(dependency->GetChild().get(), dependency.get()); - } else { - m_Members.emplace(compositeKey, MemberValueType{{dependency->GetChild().get(), dependency.get()}}); - } + auto it = m_Members.find(compositeKey); + + // The dependency must be compatible with the group, i.e. its parent config must be known in the group already. + VERIFY(it != m_Members.end()); + + it->second.emplace(dependency->GetChild().get(), dependency.get()); } /** @@ -196,10 +194,6 @@ void DependencyGroup::RemoveDependency(const Dependency::Ptr& dependency) // This will also remove the child Checkable from the multimap container // entirely if this was the last child of it. it->second.erase(childrenIt); - // If the composite key has no more children left, we can remove it entirely as well. - if (it->second.empty()) { - m_Members.erase(it); - } return; } } diff --git a/lib/icinga/dependency.cpp b/lib/icinga/dependency.cpp index 52456ff4bd5..cf4af862710 100644 --- a/lib/icinga/dependency.cpp +++ b/lib/icinga/dependency.cpp @@ -123,7 +123,7 @@ class DependencyCycleChecker } // Explicitly configured dependency objects - for (const auto& dep : checkable->GetDependencies()) { + for (const auto& dep : checkable->GetDependencies(/* includePending = */ true)) { m_Stack.emplace_back(dep); AssertNoCycle(dep->GetParent()); m_Stack.pop_back(); diff --git a/lib/icinga/dependency.hpp b/lib/icinga/dependency.hpp index dab2bc4b8db..26a7dffe01b 100644 --- a/lib/icinga/dependency.hpp +++ b/lib/icinga/dependency.hpp @@ -132,7 +132,6 @@ class DependencyGroup final : public SharedObject using MemberValueType = std::unordered_multimap; using MembersMap = std::map; - explicit DependencyGroup(String name); DependencyGroup(String name, const std::set& dependencies); static DependencyGroup::Ptr Register(const DependencyGroup::Ptr& dependencyGroup); @@ -151,7 +150,7 @@ class DependencyGroup final : public SharedObject return !m_RedundancyGroupName.IsEmpty(); } - bool IsEmpty() const; + bool HasChildren() const; void AddDependency(const Dependency::Ptr& dependency); void RemoveDependency(const Dependency::Ptr& dependency); std::vector GetDependenciesForChild(const Checkable* child) const; From 864e2aaae0009e34171b10822b50b4ba1f986942 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Thu, 13 Mar 2025 16:37:26 +0100 Subject: [PATCH 38/40] Drop superfluous mutex lock & don't manually unpack `std::tuple` --- lib/icinga/dependency-group.cpp | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/lib/icinga/dependency-group.cpp b/lib/icinga/dependency-group.cpp index 29d628832fb..0ffe33741fb 100644 --- a/lib/icinga/dependency-group.cpp +++ b/lib/icinga/dependency-group.cpp @@ -138,7 +138,6 @@ std::vector DependencyGroup::GetDependenciesForChild(const Chec */ void DependencyGroup::LoadParents(std::set& parents) const { - std::lock_guard lock(m_Mutex); for (auto& [compositeKey, children] : m_Members) { parents.insert(std::get<0>(compositeKey)); } @@ -203,7 +202,7 @@ void DependencyGroup::RemoveDependency(const Dependency::Ptr& dependency) /** * Copy the dependency objects of the current dependency group to the provided dependency group (destination). * - * @param dest The dependency group to move the dependencies to. + * @param dest The dependency group to copy the dependencies to. */ void DependencyGroup::CopyDependenciesTo(const DependencyGroup::Ptr& dest) { @@ -274,12 +273,9 @@ String DependencyGroup::GetCompositeKey() // not achievable using pointers. using StringTuple = std::tuple; std::vector compositeKeys; - { - std::lock_guard lock(m_Mutex); - for (auto& [compositeKey, _] : m_Members) { - auto [parent, tp, stateFilter, ignoreSoftStates] = compositeKey; - compositeKeys.emplace_back(parent->GetName(), tp ? tp->GetName() : "", stateFilter, ignoreSoftStates); - } + for (auto& [compositeKey, _] : m_Members) { + auto [parent, tp, stateFilter, ignoreSoftStates] = compositeKey; + compositeKeys.emplace_back(parent->GetName(), tp ? tp->GetName() : "", stateFilter, ignoreSoftStates); } // IMPORTANT: The order of the composite keys must be sorted to ensure the deterministic hash value. @@ -287,11 +283,10 @@ String DependencyGroup::GetCompositeKey() Array::Ptr data(new Array{GetRedundancyGroupName()}); for (auto& compositeKey : compositeKeys) { - auto [parent, tp, stateFilter, ignoreSoftStates] = compositeKey; - data->Add(std::move(parent)); - data->Add(std::move(tp)); - data->Add(stateFilter); - data->Add(ignoreSoftStates); + // std::apply is used to unpack the composite key tuple and add its elements to the data array. + // It's like manually expanding the tuple into x variables and then adding them one by one to the array. + // See https://en.cppreference.com/w/cpp/language/fold for more information. + std::apply([&data](auto&&... args) { (data->Add(std::move(args)), ...); }, std::move(compositeKey)); } return PackObject(data); From 065118bc22d420d22f89c60fb1d85955e9a64394 Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Thu, 13 Mar 2025 15:02:56 +0100 Subject: [PATCH 39/40] Make DependencyGroup::State an enum The previous struct used two bools to represent three useful states. Make this more explicit by having these three states as an enum. --- lib/icinga/checkable-dependency.cpp | 2 +- lib/icinga/dependency-group.cpp | 22 ++++++++++++++++------ lib/icinga/dependency.hpp | 7 +------ lib/icingadb/icingadb-utility.cpp | 4 ++-- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index 27ea5c0ae98..65d9dd9022d 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -206,7 +206,7 @@ bool Checkable::IsReachable(DependencyType dt, int rstack) const } for (auto& dependencyGroup : GetDependencyGroups()) { - if (auto state(dependencyGroup->GetState(this, dt, rstack + 1)); !state.Reachable || !state.OK) { + if (auto state(dependencyGroup->GetState(this, dt, rstack + 1)); state != DependencyGroup::State::Ok) { Log(LogDebug, "Checkable") << "Dependency group '" << dependencyGroup->GetRedundancyGroupName() << "' have failed for checkable '" << GetName() << "': Marking as unreachable."; diff --git a/lib/icinga/dependency-group.cpp b/lib/icinga/dependency-group.cpp index 0ffe33741fb..aa7d5fc06ac 100644 --- a/lib/icinga/dependency-group.cpp +++ b/lib/icinga/dependency-group.cpp @@ -325,14 +325,24 @@ DependencyGroup::State DependencyGroup::GetState(const Checkable* child, Depende if (IsRedundancyGroup()) { // The state of a redundancy group is determined by the best state of any parent. If any parent ist reachable, - // the redundancy group is reachable, analogously for availability. Note that an unreachable group cannot be - // available as reachable = 0 implies available = 0. - return {reachable > 0, available > 0}; + // the redundancy group is reachable, analogously for availability. + if (reachable == 0) { + return State::Unreachable; + } else if (available == 0) { + return State::Failed; + } else { + return State::Ok; + } } else { // For dependencies without a redundancy group, dependencies.size() will be 1 in almost all cases. It will only // contain more elements if there are duplicate dependency config objects between two checkables. In this case, - // all of them have to be reachable or available as they don't provide redundancy. Note that unreachable implies - // unavailable here as well as only reachable parents count towards the number of available parents. - return {reachable == dependencies.size(), available == dependencies.size()}; + // all of them have to be reachable/available as they don't provide redundancy. + if (reachable < dependencies.size()) { + return State::Unreachable; + } else if (available < dependencies.size()) { + return State::Failed; + } else { + return State::Ok; + } } } diff --git a/lib/icinga/dependency.hpp b/lib/icinga/dependency.hpp index 26a7dffe01b..b4e206b7f2c 100644 --- a/lib/icinga/dependency.hpp +++ b/lib/icinga/dependency.hpp @@ -163,12 +163,7 @@ class DependencyGroup final : public SharedObject const String& GetRedundancyGroupName() const; String GetCompositeKey(); - struct State - { - bool Reachable; // Whether the dependency group is reachable. - bool OK; // Whether the dependency group is reachable and OK. - }; - + enum class State { Ok, Failed, Unreachable }; State GetState(const Checkable* child, DependencyType dt = DependencyState, int rstack = 0) const; static boost::signals2::signal OnChildRegistered; diff --git a/lib/icingadb/icingadb-utility.cpp b/lib/icingadb/icingadb-utility.cpp index b93aff67a24..89e5a5031fd 100644 --- a/lib/icingadb/icingadb-utility.cpp +++ b/lib/icingadb/icingadb-utility.cpp @@ -213,8 +213,8 @@ Dictionary::Ptr IcingaDB::SerializeRedundancyGroupState(const Checkable::Ptr& ch {"id", redundancyGroup->GetIcingaDBIdentifier()}, {"environment_id", m_EnvironmentId}, {"redundancy_group_id", redundancyGroup->GetIcingaDBIdentifier()}, - {"failed", !state.Reachable || !state.OK}, - {"is_reachable", state.Reachable}, + {"failed", state != DependencyGroup::State::Ok}, + {"is_reachable", state != DependencyGroup::State::Unreachable}, {"last_state_change", TimestampToMilliseconds(Utility::GetTime())}, }; } From bc2c750551faed595f4983a02738022da1d9168a Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Wed, 26 Mar 2025 10:48:37 +0100 Subject: [PATCH 40/40] IcingaDB: Don't stream runtime state updates to Redis --- lib/icingadb/icingadb-objects.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index bd8e7432248..40580a35855 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -3383,10 +3383,12 @@ void IcingaDB::DeleteState(const String& id, RedisKey redisKey, bool hasChecksum hdels.emplace_back(RedisConnection::Query{"HDEL", m_PrefixConfigObject + redisKeyWithoutPrefix, id}); m_Rcon->FireAndForgetQueries(std::move(hdels), Prio::RuntimeStateSync); - m_Rcon->FireAndForgetQueries({{ + // TODO: This is currently purposefully commented out due to how Icinga DB (Go) handles runtime state + // upsert and delete events. See https://github.com/Icinga/icingadb/pull/894 for more details. + /*m_Rcon->FireAndForgetQueries({{ "XADD", "icinga:runtime:state", "MAXLEN", "~", "1000000", "*", "redis_key", m_PrefixConfigObject + redisKeyWithoutPrefix, "id", id, "runtime_type", "delete" - }}, Prio::RuntimeStateStream, {0, 1}); + }}, Prio::RuntimeStateStream, {0, 1});*/ } /**