diff --git a/lib/icinga/checkable-notification.cpp b/lib/icinga/checkable-notification.cpp index 568ff6c52da..dae5574e8fe 100644 --- a/lib/icinga/checkable-notification.cpp +++ b/lib/icinga/checkable-notification.cpp @@ -2,11 +2,13 @@ #include "icinga/checkable.hpp" #include "icinga/icingaapplication.hpp" +#include "base/dictionary.hpp" #include "base/objectlock.hpp" #include "base/logger.hpp" #include "base/exception.hpp" #include "base/context.hpp" #include "base/convert.hpp" +#include "remote/apilistener.hpp" using namespace icinga; @@ -52,17 +54,47 @@ void Checkable::SendNotifications(NotificationType type, const CheckResult::Ptr& return; for (const Notification::Ptr& notification : notifications) { - try { - if (!notification->IsPaused()) { - notification->BeginExecuteNotification(type, cr, force, false, author, text); - } else { - Log(LogNotice, "Notification") - << "Notification '" << notification->GetName() << "': HA cluster active, this endpoint does not have the authority (paused=true). Skipping."; + if (ApiListener::UpdatedObjectAuthority()) { + try { + if (!notification->IsPaused()) { + auto stashedNotifications (notification->GetStashedNotifications()); + + if (stashedNotifications->GetLength()) { + Log(LogNotice, "Notification") + << "Notification '" << notification->GetName() << "': there are some stashed notifications. Stashing notification to preserve order."; + + stashedNotifications->Add(new Dictionary({ + {"type", type}, + {"cr", cr}, + {"force", force}, + {"reminder", false}, + {"author", author}, + {"text", text} + })); + } else { + notification->BeginExecuteNotification(type, cr, force, false, author, text); + } + } else { + Log(LogNotice, "Notification") + << "Notification '" << notification->GetName() << "': HA cluster active, this endpoint does not have the authority (paused=true). Skipping."; + } + } catch (const std::exception& ex) { + Log(LogWarning, "Checkable") + << "Exception occurred during notification '" << notification->GetName() << "' for checkable '" + << GetName() << "': " << DiagnosticInformation(ex, false); } - } catch (const std::exception& ex) { - Log(LogWarning, "Checkable") - << "Exception occurred during notification '" << notification->GetName() << "' for checkable '" - << GetName() << "': " << DiagnosticInformation(ex, false); + } else { + Log(LogNotice, "Notification") + << "Notification '" << notification->GetName() << "': object authority hasn't been updated, yet. Stashing notification."; + + notification->GetStashedNotifications()->Add(new Dictionary({ + {"type", type}, + {"cr", cr}, + {"force", force}, + {"reminder", false}, + {"author", author}, + {"text", text} + })); } } } diff --git a/lib/icinga/notification.ti b/lib/icinga/notification.ti index f2aa77ed12d..0212a144a58 100644 --- a/lib/icinga/notification.ti +++ b/lib/icinga/notification.ti @@ -80,6 +80,10 @@ class Notification : CustomVarObject < NotificationNameComposer default {{{ return false; }}} }; + [state, no_user_view, no_user_modify] Array::Ptr stashed_notifications { + default {{{ return new Array(); }}} + }; + [state] Timestamp last_notification; [state] Timestamp next_notification; [state] int notification_number; diff --git a/lib/notification/notificationcomponent.cpp b/lib/notification/notificationcomponent.cpp index 36fe8ecaa68..aa960120166 100644 --- a/lib/notification/notificationcomponent.cpp +++ b/lib/notification/notificationcomponent.cpp @@ -10,6 +10,7 @@ #include "base/utility.hpp" #include "base/exception.hpp" #include "base/statsfunction.hpp" +#include "remote/apilistener.hpp" using namespace icinga; @@ -72,12 +73,27 @@ void NotificationComponent::NotificationTimerHandler() continue; String notificationName = notification->GetName(); + bool updatedObjectAuthority = ApiListener::UpdatedObjectAuthority(); /* Skip notification if paused, in a cluster setup & HA feature is enabled. */ - if (notification->IsPaused() && myEndpoint && GetEnableHA()) { - Log(LogNotice, "NotificationComponent") - << "Reminder notification '" << notificationName << "': HA cluster active, this endpoint does not have the authority (paused=true). Skipping."; - continue; + if (notification->IsPaused()) { + if (updatedObjectAuthority) { + auto stashedNotifications (notification->GetStashedNotifications()); + ObjectLock olock(stashedNotifications); + + if (stashedNotifications->GetLength()) { + Log(LogNotice, "NotificationComponent") + << "Notification '" << notificationName << "': HA cluster active, this endpoint does not have the authority. Dropping all stashed notifications."; + + stashedNotifications->Clear(); + } + } + + if (myEndpoint && GetEnableHA()) { + Log(LogNotice, "NotificationComponent") + << "Reminder notification '" << notificationName << "': HA cluster active, this endpoint does not have the authority (paused=true). Skipping."; + continue; + } } Checkable::Ptr checkable = notification->GetCheckable(); @@ -85,6 +101,42 @@ void NotificationComponent::NotificationTimerHandler() if (!IcingaApplication::GetInstance()->GetEnableNotifications() || !checkable->GetEnableNotifications()) continue; + bool reachable = checkable->IsReachable(DependencyNotification); + + if (reachable) { + Array::Ptr unstashedNotifications = new Array(); + + { + auto stashedNotifications (notification->GetStashedNotifications()); + ObjectLock olock(stashedNotifications); + + stashedNotifications->CopyTo(unstashedNotifications); + stashedNotifications->Clear(); + } + + ObjectLock olock(unstashedNotifications); + + for (Dictionary::Ptr unstashedNotification : unstashedNotifications) { + try { + Log(LogNotice, "NotificationComponent") + << "Attempting to send stashed notification '" << notificationName << "'."; + + notification->BeginExecuteNotification( + (NotificationType)(int)unstashedNotification->Get("type"), + (CheckResult::Ptr)unstashedNotification->Get("cr"), + (bool)unstashedNotification->Get("force"), + (bool)unstashedNotification->Get("reminder"), + (String)unstashedNotification->Get("author"), + (String)unstashedNotification->Get("text") + ); + } catch (const std::exception& ex) { + Log(LogWarning, "NotificationComponent") + << "Exception occurred during notification for object '" + << notificationName << "': " << DiagnosticInformation(ex, false); + } + } + } + if (notification->GetInterval() <= 0 && notification->GetNoMoreNotifications()) { Log(LogNotice, "NotificationComponent") << "Reminder notification '" << notificationName << "': Notification was sent out once and interval=0 disables reminder notifications."; @@ -94,8 +146,6 @@ void NotificationComponent::NotificationTimerHandler() if (notification->GetNextNotification() > now) continue; - bool reachable = checkable->IsReachable(DependencyNotification); - { ObjectLock olock(notification); notification->SetNextNotification(Utility::GetTime() + notification->GetInterval());