Add notification result store/sync

doc/08-advanced-topics.md

@@ -1170,3 +1170,16 @@ Icinga 2 parses performance data strings returned by check plugins and makes the
   warn                      | Value                 | Warning threshold value.
   min                       | Value                 | Minimum value returned by the check.
   max                       | Value                 | Maximum value returned by the check.
+
+### NotificationResult <a id="advanced-value-types-notificationresult"></a>
+
+  Name                      | Type                  | Description
+  --------------------------|-----------------------|----------------------------------
+  exit\_status              | Number                | The exit status returned by the check execution.
+  output                    | String                | The notification command output.
+  execution\_endpoint       | String                | Name of the node executing the check.
+  command                   | Value                 | Array of command with shell-escaped arguments or command line string.
+  execution\_start          | Timestamp             | Check execution start time (as a UNIX timestamp).
+  execution\_end            | Timestamp             | Check execution end time (as a UNIX timestamp).
+  active                    | Boolean               | Whether the result is from an active or passive check.
+

lib/compat/compatlogger.cpp

@@ -50,7 +50,7 @@ void CompatLogger::Start(bool runtimeCreated)
 		<< "The CompatLogger feature is DEPRECATED and will be removed in Icinga v2.11.";
 
 	Checkable::OnNewCheckResult.connect(std::bind(&CompatLogger::CheckResultHandler, this, _1, _2));
-	Checkable::OnNotificationSentToUser.connect(std::bind(&CompatLogger::NotificationSentHandler, this, _1, _2, _3, _4, _5, _6, _7, _8));
+	Checkable::OnNotificationSentToUser.connect(std::bind(&CompatLogger::NotificationSentHandler, this, _1, _2, _3, _4, _5, _6, _7, _8, _9));
 	Downtime::OnDowntimeTriggered.connect(std::bind(&CompatLogger::TriggerDowntimeHandler, this, _1));
 	Downtime::OnDowntimeRemoved.connect(std::bind(&CompatLogger::RemoveDowntimeHandler, this, _1));
 	Checkable::OnEventCommandExecuted.connect(std::bind(&CompatLogger::EventCommandHandler, this, _1));
@@ -230,7 +230,7 @@ void CompatLogger::RemoveDowntimeHandler(const Downtime::Ptr& downtime)
  * @threadsafety Always.
  */
 void CompatLogger::NotificationSentHandler(const Notification::Ptr& notification, const Checkable::Ptr& checkable,
-	const User::Ptr& user, NotificationType notification_type, CheckResult::Ptr const& cr,
+	const User::Ptr& user, NotificationType notification_type, const CheckResult::Ptr& cr, const NotificationResult::Ptr& nr,
 	const String& author, const String& comment_text, const String& command_name)
 {
 	Host::Ptr host;

lib/compat/compatlogger.hpp

@@ -36,8 +36,8 @@ class CompatLogger final : public ObjectImpl<CompatLogger>
 
 	void CheckResultHandler(const Checkable::Ptr& service, const CheckResult::Ptr& cr);
 	void NotificationSentHandler(const Notification::Ptr& notification, const Checkable::Ptr& service,
-		const User::Ptr& user, NotificationType notification_type, CheckResult::Ptr const& cr,
-		const String& author, const String& comment_text, const String& command_name);
+		const User::Ptr& user, NotificationType notification_type, const CheckResult::Ptr& cr,
+		const NotificationResult::Ptr& nr, const String& author, const String& comment_text, const String& command_name);
 	void FlappingChangedHandler(const Checkable::Ptr& checkable);
 	void EnableFlappingChangedHandler(const Checkable::Ptr& checkable);
 	void TriggerDowntimeHandler(const Downtime::Ptr& downtime);

lib/db_ido/dbevents.cpp

@@ -56,7 +56,7 @@ void DbEvents::StaticInitialize()
 	Checkable::OnStateChange.connect(std::bind(&DbEvents::AddStateChangeHistory, _1, _2, _3));
 
 	Checkable::OnNewCheckResult.connect(std::bind(&DbEvents::AddCheckResultLogHistory, _1, _2));
-	Checkable::OnNotificationSentToUser.connect(std::bind(&DbEvents::AddNotificationSentLogHistory, _1, _2, _3, _4, _5, _6, _7));
+	Checkable::OnNotificationSentToUser.connect(std::bind(&DbEvents::AddNotificationSentLogHistory, _1, _2, _3, _4, _5, _6, _7, _8));
 	Checkable::OnFlappingChanged.connect(std::bind(&DbEvents::AddFlappingChangedLogHistory, _1));
 	Checkable::OnEnableFlappingChanged.connect(std::bind(&DbEvents::AddEnableFlappingChangedLogHistory, _1));
 	Downtime::OnDowntimeTriggered.connect(std::bind(&DbEvents::AddTriggerDowntimeLogHistory, _1));
@@ -1061,7 +1061,7 @@ void DbEvents::AddRemoveDowntimeLogHistory(const Downtime::Ptr& downtime)
 }
 
 void DbEvents::AddNotificationSentLogHistory(const Notification::Ptr& notification, const Checkable::Ptr& checkable, const User::Ptr& user,
-	NotificationType notification_type, const CheckResult::Ptr& cr,
+	NotificationType notification_type, const CheckResult::Ptr& cr, const NotificationResult::Ptr& nr,
 	const String& author, const String& comment_text)
 {
 	CheckCommand::Ptr commandObj = checkable->GetCheckCommand();

lib/db_ido/dbevents.hpp

@@ -94,8 +94,8 @@ class DbEvents
 	static void AddTriggerDowntimeLogHistory(const Downtime::Ptr& downtime);
 	static void AddRemoveDowntimeLogHistory(const Downtime::Ptr& downtime);
 	static void AddNotificationSentLogHistory(const Notification::Ptr& notification, const Checkable::Ptr& checkable,
-		const User::Ptr& user, NotificationType notification_type, const CheckResult::Ptr& cr, const String& author,
-		const String& comment_text);
+		const User::Ptr& user, NotificationType notification_type, const CheckResult::Ptr& cr, const NotificationResult::Ptr& nr,
+		const String& author, const String& comment_text);
 
 	static void AddFlappingChangedLogHistory(const Checkable::Ptr& checkable);
 	static void AddEnableFlappingChangedLogHistory(const Checkable::Ptr& checkable);

lib/icinga/CMakeLists.txt

@@ -13,6 +13,7 @@ mkclass_target(host.ti host-ti.cpp host-ti.hpp)
 mkclass_target(icingaapplication.ti icingaapplication-ti.cpp icingaapplication-ti.hpp)
 mkclass_target(customvarobject.ti customvarobject-ti.cpp customvarobject-ti.hpp)
 mkclass_target(notificationcommand.ti notificationcommand-ti.cpp notificationcommand-ti.hpp)
+mkclass_target(notificationresult.ti notificationresult-ti.cpp notificationresult-ti.hpp)
 mkclass_target(notification.ti notification-ti.cpp notification-ti.hpp)
 mkclass_target(scheduleddowntime.ti scheduleddowntime-ti.cpp scheduleddowntime-ti.hpp)
 mkclass_target(servicegroup.ti servicegroup-ti.cpp servicegroup-ti.hpp)
@@ -51,6 +52,7 @@ set(icinga_SOURCES
   macroresolver.hpp
   notification.cpp notification.hpp notification-ti.hpp notification-apply.cpp
   notificationcommand.cpp notificationcommand.hpp notificationcommand-ti.hpp
+  notificationresult.cpp notificationresult.hpp notificationresult-ti.hpp
   objectutils.cpp objectutils.hpp
   pluginutility.cpp pluginutility.hpp
   scheduleddowntime.cpp scheduleddowntime.hpp scheduleddowntime-ti.hpp scheduleddowntime-apply.cpp

lib/icinga/checkable-notification.cpp

@@ -14,8 +14,8 @@ boost::signals2::signal<void (const Notification::Ptr&, const Checkable::Ptr&, c
 	const NotificationType&, const CheckResult::Ptr&, const String&, const String&,
 	const MessageOrigin::Ptr&)> Checkable::OnNotificationSentToAllUsers;
 boost::signals2::signal<void (const Notification::Ptr&, const Checkable::Ptr&, const User::Ptr&,
-	const NotificationType&, const CheckResult::Ptr&, const String&, const String&, const String&,
-	const MessageOrigin::Ptr&)> Checkable::OnNotificationSentToUser;
+	const NotificationType&, const CheckResult::Ptr&, const NotificationResult::Ptr&, const String&,
+	const String&, const String&, const MessageOrigin::Ptr&)> Checkable::OnNotificationSentToUser;
 
 void Checkable::ResetNotificationNumbers()
 {

lib/icinga/checkable.hpp

@@ -116,8 +116,8 @@ class Checkable : public ObjectImpl<Checkable>
 	static boost::signals2::signal<void (const Checkable::Ptr&, NotificationType, const CheckResult::Ptr&,
 		const String&, const String&, const MessageOrigin::Ptr&)> OnNotificationsRequested;
 	static boost::signals2::signal<void (const Notification::Ptr&, const Checkable::Ptr&, const User::Ptr&,
-		const NotificationType&, const CheckResult::Ptr&, const String&, const String&, const String&,
-		const MessageOrigin::Ptr&)> OnNotificationSentToUser;
+		const NotificationType&, const CheckResult::Ptr&, const NotificationResult::Ptr&, const String&,
+		const String&, const String&, const MessageOrigin::Ptr&)> OnNotificationSentToUser;
 	static boost::signals2::signal<void (const Notification::Ptr&, const Checkable::Ptr&, const std::set<User::Ptr>&,
 		const NotificationType&, const CheckResult::Ptr&, const String&,
 		const String&, const MessageOrigin::Ptr&)> OnNotificationSentToAllUsers;

lib/icinga/clusterevents.cpp

@@ -624,7 +624,7 @@ Value ClusterEvents::SendNotificationsAPIHandler(const MessageOrigin::Ptr& origi
 }
 
 void ClusterEvents::NotificationSentUserHandler(const Notification::Ptr& notification, const Checkable::Ptr& checkable, const User::Ptr& user,
-	NotificationType notificationType, const CheckResult::Ptr& cr, const String& author, const String& commentText, const String& command,
+	NotificationType notificationType, const CheckResult::Ptr& cr, const NotificationResult::Ptr& nr, const String& author, const String& commentText, const String& command,
 	const MessageOrigin::Ptr& origin)
 {
 	ApiListener::Ptr listener = ApiListener::GetInstance();
@@ -644,6 +644,7 @@ void ClusterEvents::NotificationSentUserHandler(const Notification::Ptr& notific
 	params->Set("user", user->GetName());
 	params->Set("type", notificationType);
 	params->Set("cr", Serialize(cr));
+	params->Set("nr", Serialize(nr));
 	params->Set("author", author);
 	params->Set("text", commentText);
 	params->Set("command", command);
@@ -705,6 +706,14 @@ Value ClusterEvents::NotificationSentUserAPIHandler(const MessageOrigin::Ptr& or
 		}
 	}
 
+	NotificationResult::Ptr nr;
+	if (params->Contains("nr")) {
+		nr = new NotificationResult();
+		Dictionary::Ptr vnr = params->Get("nr");
+
+		Deserialize(nr, vnr, true);
+	}
+
 	NotificationType type = static_cast<NotificationType>(static_cast<int>(params->Get("type")));
 	String author = params->Get("author");
 	String text = params->Get("text");
@@ -721,7 +730,7 @@ Value ClusterEvents::NotificationSentUserAPIHandler(const MessageOrigin::Ptr& or
 
 	String command = params->Get("command");
 
-	Checkable::OnNotificationSentToUser(notification, checkable, user, type, cr, author, text, command, origin);
+	Checkable::OnNotificationSentToUser(notification, checkable, user, type, cr, nr, author, text, command, origin);
 
 	return Empty;
 }

lib/icinga/clusterevents.hpp

@@ -51,7 +51,8 @@ class ClusterEvents
 	static Value SendNotificationsAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params);
 
 	static void NotificationSentUserHandler(const Notification::Ptr& notification, const Checkable::Ptr& checkable, const User::Ptr& user,
-		NotificationType notificationType, const CheckResult::Ptr& cr, const String& author, const String& commentText, const String& command, const MessageOrigin::Ptr& origin);
+		NotificationType notificationType, const CheckResult::Ptr& cr, const NotificationResult::Ptr& nr,
+		const String& author, const String& commentText, const String& command, const MessageOrigin::Ptr& origin);
 	static Value NotificationSentUserAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params);
 
 	static void NotificationSentToAllUsersHandler(const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set<User::Ptr>& users,

lib/icinga/notification.cpp

@@ -22,6 +22,7 @@ std::map<String, int> Notification::m_StateFilterMap;
 std::map<String, int> Notification::m_TypeFilterMap;
 
 boost::signals2::signal<void (const Notification::Ptr&, const MessageOrigin::Ptr&)> Notification::OnNextNotificationChanged;
+boost::signals2::signal<void (const Notification::Ptr&, const NotificationResult::Ptr&, const MessageOrigin::Ptr&)> Notification::OnNewNotificationResult;
 
 String NotificationNameComposer::MakeName(const String& shortName, const Object::Ptr& context) const
 {
@@ -531,10 +532,14 @@ void Notification::ExecuteNotificationHelper(NotificationType type, const User::
 	String commandName = command->GetName();
 
 	try {
-		command->Execute(this, user, cr, type, author, text);
+		NotificationResult::Ptr nr = new NotificationResult();
+
+		nr->SetExecutionStart(Utility::GetTime());
+
+		command->Execute(this, user, cr, nr, type, author, text);
 
 		/* required by compatlogger */
-		Service::OnNotificationSentToUser(this, GetCheckable(), user, type, cr, author, text, commandName, nullptr);
+		Checkable::OnNotificationSentToUser(this, GetCheckable(), user, type, cr, nr, author, text, command->GetName(), nullptr);
 
 		Log(LogInformation, "Notification")
 			<< "Completed sending '" << NotificationTypeToStringInternal(type)
@@ -550,6 +555,36 @@ void Notification::ExecuteNotificationHelper(NotificationType type, const User::
 	}
 }
 
+void Notification::ProcessNotificationResult(const NotificationResult::Ptr& nr, const MessageOrigin::Ptr& origin)
+{
+	if (!nr)
+		return;
+
+	double now = Utility::GetTime();
+
+	if (nr->GetExecutionStart() == 0)
+		nr->SetExecutionStart(now);
+
+	if (nr->GetExecutionEnd() == 0)
+		nr->SetExecutionEnd(now);
+
+	/* Determine the execution endpoint from a locally executed check. */
+	if (!origin || origin->IsLocal())
+		nr->SetExecutionEndpoint(IcingaApplication::GetInstance()->GetNodeName());
+
+	if (!IsActive())
+		return;
+
+	{
+		ObjectLock olock(this);
+
+		SetLastNotificationResult(nr);
+	}
+
+	/* Notify cluster, API and feature events. */
+	OnNewNotificationResult(this, nr, origin);
+}
+
 int icinga::ServiceStateToFilter(ServiceState state)
 {
 	switch (state) {

lib/icinga/notification.hpp

@@ -10,6 +10,7 @@
 #include "icinga/usergroup.hpp"
 #include "icinga/timeperiod.hpp"
 #include "icinga/checkresult.hpp"
+#include "icinga/notificationresult.hpp"
 #include "remote/endpoint.hpp"
 #include "remote/messageorigin.hpp"
 #include "base/array.hpp"
@@ -82,10 +83,13 @@ class Notification final : public ObjectImpl<Notification>
 
 	Endpoint::Ptr GetCommandEndpoint() const;
 
+	void ProcessNotificationResult(const NotificationResult::Ptr& nr, const MessageOrigin::Ptr& origin = nullptr);
+
 	static String NotificationTypeToString(NotificationType type);
 	static String NotificationFilterToString(int filter, const std::map<String, int>& filterMap);
 
 	static boost::signals2::signal<void (const Notification::Ptr&, const MessageOrigin::Ptr&)> OnNextNotificationChanged;
+	static boost::signals2::signal<void (const Notification::Ptr&, const NotificationResult::Ptr&, const MessageOrigin::Ptr&)> OnNewNotificationResult;
 
 	void Validate(int types, const ValidationUtils& utils) override;
 

lib/icinga/notification.ti

@@ -1,6 +1,8 @@
 /* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
 
 #include "icinga/customvarobject.hpp"
+#include "icinga/notificationresult.hpp"
+#include "base/array.hpp"
 #impl_include "icinga/notificationcommand.hpp"
 #impl_include "icinga/service.hpp"
 
@@ -16,6 +18,7 @@ public:
 	virtual String MakeName(const String& shortName, const Object::Ptr& context) const;
 	virtual Dictionary::Ptr ParseName(const String& name) const;
 };
+
 }}}
 
 class Notification : CustomVarObject < NotificationNameComposer
@@ -81,6 +84,7 @@ class Notification : CustomVarObject < NotificationNameComposer
 	[state] Timestamp next_notification;
 	[state] int notification_number;
 	[state] Timestamp last_problem_notification;
+	[state] NotificationResult::Ptr last_notification_result;
 
 	[config, navigation] name(Endpoint) command_endpoint (CommandEndpointRaw) {
 		navigate {{{

lib/icinga/notificationcommand.cpp

@@ -8,14 +8,15 @@ using namespace icinga;
 REGISTER_TYPE(NotificationCommand);
 
 Dictionary::Ptr NotificationCommand::Execute(const Notification::Ptr& notification,
-	const User::Ptr& user, const CheckResult::Ptr& cr, const NotificationType& type,
-	const String& author, const String& comment, const Dictionary::Ptr& resolvedMacros,
-	bool useResolvedMacros)
+	const User::Ptr& user, const CheckResult::Ptr& cr, const NotificationResult::Ptr& nr,
+	const NotificationType& type, const String& author, const String& comment,
+	const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
 {
 	return GetExecute()->Invoke({
 		notification,
 		user,
 		cr,
+		nr,
 		type,
 		author,
 		comment,

lib/icinga/notificationcommand.hpp

@@ -23,8 +23,8 @@ class NotificationCommand final : public ObjectImpl<NotificationCommand>
 	DECLARE_OBJECTNAME(NotificationCommand);
 
 	virtual Dictionary::Ptr Execute(const intrusive_ptr<Notification>& notification,
-		const User::Ptr& user, const CheckResult::Ptr& cr, const NotificationType& type,
-		const String& author, const String& comment,
+		const User::Ptr& user, const CheckResult::Ptr& cr, const NotificationResult::Ptr& nr,
+		const NotificationType& type, const String& author, const String& comment,
 		const Dictionary::Ptr& resolvedMacros = nullptr,
 		bool useResolvedMacros = false);
 };

lib/icinga/notificationresult.cpp

@@ -0,0 +1,13 @@
+/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
+
+#include "icinga/notificationresult.hpp"
+#include "icinga/notificationresult-ti.cpp"
+
+using namespace icinga;
+
+REGISTER_TYPE(NotificationResult);
+
+double NotificationResult::CalculateExecutionTime() const
+{
+	return GetExecutionEnd() - GetExecutionStart();
+}

lib/icinga/notificationresult.hpp

@@ -0,0 +1,27 @@
+/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
+
+#ifndef NOTIFICATIONRESULT_H
+#define NOTIFICATIONRESULT_H
+
+#include "icinga/i2-icinga.hpp"
+#include "icinga/notificationresult-ti.hpp"
+
+namespace icinga
+{
+
+/**
+ * A notification result.
+ *
+ * @ingroup icinga
+ */
+class NotificationResult final : public ObjectImpl<NotificationResult>
+{
+public:
+	DECLARE_OBJECT(NotificationResult);
+
+	double CalculateExecutionTime() const;
+};
+
+}
+
+#endif /* NOTIFICATIONRESULT_H */

lib/icinga/notificationresult.ti

@@ -0,0 +1,24 @@
+/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
+
+library icinga;
+
+namespace icinga
+{
+
+class NotificationResult
+{
+	[state] Timestamp execution_start;
+	[state] Timestamp execution_end;
+
+	[state] Value command;
+	[state] int exit_status;
+	[state] String output;
+
+	[state] bool active {
+		default {{{ return true; }}}
+	};
+
+	[state] String execution_endpoint;
+};
+
+}