Migrate SM session metrics to mongoose_instrument

big_tests/tests/graphql_metric_SUITE.erl

@@ -374,8 +374,8 @@ check_node_result_is_valid(ResList, MetricsAreGlobal) ->
             false -> [metric_host_type(), <<"c2s_element_in">>, <<"stanza_count">>]
         end,
     check_spiral_dict(maps:get(SentName, Map)),
-    [#{<<"key">> := <<"value">>,<<"value">> := V}] =
-        maps:get([<<"global">>,<<"uniqueSessionCount">>], Map),
+    [#{<<"key">> := <<"value">>,<<"value">> := V} | _] =
+        maps:get([<<"global">>,<<"sm_unique_sessions">>,<<"count">>], Map),
     ?assert(is_integer(V)),
     HistObjects = maps:get([<<"global">>, <<"xmpp_stanza_size_received">>, <<"byte_size">>], Map),
     check_histogram(kv_objects_to_map(HistObjects)).

big_tests/tests/metrics_api_SUITE.erl

@@ -18,6 +18,7 @@
 
 -import(distributed_helper, [mim/0, mim2/0, rpc/4]).
 -import(rest_helper, [assert_status/2, make_request/1]).
+-import(mongooseimctl_helper, [rpc_call/3]).
 
 -include_lib("eunit/include/eunit.hrl").
 
@@ -58,11 +59,16 @@ init_per_suite(Config) ->
     HostType = host_type(),
     Config1 = dynamic_modules:save_modules(HostType, Config),
     dynamic_modules:ensure_stopped(HostType, [mod_offline]),
-    escalus:init_per_suite(Config1).
+    Config2 = mongoose_helper:backup_and_set_config_option(Config1,
+                                                           [instrumentation, probe_interval], 1),
+    restart_sm_probes(),
+    escalus:init_per_suite(Config2).
 
 end_per_suite(Config) ->
     dynamic_modules:restore_modules(Config),
-    escalus:end_per_suite(Config).
+    escalus:end_per_suite(Config),
+    mongoose_helper:restore_config_option(Config, [instrumentation, probe_interval]),
+    restart_sm_probes().
 
 init_per_group(GroupName, Config) ->
     metrics_helper:prepare_by_all_metrics_are_global(Config, GroupName =:= all_metrics_are_global).
@@ -225,17 +231,13 @@ one_presence_error(Config) ->
         {'c2s_element_out.presence_error_count', 1}]).
 
 session_counters(Config) ->
-    Names = [totalSessionCount, uniqueSessionCount, nodeSessionCount],
     escalus:story
       (Config, [{alice, 2}, {bob, 1}],
        fun(_User11, _User12, _User2) ->
-            %% Force update
-            lists:foreach(fun metrics_helper:sample/1, Names),
             timer:sleep(timer:seconds(1)),
-
-            ?assertEqual(3, fetch_global_gauge_value(totalSessionCount, Config)),
-            ?assertEqual(2, fetch_global_gauge_value(uniqueSessionCount, Config)),
-            ?assertEqual(3, fetch_global_gauge_value(nodeSessionCount, Config))
+            ?assertEqual(3, fetch_global_gauge_value('sm_total_sessions.count', Config)),
+            ?assertEqual(2, fetch_global_gauge_value('sm_unique_sessions.count', Config)),
+            ?assertEqual(3, fetch_global_gauge_value('sm_node_sessions.count', Config))
        end).
 
 node_uptime(Config) ->
@@ -491,3 +493,7 @@ request(Method, Path) ->
 request(Method, Path, Server) ->
     make_request(#{role => admin, method => Method, path => iolist_to_binary(Path),
                    return_headers => true, return_maps => true, server => Server}).
+
+restart_sm_probes() ->
+    rpc_call(ejabberd_sm, stop_probes, []),
+    rpc_call(ejabberd_sm, start_probes, []).

big_tests/tests/metrics_session_SUITE.erl

@@ -25,6 +25,7 @@
                          wait_for_counter/2,
                          wait_for_counter/3]).
 -import(domain_helper, [host_type/0]).
+-import(mongooseimctl_helper, [rpc_call/3]).
 
 %%--------------------------------------------------------------------
 %% Suite configuration
@@ -39,7 +40,8 @@ groups() ->
                             login_many,
                             auth_failed]},
      {session_global, [sequence], [session_global,
-                                   session_unique]}].
+                                   session_unique,
+                                   session_node]}].
 
 suite() ->
     [{require, ejabberd_node} | escalus:suite()].
@@ -50,13 +52,21 @@ suite() ->
 
 init_per_suite(Config) ->
     instrument_helper:start([{sm_session, #{host_type => host_type()}},
-                             {c2s_auth_failed, #{host_type => host_type()}}]),
-    escalus:init_per_suite(Config).
+                             {c2s_auth_failed, #{host_type => host_type()}},
+                             {sm_total_sessions, #{}},
+                             {sm_unique_sessions, #{}},
+                             {sm_node_sessions, #{}}]),
+    Config1 = mongoose_helper:backup_and_set_config_option(Config,
+                                                           [instrumentation, probe_interval], 1),
+    restart_sm_probes(),
+    escalus:init_per_suite(Config1).
 
 end_per_suite(Config) ->
     escalus_fresh:clean(),
     escalus:end_per_suite(Config),
-    instrument_helper:stop().
+    mongoose_helper:restore_config_option(Config, [instrumentation, probe_interval]),
+    instrument_helper:stop(),
+    restart_sm_probes().
 
 init_per_group(_GroupName, Config) ->
     escalus:create_users(Config, escalus:get_users([alice, bob])).
@@ -104,16 +114,20 @@ auth_failed(Config) ->
 
 session_global(Config) ->
     escalus:story(Config, [{alice, 1}], fun(_Alice) ->
-        metrics_helper:sample(totalSessionCount),
-        wait_for_counter(global, 1, totalSessionCount)
+        assert_sm_total_sessions(1)
         end).
 
 session_unique(Config) ->
     escalus:story(Config, [{alice, 2}], fun(_Alice1, _Alice2) ->
-        metrics_helper:sample(uniqueSessionCount),
-        metrics_helper:sample(totalSessionCount),
-        wait_for_counter(global, 1, uniqueSessionCount),
-        wait_for_counter(global, 2, totalSessionCount)
+        assert_sm_unique_sessions(1),
+        assert_sm_total_sessions(2)
+        end).
+
+session_node(Config) ->
+    escalus:story(Config, [{alice, 2}, {bob, 1}], fun(_Alice1, _Alice2, _Bob) ->
+        assert_sm_node_sessions(3),
+        assert_sm_unique_sessions(2),
+        assert_sm_total_sessions(3)
         end).
 
 %% Instrumentation events
@@ -139,3 +153,21 @@ assert_c2s_auth_failed(UserSpec) ->
     UserName = proplists:get_value(username, UserSpec),
     F = fun(M) -> M =:= #{count => 1, server => Server, username => UserName} end,
     instrument_helper:assert(c2s_auth_failed, #{host_type => host_type()}, F).
+
+assert_sm_total_sessions(ExpectedCount) ->
+    assert_sm_sessions(ExpectedCount, sm_total_sessions).
+
+assert_sm_unique_sessions(ExpectedCount) ->
+    assert_sm_sessions(ExpectedCount, sm_unique_sessions).
+
+assert_sm_node_sessions(ExpectedCount) ->
+    assert_sm_sessions(ExpectedCount, sm_node_sessions).
+
+assert_sm_sessions(ExpectedCount, Event) ->
+    Measurements = instrument_helper:wait_for_new(Event, #{}),
+    F = fun(#{count := Count}) -> ExpectedCount =:= Count end,
+    instrument_helper:assert(Event, #{}, Measurements, F).
+
+restart_sm_probes() ->
+    rpc_call(ejabberd_sm, stop_probes, []),
+    rpc_call(ejabberd_sm, start_probes, []).

doc/operation-and-maintenance/Logging-&-monitoring.md

@@ -116,7 +116,7 @@ We recommend the following metrics as a baseline for tracking your MongooseIM in
 For time-based metrics, you can choose to display multiple calculated values for a reporting period - we recommend tracking at least `max`, `median` and `mean`.
 
 ```
-Session count:                   <prefix>.global.totalSessionCount.value
+Session count:                   <prefix>.global.sm_total_sessions.count
 Outgoing XMPP messages:          <prefix>.<domain>.c2s_element_out.message_count.one
 Incoming XMPP messages:          <prefix>.<domain>.c2s_element_in.message_count.one
 Successful logins:               <prefix>.<domain>.sm_session.logins.one

doc/operation-and-maintenance/MongooseIM-metrics.md

@@ -172,10 +172,9 @@ Metrics specific to an extension, e.g. Message Archive Management, are described
 | Name | Type | Description (when it gets incremented) |
 | ---- | ---- | -------------------------------------- |
 | `[global, routingErrors]` | spiral | It is not possible to route a stanza (all routing handlers failed). |
-| `[global, nodeSessionCount]` | value | A number of sessions connected to a given MongooseIM node. |
-| `[global, totalSessionCount]` | value | A number of sessions connected to a MongooseIM cluster. |
-| `[global, uniqueSessionCount]` | value | A number of unique users connected to a MongooseIM cluster (e.g. 3 sessions of the same user will be counted as 1 in this metric). |
-| `[global, cache, unique_sessions_number]` | gauge | A cached value of `uniqueSessionCount`. It is automatically updated when a unique session count is calculated. |
+| `[global, sm_node_sessions, count]` | gauge | A number of sessions connected to a given MongooseIM node. |
+| `[global, sm_total_sessions, count]` | gauge | A number of sessions connected to a MongooseIM cluster. |
+| `[global, sm_unique_sessions, count]` | gauge | A number of unique users connected to a MongooseIM cluster (e.g. 3 sessions of the same user will be counted as 1 in this metric). |
 | `[global, nodeUpTime]` | value | Node uptime. |
 | `[global, clusterSize]` | value | A number of nodes in a MongooseIM cluster seen by a given MongooseIM node (based on Mnesia). For CETS use `global.cets.system.joined_nodes` instead. |
 | `[global, tcpPortsUsed]` | value | A number of open tcp connections. This should relate to the number of connected sessions and databases, as well as federations and http requests, in order to detect connection leaks. |

doc/rest-api/Administration-backend_swagger.yml

@@ -826,9 +826,9 @@ paths:
                   items:
                     type: string
                   example:
-                    - "nodeSessionCount"
-                    - "totalSessionCount"
-                    - "uniqueSessionCount"
+                    - "sm_node_sessions"
+                    - "sm_total_sessions"
+                    - "sm_unique_sessions"
   /metrics/all:
     get:
       description: Returns a list of metrics aggregated for all host types

src/ejabberd_sm.erl

@@ -27,6 +27,7 @@
 
 -behaviour(gen_server).
 -behaviour(gen_iq_component).
+-behaviour(mongoose_instrument_probe).
 
 
 %% API
@@ -63,7 +64,10 @@
          session_cleanup/1,
          sessions_cleanup/1,
          terminate_session/2,
-         sm_backend/0
+         sm_backend/0,
+         probe/2,
+         start_probes/0,
+         stop_probes/0
         ]).
 
 %% Hook handlers
@@ -85,7 +89,8 @@
 -export([do_route/4]).
 
 -ignore_xref([do_filter/3, do_route/4, get_unique_sessions_number/0,
-              get_user_present_pids/2, start_link/0, user_resources/2, sm_backend/0]).
+              get_user_present_pids/2, start_link/0, user_resources/2, sm_backend/0,
+              start_probes/0, stop_probes/0]).
 
 -include("mongoose.hrl").
 -include("jlib.hrl").
@@ -121,7 +126,6 @@
 
 %% default value for the maximum number of user connections
 -define(MAX_USER_SESSIONS, 100).
--define(UNIQUE_COUNT_CACHE, [cache, unique_sessions_number]).
 
 %%====================================================================
 %% API
@@ -312,15 +316,7 @@
 
 -spec get_unique_sessions_number() -> integer().
 get_unique_sessions_number() ->
-    try
-        C = ejabberd_sm_backend:unique_count(),
-        mongoose_metrics:update(global, ?UNIQUE_COUNT_CACHE, C),
-        C
-    catch
-        _:_ ->
-            get_cached_unique_count()
-    end.
-
+    ejabberd_sm_backend:unique_count().
 
 -spec get_total_sessions_number() -> integer().
 get_total_sessions_number() ->
@@ -483,15 +479,12 @@
                           mongoose_instrument:set_up(instrumentation(HostType))
                   end,
                   ?ALL_HOST_TYPES),
-    %% Create metrics after backend has started, otherwise probe could have null value
-    create_metrics(),
+    %% Set up global metrics here to avoid registering global hooks
+    mongoose_instrument:set_up(instrumentation(global)),
     {ok, #state{}}.
 
-create_metrics() ->
-    mongoose_metrics:ensure_metric(global, ?UNIQUE_COUNT_CACHE, gauge),
-    mongoose_metrics:create_probe_metric(global, totalSessionCount, mongoose_metrics_probe_total_sessions),
-    mongoose_metrics:create_probe_metric(global, uniqueSessionCount, mongoose_metrics_probe_unique_sessions),
-    mongoose_metrics:create_probe_metric(global, nodeSessionCount, mongoose_metrics_probe_node_sessions).
+start_probes() ->
+    mongoose_instrument:set_up(global_probes()).
 
 -spec hooks(binary()) -> [gen_hook:hook_tuple()].
 hooks(HostType) ->
@@ -573,9 +566,13 @@
 %%--------------------------------------------------------------------
 -spec terminate(_, state()) -> 'ok'.
 terminate(_Reason, _State) ->
-    [mongoose_instrument:tear_down(instrumentation(HostType)) || HostType <- ?ALL_HOST_TYPES],
+    [mongoose_instrument:tear_down(instrumentation(HostType))
+     || HostType <- [global | ?ALL_HOST_TYPES]],
     ok.
 
+stop_probes() ->
+    mongoose_instrument:tear_down(global_probes()).
+
 %%--------------------------------------------------------------------
 %% Func: code_change(OldVsn, State, Extra) -> {ok, NewState}
 %% Description: Convert process state when code is changed
@@ -979,25 +976,35 @@
     Resources = get_user_resources(JID),
     lists:sort(Resources).
 
--spec get_cached_unique_count() -> non_neg_integer().
-get_cached_unique_count() ->
-    case mongoose_metrics:get_metric_value(global, ?UNIQUE_COUNT_CACHE) of
-        {ok, DataPoints} ->
-            proplists:get_value(value, DataPoints);
-        _ ->
-            0
-    end.
-
 %% It is used from big tests
 -spec sm_backend() -> backend().
 sm_backend() ->
     mongoose_backend:get_backend_module(global, ?MODULE).
 
--spec instrumentation(mongooseim:host_type()) -> [mongoose_instrument:spec()].
+-spec instrumentation(mongooseim:host_type_or_global()) -> [mongoose_instrument:spec()].
+instrumentation(global) ->
+    global_probes();
 instrumentation(HostType) ->
     [{sm_session, #{host_type => HostType},
       #{metrics => #{logins => spiral, logouts => spiral, count => counter}}},
      {sm_presence_subscription, #{host_type => HostType},
       #{metrics => #{subscription_count => spiral, unsubscription_count => spiral}}},
      {sm_message_bounced, #{host_type => HostType},
       #{metrics => #{count => spiral}}}].
+
+global_probes() ->
+    [{sm_total_sessions, #{},
+      #{probe => #{module => ?MODULE}, metrics => #{count => gauge}}},
+     {sm_unique_sessions, #{},
+      #{probe => #{module => ?MODULE}, metrics => #{count => gauge}}},
+     {sm_node_sessions, #{},
+      #{probe => #{module => ?MODULE}, metrics => #{count => gauge}}}].
+
+-spec probe(mongoose_instrument:event_name(), mongoose_instrument:labels()) ->
+    mongoose_instrument:measurements().
+probe(sm_total_sessions, #{}) ->
+    #{count => get_total_sessions_number()};
+probe(sm_unique_sessions, #{}) ->
+    #{count => get_unique_sessions_number()};
+probe(sm_node_sessions, #{}) ->
+    #{count => get_node_sessions_number()}.

src/metrics/mongoose_metrics.erl

@@ -21,7 +21,6 @@
 %% API
 -export([init/0,
          init_mongooseim_metrics/0,
-         create_probe_metric/3,
          update/3,
          ensure_metric/3,
          get_metric_value/1,
@@ -74,12 +73,6 @@ init_subscriptions() ->
                 subscribe_to_all(Name, Interval)
         end, Reporters).
 
--spec create_probe_metric(mongooseim:host_type_or_global(), atom(), module()) ->
-    ok | {ok, already_present} | {error, any()}.
-create_probe_metric(HostType, Name, Module) ->
-    {Metric, Spec} = ?PROBE(Name, Module),
-    ensure_metric(HostType, Metric, Spec).
-
 -spec update(HostType :: mongooseim:host_type_or_global(), Name :: term() | list(),
              Change :: term()) -> any().
 update(HostType, Name, Change) when is_list(Name) ->