Instrumentation PoC (WIP)

big_tests/tests/mam_SUITE.erl

@@ -859,7 +859,8 @@ init_per_testcase(C=muc_archive_request, Config) ->
     Config2 = %% Check that metric is incremented on MUC flushed
         case ?config(configuration, Config1) of
             rdbms_async_pool ->
-                MongooseMetrics = [{['_', 'modMucMamFlushed'], changed}],
+                HostTypePrefix = domain_helper:make_metrics_prefix(host_type()),
+                MongooseMetrics = [{[HostTypePrefix, mod_mam_muc_flushed, count], changed}],
                 [{mongoose_metrics, MongooseMetrics} | Config1];
             _ ->
                 Config1
@@ -3424,7 +3425,7 @@ metric_incremented_on_archive_request(ConfigIn) ->
         end,
     HostType = domain_helper:host_type(mim),
     HostTypePrefix = domain_helper:make_metrics_prefix(HostType),
-    MongooseMetrics = [{[HostTypePrefix, backends, mod_mam_pm, lookup], changed}],
+    MongooseMetrics = [{[HostTypePrefix, mod_mam_pm_lookup, count], changed}],
     Config = [{mongoose_metrics, MongooseMetrics} | ConfigIn],
     escalus_fresh:story(Config, [{alice, 1}], F).
 

big_tests/tests/muc_helper.erl

@@ -317,7 +317,12 @@ story_with_room(Config, RoomOpts, [{Owner, _}|_] = UserSpecs, StoryFun) ->
         StoryFun2 = fun(Args) -> apply(StoryFun, [Config2 | Args]) end,
         escalus_story:story_with_client_list(Config2, UserSpecs, StoryFun2)
     after
-        mam_helper:destroy_room(Config2)
+        case dynamic_modules:get_current_modules(domain_helper:host_type()) of
+            #{mod_mam_muc := _} ->
+                mam_helper:destroy_room(Config2);
+            #{} ->
+                ok
+        end
     end.
 
 %%--------------------------------------------------------------------

rebar.config

@@ -68,6 +68,8 @@
   {exometer_report_statsd, {git, "https://github.com/esl/exometer_report_statsd.git", {branch, "master"}}},
   {syslogger, "0.3.0"},
   {flatlog, "0.1.2"},
+  {prometheus, "4.11.0"},
+  {prometheus_cowboy, "0.1.8"},
 
   %%% Stateless libraries
   {opuntia, "1.1.0"},
@@ -221,7 +223,7 @@
     {plt_extra_apps, [jid, cowboy, cowlib, lasse, p1_utils, ranch, gen_fsm_compat, epgsql, cqerl,
                       common_test, tools, amqp_client, jiffy, erl_csv, inets, compiler, jsx,
                       mysql, eredis, erlcloud, eodbc, telemetry,
-                      nksip, nklib, nkservice, nkpacket]}]}.
+                      nksip, nklib, nkservice, nkpacket, prometheus]}]}.
 
 {cover_print_enabled, true}.
 {cover_export_enabled, true}.

rebar.lock

@@ -1,5 +1,6 @@
 {"1.2.0",
-[{<<"amqp_client">>,{pkg,<<"amqp_client">>,<<"3.12.6">>},0},
+[{<<"accept">>,{pkg,<<"accept">>,<<"0.3.5">>},2},
+ {<<"amqp_client">>,{pkg,<<"amqp_client">>,<<"3.12.6">>},0},
  {<<"backoff">>,{pkg,<<"backoff">>,<<"1.1.3">>},1},
  {<<"base16">>,{pkg,<<"base16">>,<<"2.0.1">>},0},
  {<<"bbmustache">>,{pkg,<<"bbmustache">>,<<"1.12.2">>},0},
@@ -102,7 +103,11 @@
   0},
  {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.4.0">>},1},
  {<<"pooler">>,{pkg,<<"pooler">>,<<"1.5.3">>},1},
+ {<<"prometheus">>,{pkg,<<"prometheus">>,<<"4.11.0">>},0},
+ {<<"prometheus_cowboy">>,{pkg,<<"prometheus_cowboy">>,<<"0.1.8">>},0},
+ {<<"prometheus_httpd">>,{pkg,<<"prometheus_httpd">>,<<"2.1.11">>},1},
  {<<"proper">>,{pkg,<<"proper">>,<<"1.4.0">>},0},
+ {<<"quantile_estimator">>,{pkg,<<"quantile_estimator">>,<<"0.2.1">>},1},
  {<<"quickrand">>,{pkg,<<"quickrand">>,<<"2.0.5">>},1},
  {<<"rabbit_common">>,{pkg,<<"rabbit_common">>,<<"3.12.6">>},1},
  {<<"ranch">>,{pkg,<<"ranch">>,<<"2.1.0">>},0},
@@ -130,6 +135,7 @@
  {<<"worker_pool">>,{pkg,<<"worker_pool">>,<<"6.0.1">>},0}]}.
 [
 {pkg_hash,[
+ {<<"accept">>, <<"B33B127ABCA7CC948BBE6CAA4C263369ABF1347CFA9D8E699C6D214660F10CD1">>},
  {<<"amqp_client">>, <<"B0050183BB4CFBD5B3F9A3276689DFA135A196AEDC3584110F231B2CE04B0426">>},
  {<<"backoff">>, <<"DE762C05ED6DFAE862D83DC9E58AE936792B01302B3595F5CFFE86F2D8E6C1DD">>},
  {<<"base16">>, <<"F0549F732E03BE8124ED0D19FD5EE52146CC8BE24C48CBC3F23AB44B157F11A2">>},
@@ -175,7 +181,11 @@
  {<<"p1_utils">>, <<"2D39B5015A567BBD2CC7033EEB93A7C60D8C84EFE1EF69A3473FAA07FA268187">>},
  {<<"parse_trans">>, <<"BB87AC362A03CA674EBB7D9D498F45C03256ADED7214C9101F7035EF44B798C7">>},
  {<<"pooler">>, <<"898CD1FA301FC42D4A8ED598CE139B71CA85B54C16AB161152B5CC5FBDCFA1A8">>},
+ {<<"prometheus">>, <<"B95F8DE8530F541BD95951E18E355A840003672E5EDA4788C5FA6183406BA29A">>},
+ {<<"prometheus_cowboy">>, <<"CFCE0BC7B668C5096639084FCD873826E6220EA714BF60A716F5BD080EF2A99C">>},
+ {<<"prometheus_httpd">>, <<"F616ED9B85B536B195D94104063025A91F904A4CFC20255363F49A197D96C896">>},
  {<<"proper">>, <<"89A44B8C39D28BB9B4BE8E4D715D534905B325470F2E0EC5E004D12484A79434">>},
+ {<<"quantile_estimator">>, <<"EF50A361F11B5F26B5F16D0696E46A9E4661756492C981F7B2229EF42FF1CD15">>},
  {<<"quickrand">>, <<"06FCAD85CB47D5C85C51D6BC9C84A082501BA098A89D64AD0A2F69599E034C04">>},
  {<<"rabbit_common">>, <<"FEBD37E11483F94B614CD636C1EDBCE1099FF64866598F75D0A624D3A60437C6">>},
  {<<"ranch">>, <<"2261F9ED9574DCFCC444106B9F6DA155E6E540B2F82BA3D42B339B93673B72A3">>},
@@ -194,6 +204,7 @@
  {<<"uuid">>, <<"60FAEEB7EDFD40847ED13CB0DD1044BAABE4E79A00C0CA9C4D13A073914B1016">>},
  {<<"worker_pool">>, <<"CA262C2DFB3B4AF661B206C82065D86F83922B7227508AA6E0BC34D3E5AE5135">>}]},
 {pkg_hash_ext,[
+ {<<"accept">>, <<"11B18C220BCC2EAB63B5470C038EF10EB6783BCB1FCDB11AA4137DEFA5AC1BB8">>},
  {<<"amqp_client">>, <<"B856F6404E7AF98C90DA870C8CE50D1380F13F2CEE02F16564B5CC5142BAE308">>},
  {<<"backoff">>, <<"30CEAD738D20E4C8D36CD37857DD5E23AEBA57CB868BF64766D47D371422BDFF">>},
  {<<"base16">>, <<"06EA2D48343282E712160BA89F692B471DB8B36ABE8394F3445FF9032251D772">>},
@@ -239,7 +250,11 @@
  {<<"p1_utils">>, <<"9219214428F2C6E5D3187FF8EB9A8783695C2427420BE9A259840E07ADA32847">>},
  {<<"parse_trans">>, <<"F99E368830BEA44552224E37E04943A54874F08B8590485DE8D13832B63A2DC3">>},
  {<<"pooler">>, <<"058D85C5081289B90E97E4DDDBC3BB5A3B4A19A728AB3BC88C689EFCC36A07C7">>},
+ {<<"prometheus">>, <<"719862351AABF4DF7079B05DC085D2BBCBE3AC0AC3009E956671B1D5AB88247D">>},
+ {<<"prometheus_cowboy">>, <<"BA286BECA9302618418892D37BCD5DC669A6CC001F4EB6D6AF85FF81F3F4F34C">>},
+ {<<"prometheus_httpd">>, <<"0BBE831452CFDF9588538EB2F570B26F30C348ADAE5E95A7D87F35A5910BCF92">>},
  {<<"proper">>, <<"18285842185BD33EFBDA97D134A5CB5A0884384DB36119FEE0E3CFA488568CBB">>},
+ {<<"quantile_estimator">>, <<"282A8A323CA2A845C9E6F787D166348F776C1D4A41EDE63046D72D422E3DA946">>},
  {<<"quickrand">>, <<"252CF0493570EBF1A58985CB71990982CDDCD4396B6427F1E10CF58924C1C052">>},
  {<<"rabbit_common">>, <<"D85282C8C9BE456B42AA4B265EDE68D176CA8A28DFCF1D521BE19267167C0DC3">>},
  {<<"ranch">>, <<"244EE3FA2A6175270D8E1FC59024FD9DBC76294A321057DE8F803B1479E76916">>},

rel/fed1.vars-toml.config

@@ -12,6 +12,7 @@
 {http_graphql_api_user_endpoint_port, 5566}.
 {http_api_endpoint_port, 5294}.
 {http_api_client_endpoint_port, 8095}.
+{http_prometheus_endpoint_port, 9095}.
 
 %% This node is for s2s testing.
 %% "localhost" host should NOT be defined.

rel/files/mongooseim.toml

@@ -117,6 +117,16 @@
     host = "_"
     path = "/api/graphql"
     schema_endpoint = "user"
+{{#http_prometheus_endpoint}}
+
+[[listen.http]]
+  {{{http_prometheus_endpoint}}}
+  transport.num_acceptors = 10
+
+  [[listen.http.handlers.mongoose_prometheus_handler]]
+    host = "_"
+    path = "/metrics"
+{{/http_prometheus_endpoint}}
 
 [[listen.c2s]]
   port = {{{c2s_port}}}
@@ -259,6 +269,8 @@
 
 [modules.mod_carboncopy]
 
+{{{instrumentation}}}
+
 [shaper.normal]
   max_rate = 1000
 

rel/mim1.vars-toml.config

@@ -16,6 +16,7 @@
 {http_graphql_api_user_endpoint_port, 5561}.
 {http_api_endpoint_port, 8088}.
 {http_api_client_endpoint_port, 8089}.
+{http_prometheus_endpoint_port, 9091}.
 
 {hosts, "\"localhost\", \"anonymous.localhost\", \"localhost.bis\""}.
 {host_types, "\"test type\", \"dummy auth\", \"anonymous\""}.

rel/mim2.vars-toml.config

@@ -14,6 +14,7 @@
 {http_graphql_api_admin_endpoint_port, 5552}.
 {http_graphql_api_domain_admin_endpoint_port, 5542}.
 {http_graphql_api_user_endpoint_port, 5562}.
+{http_prometheus_endpoint_port, 9092}.
 
 {hosts, "\"localhost\", \"anonymous.localhost\", \"localhost.bis\""}.
 {host_types, "\"test type\", \"dummy auth\""}.

rel/mim3.vars-toml.config

@@ -14,6 +14,7 @@
 {http_graphql_api_user_endpoint_port, 5563}.
 {http_api_endpoint_port, 8092}.
 {http_api_client_endpoint_port, 8193}.
+{http_prometheus_endpoint_port, 9093}.
 
 "./vars-toml.config".
 

rel/prod.vars-toml.config

@@ -13,6 +13,7 @@
 {http_graphql_api_user_endpoint_port, 5561}.
 {http_api_endpoint_port, 8088}.
 {http_api_client_endpoint_port, 8089}.
+{http_prometheus_endpoint_port, 9091}.
 
 {hosts, "\"localhost\""}.
 {default_server_domain, "\"localhost\""}.

rel/reg1.vars-toml.config

@@ -13,6 +13,7 @@
 {http_graphql_api_user_endpoint_port, 5564}.
 {http_api_endpoint_port, 8074}.
 {http_api_client_endpoint_port, 8075}.
+{http_prometheus_endpoint_port, 9094}.
 
 %% This node is for global distribution testing.
 %% reg is short for region.

rel/vars-toml.config

@@ -15,10 +15,16 @@
 {http_api_endpoint, "ip_address = \"127.0.0.1\"
   port = {{http_api_endpoint_port}}"}.
 {http_api_client_endpoint, "port = {{ http_api_client_endpoint_port }}"}.
+{http_prometheus_endpoint, "port = {{ http_prometheus_endpoint_port }}"}.
 {s2s_use_starttls, "\"optional\""}.
 {s2s_certfile, "\"priv/ssl/fake_server.pem\""}.
 {internal_databases, "[internal_databases.mnesia]"}.
 
+%% This is temporary, and most likely we will only enable Prometheus by default
+{instrumentation, "[instrumentation.exometer]
+
+[instrumentation.prometheus]"}.
+
 "./configure.vars.config".
 
 %% Defined by appending configure.vars.config

src/config/mongoose_config_spec.erl

@@ -95,6 +95,7 @@ root() ->
                  <<"internal_databases">> => internal_databases(),
                  <<"services">> => services(),
                  <<"modules">> => Modules#section{include = always},
+                 <<"instrumentation">> => instrumentation(),
                  <<"shaper">> => shaper(),
                  <<"acl">> => acl(),
                  <<"access">> => access(),
@@ -784,6 +785,14 @@ iqdisc() ->
 process_iqdisc(#{type := Type, workers := N}) -> {queues = Type, N};
 process_iqdisc(#{type := Type}) -> Type.
 
+%% path: instrumentation
+instrumentation() ->
+    #section{items = #{default => #section{}},
+             validate_keys = {module, mongoose_instrument},
+             wrap = global_config,
+             include = always
+            }.
+
 %% path: shaper
 shaper() ->
     #section{

src/gen_mod.erl

@@ -72,6 +72,7 @@
 -callback hooks(HostType :: host_type()) -> gen_hook:hook_list().
 -callback supported_features() -> [module_feature()].
 -callback config_spec() -> mongoose_config_spec:config_section().
+-callback instrumentation(host_type()) -> [mongoose_instrument:spec()].
 
 %% Optional callback specifying module dependencies.
 %% The dependent module can specify parameters with which the dependee should be
@@ -86,7 +87,7 @@
 %% function).
 -callback deps(host_type(), module_opts()) -> gen_mod_deps:deps().
 
--optional_callbacks([hooks/1, config_spec/0, supported_features/0, deps/2]).
+-optional_callbacks([hooks/1, config_spec/0, supported_features/0, instrumentation/1, deps/2]).
 
 %% @doc This function should be called by mongoose_modules only.
 %% To start a new module at runtime, use mongoose_modules:ensure_module/3 instead.
@@ -101,6 +102,7 @@ start_module_for_host_type(HostType, Module, Opts) ->
         lists:map(fun mongoose_service:assert_loaded/1,
                   get_required_services(HostType, Module, Opts)),
         check_dynamic_domains_support(HostType, Module),
+        run_for_instrumentation(HostType, fun mongoose_instrument:set_up/1, Module),
         Res = Module:start(HostType, Opts),
         run_for_hooks(HostType, fun gen_hook:add_handlers/1, Module),
         {links, LinksAfter} = erlang:process_info(self(), links),
@@ -152,6 +154,12 @@ run_for_hooks(HostType, Fun, Module) ->
         false -> ok
     end.
 
+run_for_instrumentation(HostType, Fun, Module) ->
+    case erlang:function_exported(Module, instrumentation, 1) of
+        true -> Fun(Module:instrumentation(HostType));
+        false -> ok
+    end.
+
 check_dynamic_domains_support(HostType, Module) ->
     case lists:member(HostType, ?MYHOSTS) of
         true -> ok;
@@ -197,7 +205,7 @@ stop_module_for_host_type(HostType, Module) ->
         {wait, Process} ->
             wait_for_process(Process);
         _ ->
-            ok
+            run_for_instrumentation(HostType, fun mongoose_instrument:tear_down/1, Module)
     catch Class:Reason:Stacktrace ->
             ?LOG_ERROR(#{what => module_stopping_failed,
                          host_type => HostType, stop_module => Module,

src/mam/mod_mam_elasticsearch_arch.erl

@@ -97,7 +97,7 @@
             ?LOG_ERROR(#{what => archive_message_failed,
                          user => Owner, server => Host, remote => Remote,
                          message_id => MessageId, reason => Reason}),
-            mongoose_metrics:update(Host, modMamDropped, 1),
+            mongoose_instrument:execute(mod_mam_pm_dropped, #{host_type => Host}, #{count => 1}),
             {ok, Err}
     end.