diff --git a/big_tests/tests/mam_SUITE.erl b/big_tests/tests/mam_SUITE.erl index 00b375a973..3fec183b4f 100644 --- a/big_tests/tests/mam_SUITE.erl +++ b/big_tests/tests/mam_SUITE.erl @@ -859,7 +859,8 @@ init_per_testcase(C=muc_archive_request, Config) -> Config2 = %% Check that metric is incremented on MUC flushed case ?config(configuration, Config1) of rdbms_async_pool -> - MongooseMetrics = [{['_', 'modMucMamFlushed'], changed}], + HostTypePrefix = domain_helper:make_metrics_prefix(host_type()), + MongooseMetrics = [{[HostTypePrefix, mod_mam_muc_flushed, count], changed}], [{mongoose_metrics, MongooseMetrics} | Config1]; _ -> Config1 @@ -3424,7 +3425,7 @@ metric_incremented_on_archive_request(ConfigIn) -> end, HostType = domain_helper:host_type(mim), HostTypePrefix = domain_helper:make_metrics_prefix(HostType), - MongooseMetrics = [{[HostTypePrefix, backends, mod_mam_pm, lookup], changed}], + MongooseMetrics = [{[HostTypePrefix, mod_mam_pm_lookup, count], changed}], Config = [{mongoose_metrics, MongooseMetrics} | ConfigIn], escalus_fresh:story(Config, [{alice, 1}], F). diff --git a/big_tests/tests/muc_helper.erl b/big_tests/tests/muc_helper.erl index 783a1d4ad4..9ea1305d3f 100644 --- a/big_tests/tests/muc_helper.erl +++ b/big_tests/tests/muc_helper.erl @@ -317,7 +317,12 @@ story_with_room(Config, RoomOpts, [{Owner, _}|_] = UserSpecs, StoryFun) -> StoryFun2 = fun(Args) -> apply(StoryFun, [Config2 | Args]) end, escalus_story:story_with_client_list(Config2, UserSpecs, StoryFun2) after - mam_helper:destroy_room(Config2) + case dynamic_modules:get_current_modules(domain_helper:host_type()) of + #{mod_mam_muc := _} -> + mam_helper:destroy_room(Config2); + #{} -> + ok + end end. %%-------------------------------------------------------------------- diff --git a/rebar.config b/rebar.config index 1b45b33b2a..c8948cd51e 100644 --- a/rebar.config +++ b/rebar.config @@ -68,6 +68,8 @@ {exometer_report_statsd, {git, "https://github.com/esl/exometer_report_statsd.git", {branch, "master"}}}, {syslogger, "0.3.0"}, {flatlog, "0.1.2"}, + {prometheus, "4.11.0"}, + {prometheus_cowboy, "0.1.8"}, %%% Stateless libraries {opuntia, "1.1.0"}, @@ -221,7 +223,7 @@ {plt_extra_apps, [jid, cowboy, cowlib, lasse, p1_utils, ranch, gen_fsm_compat, epgsql, cqerl, common_test, tools, amqp_client, jiffy, erl_csv, inets, compiler, jsx, mysql, eredis, erlcloud, eodbc, telemetry, - nksip, nklib, nkservice, nkpacket]}]}. + nksip, nklib, nkservice, nkpacket, prometheus]}]}. {cover_print_enabled, true}. {cover_export_enabled, true}. diff --git a/rebar.lock b/rebar.lock index ed13fdb11b..9e7e748b01 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,5 +1,6 @@ {"1.2.0", -[{<<"amqp_client">>,{pkg,<<"amqp_client">>,<<"3.12.6">>},0}, +[{<<"accept">>,{pkg,<<"accept">>,<<"0.3.5">>},2}, + {<<"amqp_client">>,{pkg,<<"amqp_client">>,<<"3.12.6">>},0}, {<<"backoff">>,{pkg,<<"backoff">>,<<"1.1.3">>},1}, {<<"base16">>,{pkg,<<"base16">>,<<"2.0.1">>},0}, {<<"bbmustache">>,{pkg,<<"bbmustache">>,<<"1.12.2">>},0}, @@ -102,7 +103,11 @@ 0}, {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.4.0">>},1}, {<<"pooler">>,{pkg,<<"pooler">>,<<"1.5.3">>},1}, + {<<"prometheus">>,{pkg,<<"prometheus">>,<<"4.11.0">>},0}, + {<<"prometheus_cowboy">>,{pkg,<<"prometheus_cowboy">>,<<"0.1.8">>},0}, + {<<"prometheus_httpd">>,{pkg,<<"prometheus_httpd">>,<<"2.1.11">>},1}, {<<"proper">>,{pkg,<<"proper">>,<<"1.4.0">>},0}, + {<<"quantile_estimator">>,{pkg,<<"quantile_estimator">>,<<"0.2.1">>},1}, {<<"quickrand">>,{pkg,<<"quickrand">>,<<"2.0.5">>},1}, {<<"rabbit_common">>,{pkg,<<"rabbit_common">>,<<"3.12.6">>},1}, {<<"ranch">>,{pkg,<<"ranch">>,<<"2.1.0">>},0}, @@ -130,6 +135,7 @@ {<<"worker_pool">>,{pkg,<<"worker_pool">>,<<"6.0.1">>},0}]}. [ {pkg_hash,[ + {<<"accept">>, <<"B33B127ABCA7CC948BBE6CAA4C263369ABF1347CFA9D8E699C6D214660F10CD1">>}, {<<"amqp_client">>, <<"B0050183BB4CFBD5B3F9A3276689DFA135A196AEDC3584110F231B2CE04B0426">>}, {<<"backoff">>, <<"DE762C05ED6DFAE862D83DC9E58AE936792B01302B3595F5CFFE86F2D8E6C1DD">>}, {<<"base16">>, <<"F0549F732E03BE8124ED0D19FD5EE52146CC8BE24C48CBC3F23AB44B157F11A2">>}, @@ -175,7 +181,11 @@ {<<"p1_utils">>, <<"2D39B5015A567BBD2CC7033EEB93A7C60D8C84EFE1EF69A3473FAA07FA268187">>}, {<<"parse_trans">>, <<"BB87AC362A03CA674EBB7D9D498F45C03256ADED7214C9101F7035EF44B798C7">>}, {<<"pooler">>, <<"898CD1FA301FC42D4A8ED598CE139B71CA85B54C16AB161152B5CC5FBDCFA1A8">>}, + {<<"prometheus">>, <<"B95F8DE8530F541BD95951E18E355A840003672E5EDA4788C5FA6183406BA29A">>}, + {<<"prometheus_cowboy">>, <<"CFCE0BC7B668C5096639084FCD873826E6220EA714BF60A716F5BD080EF2A99C">>}, + {<<"prometheus_httpd">>, <<"F616ED9B85B536B195D94104063025A91F904A4CFC20255363F49A197D96C896">>}, {<<"proper">>, <<"89A44B8C39D28BB9B4BE8E4D715D534905B325470F2E0EC5E004D12484A79434">>}, + {<<"quantile_estimator">>, <<"EF50A361F11B5F26B5F16D0696E46A9E4661756492C981F7B2229EF42FF1CD15">>}, {<<"quickrand">>, <<"06FCAD85CB47D5C85C51D6BC9C84A082501BA098A89D64AD0A2F69599E034C04">>}, {<<"rabbit_common">>, <<"FEBD37E11483F94B614CD636C1EDBCE1099FF64866598F75D0A624D3A60437C6">>}, {<<"ranch">>, <<"2261F9ED9574DCFCC444106B9F6DA155E6E540B2F82BA3D42B339B93673B72A3">>}, @@ -194,6 +204,7 @@ {<<"uuid">>, <<"60FAEEB7EDFD40847ED13CB0DD1044BAABE4E79A00C0CA9C4D13A073914B1016">>}, {<<"worker_pool">>, <<"CA262C2DFB3B4AF661B206C82065D86F83922B7227508AA6E0BC34D3E5AE5135">>}]}, {pkg_hash_ext,[ + {<<"accept">>, <<"11B18C220BCC2EAB63B5470C038EF10EB6783BCB1FCDB11AA4137DEFA5AC1BB8">>}, {<<"amqp_client">>, <<"B856F6404E7AF98C90DA870C8CE50D1380F13F2CEE02F16564B5CC5142BAE308">>}, {<<"backoff">>, <<"30CEAD738D20E4C8D36CD37857DD5E23AEBA57CB868BF64766D47D371422BDFF">>}, {<<"base16">>, <<"06EA2D48343282E712160BA89F692B471DB8B36ABE8394F3445FF9032251D772">>}, @@ -239,7 +250,11 @@ {<<"p1_utils">>, <<"9219214428F2C6E5D3187FF8EB9A8783695C2427420BE9A259840E07ADA32847">>}, {<<"parse_trans">>, <<"F99E368830BEA44552224E37E04943A54874F08B8590485DE8D13832B63A2DC3">>}, {<<"pooler">>, <<"058D85C5081289B90E97E4DDDBC3BB5A3B4A19A728AB3BC88C689EFCC36A07C7">>}, + {<<"prometheus">>, <<"719862351AABF4DF7079B05DC085D2BBCBE3AC0AC3009E956671B1D5AB88247D">>}, + {<<"prometheus_cowboy">>, <<"BA286BECA9302618418892D37BCD5DC669A6CC001F4EB6D6AF85FF81F3F4F34C">>}, + {<<"prometheus_httpd">>, <<"0BBE831452CFDF9588538EB2F570B26F30C348ADAE5E95A7D87F35A5910BCF92">>}, {<<"proper">>, <<"18285842185BD33EFBDA97D134A5CB5A0884384DB36119FEE0E3CFA488568CBB">>}, + {<<"quantile_estimator">>, <<"282A8A323CA2A845C9E6F787D166348F776C1D4A41EDE63046D72D422E3DA946">>}, {<<"quickrand">>, <<"252CF0493570EBF1A58985CB71990982CDDCD4396B6427F1E10CF58924C1C052">>}, {<<"rabbit_common">>, <<"D85282C8C9BE456B42AA4B265EDE68D176CA8A28DFCF1D521BE19267167C0DC3">>}, {<<"ranch">>, <<"244EE3FA2A6175270D8E1FC59024FD9DBC76294A321057DE8F803B1479E76916">>}, diff --git a/rel/fed1.vars-toml.config b/rel/fed1.vars-toml.config index 34c4b2420f..6abdd550e4 100644 --- a/rel/fed1.vars-toml.config +++ b/rel/fed1.vars-toml.config @@ -12,6 +12,7 @@ {http_graphql_api_user_endpoint_port, 5566}. {http_api_endpoint_port, 5294}. {http_api_client_endpoint_port, 8095}. +{http_prometheus_endpoint_port, 9095}. %% This node is for s2s testing. %% "localhost" host should NOT be defined. diff --git a/rel/files/mongooseim.toml b/rel/files/mongooseim.toml index 8e9a95c4cc..8d6ff099db 100644 --- a/rel/files/mongooseim.toml +++ b/rel/files/mongooseim.toml @@ -117,6 +117,16 @@ host = "_" path = "/api/graphql" schema_endpoint = "user" +{{#http_prometheus_endpoint}} + +[[listen.http]] + {{{http_prometheus_endpoint}}} + transport.num_acceptors = 10 + + [[listen.http.handlers.mongoose_prometheus_handler]] + host = "_" + path = "/metrics" +{{/http_prometheus_endpoint}} [[listen.c2s]] port = {{{c2s_port}}} @@ -259,6 +269,8 @@ [modules.mod_carboncopy] +{{{instrumentation}}} + [shaper.normal] max_rate = 1000 diff --git a/rel/mim1.vars-toml.config b/rel/mim1.vars-toml.config index 6c909e77e5..8b0769a716 100644 --- a/rel/mim1.vars-toml.config +++ b/rel/mim1.vars-toml.config @@ -16,6 +16,7 @@ {http_graphql_api_user_endpoint_port, 5561}. {http_api_endpoint_port, 8088}. {http_api_client_endpoint_port, 8089}. +{http_prometheus_endpoint_port, 9091}. {hosts, "\"localhost\", \"anonymous.localhost\", \"localhost.bis\""}. {host_types, "\"test type\", \"dummy auth\", \"anonymous\""}. diff --git a/rel/mim2.vars-toml.config b/rel/mim2.vars-toml.config index ef2992d590..67033bd9de 100644 --- a/rel/mim2.vars-toml.config +++ b/rel/mim2.vars-toml.config @@ -14,6 +14,7 @@ {http_graphql_api_admin_endpoint_port, 5552}. {http_graphql_api_domain_admin_endpoint_port, 5542}. {http_graphql_api_user_endpoint_port, 5562}. +{http_prometheus_endpoint_port, 9092}. {hosts, "\"localhost\", \"anonymous.localhost\", \"localhost.bis\""}. {host_types, "\"test type\", \"dummy auth\""}. diff --git a/rel/mim3.vars-toml.config b/rel/mim3.vars-toml.config index 9d00c27336..a2e91723eb 100644 --- a/rel/mim3.vars-toml.config +++ b/rel/mim3.vars-toml.config @@ -14,6 +14,7 @@ {http_graphql_api_user_endpoint_port, 5563}. {http_api_endpoint_port, 8092}. {http_api_client_endpoint_port, 8193}. +{http_prometheus_endpoint_port, 9093}. "./vars-toml.config". diff --git a/rel/prod.vars-toml.config b/rel/prod.vars-toml.config index 666ef865a3..88bbf033be 100644 --- a/rel/prod.vars-toml.config +++ b/rel/prod.vars-toml.config @@ -13,6 +13,7 @@ {http_graphql_api_user_endpoint_port, 5561}. {http_api_endpoint_port, 8088}. {http_api_client_endpoint_port, 8089}. +{http_prometheus_endpoint_port, 9091}. {hosts, "\"localhost\""}. {default_server_domain, "\"localhost\""}. diff --git a/rel/reg1.vars-toml.config b/rel/reg1.vars-toml.config index da485e138c..6b138f9f68 100644 --- a/rel/reg1.vars-toml.config +++ b/rel/reg1.vars-toml.config @@ -13,6 +13,7 @@ {http_graphql_api_user_endpoint_port, 5564}. {http_api_endpoint_port, 8074}. {http_api_client_endpoint_port, 8075}. +{http_prometheus_endpoint_port, 9094}. %% This node is for global distribution testing. %% reg is short for region. diff --git a/rel/vars-toml.config b/rel/vars-toml.config index ad83878db3..e76a2280ed 100644 --- a/rel/vars-toml.config +++ b/rel/vars-toml.config @@ -15,10 +15,16 @@ {http_api_endpoint, "ip_address = \"127.0.0.1\" port = {{http_api_endpoint_port}}"}. {http_api_client_endpoint, "port = {{ http_api_client_endpoint_port }}"}. +{http_prometheus_endpoint, "port = {{ http_prometheus_endpoint_port }}"}. {s2s_use_starttls, "\"optional\""}. {s2s_certfile, "\"priv/ssl/fake_server.pem\""}. {internal_databases, "[internal_databases.mnesia]"}. +%% This is temporary, and most likely we will only enable Prometheus by default +{instrumentation, "[instrumentation.exometer] + +[instrumentation.prometheus]"}. + "./configure.vars.config". %% Defined by appending configure.vars.config diff --git a/src/config/mongoose_config_spec.erl b/src/config/mongoose_config_spec.erl index 866fa351d3..88e9328b5c 100644 --- a/src/config/mongoose_config_spec.erl +++ b/src/config/mongoose_config_spec.erl @@ -95,6 +95,7 @@ root() -> <<"internal_databases">> => internal_databases(), <<"services">> => services(), <<"modules">> => Modules#section{include = always}, + <<"instrumentation">> => instrumentation(), <<"shaper">> => shaper(), <<"acl">> => acl(), <<"access">> => access(), @@ -784,6 +785,14 @@ iqdisc() -> process_iqdisc(#{type := Type, workers := N}) -> {queues = Type, N}; process_iqdisc(#{type := Type}) -> Type. +%% path: instrumentation +instrumentation() -> + #section{items = #{default => #section{}}, + validate_keys = {module, mongoose_instrument}, + wrap = global_config, + include = always + }. + %% path: shaper shaper() -> #section{ diff --git a/src/gen_mod.erl b/src/gen_mod.erl index 4263d8719c..30a79bfcd7 100644 --- a/src/gen_mod.erl +++ b/src/gen_mod.erl @@ -72,6 +72,7 @@ -callback hooks(HostType :: host_type()) -> gen_hook:hook_list(). -callback supported_features() -> [module_feature()]. -callback config_spec() -> mongoose_config_spec:config_section(). +-callback instrumentation(host_type()) -> [mongoose_instrument:spec()]. %% Optional callback specifying module dependencies. %% The dependent module can specify parameters with which the dependee should be @@ -86,7 +87,7 @@ %% function). -callback deps(host_type(), module_opts()) -> gen_mod_deps:deps(). --optional_callbacks([hooks/1, config_spec/0, supported_features/0, deps/2]). +-optional_callbacks([hooks/1, config_spec/0, supported_features/0, instrumentation/1, deps/2]). %% @doc This function should be called by mongoose_modules only. %% To start a new module at runtime, use mongoose_modules:ensure_module/3 instead. @@ -101,6 +102,7 @@ start_module_for_host_type(HostType, Module, Opts) -> lists:map(fun mongoose_service:assert_loaded/1, get_required_services(HostType, Module, Opts)), check_dynamic_domains_support(HostType, Module), + run_for_instrumentation(HostType, fun mongoose_instrument:set_up/1, Module), Res = Module:start(HostType, Opts), run_for_hooks(HostType, fun gen_hook:add_handlers/1, Module), {links, LinksAfter} = erlang:process_info(self(), links), @@ -152,6 +154,12 @@ run_for_hooks(HostType, Fun, Module) -> false -> ok end. +run_for_instrumentation(HostType, Fun, Module) -> + case erlang:function_exported(Module, instrumentation, 1) of + true -> Fun(Module:instrumentation(HostType)); + false -> ok + end. + check_dynamic_domains_support(HostType, Module) -> case lists:member(HostType, ?MYHOSTS) of true -> ok; @@ -197,7 +205,7 @@ stop_module_for_host_type(HostType, Module) -> {wait, Process} -> wait_for_process(Process); _ -> - ok + run_for_instrumentation(HostType, fun mongoose_instrument:tear_down/1, Module) catch Class:Reason:Stacktrace -> ?LOG_ERROR(#{what => module_stopping_failed, host_type => HostType, stop_module => Module, diff --git a/src/mam/mod_mam_elasticsearch_arch.erl b/src/mam/mod_mam_elasticsearch_arch.erl index 578b88129e..edbae3948b 100644 --- a/src/mam/mod_mam_elasticsearch_arch.erl +++ b/src/mam/mod_mam_elasticsearch_arch.erl @@ -97,7 +97,7 @@ archive_message(_Result, ?LOG_ERROR(#{what => archive_message_failed, user => Owner, server => Host, remote => Remote, message_id => MessageId, reason => Reason}), - mongoose_metrics:update(Host, modMamDropped, 1), + mongoose_instrument:execute(mod_mam_pm_dropped, #{host_type => Host}, #{count => 1}), {ok, Err} end. diff --git a/src/mam/mod_mam_muc.erl b/src/mam/mod_mam_muc.erl index 98de0354a3..d60f77466c 100644 --- a/src/mam/mod_mam_muc.erl +++ b/src/mam/mod_mam_muc.erl @@ -28,6 +28,7 @@ %%% @end %%%------------------------------------------------------------------- -module(mod_mam_muc). +-behaviour(gen_mod). %% ---------------------------------------------------------------------- %% Exports @@ -37,7 +38,7 @@ archive_id/2]). %% gen_mod handlers --export([start/2, stop/1, supported_features/0]). +-export([start/2, stop/1, supported_features/0, hooks/1, instrumentation/1]). %% ejabberd room handlers -export([disco_muc_features/3, @@ -54,8 +55,7 @@ -export([lookup_messages/2]). -export([archive_id_int/2]). --ignore_xref([archive_id/2, archive_message_for_ct/1, archive_size/2, delete_archive/2, - start/2, stop/1, supported_features/0]). +-ignore_xref([archive_id/2, archive_message_for_ct/1, archive_size/2, delete_archive/2]). -include_lib("mongoose.hrl"). -include_lib("jlib.hrl"). @@ -117,15 +117,12 @@ archive_id(MucHost, RoomName) when is_binary(MucHost), is_binary(RoomName) -> -spec start(host_type(), gen_mod:module_opts()) -> any(). start(HostType, Opts) -> ?LOG_DEBUG(#{what => mam_muc_starting}), - ensure_metrics(HostType), - gen_hook:add_handlers(hooks(HostType)), add_iq_handlers(HostType, Opts), ok. -spec stop(host_type()) -> any(). stop(HostType) -> ?LOG_DEBUG(#{what => mam_muc_stopping}), - gen_hook:delete_handlers(hooks(HostType)), remove_iq_handlers(HostType), ok. @@ -229,7 +226,8 @@ room_process_mam_iq(Acc, From, To, IQ, #{host_type := HostType}) -> handle_error_iq(Acc, HostType, To, Action, handle_mam_iq(HostType, Action, From, To, IQ)); {error, max_delay_reached} -> - mongoose_metrics:update(HostType, modMucMamDroppedIQ, 1), + mongoose_instrument:execute(mod_mam_muc_dropped_iq, + #{host_type => HostType}, #{count => 1}), {Acc, return_max_delay_reached_error_iq(IQ)} end; {error, Reason} -> @@ -488,21 +486,25 @@ get_behaviour(HostType, ArcID, LocJID = #jid{}, RemJID = #jid{}) -> AlwaysJIDs :: [jid:literal_jid()], NeverJIDs :: [jid:literal_jid()]) -> any(). set_prefs(HostType, ArcID, ArcJID, DefaultMode, AlwaysJIDs, NeverJIDs) -> - mongoose_hooks:mam_muc_set_prefs(HostType, ArcID, ArcJID, DefaultMode, - AlwaysJIDs, NeverJIDs). + Result = mongoose_hooks:mam_muc_set_prefs(HostType, ArcID, ArcJID, DefaultMode, + AlwaysJIDs, NeverJIDs), + mongoose_instrument:execute(mod_mam_muc_set_prefs, #{host_type => HostType}, #{count => 1}), + Result. %% @doc Load settings from the database. -spec get_prefs(HostType :: host_type(), ArcID :: mod_mam:archive_id(), ArcJID :: jid:jid(), GlobalDefaultMode :: mod_mam:archive_behaviour()) -> mod_mam:preference() | {error, Reason :: term()}. get_prefs(HostType, ArcID, ArcJID, GlobalDefaultMode) -> - mongoose_hooks:mam_muc_get_prefs(HostType, GlobalDefaultMode, ArcID, ArcJID). + Result = mongoose_hooks:mam_muc_get_prefs(HostType, GlobalDefaultMode, ArcID, ArcJID), + mongoose_instrument:execute(mod_mam_muc_get_prefs, #{host_type => HostType}, #{count => 1}), + Result. -spec remove_archive(host_type(), mod_mam:archive_id() | undefined, jid:jid()) -> ok. remove_archive(HostType, ArcID, ArcJID = #jid{}) -> mongoose_hooks:mam_muc_remove_archive(HostType, ArcID, ArcJID), - ok. + mongoose_instrument:execute(mod_mam_muc_remove_archive, #{host_type => HostType}, #{count => 1}). %% See description in mod_mam_pm. -spec lookup_messages(HostType :: host_type(), Params :: map()) -> @@ -523,20 +525,25 @@ lookup_messages_without_policy_violation_check(HostType, true -> %% Use of disabled full text search {error, 'not-supported'}; false -> - StartT = erlang:monotonic_time(microsecond), - R = mongoose_hooks:mam_muc_lookup_messages(HostType, Params), - Diff = erlang:monotonic_time(microsecond) - StartT, - mongoose_metrics:update(HostType, [backends, ?MODULE, lookup], Diff), - R + mongoose_instrument:span(mod_mam_muc_lookup, #{host_type => HostType}, + fun mongoose_hooks:mam_muc_lookup_messages/2, [HostType, Params], + fun measure_lookup/2) end. +measure_lookup(Time, {ok, {_TotalCount, _Offset, MessageRows}}) -> + #{count => 1, time => Time, size => length(MessageRows)}; +measure_lookup(_, _OtherResult) -> + #{}. + archive_message_for_ct(Params = #{local_jid := RoomJid}) -> HostType = mod_muc_light_utils:room_jid_to_host_type(RoomJid), archive_message(HostType, Params). -spec archive_message(host_type(), mod_mam:archive_message_params()) -> ok | {error, timeout}. archive_message(HostType, Params) -> - mongoose_hooks:mam_muc_archive_message(HostType, Params). + mongoose_instrument:span(mod_mam_muc_archive_message, #{host_type => HostType}, + fun mongoose_hooks:mam_muc_archive_message/2, [HostType, Params], + fun(Time, _Result) -> #{time => Time, count => 1} end). %% ---------------------------------------------------------------------- %% Helpers @@ -579,7 +586,7 @@ message_row_to_ext_id(#{id := MessID}) -> -spec handle_error_iq(mongoose_acc:t(), host_type(), jid:jid(), atom(), {error, term(), jlib:iq()} | jlib:iq() | ignore) -> {mongoose_acc:t(), jlib:iq() | ignore}. handle_error_iq(Acc, HostType, _To, _Action, {error, _Reason, IQ}) -> - mongoose_metrics:update(HostType, modMucMamDroppedIQ, 1), + mongoose_instrument:execute(mod_mam_muc_dropped_iq, #{host_type => HostType}, #{count => 1}), {Acc, IQ}; handle_error_iq(Acc, _HostType, _To, _Action, IQ) -> {Acc, IQ}. @@ -653,8 +660,7 @@ hooks(HostType) -> [{disco_muc_features, HostType, fun ?MODULE:disco_muc_features/3, #{}, 99}, {filter_room_packet, HostType, fun ?MODULE:filter_room_packet/3, #{}, 60}, {forget_room, HostType, fun ?MODULE:forget_room/3, #{}, 90}, - {get_personal_data, HostType, fun ?MODULE:get_personal_data/3, #{}, 50} - | mongoose_metrics_mam_hooks:get_mam_muc_hooks(HostType)]. + {get_personal_data, HostType, fun ?MODULE:get_personal_data/3, #{}, 50}]. add_iq_handlers(HostType, Opts) -> IQDisc = gen_mod:get_opt(iqdisc, Opts, parallel), @@ -678,20 +684,19 @@ remove_iq_handlers(HostType) -> ?NS_MAM_06, mod_muc_iq), ok. -ensure_metrics(HostType) -> - mongoose_metrics:ensure_metric(HostType, [backends, ?MODULE, lookup], histogram), - lists:foreach(fun(Name) -> - mongoose_metrics:ensure_metric(HostType, Name, spiral) - end, - spirals()). - -spirals() -> - [modMucMamPrefsSets, - modMucMamPrefsGets, - modMucMamArchiveRemoved, - modMucMamLookups, - modMucMamForwarded, - modMucMamArchived, - modMucMamFlushed, - modMucMamDropped, - modMucMamDroppedIQ]. +-spec instrumentation(host_type()) -> [mongoose_instrument:spec()]. +instrumentation(HostType) -> + [{mod_mam_muc_archive_message, #{host_type => HostType}, + #{metrics => #{count => spiral, time => histogram}}}, + {mod_mam_muc_lookup, #{host_type => HostType}, + #{metrics => #{count => spiral, size => histogram, time => histogram}}}, + {mod_mam_muc_dropped_iq, #{host_type => HostType}, + #{metrics => #{count => spiral}}}, + {mod_mam_muc_dropped, #{host_type => HostType}, + #{metrics => #{count => spiral}}}, + {mod_mam_muc_remove_archive, #{host_type => HostType}, + #{metrics => #{count => spiral}}}, + {mod_mam_muc_get_prefs, #{host_type => HostType}, + #{metrics => #{count => spiral}}}, + {mod_mam_muc_set_prefs, #{host_type => HostType}, + #{metrics => #{count => spiral}}}]. diff --git a/src/mam/mod_mam_muc_elasticsearch_arch.erl b/src/mam/mod_mam_muc_elasticsearch_arch.erl index 4173bd1581..242fe243c7 100644 --- a/src/mam/mod_mam_muc_elasticsearch_arch.erl +++ b/src/mam/mod_mam_muc_elasticsearch_arch.erl @@ -97,7 +97,7 @@ archive_message(_Result, Params, #{host_type := HostType}) -> #{what => archive_muc_message_failed, reason => Reason, server => HostType, room => Room, source => SourceBinJid, message_id => MessageId})), - mongoose_metrics:update(HostType, modMamDropped, 1), + mongoose_instrument:execute(mod_mam_dropped, #{host_type => HostType}, #{count => 1}), {ok, Err} end. diff --git a/src/mam/mod_mam_muc_rdbms_arch_async.erl b/src/mam/mod_mam_muc_rdbms_arch_async.erl index 5e448c6158..c5f17c18d9 100644 --- a/src/mam/mod_mam_muc_rdbms_arch_async.erl +++ b/src/mam/mod_mam_muc_rdbms_arch_async.erl @@ -4,11 +4,8 @@ -include("mongoose_logger.hrl"). --define(PER_MESSAGE_FLUSH_TIME, [mod_mam_muc_rdbms_async_pool_writer, per_message_flush_time]). --define(FLUSH_TIME, [mod_mam_muc_rdbms_async_pool_writer, flush_time]). - -behaviour(gen_mod). --export([start/2, stop/1, hooks/1, supported_features/0]). +-export([start/2, stop/1, hooks/1, instrumentation/1, supported_features/0]). -export([archive_muc_message/3, mam_muc_archive_sync/3]). -export([flush/2]). -ignore_xref([flush/2]). @@ -37,8 +34,6 @@ mam_muc_archive_sync(Result, _Params, #{host_type := HostType}) -> start(HostType, Opts) -> {PoolOpts, Extra} = mod_mam_rdbms_arch_async:make_pool_opts(muc, Opts), mod_mam_rdbms_arch_async:prepare_insert_queries(muc, Extra), - mongoose_metrics:ensure_metric(HostType, ?PER_MESSAGE_FLUSH_TIME, histogram), - mongoose_metrics:ensure_metric(HostType, ?FLUSH_TIME, histogram), mongoose_async_pools:start_pool(HostType, muc_mam, PoolOpts). -spec stop(mongooseim:host_type()) -> any(). @@ -52,16 +47,24 @@ hooks(HostType) -> {mam_muc_archive_message, HostType, fun ?MODULE:archive_muc_message/3, #{}, 50} ]. +-spec instrumentation(mongooseim:host_type()) -> [mongoose_instrument:spec()]. +instrumentation(HostType) -> + [{mod_mam_muc_flushed, #{host_type => HostType}, + #{metrics => #{time_per_message => histogram, time => histogram, count => spiral}}}]. + -spec supported_features() -> [atom()]. supported_features() -> [dynamic_domains]. %%% flush callbacks flush(Acc, Extra = #{host_type := HostType, queue_length := MessageCount}) -> - {FlushTime, Result} = timer:tc(fun do_flush_muc/2, [Acc, Extra]), - mongoose_metrics:update(HostType, ?PER_MESSAGE_FLUSH_TIME, round(FlushTime / MessageCount)), - mongoose_metrics:update(HostType, ?FLUSH_TIME, FlushTime), - Result. + mongoose_instrument:span(mod_mam_muc_flushed, #{host_type => HostType}, + fun do_flush_muc/2, [Acc, Extra], + fun(Time, _Result) -> + #{time => Time, + time_per_message => round(Time / MessageCount), + count => MessageCount} + end). %% mam workers callbacks do_flush_muc(Acc, #{host_type := HostType, queue_length := MessageCount, @@ -84,7 +87,7 @@ do_flush_muc(Acc, #{host_type := HostType, queue_length := MessageCount, process_batch_result({updated, _Count}, _, _, _) -> ok; process_batch_result({error, Reason}, Rows, HostType, MessageCount) -> - mongoose_metrics:update(HostType, modMucMamDropped, MessageCount), + mongoose_instrument:execute(mod_mam_muc_dropped, #{host_type => HostType}, #{count => MessageCount}), Keys = [ maps:with([message_id, archive_id], Row) || Row <- Rows ], ?LOG_ERROR(#{what => archive_muc_batch_messages_failed, text => <<"A batch of muc messages failed to archive, modMucMamDropped metric updated">>, @@ -97,7 +100,7 @@ process_list_results(Results, HostType) -> process_single_result({{updated, _Count}, _}, _HostType) -> ok; process_single_result({{error, Reason}, #{message_id := MsgId, archive_id := ArcId}}, HostType) -> - mongoose_metrics:update(HostType, modMucMamDropped, 1), + mongoose_instrument:execute(mod_mam_muc_dropped, #{host_type => HostType}, #{count => 1}), ?LOG_ERROR(#{what => archive_muc_single_message_failed, text => <<"Single muc message failed to archive, modMucMamDropped metric updated">>, message_id => MsgId, archive_id => ArcId, reason => Reason}). diff --git a/src/mam/mod_mam_pm.erl b/src/mam/mod_mam_pm.erl index fa6b2e9c1b..a92abb868a 100644 --- a/src/mam/mod_mam_pm.erl +++ b/src/mam/mod_mam_pm.erl @@ -40,7 +40,7 @@ archive_id/2]). %% gen_mod handlers --export([start/2, stop/1, supported_features/0]). +-export([start/2, stop/1, supported_features/0, hooks/1, instrumentation/1]). %% hook handlers -export([disco_local_features/3, @@ -109,15 +109,12 @@ archive_id(Server, User) -spec start(host_type(), gen_mod:module_opts()) -> any(). start(HostType, Opts) -> ?LOG_INFO(#{what => mam_starting, host_type => HostType}), - ensure_metrics(HostType), - gen_hook:add_handlers(hooks(HostType)), add_iq_handlers(HostType, Opts), ok. -spec stop(host_type()) -> any(). stop(HostType) -> ?LOG_INFO(#{what => mam_stopping, host_type => HostType}), - gen_hook:delete_handlers(hooks(HostType)), remove_iq_handlers(HostType), ok. @@ -150,11 +147,13 @@ process_mam_iq(Acc, From, To, IQ, _Extra) -> ?LOG_WARNING(#{what => mam_max_delay_reached, text => <<"Return max_delay_reached error IQ from MAM">>, action => Action, acc => Acc}), - mongoose_metrics:update(HostType, modMamDroppedIQ, 1), + mongoose_instrument:execute(mod_mam_pm_dropped_iq, + #{host_type => HostType}, #{count => 1}), {Acc, return_max_delay_reached_error_iq(IQ)} end; false -> - mongoose_metrics:update(HostType, modMamDroppedIQ, 1), + mongoose_instrument:execute(mod_mam_pm_dropped_iq, + #{host_type => HostType}, #{count => 1}), {Acc, return_action_not_allowed_error_iq(IQ)} end. @@ -565,20 +564,24 @@ get_behaviour(HostType, ArcID, LocJID=#jid{}, RemJID=#jid{}) -> DefaultMode :: atom(), AlwaysJIDs :: [jid:literal_jid()], NeverJIDs :: [jid:literal_jid()]) -> any(). set_prefs(HostType, ArcID, ArcJID, DefaultMode, AlwaysJIDs, NeverJIDs) -> - mongoose_hooks:mam_set_prefs(HostType, ArcID, ArcJID, DefaultMode, - AlwaysJIDs, NeverJIDs). + Result = mongoose_hooks:mam_set_prefs(HostType, ArcID, ArcJID, DefaultMode, + AlwaysJIDs, NeverJIDs), + mongoose_instrument:execute(mod_mam_pm_set_prefs, #{host_type => HostType}, #{count => 1}), + Result. %% @doc Load settings from the database. -spec get_prefs(HostType :: host_type(), ArcID :: mod_mam:archive_id(), ArcJID :: jid:jid(), GlobalDefaultMode :: mod_mam:archive_behaviour() ) -> mod_mam:preference() | {error, Reason :: term()}. get_prefs(HostType, ArcID, ArcJID, GlobalDefaultMode) -> - mongoose_hooks:mam_get_prefs(HostType, GlobalDefaultMode, ArcID, ArcJID). + Result = mongoose_hooks:mam_get_prefs(HostType, GlobalDefaultMode, ArcID, ArcJID), + mongoose_instrument:execute(mod_mam_pm_get_prefs, #{host_type => HostType}, #{count => 1}), + Result. --spec remove_archive_hook(host_type(), mod_mam:archive_id(), jid:jid()) -> 'ok'. +-spec remove_archive_hook(host_type(), mod_mam:archive_id(), jid:jid()) -> ok. remove_archive_hook(HostType, ArcID, ArcJID=#jid{}) -> mongoose_hooks:mam_remove_archive(HostType, ArcID, ArcJID), - ok. + mongoose_instrument:execute(mod_mam_pm_remove_archive, #{host_type => HostType}, #{count => 1}). -spec lookup_messages(HostType :: host_type(), Params :: map()) -> {ok, mod_mam:lookup_result()} @@ -598,13 +601,20 @@ lookup_messages_without_policy_violation_check( true -> %% Use of disabled full text search {error, 'not-supported'}; false -> - StartT = erlang:monotonic_time(microsecond), - R = mongoose_hooks:mam_lookup_messages(HostType, Params), - Diff = erlang:monotonic_time(microsecond) - StartT, - mongoose_metrics:update(HostType, [backends, ?MODULE, lookup], Diff), - R + mongoose_instrument:span(mod_mam_pm_lookup, #{host_type => HostType}, + fun mongoose_hooks:mam_lookup_messages/2, [HostType, Params], + fun(Time, Result) -> measure_lookup(Params, Time, Result) end) end. +measure_lookup(Params, Time, {ok, {_TotalCount, _Offset, MessageRows}}) -> + M = case Params of + #{is_simple := true} -> #{simple => 1}; + #{} -> #{} + end, + M#{count => 1, time => Time, size => length(MessageRows)}; +measure_lookup(_, _, _OtherResult) -> + #{}. + archive_message_from_ct(Params = #{local_jid := JID}) -> HostType = jid_to_host_type(JID), archive_message(HostType, Params). @@ -612,11 +622,9 @@ archive_message_from_ct(Params = #{local_jid := JID}) -> -spec archive_message(host_type(), mod_mam:archive_message_params()) -> ok | {error, timeout}. archive_message(HostType, Params) -> - StartT = erlang:monotonic_time(microsecond), - R = mongoose_hooks:mam_archive_message(HostType, Params), - Diff = erlang:monotonic_time(microsecond) - StartT, - mongoose_metrics:update(HostType, [backends, ?MODULE, archive], Diff), - R. + mongoose_instrument:span(mod_mam_pm_archive_message, #{host_type => HostType}, + fun mongoose_hooks:mam_archive_message/2, [HostType, Params], + fun(Time, _Result) -> #{time => Time, count => 1} end). %% ---------------------------------------------------------------------- %% Helpers @@ -636,7 +644,7 @@ message_row_to_ext_id(#{id := MessID}) -> mod_mam_utils:mess_id_to_external_binary(MessID). handle_error_iq(HostType, Acc, _To, _Action, {error, _Reason, IQ}) -> - mongoose_metrics:update(HostType, modMamDroppedIQ, 1), + mongoose_instrument:execute(mod_mam_pm_dropped_iq, #{host_type => HostType}, #{count => 1}), {Acc, IQ}; handle_error_iq(_Host, Acc, _To, _Action, IQ) -> {Acc, IQ}. @@ -711,7 +719,6 @@ hooks(HostType) -> {amp_determine_strategy, HostType, fun ?MODULE:determine_amp_strategy/3, #{}, 20}, {sm_filter_offline_message, HostType, fun ?MODULE:sm_filter_offline_message/3, #{}, 50}, {get_personal_data, HostType, fun ?MODULE:get_personal_data/3, #{}, 50} - | mongoose_metrics_mam_hooks:get_mam_hooks(HostType) ]. add_iq_handlers(HostType, Opts) -> @@ -732,22 +739,19 @@ remove_iq_handlers(HostType) -> || Namespace <- [?NS_MAM_04, ?NS_MAM_06]], ok. -ensure_metrics(HostType) -> - mongoose_metrics:ensure_metric(HostType, [backends, ?MODULE, lookup], histogram), - mongoose_metrics:ensure_metric(HostType, [modMamLookups, simple], spiral), - mongoose_metrics:ensure_metric(HostType, [backends, ?MODULE, archive], histogram), - lists:foreach(fun(Name) -> - mongoose_metrics:ensure_metric(HostType, Name, spiral) - end, - spirals()). - -spirals() -> - [modMamPrefsSets, - modMamPrefsGets, - modMamArchiveRemoved, - modMamLookups, - modMamForwarded, - modMamArchived, - modMamFlushed, - modMamDropped, - modMamDroppedIQ]. +-spec instrumentation(host_type()) -> [mongoose_instrument:spec()]. +instrumentation(HostType) -> + [{mod_mam_pm_archive_message, #{host_type => HostType}, + #{metrics => #{count => spiral, time => histogram}}}, + {mod_mam_pm_lookup, #{host_type => HostType}, + #{metrics => #{count => spiral, simple => spiral, size => histogram, time => histogram}}}, + {mod_mam_pm_dropped_iq, #{host_type => HostType}, + #{metrics => #{count => spiral}}}, + {mod_mam_pm_dropped, #{host_type => HostType}, + #{metrics => #{count => spiral}}}, + {mod_mam_pm_remove_archive, #{host_type => HostType}, + #{metrics => #{count => spiral}}}, + {mod_mam_pm_get_prefs, #{host_type => HostType}, + #{metrics => #{count => spiral}}}, + {mod_mam_pm_set_prefs, #{host_type => HostType}, + #{metrics => #{count => spiral}}}]. diff --git a/src/mam/mod_mam_rdbms_arch_async.erl b/src/mam/mod_mam_rdbms_arch_async.erl index 25ad646e31..ef971127b2 100644 --- a/src/mam/mod_mam_rdbms_arch_async.erl +++ b/src/mam/mod_mam_rdbms_arch_async.erl @@ -4,12 +4,9 @@ -include("mongoose_logger.hrl"). --define(PER_MESSAGE_FLUSH_TIME, [mod_mam_rdbms_async_pool_writer, per_message_flush_time]). --define(FLUSH_TIME, [mod_mam_rdbms_async_pool_writer, flush_time]). - -behaviour(gen_mod). --export([start/2, stop/1, hooks/1, supported_features/0]). +-export([start/2, stop/1, hooks/1, instrumentation/1, supported_features/0]). -export([archive_pm_message/3, mam_archive_sync/3]). -export([flush/2]). @@ -35,8 +32,6 @@ mam_archive_sync(Result, _Params, #{host_type := HostType}) -> start(HostType, Opts) -> {PoolOpts, Extra} = make_pool_opts(pm, Opts), prepare_insert_queries(pm, Extra), - mongoose_metrics:ensure_metric(HostType, ?PER_MESSAGE_FLUSH_TIME, histogram), - mongoose_metrics:ensure_metric(HostType, ?FLUSH_TIME, histogram), mongoose_async_pools:start_pool(HostType, pm_mam, PoolOpts). -spec stop(mongooseim:host_type()) -> any(). @@ -50,6 +45,11 @@ hooks(HostType) -> {mam_archive_message, HostType, fun ?MODULE:archive_pm_message/3, #{}, 50} ]. +-spec instrumentation(mongooseim:host_type()) -> [mongoose_instrument:spec()]. +instrumentation(HostType) -> + [{mod_mam_pm_flushed, #{host_type => HostType}, + #{metrics => #{time_per_message => histogram, time => histogram, count => spiral}}}]. + -spec supported_features() -> [atom()]. supported_features() -> [dynamic_domains]. @@ -86,10 +86,13 @@ multi_name(Name, Times) -> %%% flush callbacks flush(Acc, Extra = #{host_type := HostType, queue_length := MessageCount}) -> - {FlushTime, Result} = timer:tc(fun do_flush_pm/2, [Acc, Extra]), - mongoose_metrics:update(HostType, ?PER_MESSAGE_FLUSH_TIME, round(FlushTime / MessageCount)), - mongoose_metrics:update(HostType, ?FLUSH_TIME, FlushTime), - Result. + mongoose_instrument:span(mod_mam_pm_flushed, #{host_type => HostType}, + fun do_flush_pm/2, [Acc, Extra], + fun(Time, _Result) -> + #{time => Time, + time_per_message => round(Time / MessageCount), + count => MessageCount} + end). %% mam workers callbacks do_flush_pm(Acc, #{host_type := HostType, queue_length := MessageCount, @@ -112,7 +115,7 @@ do_flush_pm(Acc, #{host_type := HostType, queue_length := MessageCount, process_batch_result({updated, _Count}, _, _, _) -> ok; process_batch_result({error, Reason}, Rows, HostType, MessageCount) -> - mongoose_metrics:update(HostType, modMamDropped, MessageCount), + mongoose_instrument:execute(mod_mam_pm_dropped, #{host_type => HostType}, #{count => MessageCount}), Keys = [ maps:with([message_id, archive_id], Row) || Row <- Rows ], ?LOG_ERROR(#{what => archive_message_failed, text => <<"archive_message batch query failed">>, @@ -125,7 +128,7 @@ process_list_results(Results, HostType) -> process_single_result({{updated, _Count}, _}, _HostType) -> ok; process_single_result({{error, Reason}, #{message_id := MsgId, archive_id := ArcId}}, HostType) -> - mongoose_metrics:update(HostType, modMamDropped, 1), + mongoose_instrument:execute(mod_mam_pm_dropped, #{host_type => HostType}, #{count => 1}), ?LOG_ERROR(#{what => archive_message_failed, text => <<"archive_message batch query failed">>, message_id => MsgId, archive_id => ArcId, reason => Reason}). diff --git a/src/metrics/mongoose_instrument.erl b/src/metrics/mongoose_instrument.erl new file mode 100644 index 0000000000..0ffdbeea12 --- /dev/null +++ b/src/metrics/mongoose_instrument.erl @@ -0,0 +1,68 @@ +-module(mongoose_instrument). + +-export([set_up/1, set_up/3, + tear_down/1, tear_down/2, + span/4, span/5, + execute/3]). + +-ignore_xref([set_up/3, span/4, tear_down/2]). + +-type event_name() :: atom(). +-type labels() :: #{host_type => mongooseim:host_type()}. % to be extended +-type metrics() :: #{atom() => spiral | histogram}. % to be extended +-type measurements() :: #{atom() => integer() | atom() | binary()}. +-type spec() :: {event_name(), labels(), config()}. +-type config() :: #{metrics => metrics()}. % to be extended +-type handler_fun() :: fun((event_name(), labels(), config(), measurements()) -> any()). +-type handlers() :: {[handler_fun()], config()}. +-type execution_time() :: integer(). +-type measure_fun(Result) :: fun((execution_time(), Result) -> measurements()). + +-callback set_up(event_name(), labels(), config()) -> boolean(). +-callback handle_event(event_name(), labels(), config(), measurements()) -> any(). + +-export_type([event_name/0, labels/0, config/0, measurements/0, spec/0, handlers/0]). + +-spec set_up([spec()]) -> ok. +set_up(Specs) -> + lists:foreach(fun({EventName, Labels, Config}) -> set_up(EventName, Labels, Config) end, Specs). + +-spec tear_down([spec()]) -> ok. +tear_down(Specs) -> + lists:foreach(fun({EventName, Labels, _Config}) -> tear_down(EventName, Labels) end, Specs). + +-spec set_up(event_name(), labels(), config()) -> ok. +set_up(EventName, Labels, Config) -> + AllModules = handler_modules(), + UsedModules = lists:filter(fun(Mod) -> Mod:set_up(EventName, Labels, Config) end, AllModules), + HandlerFuns = [fun Mod:handle_event/4 || Mod <- UsedModules], + mongoose_instrument_registry:attach(EventName, Labels, {HandlerFuns, Config}). + +-spec tear_down(event_name(), labels()) -> ok. +tear_down(EventName, Labels) -> + mongoose_instrument_registry:detach(EventName, Labels). + +-spec span(event_name(), labels(), fun(() -> Result), measure_fun(Result)) -> Result. +span(Event, Labels, F, MeasureF) -> + span(Event, Labels, F, [], MeasureF). + +-spec span(event_name(), labels(), fun((...) -> Result), list(), measure_fun(Result)) -> Result. +span(Event, Labels, F, Args, MeasureF) -> + {ok, Handlers} = mongoose_instrument_registry:lookup(Event, Labels), + {Time, Result} = timer:tc(F, Args), + handle_event(Event, Labels, MeasureF(Time, Result), Handlers), + Result. + +-spec execute(event_name(), labels(), measurements()) -> ok. +execute(Event, Labels, Measurements) -> + {ok, Handlers} = mongoose_instrument_registry:lookup(Event, Labels), + handle_event(Event, Labels, Measurements, Handlers). + +-spec handle_event(event_name(), labels(), measurements(), handlers()) -> ok. +handle_event(Event, Labels, Measurements, {EventHandlers, Config}) -> + lists:foreach(fun(Handler) -> Handler(Event, Labels, Config, Measurements) end, EventHandlers). + +-spec handler_modules() -> [module()]. +handler_modules() -> + [list_to_existing_atom("mongoose_instrument_" ++ atom_to_list(Key)) + || Key <- maps:keys(mongoose_config:get_opt(instrumentation))]. diff --git a/src/metrics/mongoose_instrument_exometer.erl b/src/metrics/mongoose_instrument_exometer.erl new file mode 100644 index 0000000000..2ae22208bc --- /dev/null +++ b/src/metrics/mongoose_instrument_exometer.erl @@ -0,0 +1,45 @@ +-module(mongoose_instrument_exometer). + +-behaviour(mongoose_instrument). + +-export([set_up/3, handle_event/4]). + +-spec set_up(mongoose_instrument:event_name(), mongoose_instrument:labels(), + mongoose_instrument:config()) -> boolean(). +set_up(EventName, Labels, #{metrics := Metrics}) -> + maps:foreach(fun(MetricName, MetricType) -> + set_up_metric(EventName, Labels, MetricName, MetricType) + end, Metrics), + true; +set_up(_EventName, _Labels, #{}) -> + false. + +-spec handle_event(mongoose_instrument:event_name(), mongoose_instrument:labels(), + mongoose_instrument:config(), mongoose_instrument:measurements()) -> ok. +handle_event(EventName, Labels, #{metrics := Metrics}, Measurements) -> + maps:foreach(fun(MetricName, MetricType) -> + handle_metric_event(EventName, Labels, MetricName, MetricType, Measurements) + end, Metrics). + +set_up_metric(EventName, Labels, MetricName, MetricType) -> + %% TODO improve handling of already existing metrics + Name = exometer_metric_name(EventName, Labels, MetricName), + catch exometer:new(Name, MetricType). + +handle_metric_event(EventName, Labels, MetricName, MetricType, Measurements) -> + case Measurements of + #{MetricName := MetricValue} -> + Name = exometer_metric_name(EventName, Labels, MetricName), + update_metric(Name, MetricType, MetricValue); + #{} -> + ok + end. + +update_metric(Name, spiral, Value) when is_integer(Value), Value >= 0 -> + exometer:update(Name, Value); +update_metric(Name, histogram, Value) when is_integer(Value) -> + exometer:update(Name, Value). + +%% This logic will need extending if we add more labels +exometer_metric_name(EventName, #{host_type := HostType}, MetricName) -> + [mongoose_metrics:get_host_type_prefix(HostType), EventName, MetricName]. diff --git a/src/metrics/mongoose_instrument_prometheus.erl b/src/metrics/mongoose_instrument_prometheus.erl new file mode 100644 index 0000000000..9ba9fe9a5f --- /dev/null +++ b/src/metrics/mongoose_instrument_prometheus.erl @@ -0,0 +1,72 @@ +-module(mongoose_instrument_prometheus). + +-behaviour(mongoose_instrument). + +-export([set_up/3, handle_event/4]). + +-spec set_up(mongoose_instrument:event_name(), mongoose_instrument:labels(), + mongoose_instrument:config()) -> boolean(). +set_up(EventName, Labels, #{metrics := Metrics}) -> + LabelKeys = labels_to_keys(Labels), + maps:foreach(fun(MetricName, MetricType) -> + set_up_metric(EventName, LabelKeys, MetricName, MetricType) + end, Metrics), + true; +set_up(_EventName, _Labels, #{}) -> + false. + +-spec handle_event(mongoose_instrument:event_name(), mongoose_instrument:labels(), + mongoose_instrument:config(), mongoose_instrument:measurements()) -> ok. +handle_event(EventName, Labels, #{metrics := Metrics}, Measurements) -> + LabelValues = labels_to_values(Labels), + maps:foreach(fun(MetricName, MetricType) -> + handle_metric_event(EventName, LabelValues, MetricName, MetricType, Measurements) + end, Metrics). + +set_up_metric(EventName, LabelKeys, MetricName, MetricType) -> + MetricSpec = metric_spec(EventName, LabelKeys, MetricName), + declare_metric(MetricSpec, MetricType). + +declare_metric(MetricSpec, spiral) -> + prometheus_counter:declare(MetricSpec); +declare_metric(MetricSpec, histogram) -> + prometheus_histogram:declare([{buckets, histogram_buckets()} | MetricSpec]). + +metric_spec(EventName, LabelKeys, MetricName) -> + [{name, full_metric_name(EventName, MetricName)}, + {help, metric_help(EventName, MetricName)}, + {labels, LabelKeys}]. + +histogram_buckets() -> + histogram_buckets([], 1 bsl 30). % ~1.07 * 10^9 + +histogram_buckets(AccBuckets, Val) when Val > 0 -> + histogram_buckets([Val | AccBuckets], Val bsr 1); +histogram_buckets(AccBuckets, _Val) -> + AccBuckets. + +handle_metric_event(EventName, LabelValues, MetricName, MetricType, Measurements) -> + case Measurements of + #{MetricName := MetricValue} -> + FullName = full_metric_name(EventName, MetricName), + update_metric(FullName, LabelValues, MetricType, MetricValue); + #{} -> + ok + end. + +metric_help(EventName, MetricName) -> + lists:flatten(io_lib:format("Event: ~p, Metric: ~p", [EventName, MetricName])). + +full_metric_name(EventName, MetricName) -> + list_to_atom(atom_to_list(EventName) ++ "_" ++ atom_to_list(MetricName)). + +labels_to_keys(Labels) -> + lists:sort(maps:keys(Labels)). + +labels_to_values(Labels) -> + [V || {_K, V} <- lists:keysort(1, maps:to_list(Labels))]. + +update_metric(Name, Labels, spiral, Value) when is_integer(Value), Value >= 0 -> + prometheus_counter:inc(Name, Labels, Value); +update_metric(Name, Labels, histogram, Value) when is_integer(Value) -> + prometheus_histogram:observe(Name, Labels, Value). diff --git a/src/metrics/mongoose_instrument_registry.erl b/src/metrics/mongoose_instrument_registry.erl new file mode 100644 index 0000000000..c2217f12cf --- /dev/null +++ b/src/metrics/mongoose_instrument_registry.erl @@ -0,0 +1,27 @@ +-module(mongoose_instrument_registry). + +-export([start/0, attach/3, detach/2, lookup/2]). + +-spec start() -> ok. +start() -> + ets:new(?MODULE, [named_table, public, {read_concurrency, true}]), + ok. + +-spec attach(mongoose_instrument:event_name(), mongoose_instrument:labels(), + mongoose_instrument:handlers()) -> ok. +attach(Event, Labels, Val) -> + ets:insert_new(?MODULE, {{Event, Labels}, Val}), + ok. + +-spec detach(mongoose_instrument:event_name(), mongoose_instrument:labels()) -> ok. +detach(Event, Labels) -> + ets:delete(?MODULE, {Event, Labels}), + ok. + +-spec lookup(mongoose_instrument:event_name(), mongoose_instrument:labels()) -> + {ok, mongoose_instrument:handlers()} | {error, not_found}. +lookup(Event, Labels) -> + case ets:lookup(?MODULE, {Event, Labels}) of + [] -> {error, not_found}; + [{_, Val}] -> {ok, Val} + end. diff --git a/src/metrics/mongoose_metrics.erl b/src/metrics/mongoose_metrics.erl index 4756afeac2..d889ed9002 100644 --- a/src/metrics/mongoose_metrics.erl +++ b/src/metrics/mongoose_metrics.erl @@ -42,7 +42,8 @@ get_mnesia_running_db_nodes_count/0, remove_host_type_metrics/1, remove_all_metrics/0, - get_report_interval/0 + get_report_interval/0, + get_host_type_prefix/1 ]). -ignore_xref([get_dist_data_stats/0, get_mnesia_running_db_nodes_count/0, @@ -65,6 +66,7 @@ -spec init() -> ok. init() -> + mongoose_instrument_registry:start(), % TODO move it out of this module prepare_prefixes(), create_vm_metrics(), create_global_metrics(?GLOBAL_COUNTERS), diff --git a/src/metrics/mongoose_metrics_mam_hooks.erl b/src/metrics/mongoose_metrics_mam_hooks.erl deleted file mode 100644 index ece05900fd..0000000000 --- a/src/metrics/mongoose_metrics_mam_hooks.erl +++ /dev/null @@ -1,173 +0,0 @@ -%%%------------------------------------------------------------------- -%%% @author Piotr Nosek -%%% @doc MAM hooks for general metrics -%%% -%%% @end -%%% Created : 13 Feb 2017 by Piotr Nosek -%%%------------------------------------------------------------------- --module(mongoose_metrics_mam_hooks). - --include("mongoose.hrl"). --include("jlib.hrl"). - --export([get_mam_hooks/1, - get_mam_muc_hooks/1]). - -%%------------------- -%% Internal exports -%%------------------- --export([mam_get_prefs/3, - mam_set_prefs/3, - mam_remove_archive/3, - mam_lookup_messages/3, - mam_archive_message/3, - mam_flush_messages/3, - mam_muc_get_prefs/3, - mam_muc_set_prefs/3, - mam_muc_remove_archive/3, - mam_muc_lookup_messages/3, - mam_muc_archive_message/3, - mam_muc_flush_messages/3]). - -%%------------------- -%% Implementation -%%------------------- - -%% @doc Here will be declared which hooks should be registered when mod_mam_pm is enabled. --spec get_mam_hooks(mongooseim:host_type()) -> gen_hook:hook_list(). -get_mam_hooks(HostType) -> - [ - {mam_set_prefs, HostType, fun ?MODULE:mam_set_prefs/3, #{}, 50}, - {mam_get_prefs, HostType, fun ?MODULE:mam_get_prefs/3, #{}, 50}, - {mam_archive_message, HostType, fun ?MODULE:mam_archive_message/3, #{}, 50}, - {mam_remove_archive, HostType, fun ?MODULE:mam_remove_archive/3, #{}, 50}, - {mam_lookup_messages, HostType, fun ?MODULE:mam_lookup_messages/3, #{}, 100}, - {mam_flush_messages, HostType, fun ?MODULE:mam_flush_messages/3, #{}, 50} - ]. - -%% @doc Here will be declared which hooks should be registered when mod_mam_muc is enabled. --spec get_mam_muc_hooks(mongooseim:host_type()) -> gen_hook:hook_list(). -get_mam_muc_hooks(HostType) -> - [ - {mam_muc_set_prefs, HostType, fun ?MODULE:mam_muc_set_prefs/3, #{}, 50}, - {mam_muc_get_prefs, HostType, fun ?MODULE:mam_muc_get_prefs/3, #{}, 50}, - {mam_muc_archive_message, HostType, fun ?MODULE:mam_muc_archive_message/3, #{}, 50}, - {mam_muc_remove_archive, HostType, fun ?MODULE:mam_muc_remove_archive/3, #{}, 50}, - {mam_muc_lookup_messages, HostType, fun ?MODULE:mam_muc_lookup_messages/3, #{}, 100}, - {mam_muc_flush_messages, HostType, fun ?MODULE:mam_muc_flush_messages/3, #{}, 50} - ]. - --spec mam_get_prefs(Acc, Params, Extra) -> {ok, Acc} when - Acc :: mod_mam:preference() | {error, Reason :: term()}, - Params :: map(), - Extra :: gen_hook:extra(). -mam_get_prefs(Result, _Params, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMamPrefsGets, 1), - {ok, Result}. - --spec mam_set_prefs(Acc, Params, Extra) -> {ok, Acc} when - Acc :: term(), - Params :: map(), - Extra :: gen_hook:extra(). -mam_set_prefs(Result, _Params, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMamPrefsSets, 1), - {ok, Result}. - --spec mam_remove_archive(Acc, Params, Extra) -> {ok, Acc} when - Acc :: term(), - Params :: #{archive_id := mod_mam:archive_id() | undefined, owner := jid:jid()}, - Extra :: gen_hook:extra(). -mam_remove_archive(Acc, _Params, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMamArchiveRemoved, 1), - {ok, Acc}. - --spec mam_lookup_messages(Acc, Params, Extra) -> {ok, Acc} when - Acc :: {ok, mod_mam:lookup_result()} | {error, term()}, - Params :: mam_iq:lookup_params(), - Extra :: gen_hook:extra(). -mam_lookup_messages({ok, {_TotalCount, _Offset, MessageRows}} = Result, - #{is_simple := IsSimple}, - #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMamForwarded, length(MessageRows)), - mongoose_metrics:update(HostType, modMamLookups, 1), - case IsSimple of - true -> - mongoose_metrics:update(HostType, [modMamLookups, simple], 1); - _ -> - ok - end, - {ok, Result}; -mam_lookup_messages(Result = {error, _}, _Params, _Extra) -> - {ok, Result}. - --spec mam_archive_message(Acc, Params, Extra) -> {ok, Acc} when - Acc :: ok | {error, timeout}, - Params :: mod_mam:archive_message_params(), - Extra :: gen_hook:extra(). -mam_archive_message(Result, _Params, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMamArchived, 1), - {ok, Result}. - --spec mam_flush_messages(Acc, Params, Extra) -> {ok, Acc} when - Acc :: ok, - Params :: #{count := integer()}, - Extra :: gen_hook:extra(). -mam_flush_messages(Acc, #{count := MessageCount}, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMamFlushed, MessageCount), - {ok, Acc}. - -%% ---------------------------------------------------------------------------- -%% mod_mam_muc - --spec mam_muc_get_prefs(Acc, Params, Extra) -> {ok, Acc} when - Acc :: mod_mam:preference() | {error, Reason :: term()}, - Params :: map(), - Extra :: gen_hook:extra(). -mam_muc_get_prefs(Result, _Params, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMucMamPrefsGets, 1), - {ok, Result}. - --spec mam_muc_set_prefs(Acc, Params, Extra) -> {ok, Acc} when - Acc :: term(), - Params :: map(), - Extra :: gen_hook:extra(). -mam_muc_set_prefs(Result, _Params, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMucMamPrefsSets, 1), - {ok, Result}. - --spec mam_muc_remove_archive(Acc, Params, Extra) -> {ok, Acc} when - Acc :: term(), - Params :: #{archive_id := mod_mam:archive_id() | undefined, room := jid:jid()}, - Extra :: gen_hook:extra(). -mam_muc_remove_archive(Acc, _Params, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMucMamArchiveRemoved, 1), - {ok, Acc}. - --spec mam_muc_lookup_messages(Acc, Params, Extra) -> {ok, Acc} when - Acc :: {ok, mod_mam:lookup_result()} | {error, term()}, - Params :: mam_iq:lookup_params(), - Extra :: gen_hook:extra(). -mam_muc_lookup_messages({ok, {_TotalCount, _Offset, MessageRows}} = Result, - _Params, - #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMucMamForwarded, length(MessageRows)), - mongoose_metrics:update(HostType, modMucMamLookups, 1), - {ok, Result}; -mam_muc_lookup_messages(Result = {error, _}, _Params, _Extra) -> - {ok, Result}. - --spec mam_muc_archive_message(Acc, Params, Extra) -> {ok, Acc} when - Acc :: ok | {error, timeout}, - Params :: mod_mam:archive_message_params(), - Extra :: gen_hook:extra(). -mam_muc_archive_message(Result, _Params, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMucMamArchived, 1), - {ok, Result}. - --spec mam_muc_flush_messages(Acc, Params, Extra) -> {ok, Acc} when - Acc :: ok, - Params :: map(), - Extra :: gen_hook:extra(). -mam_muc_flush_messages(Acc, #{count := MessageCount}, #{host_type := HostType}) -> - mongoose_metrics:update(HostType, modMucMamFlushed, MessageCount), - {ok, Acc}. diff --git a/src/metrics/mongoose_prometheus_handler.erl b/src/metrics/mongoose_prometheus_handler.erl new file mode 100644 index 0000000000..c11207dcfe --- /dev/null +++ b/src/metrics/mongoose_prometheus_handler.erl @@ -0,0 +1,10 @@ +-module(mongoose_prometheus_handler). + +-behaviour(mongoose_http_handler). + +%% mongoose_http_handler callbacks +-export([routes/1]). + +-spec routes(mongoose_http_handler:options()) -> mongoose_http_handler:routes(). +routes(#{path := BasePath}) -> + [{[BasePath, "/[:registry]"], prometheus_cowboy2_handler, #{}}]. diff --git a/src/mongooseim.app.src b/src/mongooseim.app.src index 514ff22b05..987acd09ba 100644 --- a/src/mongooseim.app.src +++ b/src/mongooseim.app.src @@ -30,6 +30,7 @@ observer_cli, pa, public_key, + prometheus_cowboy, ranch, recon, runtime_tools, diff --git a/test/common/config_parser_helper.erl b/test/common/config_parser_helper.erl index f63a2b7262..c4ea3ecea8 100644 --- a/test/common/config_parser_helper.erl +++ b/test/common/config_parser_helper.erl @@ -28,6 +28,7 @@ options("host_types") -> {sm_backend, mnesia}, {component_backend, mnesia}, {s2s_backend, mnesia}, + {instrumentation, #{}}, {{s2s, <<"another host type">>}, default_s2s()}, {{s2s, <<"localhost">>}, default_s2s()}, {{s2s, <<"some host type">>}, default_s2s()}, @@ -92,6 +93,7 @@ options("miscellaneous") -> id => "G-12345678", secret => "Secret" }}}}, + {instrumentation, #{prometheus => #{}, exometer => #{}}}, {{s2s, <<"anonymous.localhost">>}, default_s2s()}, {{s2s, <<"localhost">>}, default_s2s()}, {sm_backend, mnesia}, @@ -125,6 +127,7 @@ options("modules") -> {sm_backend, mnesia}, {component_backend, mnesia}, {s2s_backend, mnesia}, + {instrumentation, #{}}, {{auth, <<"dummy_host">>}, default_auth()}, {{auth, <<"localhost">>}, default_auth()}, {{modules, <<"dummy_host">>}, all_modules()}, @@ -262,6 +265,7 @@ options("mongooseim-pgsql") -> {sm_backend, mnesia}, {component_backend, mnesia}, {s2s_backend, mnesia}, + {instrumentation, #{exometer => #{}}}, {{auth, <<"anonymous.localhost">>}, (default_auth())#{anonymous => #{backend => mnesia, allow_multiple_connections => true, @@ -359,6 +363,7 @@ options("outgoing_pools") -> {sm_backend, mnesia}, {component_backend, mnesia}, {s2s_backend, mnesia}, + {instrumentation, #{}}, {{auth, <<"anonymous.localhost">>}, default_auth()}, {{auth, <<"localhost">>}, default_auth()}, {{auth, <<"localhost.bis">>}, default_auth()}, @@ -386,6 +391,7 @@ options("s2s_only") -> {sm_backend, mnesia}, {component_backend, mnesia}, {s2s_backend, mnesia}, + {instrumentation, #{}}, {{auth, <<"dummy_host">>}, default_auth()}, {{auth, <<"localhost">>}, default_auth()}, {{modules, <<"dummy_host">>}, #{}}, diff --git a/test/config_parser_SUITE.erl b/test/config_parser_SUITE.erl index c740a30d92..288312aae2 100644 --- a/test/config_parser_SUITE.erl +++ b/test/config_parser_SUITE.erl @@ -57,6 +57,7 @@ all() -> {group, s2s}, {group, modules}, {group, services}, + {group, instrumentation}, {group, logs}]. groups() -> @@ -233,6 +234,7 @@ groups() -> incorrect_module]}, {services, [parallel], [service_domain_db, service_mongoose_system_metrics]}, + {instrumentation, [parallel], [instrumentation]}, {logs, [], [no_warning_about_subdomain_patterns, no_warning_for_resolvable_domain]} ]. @@ -2909,6 +2911,19 @@ service_mongoose_system_metrics(_Config) -> ?err(T(#{<<"tracking_id">> => #{<<"secret">> => 666, <<"id">> => 666}})), ?err(T(#{<<"report">> => <<"maybe">>})). +%% Instrumentation + +instrumentation(_Config) -> + P = [instrumentation], + T = fun(Opts) -> #{<<"instrumentation">> => Opts} end, + ?cfg(P, #{}, T(#{})), + ?cfg(P, #{prometheus => #{}}, T(#{<<"prometheus">> => #{}})), + ?cfg(P, #{exometer => #{}}, T(#{<<"exometer">> => #{}})), + ?err(T(#{<<"prometheus">> => #{<<"fire">> => 1}})), + ?err(T(#{<<"bad_module">> => #{}})). + +%% Logs + no_warning_about_subdomain_patterns(_Config) -> check_module_defaults(mod_vcard), check_iqdisc(mod_vcard), diff --git a/test/config_parser_SUITE_data/miscellaneous.toml b/test/config_parser_SUITE_data/miscellaneous.toml index 6ff3da56a3..026bc436f4 100644 --- a/test/config_parser_SUITE_data/miscellaneous.toml +++ b/test/config_parser_SUITE_data/miscellaneous.toml @@ -79,3 +79,7 @@ [internal_databases.mnesia] [internal_databases.cets] + +[instrumentation.prometheus] + +[instrumentation.exometer] diff --git a/test/config_parser_SUITE_data/mongooseim-pgsql.toml b/test/config_parser_SUITE_data/mongooseim-pgsql.toml index c2d88921a7..7042d7c95a 100644 --- a/test/config_parser_SUITE_data/mongooseim-pgsql.toml +++ b/test/config_parser_SUITE_data/mongooseim-pgsql.toml @@ -198,6 +198,8 @@ [modules.mod_carboncopy] +[instrumentation.exometer] + [shaper.normal] max_rate = 1000 diff --git a/test/mongoose_config_SUITE.erl b/test/mongoose_config_SUITE.erl index c2783954df..20d628cd37 100644 --- a/test/mongoose_config_SUITE.erl +++ b/test/mongoose_config_SUITE.erl @@ -186,7 +186,8 @@ minimal_config_opts() -> {auth, <<"localhost">>} => config_parser_helper:default_auth(), {modules, <<"localhost">>} => #{}, {replaced_wait_timeout, <<"localhost">>} => 2000, - {s2s, <<"localhost">>} => config_parser_helper:default_s2s()}. + {s2s, <<"localhost">>} => config_parser_helper:default_s2s(), + instrumentation => #{}}. start_slave_node(Config) -> SlaveNode = do_start_slave_node(),