diff --git a/applications/crossbar/priv/api/swagger.json b/applications/crossbar/priv/api/swagger.json index 384622b227c..30ba605be90 100644 --- a/applications/crossbar/priv/api/swagger.json +++ b/applications/crossbar/priv/api/swagger.json @@ -35805,6 +35805,37 @@ }, "type": "object" }, + "system_config.tasks.cleanup": { + "description": "Schema for tasks.cleanup system_config", + "properties": { + "cleanup_dbs": { + "default": { + "classifications": [ + "account", + "modb", + "yodb", + "ratedeck", + "resource_selectors" + ], + "databases": [ + "alerts", + "accounts", + "system_config", + "system_data", + "functions", + "system_media", + "offnet", + "port_requests", + "sip_auth", + "webhooks" + ] + }, + "description": "tasks cleanup cleanup_dbs", + "type": "object" + } + }, + "type": "object" + }, "system_config.tasks.fax_cleanup": { "description": "Schema for tasks.fax_cleanup system_config", "properties": { diff --git a/applications/crossbar/priv/couchdb/schemas/system_config.tasks.cleanup.json b/applications/crossbar/priv/couchdb/schemas/system_config.tasks.cleanup.json new file mode 100644 index 00000000000..22125c74ce8 --- /dev/null +++ b/applications/crossbar/priv/couchdb/schemas/system_config.tasks.cleanup.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "_id": "system_config.tasks.cleanup", + "description": "Schema for tasks.cleanup system_config", + "properties": { + "cleanup_dbs": { + "default": { + "classifications": [ + "account", + "modb", + "yodb", + "ratedeck", + "resource_selectors" + ], + "databases": [ + "alerts", + "accounts", + "system_config", + "system_data", + "functions", + "system_media", + "offnet", + "port_requests", + "sip_auth", + "webhooks" + ] + }, + "description": "tasks cleanup cleanup_dbs", + "type": "object" + } + }, + "type": "object" +} diff --git a/applications/tasks/src/modules/kt_cleanup.erl b/applications/tasks/src/modules/kt_cleanup.erl index cc45be81ae6..650fec114f9 100644 --- a/applications/tasks/src/modules/kt_cleanup.erl +++ b/applications/tasks/src/modules/kt_cleanup.erl @@ -2,12 +2,16 @@ %%% @copyright (C) 2013-2020, 2600Hz %%% @doc %%% @author Pierre Fenoll +%%% +%%% This Source Code Form is subject to the terms of the Mozilla Public +%%% License, v. 2.0. If a copy of the MPL was not distributed with this +%%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%%% %%% @end %%%----------------------------------------------------------------------------- -module(kt_cleanup). %% behaviour: tasks_provider - -export([init/0]). %% Triggerables @@ -15,11 +19,42 @@ -include("tasks.hrl"). +-define(CLEANUP_CAT, <<(?CONFIG_CAT)/binary, ".cleanup">>). + %% How long to pause before attempting to delete the next chunk of soft-deleted docs -define(SOFT_DELETE_PAUSE ,kapps_config:get_integer(?CONFIG_CAT, <<"soft_delete_pause_ms">>, 10 * ?MILLISECONDS_IN_SECOND) ). + +-define(DEFAULT_CLEANUP + ,kz_json:from_list( + [{<<"classifications">>, [<<"account">> + ,<<"modb">> + ,<<"yodb">> + ,<<"ratedeck">> + ,<<"resource_selectors">> + ] + } + ,{<<"databases">>, [?KZ_ALERTS_DB + ,?KZ_ACCOUNTS_DB + ,?KZ_CONFIG_DB + ,?KZ_DATA_DB + ,?KZ_MEDIA_DB + ,?KZ_OFFNET_DB + ,?KZ_PORT_REQUESTS_DB + ,?KZ_SIP_DB + ,?KZ_WEBHOOKS_DB + ,<<"functions">> + ] + } + ] + ) + ). +-define(CLEANUP_DBS + ,kapps_config:get_json(?CLEANUP_CAT, <<"cleanup_dbs">>, ?DEFAULT_CLEANUP) + ). + %%%============================================================================= %%% API %%%============================================================================= @@ -34,16 +69,21 @@ init() -> %%% Triggerables --spec cleanup_soft_deletes(kz_term:ne_binary()) -> ok. -cleanup_soft_deletes(?KZ_ACCOUNTS_DB) -> - do_cleanup(?KZ_ACCOUNTS_DB); -cleanup_soft_deletes(Account) -> - kz_datamgr:suppress_change_notice(), - case kapps_util:is_account_db(Account) of - 'true' -> cleanup_account_soft_deletes(Account); +-spec cleanup_soft_deletes(kz_term:ne_binary()) -> 'ok'. +cleanup_soft_deletes(Db) -> + CleanupDbs = ?CLEANUP_DBS, + Classification = kzs_util:db_classification(Db), + case lists:member(Db, kz_json:get_list_value(<<"databases">>, CleanupDbs, [])) + orelse lists:member(kz_term:to_binary(Classification) + ,kz_json:get_list_value(<<"classifications">>, CleanupDbs, []) + ) + of + 'true' -> + lager:debug("cleaning up database ~s(~s)", [Db, Classification]), + kz_datamgr:suppress_change_notice(), + cleanup_db(Db, Classification), + kz_datamgr:enable_change_notice(); 'false' -> - _ = kz_datamgr:enable_change_notice(), - %% no longer checking other dbs for soft deletes 'ok' end. @@ -55,26 +95,44 @@ cleanup_soft_deletes(Account) -> %% @doc %% @end %%------------------------------------------------------------------------------ --spec cleanup_account_soft_deletes(kz_term:ne_binary()) -> 'ok'. -cleanup_account_soft_deletes(Account) -> - AccountDb = kz_util:format_account_id(Account, 'encoded'), - do_cleanup(AccountDb). - --spec do_cleanup(kz_term:ne_binary()) -> 'ok'. -do_cleanup(Db) -> - View = <<"maintenance/soft_deletes">>, +-spec cleanup_db(kz_term:ne_binary(), kazoo_data:db_classification()) -> 'ok'. +cleanup_db(Db, _Classfication) -> + do_cleanup_soft_deleted(Db). + +-spec do_cleanup_soft_deleted(kz_term:ne_binary()) -> 'ok'. +do_cleanup_soft_deleted(Db) -> + do_cleanup(Db, <<"maintenance/soft_deletes">>, 10). + +-spec do_cleanup(kz_term:ne_binary(), kz_term:ne_binary(), integer()) -> 'ok'. +do_cleanup(_Db, _View, Loop) when Loop =< 0 -> + lager:debug("reached to maximum 10 loops for database ~s", [_Db]); +do_cleanup(Db, View, Loop) -> ViewOptions = [{'limit', kz_datamgr:max_bulk_insert()}], case kz_datamgr:get_results(Db, View, ViewOptions) of - {'ok', []} -> 'ok'; - {'ok', L} -> - lager:debug("removing ~b soft-deleted docs from ~s", [length(L), Db]), - _ = kz_datamgr:del_docs(Db, L), - 'ok' = timer:sleep(?SOFT_DELETE_PAUSE), - do_cleanup(Db); - {'error', 'not_found'} -> - lager:warning("db ~s or view '~s' not found", [Db, View]); + {'ok', []} -> + lager:debug("no soft delete in db ~s", [Db]); + {'ok', Result} -> + delete_soft_deleted(Db, [build_id_rev(JObj) || JObj <- Result]), + do_cleanup(Db, View, Loop - 1); {'error', _E} -> lager:debug("failed to lookup soft-deleted tokens: ~p", [_E]) end. +-spec delete_soft_deleted(kz_term:ne_binary(), kz_json:objects()) -> 'ok'. +delete_soft_deleted(Db, JObjs) -> + lager:debug("removing ~b soft-deleted docs from ~s", [length(JObjs), Db]), + _ = kz_datamgr:del_docs(Db, JObjs), + timer:sleep(?SOFT_DELETE_PAUSE). + +-spec build_id_rev(kz_json:object()) -> kz_json:object(). +build_id_rev(JObj0) -> + JObj = + kz_json:get_ne_json_value(<<"value">>, JObj0 + ,kz_json:get_ne_json_value(<<"doc">>, JObj0, JObj0) + ), + kz_json:from_list( + [{<<"_id">>, kz_doc:id(JObj)} + ,{<<"_rev">>, kz_doc:revision(JObj)} + ]). + %%% End of Module. diff --git a/core/kazoo_apps/priv/couchdb/views/maintenance.json b/core/kazoo_apps/priv/couchdb/views/maintenance.json index da0a14f8da1..266567da2e5 100644 --- a/core/kazoo_apps/priv/couchdb/views/maintenance.json +++ b/core/kazoo_apps/priv/couchdb/views/maintenance.json @@ -5,20 +5,29 @@ { "classification": "account" }, + { + "classification": "acdc" + }, + { + "classification": "aggregate" + }, { "classification": "modb" }, { - "database": "accounts" + "classification": "yodb" + }, + { + "classification": "numbers" }, { - "database": "sip_auth" + "classification": "ratedeck" }, { - "database": "system_config" + "classification": "resource_selectors" }, { - "database": "system_data" + "classification": "system" } ] }, @@ -41,7 +50,16 @@ "reduce": "_count" }, "soft_deletes": { - "map": "function(doc) { if(doc.pvt_deleted) { emit(doc.pvt_type, null); } }" + "map": [ + "function(doc) {", + " if (doc.pvt_deleted) {", + " emit(doc.pvt_type, {", + " 'id': doc._id,", + " '_rev': doc._rev", + " });", + " }", + "}" + ] } } }