Merge pull request #3616 from esl/inbox/async_removes

Inbox/async removes

big_tests/tests/inbox_extensions_SUITE.erl

@@ -78,6 +78,7 @@ groups() ->
         box_full_archive_can_be_fetched_queryid,
         box_and_archive_box_has_preference,
         box_other_does_get_fetched,
+        box_all_full_fetch,
         % archive
         archive_active_entry_gets_archived,
         archive_archived_entry_gets_active_on_request,
@@ -417,6 +418,16 @@ box_other_does_get_fetched(Config) ->
         inbox_helper:check_inbox(Bob, [], #{box => archive})
     end).
 
+box_all_full_fetch(Config) ->
+    escalus:fresh_story(Config, [{alice, 1}, {bob, 1}, {kate, 1}], fun(Alice, Bob, Kate) ->
+        % Alice sends a message to Bob and Kate
+        #{ Alice := AliceConvs } = inbox_helper:given_conversations_between(Alice, [Bob, Kate]),
+        inbox_helper:check_inbox(Alice, AliceConvs),
+        set_inbox_properties(Alice, Bob, [{box, archive}]),
+        set_inbox_properties(Alice, Kate, [{box, other}]),
+        inbox_helper:check_inbox(Alice, AliceConvs, #{box => all})
+    end).
+
 % archive
 archive_active_entry_gets_archived(Config) ->
     escalus:fresh_story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->

big_tests/tests/inbox_helper.erl

@@ -124,11 +124,13 @@ inbox_opts() ->
     config_parser_helper:default_mod_config(mod_inbox).
 
 inbox_opts(regular) ->
-    (inbox_opts())#{boxes => [<<"inbox">>, <<"archive">>, <<"other">>]};
+    DefOps = #{boxes := Boxes} = inbox_opts(),
+    DefOps#{boxes := Boxes ++ [<<"other">>]};
 inbox_opts(async_pools) ->
-    (inbox_opts())#{backend => rdbms_async,
-                    async_writer => #{pool_size => 4},
-                    boxes => [<<"inbox">>, <<"archive">>, <<"other">>]}.
+    DefOps = #{boxes := Boxes} = inbox_opts(),
+    DefOps#{backend => rdbms_async,
+            async_writer => #{pool_size => 1},
+            boxes => Boxes ++ [<<"other">>]}.
 
 skip_or_run_inbox_tests(TestCases) ->
     case (not ct_helper:is_ct_running())
@@ -148,6 +150,8 @@ maybe_run_in_parallel(Gs) ->
 insert_parallels(Gs) ->
     Fun = fun({muclight_config, Conf, Tests}) ->
                   {muclight_config, Conf, Tests};
+             ({bin, Conf, Tests}) ->
+                  {bin, Conf, Tests};
              ({regular, Conf, Tests}) ->
                   {regular, Conf, Tests};
              ({async_pools, Conf, Tests}) ->
@@ -159,7 +163,8 @@ insert_parallels(Gs) ->
 
 inbox_modules(Backend) ->
     [
-     {mod_inbox, inbox_opts(Backend)}
+     {mod_inbox, inbox_opts(Backend)},
+     {mod_inbox_commands, #{}}
     ].
 
 muclight_modules() ->
@@ -689,7 +694,8 @@ create_room_send_msg_check_inbox(Owner, MemberList, RoomName, Msg, Id) ->
     OwnerRoomJid = <<RoomJid/binary,"/", OwnerJid/binary>>,
     %% Owner sent the message so he has unread set to 0
     check_inbox(Owner, [#conv{unread = 0, from = OwnerRoomJid, to = OwnerJid, content = Msg}]),
-    foreach_check_inbox(MemberList, 1, OwnerRoomJid, Msg).
+    foreach_check_inbox(MemberList, 1, OwnerRoomJid, Msg),
+    RoomJid.
 
 verify_is_owner_aff_change(Client, Msg) ->
     verify_muc_light_aff_msg(Msg, [{Client,  owner}]).

big_tests/tests/sm_SUITE.erl

@@ -170,7 +170,8 @@ init_per_testcase(CaseName, Config) ->
 end_per_testcase(CN, Config) when CN =:= resume_expired_session_returns_correct_h;
                                   CN =:= gc_repeat_after_never_means_no_cleaning;
                                   CN =:= gc_repeat_after_timeout_does_clean ->
-    rpc(mim(), ejabberd_sup, stop_child, [stream_management_stale_h]),
+    Name = rpc(mim(), gen_mod, get_module_proc, [host_type(), stream_management_stale_h]),
+    rpc(mim(), ejabberd_sup, stop_child, [Name]),
     escalus:end_per_testcase(CN, Config);
 end_per_testcase(replies_are_processed_by_resumed_session = CN, Config) ->
     unregister_handler(),

doc/configuration/Modules.md

@@ -106,6 +106,9 @@ Implements [XEP-0363: HTTP File Upload](https://xmpp.org/extensions/xep-0363.htm
 ### [mod_inbox](../modules/mod_inbox.md)
 Implements custom inbox XEP
 
+### [mod_inbox_commands](../modules/mod_inbox_commands.md)
+Exposes administrative commands for the [inbox](../modules/mod_inbox.md)
+
 ### [mod_global_distrib](../modules/mod_global_distrib.md)
 Enables sharing a single XMPP domain between distinct datacenters (**experimental**).
 

doc/modules/mod_inbox.md

@@ -30,7 +30,23 @@ Number of workers in the pool. More than the number of available schedulers is r
 A list of supported inbox boxes by the server. This can be used by clients to classify their inbox entries in any way that fits the end-user. The strings provided here will be used verbatim in the IQ query as described in [Inbox – Filtering and Ordering](../open-extensions/inbox.md#filtering-and-ordering).
 
 !!! note
-    `inbox` and `archive` are always enabled, and therefore don't need to be specified here.
+    `inbox`, `archive`, and `bin` are reserved box names and are always enabled, therefore they don't need to –and must not– be specified in this section. `all` has a special meaning in the box query and therefore is also not allowed as a box name.
+
+    If the asynchronous backend is configured, automatic removals become moves to the `bin` box, also called "Trash bin". This is to ensure eventual consistency. Then the bin can be emptied, either on a [user request](../open-extensions/inbox.md#examples-emptying-the-trash-bin), or through an [admin API endpoint](../mod_inbox_commands#admin-endpoint).
+
+#### `modules.mod_inbox.bin_ttl`
+* **Syntax:** non-negative integer, expressed in days.
+* **Default:** `30`
+* **Example:** `modules.mod_inbox.bin_ttl = 7`
+
+How old entries in the bin can be before the automatic bin cleaner collects them. A value of `7` would mean that entries that have been in the bin for more than 7 days will be cleaned on the next bin collection.
+
+#### `modules.mod_inbox.bin_clean_after`
+* **Syntax:** non-negative integer, expressed in hours
+* **Default:** `1`
+* **Example:** `modules.mod_inbox.bin_clean_after = 24`
+
+How often the automatic garbage collection runs over the bin.
 
 ### `modules.mod_inbox.reset_markers`
 * **Syntax:** array of strings, out of `"displayed"`, `"received"`, `"acknowledged"`
@@ -93,6 +109,7 @@ or [affiliation](https://xmpp.org/extensions/xep-0045.html#affil) change.
 
 ```toml
 [modules.mod_inbox]
+  backend = "rdbms_async"
   reset_markers = ["displayed"]
   aff_changes = true
   remove_on_kicked = true

doc/modules/mod_inbox_commands.md

@@ -0,0 +1,47 @@
+## Configuration
+This module contains command definitions which are loaded when the module is activated.
+There are no options to be provided, therefore the following entry in the config file is sufficient:
+
+```toml
+[modules.mod_inbox_commands]
+```
+
+## Admin endpoint
+
+### Bin flush for a user
+To clean the bin for a given user, the following admin API request can be triggered:
+
+```http
+DELETE /api/inbox/<domain>/<user>/<days>/bin,
+```
+where `<domain>` and `<user>` are the domain and name parts of the user's jid, respectively, and `<days>` is the required number of days for an entry to be considered old enough to be removed, zero allowed (which clears all).
+
+The result would be a `200` with the number of rows that were removed as the body, or a corresponding error. For example, if only one entry was cleaned:
+```http
+HTTP/1.1 200 OK
+server: Cowboy,
+date: Wed, 30 Mar 2022 14:06:20 GMT,
+content-type: application/json,
+content-length: 1
+
+1
+```
+
+### Global bin flush
+If all the bins were desired to be cleared, the following API can be used instead:
+
+```http
+DELETE /api/inbox/<host_type>/<days>/bin,
+```
+where as before, `<days>` is the required number of days for an entry to be considered old enough to be removed, and `<host_type>` is the host type where inbox is configured.
+
+The result would look analogously:
+```http
+HTTP/1.1 200 OK
+server: Cowboy,
+date: Wed, 30 Mar 2022 14:06:20 GMT,
+content-type: application/json,
+content-length: 1
+
+42
+```

doc/open-extensions/inbox.md

@@ -59,6 +59,7 @@ The inbox is fetched using regular XMPP [Data Forms]. To request the supported f
         <option label='all'><value>all</value></option>
         <option label='inbox'><value>inbox</value></option>
         <option label='archive'><value>archive</value></option>
+        <option label='bin'><value>bin</value></option>
       </field>
       <field var='archive' type='boolean'/>
     </x>
@@ -120,7 +121,7 @@ A client may specify the following parameters:
 * variable `end`: End date for the result set (value: ISO timestamp)
 * variable `order`: Order by timestamp (values: `asc`, `desc`)
 * variable `hidden_read`: Show only conversations with unread messages (values: `true`, `false`)
-* variable `box`: Indicate which box is desired. Supported are `all`, `inbox`, and `archive`. More boxes can be implemented, see [mod_inbox – Boxes](../modules/mod_inbox.md#modulesmod_inboxboxes)
+* variable `box`: Indicate which box is desired. Supported are `all`, `inbox`, `archive` and `bin`. More boxes can be implemented, see [mod_inbox – Boxes](../modules/mod_inbox.md#modulesmod_inboxboxes). If not provided, all except the bin are returned.
 * variable `archive` [deprecated, prefer `box`]: whether to query the archive inbox. `true` means querying only the archive box, `false` means querying only the active box. If the flag is not set, it is assumed all entries are requested. This is kept for backwards compatibility reasons, use the `box` flag instead.
 
 They are encoded inside a standard XMPP [Data Forms] format.
@@ -152,11 +153,11 @@ where `Max` is a non-negative integer.
 Given an entry, certain properties are defined for such an entry:
 
 ### Box
-Clients usually have two different boxes for the inbox: the regular one, simply called the inbox (or the active inbox), and an archive box, where clients can manually throw conversations they don't want displayed in the default UI.
+Clients usually have two different boxes for the inbox: the regular one, simply called the inbox (or the active inbox), and an archive box, where clients can manually throw conversations they don't want displayed in the default UI. A third box is the trash bin, where deleted entries go and are cleaned up in regular intervals.
 
 It is expected that entries will reside in the archive until they're either manually moved back to the active box, or they receive a new message: in such case the entry should jump back to the active box automatically.
 
-More boxes can be implemented, see [mod_inbox#boxes](../modules/mod_inbox.md#modulesmod_inboxboxes). Movement between boxes can be achieved through the right XMPP IQ, no automatic movements are developed as in the case of inbox-archive.
+More boxes can be implemented, see [mod_inbox#boxes](../modules/mod_inbox.md#modulesmod_inboxboxes). Movement between boxes can be achieved through the right XMPP IQ, no more automatic movements are developed as in the case of inbox-archive.
 
 ### Read
 Entries keep a count of unread messages that is incremented automatically upon receiving a new message, and (in the current implementation) set to zero upon receiving either a message by one-self, or an appropriate chat marker as defined in [XEP-0333](https://xmpp.org/extensions/xep-0333.html) (which markers reset the count is a matter of configuration, see [doc](../modules/mod_inbox.md#modulesmod_inboxreset_markers)).
@@ -187,6 +188,12 @@ The server would respond with:
       <field var='archive' type='boolean' value='false'/>
       <field var='read' type='boolean' value='false'/>
       <field var='mute' type='text-single' value='0'/>
+      <field var='box' type='list-simple' value='all'>
+        <option label='all'><value>all</value></option>
+        <option label='inbox'><value>inbox</value></option>
+        <option label='archive'><value>archive</value></option>
+        <option label='bin'><value>bin</value></option>
+      </field>
     </x>
   </query>
 </iq>
@@ -313,6 +320,23 @@ If the client had sent an invalid number (negative, or NaN), the server would an
 </iq>
 ```
 
+### Examples: emptying the trash bin
+A user can empty his trash bin, through the following request:
+```xml
+<iq id='some_unique_id' type='set'>
+  <empty-bin xmlns='erlang-solutions.com:xmpp:inbox:0'/>
+</iq>
+```
+On success, the server would return how many entries where dropped as in:
+```xml
+<iq id='some_unique_id' to='alice@localhost/res1' type='result'>
+  <empty-bin xmlns='erlang-solutions.com:xmpp:inbox:0'>
+    <num>2</num>
+  </empty-bin>
+</iq>
+```
+The server might answer with a corresponding error message, might anything go wrong.
+
 ### Examples: muting an entry
 To mute an entry for a full day (86400 seconds in a day, 604800 in a week, for example), a client can send:
 ```xml
@@ -388,7 +412,7 @@ And similarly, to set a conversation as unread:
 </iq>
 ```
 
-### Deprecated entry stanza:
+### Deprecated reset entry stanza:
 You can reset the inbox with the following stanza:
 ```xml
 <iq type='set'>

include/mod_inbox.hrl

@@ -1,5 +1,3 @@
--type content() :: binary().
-
 -type id() :: binary().
 
 -type marker() :: binary().

mkdocs.yml

@@ -112,6 +112,7 @@ nav:
       - 'mod_global_distrib': 'modules/mod_global_distrib.md'
       - 'mod_http_upload': 'modules/mod_http_upload.md'
       - 'mod_inbox': 'modules/mod_inbox.md'
+      - 'mod_inbox_commands': 'modules/mod_inbox_commands.md'
       - 'mod_jingle_sip': 'modules/mod_jingle_sip.md'
       - 'mod_keystore': 'modules/mod_keystore.md'
       - 'mod_last': 'modules/mod_last.md'

priv/mssql2012.sql

@@ -518,6 +518,12 @@ GO
 CREATE INDEX i_inbox_su_ts ON inbox(lserver, luser, timestamp);
 GO
 
+CREATE INDEX i_inbox_us_box ON inbox(lserver, luser, box);
+GO
+
+CREATE INDEX i_inbox_box ON inbox(box);
+GO
+
 CREATE TABLE dbo.pubsub_nodes (
     nidx BIGINT           IDENTITY(1,1) PRIMARY KEY,
     p_key NVARCHAR(150)   NOT NULL,

priv/mysql.sql

@@ -398,6 +398,8 @@ CREATE TABLE inbox (
     PRIMARY KEY(lserver, luser, remote_bare_jid));
 
 CREATE INDEX i_inbox USING BTREE ON inbox(lserver, luser, timestamp);
+CREATE INDEX i_inbox_us_box USING BTREE ON inbox(lserver, luser, box);
+CREATE INDEX i_inbox_box USING BTREE ON inbox(box);
 
 CREATE TABLE pubsub_nodes (
     nidx BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,

priv/pg.sql

@@ -357,9 +357,9 @@ CREATE TABLE inbox (
     unread_count INT                 NOT NULL,
     PRIMARY KEY(lserver, luser, remote_bare_jid));
 
-CREATE INDEX i_inbox_timestamp
-    ON inbox
-    USING BTREE(lserver, luser, timestamp);
+CREATE INDEX i_inbox_timestamp ON inbox USING BTREE(lserver, luser, timestamp);
+CREATE INDEX i_inbox_us_box ON inbox USING BTREE(lserver, luser, box);
+CREATE INDEX i_inbox_box ON inbox (box) WHERE (box = 'bin');
 
 CREATE TABLE pubsub_nodes (
     nidx               BIGSERIAL PRIMARY KEY,