app: add pending reply status, persist replies in the database

alembic/versions/86b01b6290da_add_reply_draft.py

@@ -0,0 +1,67 @@
+"""add reply draft
+
+Revision ID: 86b01b6290da
+Revises: 36a79ffcfbfb
+Create Date: 2019-10-17 09:45:07.643076
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '86b01b6290da'
+down_revision = '36a79ffcfbfb'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('replysendstatuses',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('name', sa.String(length=36), nullable=False),
+        sa.PrimaryKeyConstraint('id', name=op.f('pk_replysendstatuses')),
+        sa.UniqueConstraint('name', name=op.f('uq_replysendstatuses_name'))
+    )
+
+    # Set the initial in-progress send statuses: PENDING, FAILED
+    conn = op.get_bind()
+    conn.execute('''
+        INSERT INTO replysendstatuses
+        ('name')
+        VALUES
+        ('PENDING'), 
+        ('FAILED');
+    ''')
+
+    op.create_table(
+        'draftreplies',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('uuid', sa.String(length=36), nullable=False),
+        sa.Column('timestamp', sa.DateTime(), nullable=False),
+        sa.Column('source_id', sa.Integer(), nullable=False),
+        sa.Column('journalist_id', sa.Integer(), nullable=True),
+        sa.Column('file_counter', sa.Integer(), nullable=False),
+        sa.Column('content', sa.Text(), nullable=True),
+        sa.Column('send_status_id', sa.Integer(), nullable=True),
+        sa.PrimaryKeyConstraint('id', name=op.f('pk_draftreplies')),
+        sa.UniqueConstraint('uuid', name=op.f('uq_draftreplies_uuid')),
+        sa.ForeignKeyConstraint(
+            ['source_id'],
+            ['sources.id'],
+            name=op.f('fk_draftreplies_source_id_sources')),
+        sa.ForeignKeyConstraint(
+            ['journalist_id'],
+            ['users.id'],
+            name=op.f('fk_draftreplies_journalist_id_users')),
+        sa.ForeignKeyConstraint(
+            ['send_status_id'],
+            ['replysendstatuses.id'],
+            op.f('fk_draftreplies_send_status_id_replysendstatuses'),
+        )
+    )
+
+
+def downgrade():
+    op.drop_table('draftreplies')
+    op.drop_table('replysendstatuses')

create_dev_data.py

@@ -1,9 +1,12 @@
 #!/usr/bin/env python3
 import json
 import os
+from sqlalchemy.orm.exc import NoResultFound
 import sys
+
 from securedrop_client.config import Config
-from securedrop_client.db import Base, make_session_maker
+from securedrop_client.db import Base, make_session_maker, ReplySendStatus
+from securedrop_client.api_jobs.uploads import ReplySendStatusCodes
 
 sdc_home = sys.argv[1]
 session = make_session_maker(sdc_home)()
@@ -13,3 +16,12 @@
     f.write(json.dumps({
         'journalist_key_fingerprint': '65A1B5FF195B56353CC63DFFCC40EF1228271441',
     }))
+
+for reply_send_status in ReplySendStatusCodes:
+    try:
+        reply_status = session.query(ReplySendStatus).filter_by(
+                name=reply_send_status.value).one()
+    except NoResultFound:
+        reply_status = ReplySendStatus(reply_send_status.value)
+        session.add(reply_status)
+        session.commit()

securedrop_client/api_jobs/uploads.py

@@ -1,16 +1,24 @@
+from enum import Enum
 import logging
 import sdclientapi
 
 from sdclientapi import API, RequestTimeoutError
+from sqlalchemy import and_
 from sqlalchemy.orm.session import Session
 
 from securedrop_client.api_jobs.base import ApiJob
 from securedrop_client.crypto import GpgHelper
-from securedrop_client.db import Reply, Source
+from securedrop_client.db import DraftReply, Reply, ReplySendStatus, Source
 
 logger = logging.getLogger(__name__)
 
 
+class ReplySendStatusCodes(Enum):
+    """In progress (sending) replies can currently have the following statuses"""
+    PENDING = 'PENDING'
+    FAILED = 'FAILED'
+
+
 class SendReplyJob(ApiJob):
     def __init__(self, source_uuid: str, reply_uuid: str, message: str, gpg: GpgHelper) -> None:
         super().__init__()
@@ -28,28 +36,72 @@ def call_api(self, api_client: API, session: Session) -> str:
         we can return the reply uuid.
         '''
         try:
-            encrypted_reply = self.gpg.encrypt_to_source(self.source_uuid, self.message)
-            sdk_reply = self._make_call(encrypted_reply, api_client)
+            draft_reply_db_object = session.query(DraftReply).filter_by(uuid=self.reply_uuid).one()
             source = session.query(Source).filter_by(uuid=self.source_uuid).one()
+            session.commit()
+
+            encrypted_reply = self.gpg.encrypt_to_source(self.source_uuid, self.message)
+            interaction_count = source.interaction_count + 1
+            filename = '{}-{}-reply.gpg'.format(interaction_count,
+                                                source.journalist_designation)
             reply_db_object = Reply(
                 uuid=self.reply_uuid,
                 source_id=source.id,
+                filename=filename,
                 journalist_id=api_client.token_journalist_uuid,
-                filename=sdk_reply.filename,
                 content=self.message,
                 is_downloaded=True,
-                is_decrypted=True
+                is_decrypted=True,
             )
+            sdk_reply = self._make_call(encrypted_reply, api_client)
+
+            # Update filename and file_counter in case they changed on the server
+            new_file_counter = int(sdk_reply.filename.split('-')[0])
+            reply_db_object.file_counter = new_file_counter
+            reply_db_object.filename = sdk_reply.filename
+
+            draft_file_counter = draft_reply_db_object.file_counter
+            draft_timestamp = draft_reply_db_object.timestamp
+
+            # If there were replies also in draft state sent after this one,
+            # re-position them after this successfully sent reply.
+            for draft_reply in session.query(DraftReply) \
+                                      .filter(and_(DraftReply.source_id == source.id,
+                                                   DraftReply.timestamp > draft_timestamp,
+                                                   DraftReply.file_counter == draft_file_counter)) \
+                                      .all():
+                draft_reply.file_counter = new_file_counter
+                session.add(draft_reply)
+
+            # Delete draft, add reply to replies table.
             session.add(reply_db_object)
+            session.delete(draft_reply_db_object)
             session.commit()
+
             return reply_db_object.uuid
         except RequestTimeoutError as e:
             message = "Failed to send reply for source {id} due to Exception: {error}".format(
                 id=self.source_uuid, error=e)
+
+            # Update draft reply send status to FAILED
+            reply_status = session.query(ReplySendStatus).filter_by(
+                name=ReplySendStatusCodes.FAILED.value).one()
+            draft_reply_db_object.send_status_id = reply_status.id
+            session.add(draft_reply_db_object)
+            session.commit()
+
             raise SendReplyJobTimeoutError(message, self.reply_uuid)
         except Exception as e:
             message = "Failed to send reply for source {id} due to Exception: {error}".format(
                 id=self.source_uuid, error=e)
+
+            # Update draft reply send status to FAILED
+            reply_status = session.query(ReplySendStatus).filter_by(
+                name=ReplySendStatusCodes.FAILED.value).one()
+            draft_reply_db_object.send_status_id = reply_status.id
+            session.add(draft_reply_db_object)
+            session.commit()
+
             raise SendReplyJobError(message, self.reply_uuid)
 
     def _make_call(self, encrypted_reply: str, api_client: API) -> sdclientapi.Reply:

securedrop_client/db.py

@@ -1,3 +1,4 @@
+import datetime
 import os
 
 from typing import Any, List, Union  # noqa: F401
@@ -54,7 +55,11 @@ def collection(self) -> List:
         collection.extend(self.messages)
         collection.extend(self.files)
         collection.extend(self.replies)
-        collection.sort(key=lambda x: x.file_counter)
+        collection.extend(self.draftreplies)
+        # Sort first by the file_counter, then by timestamp (used only for draft replies).
+        collection.sort(key=lambda x: (x.file_counter,
+                                       getattr(x, "timestamp",
+                                               datetime.datetime(datetime.MINYEAR, 1, 1))))
         return collection
 
 
@@ -214,6 +219,55 @@ def __repr__(self) -> str:
         return '<Reply {}>'.format(self.filename)
 
 
+class DraftReply(Base):
+
+    __tablename__ = 'draftreplies'
+
+    id = Column(Integer, primary_key=True)
+    uuid = Column(String(36), unique=True, nullable=False)
+    timestamp = Column(DateTime, nullable=False)
+    source_id = Column(Integer, ForeignKey('sources.id'), nullable=False)
+    source = relationship("Source",
+                          backref=backref("draftreplies", order_by=id,
+                                          cascade="delete"))
+    journalist_id = Column(Integer, ForeignKey('users.id'))
+    journalist = relationship(
+        "User", backref=backref('draftreplies', order_by=id))
+
+    # Tracks where in this conversation the draft reply was sent.
+    # This points to the file_counter of the previous conversation item.
+    file_counter = Column(Integer, nullable=False)
+    content = Column(Text)
+
+    # This tracks the sending status of the reply.
+    send_status_id = Column(
+        Integer,
+        ForeignKey('replysendstatuses.id')
+    )
+    send_status = relationship("ReplySendStatus")
+
+    def __init__(self, **kwargs: Any) -> None:
+        super().__init__(**kwargs)
+
+    def __repr__(self) -> str:
+        return '<DraftReply {}>'.format(self.uuid)
+
+
+class ReplySendStatus(Base):
+
+    __tablename__ = 'replysendstatuses'
+
+    id = Column(Integer, primary_key=True)
+    name = Column(String(36), unique=True, nullable=False)
+
+    def __init__(self, name: str) -> None:
+        super().__init__()
+        self.name = name
+
+    def __repr__(self) -> str:
+        return '<Reply status {}>'.format(self.name)
+
+
 class User(Base):
 
     __tablename__ = 'users'