Production Deploy 3/6/2024

README.md

@@ -299,6 +299,36 @@ brew services start postgresql@15
 brew services start redis
 ```
 
+### Final environment setup
+
+There's one final thing to adjust in the newly created `.env` file.  This
+project has support for end-to-end (E2E) tests and has some additional checks
+for the presence of an E2E test user so that it can be authenticated properly.
+
+In the `.env` file, you should see this section:
+
+```
+#############################################################
+
+# E2E Testing
+
+NOTIFY_E2E_TEST_EMAIL=example@fake.gov
+NOTIFY_E2E_TEST_PASSWORD="don't write secrets to the sample file"
+```
+
+You can leave the email address alone or change it to something else to your
+liking.
+
+**You should absolutely change the `NOTIFY_E2E_TEST_PASSWORD` environment
+variable to something else, preferably a lengthy passphrase.**
+
+With those two environment variable set, the database migrations will run
+properly and an E2E test user will be ready to go for use in the admin project.
+
+_Note:  Whatever you set these two environment variables to, you'll need to
+match their values on the admin side.  Please see the admin README and
+documentation for more details._
+
 ## Running the Project and Routine Maintenance
 
 The first time you run the project you'll need to run the project setup from the
@@ -410,6 +440,7 @@ instructions above for more details.
   - [CI testing](./docs/all.md#ci-testing)
   - [Manual testing](./docs/all.md#manual-testing)
   - [To run a local OWASP scan](./docs/all.md#to-run-a-local-owasp-scan)
+  - [End-to-end testing](./docs/all.md#end-to-end-testing)
 - [Deploying](./docs/all.md#deploying)
   - [Egress Proxy](./docs/all.md#egress-proxy)
   - [Managing environment variables](./docs/all.md#managing-environment-variables)
@@ -421,6 +452,7 @@ instructions above for more details.
   - [Migrations](./docs/all.md#migrations)
   - [Purging user data](./docs/all.md#purging-user-data)
 - [One-off tasks](./docs/all.md#one-off-tasks)
+- [Test Loading Commands](./docs/all.md#commands-for-test-loading-the-local-dev-database)
 - [How messages are queued and sent](./docs/all.md#how-messages-are-queued-and-sent)
 - [Writing public APIs](./docs/all.md#writing-public-apis)
   - [Overview](./docs/all.md#overview)

app/billing/billing_schemas.py

@@ -1,5 +1,7 @@
 from datetime import datetime
 
+from app.enums import NotificationType
+
 create_or_update_free_sms_fragment_limit_schema = {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST annual billing schema",
@@ -25,7 +27,7 @@ def serialize_ft_billing_remove_emails(rows):
             "charged_units": row.charged_units,
         }
         for row in rows
-        if row.notification_type != "email"
+        if row.notification_type != NotificationType.EMAIL
     ]
 
 

app/celery/nightly_tasks.py

@@ -25,14 +25,15 @@
 from app.dao.service_data_retention_dao import (
     fetch_service_data_retention_for_all_services_by_notification_type,
 )
-from app.models import EMAIL_TYPE, SMS_TYPE, FactProcessingTime
+from app.enums import NotificationType
+from app.models import FactProcessingTime
 from app.utils import get_midnight_in_utc
 
 
 @notify_celery.task(name="remove_sms_email_jobs")
 @cronitor("remove_sms_email_jobs")
 def remove_sms_email_csv_files():
-    _remove_csv_files([EMAIL_TYPE, SMS_TYPE])
+    _remove_csv_files([NotificationType.EMAIL, NotificationType.SMS])
 
 
 def _remove_csv_files(job_types):
@@ -69,13 +70,13 @@ def delete_notifications_older_than_retention():
 @notify_celery.task(name="delete-sms-notifications")
 @cronitor("delete-sms-notifications")
 def delete_sms_notifications_older_than_retention():
-    _delete_notifications_older_than_retention_by_type("sms")
+    _delete_notifications_older_than_retention_by_type(NotificationType.SMS)
 
 
 @notify_celery.task(name="delete-email-notifications")
 @cronitor("delete-email-notifications")
 def delete_email_notifications_older_than_retention():
-    _delete_notifications_older_than_retention_by_type("email")
+    _delete_notifications_older_than_retention_by_type(NotificationType.EMAIL)
 
 
 def _delete_notifications_older_than_retention_by_type(notification_type):

app/celery/process_ses_receipts_tasks.py

@@ -20,7 +20,8 @@
     get_service_complaint_callback_api_for_service,
     get_service_delivery_status_callback_api_for_service,
 )
-from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING, Complaint
+from app.enums import CallbackType, NotificationStatus
+from app.models import Complaint
 
 
 @notify_celery.task(
@@ -76,7 +77,10 @@ def process_ses_results(self, response):
                 f"SES bounce for notification ID {notification.id}: {bounce_message}"
             )
 
-        if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}:
+        if notification.status not in {
+            NotificationStatus.SENDING,
+            NotificationStatus.PENDING,
+        }:
             notifications_dao._duplicate_update_warning(
                 notification, notification_status
             )
@@ -166,22 +170,22 @@ def get_aws_responses(ses_message):
         "Permanent": {
             "message": "Hard bounced",
             "success": False,
-            "notification_status": "permanent-failure",
+            "notification_status": NotificationStatus.PERMANENT_FAILURE,
         },
         "Temporary": {
             "message": "Soft bounced",
             "success": False,
-            "notification_status": "temporary-failure",
+            "notification_status": NotificationStatus.TEMPORARY_FAILURE,
         },
         "Delivery": {
             "message": "Delivered",
             "success": True,
-            "notification_status": "delivered",
+            "notification_status": NotificationStatus.DELIVERED,
         },
         "Complaint": {
             "message": "Complaint",
             "success": True,
-            "notification_status": "delivered",
+            "notification_status": NotificationStatus.DELIVERED,
         },
     }[status]
 
@@ -205,7 +209,7 @@ def handle_complaint(ses_message):
         )
         return
     notification = dao_get_notification_history_by_reference(reference)
-    ses_complaint = ses_message.get("complaint", None)
+    ses_complaint = ses_message.get(CallbackType.COMPLAINT, None)
 
     complaint = Complaint(
         notification_id=notification.id,
@@ -237,7 +241,7 @@ def remove_emails_from_bounce(bounce_dict):
 
 def remove_emails_from_complaint(complaint_dict):
     remove_mail_headers(complaint_dict)
-    complaint_dict["complaint"].pop("complainedRecipients")
+    complaint_dict[CallbackType.COMPLAINT].pop("complainedRecipients")
     return complaint_dict["mail"].pop("destination")
 
 

app/celery/provider_tasks.py

@@ -15,13 +15,8 @@
     update_notification_status_by_id,
 )
 from app.delivery import send_to_providers
+from app.enums import NotificationStatus
 from app.exceptions import NotificationTechnicalFailureException
-from app.models import (
-    NOTIFICATION_DELIVERED,
-    NOTIFICATION_FAILED,
-    NOTIFICATION_TECHNICAL_FAILURE,
-    NOTIFICATION_TEMPORARY_FAILURE,
-)
 
 # This is the amount of time to wait after sending an sms message before we check the aws logs and look for delivery
 # receipts
@@ -67,12 +62,12 @@ def check_sms_delivery_receipt(self, message_id, notification_id, sent_at):
             raise self.retry(exc=ntfe)
 
     if status == "success":
-        status = NOTIFICATION_DELIVERED
+        status = NotificationStatus.DELIVERED
     elif status == "failure":
-        status = NOTIFICATION_FAILED
+        status = NotificationStatus.FAILED
     # if status is not success or failure the client raised an exception and this method will retry
 
-    if status == NOTIFICATION_DELIVERED:
+    if status == NotificationStatus.DELIVERED:
         sanitize_successful_notification_by_id(
             notification_id, carrier=carrier, provider_response=provider_response
         )
@@ -126,7 +121,8 @@ def deliver_sms(self, notification_id):
         )
     except Exception as e:
         update_notification_status_by_id(
-            notification_id, NOTIFICATION_TEMPORARY_FAILURE
+            notification_id,
+            NotificationStatus.TEMPORARY_FAILURE,
         )
         if isinstance(e, SmsClientResponseException):
             current_app.logger.warning(
@@ -151,7 +147,8 @@ def deliver_sms(self, notification_id):
                 )
             )
             update_notification_status_by_id(
-                notification_id, NOTIFICATION_TECHNICAL_FAILURE
+                notification_id,
+                NotificationStatus.TECHNICAL_FAILURE,
             )
             raise NotificationTechnicalFailureException(message)
 
@@ -194,6 +191,7 @@ def deliver_email(self, notification_id):
                 )
             )
             update_notification_status_by_id(
-                notification_id, NOTIFICATION_TECHNICAL_FAILURE
+                notification_id,
+                NotificationStatus.TECHNICAL_FAILURE,
             )
             raise NotificationTechnicalFailureException(message)

app/celery/reporting_tasks.py

@@ -8,7 +8,7 @@
 from app.dao.fact_billing_dao import fetch_billing_data_for_day, update_fact_billing
 from app.dao.fact_notification_status_dao import update_fact_notification_status
 from app.dao.notifications_dao import get_service_ids_with_notifications_on_date
-from app.models import EMAIL_TYPE, SMS_TYPE
+from app.enums import NotificationType
 
 
 @notify_celery.task(name="create-nightly-billing")
@@ -80,7 +80,7 @@ def create_nightly_notification_status():
 
     yesterday = datetime.utcnow().date() - timedelta(days=1)
 
-    for notification_type in [SMS_TYPE, EMAIL_TYPE]:
+    for notification_type in (NotificationType.SMS, NotificationType.EMAIL):
         days = 4
 
         for i in range(days):

app/celery/research_mode_tasks.py

@@ -6,7 +6,7 @@
 from app.celery.process_ses_receipts_tasks import process_ses_results
 from app.config import QueueNames
 from app.dao.notifications_dao import get_notification_by_id
-from app.models import SMS_TYPE
+from app.enums import NotificationType
 
 temp_fail = "2028675303"
 perm_fail = "2028675302"
@@ -21,7 +21,7 @@ def send_sms_response(provider, reference):
     body = sns_callback(reference)
     headers = {"Content-type": "application/json"}
 
-    make_request(SMS_TYPE, provider, body, headers)
+    make_request(NotificationType.SMS, provider, body, headers)
 
 
 def send_email_response(reference, to):

app/celery/scheduled_tasks.py

@@ -34,14 +34,8 @@
 )
 from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
 from app.delivery.send_to_providers import provider_to_use
-from app.models import (
-    EMAIL_TYPE,
-    JOB_STATUS_ERROR,
-    JOB_STATUS_IN_PROGRESS,
-    JOB_STATUS_PENDING,
-    SMS_TYPE,
-    Job,
-)
+from app.enums import JobStatus, NotificationType
+from app.models import Job
 from app.notifications.process_notifications import send_notification_to_queue
 
 MAX_NOTIFICATION_FAILS = 10000
@@ -160,7 +154,7 @@ def check_db_notification_fails():
         )
     # suppress any spam coming from development tier
     if message and curr_env != "development":
-        provider = provider_to_use(EMAIL_TYPE, False)
+        provider = provider_to_use(NotificationType.EMAIL, False)
         from_address = '"{}" <{}@{}>'.format(
             "Failed Notification Count Alert",
             "test_sender",
@@ -192,11 +186,11 @@ def check_job_status():
     thirty_five_minutes_ago = datetime.utcnow() - timedelta(minutes=35)
 
     incomplete_in_progress_jobs = Job.query.filter(
-        Job.job_status == JOB_STATUS_IN_PROGRESS,
+        Job.job_status == JobStatus.IN_PROGRESS,
         between(Job.processing_started, thirty_five_minutes_ago, thirty_minutes_ago),
     )
     incomplete_pending_jobs = Job.query.filter(
-        Job.job_status == JOB_STATUS_PENDING,
+        Job.job_status == JobStatus.PENDING,
         Job.scheduled_for.isnot(None),
         between(Job.scheduled_for, thirty_five_minutes_ago, thirty_minutes_ago),
     )
@@ -211,7 +205,7 @@ def check_job_status():
     # if they haven't been re-processed in time.
     job_ids = []
     for job in jobs_not_complete_after_30_minutes:
-        job.job_status = JOB_STATUS_ERROR
+        job.job_status = JobStatus.ERROR
         dao_update_job(job)
         job_ids.append(str(job.id))
 
@@ -224,7 +218,7 @@ def check_job_status():
 def replay_created_notifications():
     # if the notification has not be send after 1 hour, then try to resend.
     resend_created_notifications_older_than = 60 * 60
-    for notification_type in (EMAIL_TYPE, SMS_TYPE):
+    for notification_type in (NotificationType.EMAIL, NotificationType.SMS):
         notifications_to_resend = notifications_not_yet_sent(
             resend_created_notifications_older_than, notification_type
         )