06/26/2024 Production Deploy

.github/actions/setup-project/action.yml

@@ -9,10 +9,10 @@ runs:
         sudo apt-get update \
         && sudo apt-get install -y --no-install-recommends \
         libcurl4-openssl-dev
-    - name: Set up Python 3.12
+    - name: Set up Python 3.12.3
       uses: actions/setup-python@v4
       with:
-        python-version: "3.12"
+        python-version: "3.12.3"
     - name: Install poetry
       shell: bash
       run: pip install --upgrade poetry

.github/workflows/checks.yml

@@ -53,7 +53,7 @@ jobs:
     - name: Check for dead code
       run: make dead-code
     - name: Run tests with coverage
-      run: poetry run coverage run --omit=*/notifications_utils/* -m pytest --maxfail=10
+      run: poetry run coverage run --omit=*/notifications_utils/*,*/migrations/* -m pytest --maxfail=10
       env:
         SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
         NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
@@ -62,7 +62,7 @@ jobs:
         NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
     - name: Check coverage threshold
       # TODO get this back up to 95
-      run: poetry run coverage report --fail-under=87
+      run: poetry run coverage report --fail-under=95
 
   validate-new-relic-config:
     runs-on: ubuntu-latest

.github/workflows/codeql.yml

@@ -0,0 +1,93 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "main", "production" ]
+  pull_request:
+    branches: [ "main", "production" ]
+  schedule:
+    - cron: '15 8 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: python
+          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      shell: bash
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/deploy-demo.yml

@@ -18,7 +18,7 @@ jobs:
 
     - name: Check for changes to Terraform
       id: changed-terraform-files
-      uses: tj-actions/changed-files@v41
+      uses: tj-actions/changed-files@v44
       with:
         files: |
           terraform/demo
@@ -76,7 +76,7 @@ jobs:
 
     - name: Check for changes to templates.json
       id: changed-templates
-      uses: tj-actions/changed-files@v41
+      uses: tj-actions/changed-files@v44
       with:
         files: |
           app/config_files/templates.json
@@ -86,7 +86,7 @@ jobs:
 
     - name: Check for changes to egress config
       id: changed-egress-config
-      uses: tj-actions/changed-files@v41
+      uses: tj-actions/changed-files@v44
       with:
         files: |
           deploy-config/egress_proxy/notify-api-demo.*.acl

.github/workflows/deploy-prod.yml

@@ -22,7 +22,7 @@ jobs:
 
     - name: Check for changes to Terraform
       id: changed-terraform-files
-      uses: tj-actions/changed-files@v41
+      uses: tj-actions/changed-files@v44
       with:
         files: |
           terraform/production
@@ -80,7 +80,7 @@ jobs:
 
     - name: Check for changes to templates.json
       id: changed-templates
-      uses: tj-actions/changed-files@v41
+      uses: tj-actions/changed-files@v44
       with:
         files: |
           app/config_files/templates.json
@@ -90,7 +90,7 @@ jobs:
 
     - name: Check for changes to egress config
       id: changed-egress-config
-      uses: tj-actions/changed-files@v41
+      uses: tj-actions/changed-files@v44
       with:
         files: |
           deploy-config/egress_proxy/notify-api-production.*.acl

.github/workflows/deploy.yml

@@ -23,7 +23,7 @@ jobs:
 
     - name: Check for changes to Terraform
       id: changed-terraform-files
-      uses: tj-actions/changed-files@v41
+      uses: tj-actions/changed-files@v44
       with:
         files: |
           terraform/staging
@@ -81,7 +81,7 @@ jobs:
 
     - name: Check for changes to templates.json
       id: changed-templates
-      uses: tj-actions/changed-files@v41
+      uses: tj-actions/changed-files@v44
       with:
         files: |
           app/config_files/templates.json
@@ -91,7 +91,7 @@ jobs:
 
     - name: Check for changes to egress config
       id: changed-egress-config
-      uses: tj-actions/changed-files@v41
+      uses: tj-actions/changed-files@v44
       with:
         files: |
           deploy-config/egress_proxy/notify-api-staging.*.acl

Makefile

@@ -81,7 +81,8 @@ test: ## Run tests and create coverage report
 	poetry run black .
 	poetry run flake8 .
 	poetry run isort --check-only ./app ./tests
-	poetry run coverage run -m pytest --maxfail=10
+	poetry run coverage run --omit=*/notifications_utils/*,*/migrations/* -m pytest --maxfail=10
+
 	poetry run coverage report -m --fail-under=95
 	poetry run coverage html -d .coverage_cache
 

app/__init__.py

@@ -117,7 +117,6 @@ def create_app(application):
     document_download_client.init_app(application)
 
     register_blueprint(application)
-    register_v2_blueprints(application)
 
     # avoid circular imports by importing this file later
     from app.commands import setup_commands
@@ -252,34 +251,6 @@ def register_blueprint(application):
     application.register_blueprint(upload_blueprint)
 
 
-def register_v2_blueprints(application):
-    from app.authentication.auth import requires_auth
-    from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint
-    from app.v2.notifications import (  # noqa
-        get_notifications,
-        post_notifications,
-        v2_notification_blueprint,
-    )
-    from app.v2.template import (  # noqa
-        get_template,
-        post_template,
-        v2_template_blueprint,
-    )
-    from app.v2.templates.get_templates import v2_templates_blueprint
-
-    v2_notification_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_notification_blueprint)
-
-    v2_templates_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_templates_blueprint)
-
-    v2_template_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_template_blueprint)
-
-    v2_inbound_sms_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_inbound_sms_blueprint)
-
-
 def init_app(app):
     @app.before_request
     def record_request_details():

app/aws/s3.py

@@ -12,7 +12,7 @@
 
 # Temporarily extend cache to 7 days
 ttl = 60 * 60 * 24 * 7
-JOBS = ExpiringDict(max_len=1000, max_age_seconds=ttl)
+JOBS = ExpiringDict(max_len=20000, max_age_seconds=ttl)
 
 
 JOBS_CACHE_HITS = "JOBS_CACHE_HITS"

app/celery/nightly_tasks.py

@@ -27,7 +27,7 @@
 )
 from app.enums import NotificationType
 from app.models import FactProcessingTime
-from app.utils import get_midnight_in_utc
+from app.utils import get_midnight_in_utc, utc_now
 
 
 @notify_celery.task(name="remove_sms_email_jobs")
@@ -46,7 +46,7 @@ def _remove_csv_files(job_types):
 
 @notify_celery.task(name="cleanup-unfinished-jobs")
 def cleanup_unfinished_jobs():
-    now = datetime.utcnow()
+    now = utc_now()
     jobs = dao_get_unfinished_jobs()
     for job in jobs:
         # The query already checks that the processing_finished time is null, so here we are saying
@@ -88,7 +88,7 @@ def _delete_notifications_older_than_retention_by_type(notification_type):
 
     for f in flexible_data_retention:
         day_to_delete_backwards_from = get_midnight_in_utc(
-            datetime.utcnow()
+            utc_now()
         ).date() - timedelta(days=f.days_of_retention)
 
         delete_notifications_for_service_and_type.apply_async(
@@ -100,7 +100,7 @@ def _delete_notifications_older_than_retention_by_type(notification_type):
             },
         )
 
-    seven_days_ago = get_midnight_in_utc(datetime.utcnow()).date() - timedelta(days=7)
+    seven_days_ago = get_midnight_in_utc(utc_now()).date() - timedelta(days=7)
 
     service_ids_with_data_retention = {x.service_id for x in flexible_data_retention}
 
@@ -136,14 +136,14 @@ def _delete_notifications_older_than_retention_by_type(notification_type):
 def delete_notifications_for_service_and_type(
     service_id, notification_type, datetime_to_delete_before
 ):
-    start = datetime.utcnow()
+    start = utc_now()
     num_deleted = move_notifications_to_notification_history(
         notification_type,
         service_id,
         datetime_to_delete_before,
     )
     if num_deleted:
-        end = datetime.utcnow()
+        end = utc_now()
         current_app.logger.info(
             f"delete-notifications-for-service-and-type: "
             f"service: {service_id}, "
@@ -158,7 +158,7 @@ def delete_notifications_for_service_and_type(
 def timeout_notifications():
     notifications = ["dummy value so len() > 0"]
 
-    cutoff_time = datetime.utcnow() - timedelta(
+    cutoff_time = utc_now() - timedelta(
         seconds=current_app.config.get("SENDING_NOTIFICATIONS_TIMEOUT_PERIOD")
     )
 
@@ -179,11 +179,11 @@ def timeout_notifications():
 @cronitor("delete-inbound-sms")
 def delete_inbound_sms():
     try:
-        start = datetime.utcnow()
+        start = utc_now()
         deleted = delete_inbound_sms_older_than_retention()
         current_app.logger.info(
             "Delete inbound sms job started {} finished {} deleted {} inbound sms notifications".format(
-                start, datetime.utcnow(), deleted
+                start, utc_now(), deleted
             )
         )
     except SQLAlchemyError:
@@ -197,7 +197,7 @@ def save_daily_notification_processing_time(local_date=None):
     # local_date is a string in the format of "YYYY-MM-DD"
     if local_date is None:
         # if a date is not provided, we run against yesterdays data
-        local_date = (datetime.utcnow() - timedelta(days=1)).date()
+        local_date = (utc_now() - timedelta(days=1)).date()
     else:
         local_date = datetime.strptime(local_date, "%Y-%m-%d").date()
 

app/celery/process_ses_receipts_tasks.py

@@ -1,4 +1,4 @@
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 import iso8601
 from celery.exceptions import Retry
@@ -22,6 +22,7 @@
 )
 from app.enums import CallbackType, NotificationStatus
 from app.models import Complaint
+from app.utils import utc_now
 
 
 @notify_celery.task(
@@ -57,7 +58,7 @@ def process_ses_results(self, response):
             message_time = iso8601.parse_date(ses_message["mail"]["timestamp"]).replace(
                 tzinfo=None
             )
-            if datetime.utcnow() - message_time < timedelta(minutes=5):
+            if utc_now() - message_time < timedelta(minutes=5):
                 current_app.logger.info(
                     f"Notification not found for reference: {reference}"
                     f"(while attempting update to {notification_status}). "