increase code coverage to 93 percent

.github/workflows/checks.yml

@@ -63,7 +63,7 @@ jobs:
         NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
     - name: Check coverage threshold
       # TODO get this back up to 95
-      run: poetry run coverage report -m --fail-under=91
+      run: poetry run coverage report -m --fail-under=93
 
   validate-new-relic-config:
     runs-on: ubuntu-latest

Makefile

@@ -84,7 +84,7 @@ test: ## Run tests and create coverage report
 	poetry run coverage run --omit=*/migrations/*,*/tests/* -m pytest --maxfail=10
 
     ## TODO set this back to 95 asap
-	poetry run coverage report -m --fail-under=91
+	poetry run coverage report -m --fail-under=93
 	poetry run coverage html -d .coverage_cache
 
 .PHONY: py-lock

app/aws/s3.py

@@ -466,23 +466,7 @@ def get_personalisation_from_s3(service_id, job_id, job_row_number):
 
     set_job_cache(job_cache, f"{job_id}_personalisation", extract_personalisation(job))
 
-    # If we can find the quick dictionary, use it
-    if job_cache.get(f"{job_id}_personalisation") is not None:
-        personalisation_to_return = job_cache.get(f"{job_id}_personalisation")[0].get(
-            job_row_number
-        )
-        if personalisation_to_return:
-            return personalisation_to_return
-        else:
-            current_app.logger.warning(
-                f"Was unable to retrieve personalisation from lookup dictionary for job {job_id}"
-            )
-            return {}
-    else:
-        current_app.logger.error(
-            f"Was unable to construct lookup dictionary for job {job_id}"
-        )
-        return {}
+    return job_cache.get(f"{job_id}_personalisation")[0].get(job_row_number)
 
 
 def get_job_metadata_from_s3(service_id, job_id):

app/commands.py

@@ -24,12 +24,6 @@
     dao_create_or_update_annual_billing_for_year,
     set_default_free_allowance_for_service,
 )
-from app.dao.fact_billing_dao import (
-    delete_billing_data_for_service_for_day,
-    fetch_billing_data_for_day,
-    get_service_ids_that_need_billing_populated,
-    update_fact_billing,
-)
 from app.dao.jobs_dao import dao_get_job_by_id
 from app.dao.organization_dao import (
     dao_add_service_to_organization,
@@ -63,7 +57,7 @@
     TemplateHistory,
     User,
 )
-from app.utils import get_midnight_in_utc, utc_now
+from app.utils import utc_now
 from notifications_utils.recipients import RecipientCSV
 from notifications_utils.template import SMSMessageTemplate
 from tests.app.db import (
@@ -167,6 +161,7 @@ def purge_functional_test_data(user_email_prefix):
                 delete_model_user(usr)
 
 
+# TODO maintainability what is the purpose of this command?  Who would use it and why?
 @notify_command(name="insert-inbound-numbers")
 @click.option(
     "-f",
@@ -175,7 +170,6 @@ def purge_functional_test_data(user_email_prefix):
     help="""Full path of the file to upload, file is a contains inbound numbers, one number per line.""",
 )
 def insert_inbound_numbers_from_file(file_name):
-    # TODO maintainability what is the purpose of this command?  Who would use it and why?
 
     current_app.logger.info(f"Inserting inbound numbers from {file_name}")
     with open(file_name) as file:
@@ -195,48 +189,49 @@ def setup_commands(application):
     application.cli.add_command(command_group)
 
 
-@notify_command(name="rebuild-ft-billing-for-day")
-@click.option("-s", "--service_id", required=False, type=click.UUID)
-@click.option(
-    "-d",
-    "--day",
-    help="The date to recalculate, as YYYY-MM-DD",
-    required=True,
-    type=click_dt(format="%Y-%m-%d"),
-)
-def rebuild_ft_billing_for_day(service_id, day):
-    # TODO maintainability what is the purpose of this command?  Who would use it and why?
-
-    """
-    Rebuild the data in ft_billing for the given service_id and date
-    """
-
-    def rebuild_ft_data(process_day, service):
-        deleted_rows = delete_billing_data_for_service_for_day(process_day, service)
-        current_app.logger.info(
-            f"deleted {deleted_rows} existing billing rows for {service} on {process_day}"
-        )
-        transit_data = fetch_billing_data_for_day(
-            process_day=process_day, service_id=service
-        )
-        # transit_data = every row that should exist
-        for data in transit_data:
-            # upsert existing rows
-            update_fact_billing(data, process_day)
-        current_app.logger.info(
-            f"added/updated {len(transit_data)} billing rows for {service} on {process_day}"
-        )
-
-    if service_id:
-        # confirm the service exists
-        dao_fetch_service_by_id(service_id)
-        rebuild_ft_data(day, service_id)
-    else:
-        services = get_service_ids_that_need_billing_populated(
-            get_midnight_in_utc(day), get_midnight_in_utc(day + timedelta(days=1))
-        )
-        for row in services:
-            rebuild_ft_data(day, row.service_id)
+# TODO maintainability what is the purpose of this command?  Who would use it and why?
+# COMMENTING OUT UNTIL WE DETERMINE IF WE NEED IT OR NOT
+# @notify_command(name="rebuild-ft-billing-for-day")
+# @click.option("-s", "--service_id", required=False, type=click.UUID)
+# @click.option(
+#     "-d",
+#     "--day",
+#     help="The date to recalculate, as YYYY-MM-DD",
+#     required=True,
+#     type=click_dt(format="%Y-%m-%d"),
+# )
+# def rebuild_ft_billing_for_day(service_id, day):
+
+#     """
+#     Rebuild the data in ft_billing for the given service_id and date
+#     """
+
+#     def rebuild_ft_data(process_day, service):
+#         deleted_rows = delete_billing_data_for_service_for_day(process_day, service)
+#         current_app.logger.info(
+#             f"deleted {deleted_rows} existing billing rows for {service} on {process_day}"
+#         )
+#         transit_data = fetch_billing_data_for_day(
+#             process_day=process_day, service_id=service
+#         )
+#         # transit_data = every row that should exist
+#         for data in transit_data:
+#             # upsert existing rows
+#             update_fact_billing(data, process_day)
+#         current_app.logger.info(
+#             f"added/updated {len(transit_data)} billing rows for {service} on {process_day}"
+#         )
+
+#     if service_id:
+#         # confirm the service exists
+#         dao_fetch_service_by_id(service_id)
+#         rebuild_ft_data(day, service_id)
+#     else:
+#         services = get_service_ids_that_need_billing_populated(
+#             get_midnight_in_utc(day), get_midnight_in_utc(day + timedelta(days=1))
+#         )
+#         for row in services:
+#             rebuild_ft_data(day, row.service_id)
 
 
 @notify_command(name="bulk-invite-user-to-service")
@@ -472,29 +467,30 @@ def associate_services_to_organizations():
     current_app.logger.info("finished associating services to organizations")
 
 
-@notify_command(name="populate-service-volume-intentions")
-@click.option(
-    "-f",
-    "--file_name",
-    required=True,
-    help="Pipe delimited file containing service_id, SMS, email",
-)
-def populate_service_volume_intentions(file_name):
-    # [0] service_id
-    # [1] SMS:: volume intentions for service
-    # [2] Email:: volume intentions for service
-
-    # TODO maintainability what is the purpose of this command? Who would use it and why?
-
-    with open(file_name, "r") as f:
-        for line in itertools.islice(f, 1, None):
-            columns = line.split(",")
-            current_app.logger.info(columns)
-            service = dao_fetch_service_by_id(columns[0])
-            service.volume_sms = columns[1]
-            service.volume_email = columns[2]
-            dao_update_service(service)
-    current_app.logger.info("populate-service-volume-intentions complete")
+# TODO maintainability what is the purpose of this command? Who would use it and why?
+# COMMENTING OUT UNTIL WE DETERMINE IF WE NEED IT OR NOT
+# @notify_command(name="populate-service-volume-intentions")
+# @click.option(
+#     "-f",
+#     "--file_name",
+#     required=True,
+#     help="Pipe delimited file containing service_id, SMS, email",
+# )
+# def populate_service_volume_intentions(file_name):
+#     # [0] service_id
+#     # [1] SMS:: volume intentions for service
+#     # [2] Email:: volume intentions for service
+
+
+#     with open(file_name, "r") as f:
+#         for line in itertools.islice(f, 1, None):
+#             columns = line.split(",")
+#             current_app.logger.info(columns)
+#             service = dao_fetch_service_by_id(columns[0])
+#             service.volume_sms = columns[1]
+#             service.volume_email = columns[2]
+#             dao_update_service(service)
+#     current_app.logger.info("populate-service-volume-intentions complete")
 
 
 @notify_command(name="populate-go-live")

app/delivery/send_to_providers.py

@@ -98,17 +98,7 @@ def send_sms_to_provider(notification):
 
                 # TODO This is temporary to test the capability of validating phone numbers
                 # The future home of the validation is TBD
-                if "+" not in recipient:
-                    recipient_lookup = f"+{recipient}"
-                else:
-                    recipient_lookup = recipient
-                if recipient_lookup in current_app.config[
-                    "SIMULATED_SMS_NUMBERS"
-                ] and os.getenv("NOTIFY_ENVIRONMENT") in ["development", "test"]:
-                    current_app.logger.info(hilite("#validate-phone-number fired"))
-                    aws_pinpoint_client.validate_phone_number("01", recipient)
-                else:
-                    current_app.logger.info(hilite("#validate-phone-number not fired"))
+                _experimentally_validate_phone_numbers(recipient)
 
                 sender_numbers = get_sender_numbers(notification)
                 if notification.reply_to_text not in sender_numbers:
@@ -145,6 +135,18 @@ def send_sms_to_provider(notification):
     return message_id
 
 
+def _experimentally_validate_phone_numbers(recipient):
+    if "+" not in recipient:
+        recipient_lookup = f"+{recipient}"
+    else:
+        recipient_lookup = recipient
+    if recipient_lookup in current_app.config["SIMULATED_SMS_NUMBERS"] and os.getenv(
+        "NOTIFY_ENVIRONMENT"
+    ) in ["development", "test"]:
+        current_app.logger.info(hilite("#validate-phone-number fired"))
+        aws_pinpoint_client.validate_phone_number("01", recipient)
+
+
 def _get_verify_code(notification):
     key = f"2facode-{notification.id}".replace(" ", "")
     recipient = redis_store.get(key)

app/service/rest.py

@@ -453,16 +453,16 @@ def get_all_notifications_for_service(service_id):
         data = notifications_filter_schema.load(MultiDict(request.get_json()))
         current_app.logger.debug(f"use POST, request {request.get_json()} data {data}")
 
-    if data.get("to"):
-        notification_type = (
-            data.get("template_type")[0] if data.get("template_type") else None
-        )
-        return search_for_notification_by_to_field(
-            service_id=service_id,
-            search_term=data["to"],
-            statuses=data.get("status"),
-            notification_type=notification_type,
-        )
+    # if data.get("to"):
+    #     notification_type = (
+    #         data.get("template_type")[0] if data.get("template_type") else None
+    #     )
+    #     return search_for_notification_by_to_field(
+    #         service_id=service_id,
+    #         search_term=data["to"],
+    #         statuses=data.get("status"),
+    #         notification_type=notification_type,
+    #     )
     page = data["page"] if "page" in data else 1
     page_size = (
         data["page_size"]
@@ -583,51 +583,51 @@ def get_notification_for_service(service_id, notification_id):
     )
 
 
-def search_for_notification_by_to_field(
-    service_id, search_term, statuses, notification_type
-):
-    results = notifications_dao.dao_get_notifications_by_recipient_or_reference(
-        service_id=service_id,
-        search_term=search_term,
-        statuses=statuses,
-        notification_type=notification_type,
-        page=1,
-        page_size=current_app.config["PAGE_SIZE"],
-    )
-
-    # We try and get the next page of results to work out if we need provide a pagination link to the next page
-    # in our response. Note, this was previously be done by having
-    # notifications_dao.dao_get_notifications_by_recipient_or_reference use count=False when calling
-    # Flask-Sqlalchemys `paginate'. But instead we now use this way because it is much more performant for
-    # services with many results (unlike using Flask SqlAlchemy `paginate` with `count=True`, this approach
-    # doesn't do an additional query to count all the results of which there could be millions but instead only
-    # asks for a single extra page of results).
-    next_page_of_pagination = notifications_dao.dao_get_notifications_by_recipient_or_reference(
-        service_id=service_id,
-        search_term=search_term,
-        statuses=statuses,
-        notification_type=notification_type,
-        page=2,
-        page_size=current_app.config["PAGE_SIZE"],
-        error_out=False,  # False so that if there are no results, it doesn't end in aborting with a 404
-    )
-
-    return (
-        jsonify(
-            notifications=notification_with_template_schema.dump(
-                results.items, many=True
-            ),
-            links=get_prev_next_pagination_links(
-                1,
-                len(next_page_of_pagination.items),
-                ".get_all_notifications_for_service",
-                statuses=statuses,
-                notification_type=notification_type,
-                service_id=service_id,
-            ),
-        ),
-        200,
-    )
+# def search_for_notification_by_to_field(
+#     service_id, search_term, statuses, notification_type
+# ):
+#     results = notifications_dao.dao_get_notifications_by_recipient_or_reference(
+#         service_id=service_id,
+#         search_term=search_term,
+#         statuses=statuses,
+#         notification_type=notification_type,
+#         page=1,
+#         page_size=current_app.config["PAGE_SIZE"],
+#     )
+
+#     # We try and get the next page of results to work out if we need provide a pagination link to the next page
+#     # in our response. Note, this was previously be done by having
+#     # notifications_dao.dao_get_notifications_by_recipient_or_reference use count=False when calling
+#     # Flask-Sqlalchemys `paginate'. But instead we now use this way because it is much more performant for
+#     # services with many results (unlike using Flask SqlAlchemy `paginate` with `count=True`, this approach
+#     # doesn't do an additional query to count all the results of which there could be millions but instead only
+#     # asks for a single extra page of results).
+#     next_page_of_pagination = notifications_dao.dao_get_notifications_by_recipient_or_reference(
+#         service_id=service_id,
+#         search_term=search_term,
+#         statuses=statuses,
+#         notification_type=notification_type,
+#         page=2,
+#         page_size=current_app.config["PAGE_SIZE"],
+#         error_out=False,  # False so that if there are no results, it doesn't end in aborting with a 404
+#     )
+
+#     return (
+#         jsonify(
+#             notifications=notification_with_template_schema.dump(
+#                 results.items, many=True
+#             ),
+#             links=get_prev_next_pagination_links(
+#                 1,
+#                 len(next_page_of_pagination.items),
+#                 ".get_all_notifications_for_service",
+#                 statuses=statuses,
+#                 notification_type=notification_type,
+#                 service_id=service_id,
+#             ),
+#         ),
+#         200,
+#     )
 
 
 @service_blueprint.route("/<uuid:service_id>/notifications/monthly", methods=["GET"])