From 5fdb17279a909b34356f81ea6dfa34b168c199a5 Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Tue, 14 May 2024 16:04:07 -0700 Subject: [PATCH 01/29] Debugging print --- auth-api/src/auth_api/__init__.py | 2 ++ auth-api/src/auth_api/services/gcp_queue/gcp_queue.py | 1 + 2 files changed, 3 insertions(+) diff --git a/auth-api/src/auth_api/__init__.py b/auth-api/src/auth_api/__init__.py index 3680b12c6d..0fc7b0c356 100644 --- a/auth-api/src/auth_api/__init__.py +++ b/auth-api/src/auth_api/__init__.py @@ -58,6 +58,8 @@ def create_app(run_mode=os.getenv('FLASK_ENV', 'production')): ma.init_app(app) mail.init_app(app) queue.init_app(app) + print('INIT QUEUE') + print(app.config.get('GCP_AUTH_KEY')[1:10]) endpoints.init_app(app) if os.getenv('FLASK_ENV', 'production') != 'testing': diff --git a/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py b/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py index df51d20c7d..601d67092a 100644 --- a/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py +++ b/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py @@ -67,6 +67,7 @@ def __init__(self, app: Flask = None): def init_app(self, app: Flask): """Initialize the application.""" self.gcp_auth_key = app.config.get('GCP_AUTH_KEY') + print(app.config.get('GCP_AUTH_KEY')[0:10]) if self.gcp_auth_key: try: audience = app.config.get( From d28b81349760583f71834c5435223c604a7066ea Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Tue, 14 May 2024 16:12:20 -0700 Subject: [PATCH 02/29] more logging to figure out an issue --- auth-api/src/auth_api/services/gcp_queue/gcp_queue.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py b/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py index 601d67092a..a979faef89 100644 --- a/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py +++ b/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py @@ -83,14 +83,20 @@ def init_app(self, app: Flask): credentials = jwt.Credentials.from_service_account_info(self.service_account_info, audience=audience) self.credentials_pub = credentials.with_claims(audience=publisher_audience) except Exception as error: # noqa: B902 + print('error creating connection?') + print(error) raise Exception('Unable to create a connection', error) from error # pylint: disable=W0719 @property def publisher(self): """Returns the publisher.""" if not self._publisher and self.credentials_pub: + print('Choosing credentials') self._publisher = pubsub_v1.PublisherClient(credentials=self.credentials_pub) else: + print('no cred!') + print(self._printer) + print(self.credentials_pub) self._publisher = pubsub_v1.PublisherClient() return self._publisher From c6040e64ce61901643135d9a18e147118587e57e Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Tue, 14 May 2024 16:14:06 -0700 Subject: [PATCH 03/29] more debug --- auth-api/src/auth_api/services/gcp_queue/gcp_queue.py | 1 + 1 file changed, 1 insertion(+) diff --git a/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py b/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py index a979faef89..a1d4a504ba 100644 --- a/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py +++ b/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py @@ -97,6 +97,7 @@ def publisher(self): print('no cred!') print(self._printer) print(self.credentials_pub) + print(self.gcp_auth_key) self._publisher = pubsub_v1.PublisherClient() return self._publisher From 5ca2a8af7639a1c012386ed3a87dd64557317512 Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Tue, 14 May 2024 16:14:26 -0700 Subject: [PATCH 04/29] more debug --- auth-api/src/auth_api/services/gcp_queue/gcp_queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py b/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py index a1d4a504ba..96a6f4ca7b 100644 --- a/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py +++ b/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py @@ -97,7 +97,7 @@ def publisher(self): print('no cred!') print(self._printer) print(self.credentials_pub) - print(self.gcp_auth_key) + print(self.gcp_auth_key[0:20]) self._publisher = pubsub_v1.PublisherClient() return self._publisher From de25dc7cb8c19dc0ed4a1e426e78037ad9ce309a Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Tue, 14 May 2024 16:15:47 -0700 Subject: [PATCH 05/29] Disable update env --- auth-api/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/auth-api/Makefile b/auth-api/Makefile index 69644203e3..699cc2455f 100644 --- a/auth-api/Makefile +++ b/auth-api/Makefile @@ -93,15 +93,15 @@ mac-cov: local-test ## Run the coverage report and display in a browser window ( ################################################################################# cd: ## CD flow ifeq ($(TAG_NAME), test) -cd: update-env +cd: oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):dev $(DOCKER_NAME):$(TAG_NAME) else ifeq ($(TAG_NAME), prod) -cd: update-env +cd: oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):$(TAG_NAME) $(DOCKER_NAME):$(TAG_NAME)-$(shell date +%F) oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):test $(DOCKER_NAME):$(TAG_NAME) else TAG_NAME=dev -cd: build update-env tag +cd: build tag endif build: ## Build the docker container From c24dc6368b608030e13173073af4ac9cb64e52bb Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Tue, 14 May 2024 16:19:55 -0700 Subject: [PATCH 06/29] remove self._publisher --- auth-api/src/auth_api/services/gcp_queue/gcp_queue.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py b/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py index 96a6f4ca7b..c4a67548a6 100644 --- a/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py +++ b/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py @@ -90,12 +90,12 @@ def init_app(self, app: Flask): @property def publisher(self): """Returns the publisher.""" - if not self._publisher and self.credentials_pub: + if self.credentials_pub: print('Choosing credentials') self._publisher = pubsub_v1.PublisherClient(credentials=self.credentials_pub) else: print('no cred!') - print(self._printer) + print(self._publisher) print(self.credentials_pub) print(self.gcp_auth_key[0:20]) self._publisher = pubsub_v1.PublisherClient() From 097c219ee63a28f83f8ccf50849b7f7c9a12f4f4 Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Tue, 14 May 2024 16:30:19 -0700 Subject: [PATCH 07/29] Fix up the queue --- auth-api/src/auth_api/__init__.py | 2 -- .../src/auth_api/services/gcp_queue/gcp_queue.py | 12 ++---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/auth-api/src/auth_api/__init__.py b/auth-api/src/auth_api/__init__.py index 0fc7b0c356..3680b12c6d 100644 --- a/auth-api/src/auth_api/__init__.py +++ b/auth-api/src/auth_api/__init__.py @@ -58,8 +58,6 @@ def create_app(run_mode=os.getenv('FLASK_ENV', 'production')): ma.init_app(app) mail.init_app(app) queue.init_app(app) - print('INIT QUEUE') - print(app.config.get('GCP_AUTH_KEY')[1:10]) endpoints.init_app(app) if os.getenv('FLASK_ENV', 'production') != 'testing': diff --git a/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py b/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py index c4a67548a6..7d27cf7c15 100644 --- a/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py +++ b/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py @@ -67,7 +67,6 @@ def __init__(self, app: Flask = None): def init_app(self, app: Flask): """Initialize the application.""" self.gcp_auth_key = app.config.get('GCP_AUTH_KEY') - print(app.config.get('GCP_AUTH_KEY')[0:10]) if self.gcp_auth_key: try: audience = app.config.get( @@ -83,21 +82,14 @@ def init_app(self, app: Flask): credentials = jwt.Credentials.from_service_account_info(self.service_account_info, audience=audience) self.credentials_pub = credentials.with_claims(audience=publisher_audience) except Exception as error: # noqa: B902 - print('error creating connection?') - print(error) raise Exception('Unable to create a connection', error) from error # pylint: disable=W0719 @property def publisher(self): """Returns the publisher.""" - if self.credentials_pub: - print('Choosing credentials') + if not self._publisher and self.credentials_pub: self._publisher = pubsub_v1.PublisherClient(credentials=self.credentials_pub) - else: - print('no cred!') - print(self._publisher) - print(self.credentials_pub) - print(self.gcp_auth_key[0:20]) + if not self._publisher: self._publisher = pubsub_v1.PublisherClient() return self._publisher From fd26d4b29f6e673e385463d2d274c56bedc8792f Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Tue, 14 May 2024 17:00:51 -0700 Subject: [PATCH 08/29] Change logging a bit, fix make files so it's rapid fire. --- queue_services/account-mailer/Makefile | 6 +++--- .../account-mailer/src/account_mailer/__init__.py | 4 +++- queue_services/auth-queue/Makefile | 6 +++--- queue_services/auth-queue/src/auth_queue/__init__.py | 4 +++- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/queue_services/account-mailer/Makefile b/queue_services/account-mailer/Makefile index 2d7772dd2f..28aeb959c3 100644 --- a/queue_services/account-mailer/Makefile +++ b/queue_services/account-mailer/Makefile @@ -87,15 +87,15 @@ mac-cov: test ## Run the coverage report and display in a browser window (mac) ################################################################################# cd: ## CD flow ifeq ($(TAG_NAME), test) -cd: update-env +cd: oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):dev $(DOCKER_NAME):$(TAG_NAME) else ifeq ($(TAG_NAME), prod) -cd: update-env +cd: oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):$(TAG_NAME) $(DOCKER_NAME):$(TAG_NAME)-$(shell date +%F) oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):test $(DOCKER_NAME):$(TAG_NAME) else TAG_NAME=dev -cd: build update-env tag +cd: build tag endif build: ## Build the docker container diff --git a/queue_services/account-mailer/src/account_mailer/__init__.py b/queue_services/account-mailer/src/account_mailer/__init__.py index b8b2acf7be..346f5b1f1f 100644 --- a/queue_services/account-mailer/src/account_mailer/__init__.py +++ b/queue_services/account-mailer/src/account_mailer/__init__.py @@ -25,12 +25,14 @@ from auth_api.services.flags import flags from auth_api.services.gcp_queue import queue from auth_api.utils.cache import cache +from auth_api.utils.util_logging import setup_logging from flask import Flask from sentry_sdk.integrations.flask import FlaskIntegration -from account_mailer import config +from account_mailer import config, _Config from account_mailer.resources.worker import bp as worker_endpoint +setup_logging(os.path.join(_Config.PROJECT_ROOT, 'logging.conf')) # important to do this first def register_endpoints(app: Flask): """Register endpoints with the flask application.""" diff --git a/queue_services/auth-queue/Makefile b/queue_services/auth-queue/Makefile index 0c3fa31ca0..2760fefe55 100644 --- a/queue_services/auth-queue/Makefile +++ b/queue_services/auth-queue/Makefile @@ -87,15 +87,15 @@ mac-cov: test ## Run the coverage report and display in a browser window (mac) ################################################################################# cd: ## CD flow ifeq ($(TAG_NAME), test) -cd: update-env +cd: oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):dev $(DOCKER_NAME):$(TAG_NAME) else ifeq ($(TAG_NAME), prod) -cd: update-env +cd: oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):$(TAG_NAME) $(DOCKER_NAME):$(TAG_NAME)-$(shell date +%F) oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):test $(DOCKER_NAME):$(TAG_NAME) else TAG_NAME=dev -cd: build update-env tag +cd: build tag endif build: ## Build the docker container diff --git a/queue_services/auth-queue/src/auth_queue/__init__.py b/queue_services/auth-queue/src/auth_queue/__init__.py index 9668577ea8..72b4687079 100644 --- a/queue_services/auth-queue/src/auth_queue/__init__.py +++ b/queue_services/auth-queue/src/auth_queue/__init__.py @@ -20,12 +20,14 @@ from auth_api.services.flags import flags from auth_api.services.gcp_queue import queue from auth_api.utils.cache import cache +from auth_api.utils.util_logging import setup_logging from flask import Flask from sentry_sdk.integrations.flask import FlaskIntegration -from auth_queue import config +from auth_queue import config, _Config from auth_queue.resources.worker import bp as worker_endpoint +setup_logging(os.path.join(_Config.PROJECT_ROOT, 'logging.conf')) # important to do this first def register_endpoints(app: Flask): """Register endpoints with the flask application.""" From 09bea0be901af8b1d1751b9e9f4697737a094a5f Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Tue, 14 May 2024 17:06:18 -0700 Subject: [PATCH 09/29] Move logging down a tad --- queue_services/account-mailer/src/account_mailer/__init__.py | 4 ++-- queue_services/auth-queue/src/auth_queue/__init__.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/queue_services/account-mailer/src/account_mailer/__init__.py b/queue_services/account-mailer/src/account_mailer/__init__.py index 346f5b1f1f..76f7167279 100644 --- a/queue_services/account-mailer/src/account_mailer/__init__.py +++ b/queue_services/account-mailer/src/account_mailer/__init__.py @@ -29,10 +29,10 @@ from flask import Flask from sentry_sdk.integrations.flask import FlaskIntegration -from account_mailer import config, _Config +from account_mailer import config from account_mailer.resources.worker import bp as worker_endpoint -setup_logging(os.path.join(_Config.PROJECT_ROOT, 'logging.conf')) # important to do this first +setup_logging(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'logging.conf')) # important to do this first def register_endpoints(app: Flask): """Register endpoints with the flask application.""" diff --git a/queue_services/auth-queue/src/auth_queue/__init__.py b/queue_services/auth-queue/src/auth_queue/__init__.py index 72b4687079..965ae0681d 100644 --- a/queue_services/auth-queue/src/auth_queue/__init__.py +++ b/queue_services/auth-queue/src/auth_queue/__init__.py @@ -24,10 +24,10 @@ from flask import Flask from sentry_sdk.integrations.flask import FlaskIntegration -from auth_queue import config, _Config +from auth_queue import config from auth_queue.resources.worker import bp as worker_endpoint -setup_logging(os.path.join(_Config.PROJECT_ROOT, 'logging.conf')) # important to do this first +setup_logging(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'logging.conf')) # important to do this first def register_endpoints(app: Flask): """Register endpoints with the flask application.""" From c49601f5ac2383ff718279142022db760c866d87 Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Wed, 15 May 2024 12:08:41 -0700 Subject: [PATCH 10/29] Move logging.conf --- .../account-mailer/{ => src/account_mailer}/logging.conf | 0 queue_services/auth-queue/{ => src/auth_queue}/logging.conf | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename queue_services/account-mailer/{ => src/account_mailer}/logging.conf (100%) rename queue_services/auth-queue/{ => src/auth_queue}/logging.conf (100%) diff --git a/queue_services/account-mailer/logging.conf b/queue_services/account-mailer/src/account_mailer/logging.conf similarity index 100% rename from queue_services/account-mailer/logging.conf rename to queue_services/account-mailer/src/account_mailer/logging.conf diff --git a/queue_services/auth-queue/logging.conf b/queue_services/auth-queue/src/auth_queue/logging.conf similarity index 100% rename from queue_services/auth-queue/logging.conf rename to queue_services/auth-queue/src/auth_queue/logging.conf From 73d17b489f6475f6de2df761f18c0c874516a9d1 Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Wed, 15 May 2024 13:25:56 -0700 Subject: [PATCH 11/29] Add in error for unknown message type, add in enums for missing sections --- auth-api/src/auth_api/services/membership.py | 6 +++--- .../account-mailer/src/account_mailer/resources/worker.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/auth-api/src/auth_api/services/membership.py b/auth-api/src/auth_api/services/membership.py index bd76951b6e..7df0b1b737 100644 --- a/auth-api/src/auth_api/services/membership.py +++ b/auth-api/src/auth_api/services/membership.py @@ -168,7 +168,7 @@ def send_notification_to_member(self, origin_url, notification_type): notification_type_for_mailer = '' data = {} if notification_type == NotificationType.ROLE_CHANGED.value: - notification_type_for_mailer = 'roleChangedNotification' + notification_type_for_mailer = QueueMessageTypes.ROLE_CHANGED_NOTIFICATION.value data = { 'accountId': org_id, 'emailAddresses': recipient, @@ -181,9 +181,9 @@ def send_notification_to_member(self, origin_url, notification_type): # TODO how to check properly if user is bceid user is_bceid_user = self._model.user.username.find('@bceid') > 0 if is_bceid_user: - notification_type_for_mailer = 'membershipApprovedNotificationForBceid' + notification_type_for_mailer = QueueMessageTypes.MEMBERSHIP_APPROVED_NOTIFICATION_FOR_BCEID.value else: - notification_type_for_mailer = 'membershipApprovedNotification' + notification_type_for_mailer = QueueMessageTypes.MEMBERSHIP_APPROVED_NOTIFICATION.value data = { 'accountId': org_id, diff --git a/queue_services/account-mailer/src/account_mailer/resources/worker.py b/queue_services/account-mailer/src/account_mailer/resources/worker.py index 0c9ef47399..8cf95b4a99 100644 --- a/queue_services/account-mailer/src/account_mailer/resources/worker.py +++ b/queue_services/account-mailer/src/account_mailer/resources/worker.py @@ -443,6 +443,7 @@ def handle_other_messages(message_type, email_msg): ) template_name = TemplateType[f'{QueueMessageTypes(message_type).name}_TEMPLATE_NAME'].value else: + current_app.logger.error('Unknown message type: %s', message_type) return kwargs = { From b7fc54197daa2982d151b0e3ac71b402d2d8c75b Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Wed, 15 May 2024 14:59:18 -0700 Subject: [PATCH 12/29] Put in detection against duplicate queue messages for account-mailer. --- ...5_b3a741249edc_duplicate_queue_messages.py | 33 +++++++++++++++++++ auth-api/src/auth_api/models/__init__.py | 1 + .../models/pubsub_message_processing.py | 27 +++++++++++++++ .../src/account_mailer/resources/worker.py | 21 ++++++++++-- 4 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 auth-api/migrations/versions/2024_05_15_b3a741249edc_duplicate_queue_messages.py create mode 100644 auth-api/src/auth_api/models/pubsub_message_processing.py diff --git a/auth-api/migrations/versions/2024_05_15_b3a741249edc_duplicate_queue_messages.py b/auth-api/migrations/versions/2024_05_15_b3a741249edc_duplicate_queue_messages.py new file mode 100644 index 0000000000..2cb3a8a858 --- /dev/null +++ b/auth-api/migrations/versions/2024_05_15_b3a741249edc_duplicate_queue_messages.py @@ -0,0 +1,33 @@ +"""Add in new table for account mailer for pubsub message processing. + +Revision ID: b3a741249edc +Revises: e2d1d6417607 +Create Date: 2024-05-15 14:52:45.780399 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'b3a741249edc' +down_revision = 'e2d1d6417607' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table('pubsub_message_processing', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('cloud_event_id', sa.String(length=250), nullable=False), + sa.Column('created', sa.DateTime(), nullable=True), + sa.Column('message_type', sa.String(length=250), nullable=False), + sa.Column('processed', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_pubsub_message_processing_id'), 'pubsub_message_processing', ['id'], unique=False) + + +def downgrade(): + op.drop_index(op.f('ix_pubsub_message_processing_id'), table_name='pubsub_message_processing') + op.drop_table('pubsub_message_processing') diff --git a/auth-api/src/auth_api/models/__init__.py b/auth-api/src/auth_api/models/__init__.py index fdb30738e2..7710f712dc 100644 --- a/auth-api/src/auth_api/models/__init__.py +++ b/auth-api/src/auth_api/models/__init__.py @@ -50,6 +50,7 @@ from .product_subscription import ProductSubscription from .product_subscriptions_status import ProductSubscriptionsStatus from .product_type_code import ProductTypeCode +from .pubsub_message_processing import PubSubMessageProcessing from .suspension_reason_code import SuspensionReasonCode from .task import Task from .user import User diff --git a/auth-api/src/auth_api/models/pubsub_message_processing.py b/auth-api/src/auth_api/models/pubsub_message_processing.py new file mode 100644 index 0000000000..1e88e2c569 --- /dev/null +++ b/auth-api/src/auth_api/models/pubsub_message_processing.py @@ -0,0 +1,27 @@ +"""This model manages pubsub message processing. + +NOTE: Only use this when it's not possible to use other indicators to track message processing. + Currently used by the account-mailer. This prevents duplicates. +""" +import datetime as dt +import pytz + +from sqlalchemy import Column, DateTime, Integer, String +from .db import db + + +class PubSubMessageProcessing(db.Model): + """PubSub Message Processing for cloud event messages.""" + + __tablename__ = 'pubsub_message_processing' + + id = Column(Integer, index=True, primary_key=True) + cloud_event_id = Column(String(250), nullable=False) + created = Column(DateTime, default=dt.datetime.now(pytz.utc)) + message_type = Column(String(250), nullable=False) + processed = Column(DateTime, nullable=True) + + @classmethod + def find_by_cloud_event_id_and_type(cls, cloud_event_id, message_type): + """Find a pubsub message processing for cloud event id and type.""" + return cls.query.filter_by(cloud_event_id=cloud_event_id, message_type=message_type).one_or_none() diff --git a/queue_services/account-mailer/src/account_mailer/resources/worker.py b/queue_services/account-mailer/src/account_mailer/resources/worker.py index 8cf95b4a99..8db0c3a774 100644 --- a/queue_services/account-mailer/src/account_mailer/resources/worker.py +++ b/queue_services/account-mailer/src/account_mailer/resources/worker.py @@ -14,9 +14,11 @@ """The unique worker functionality for this service is contained here.""" import dataclasses import json -from datetime import datetime +from datetime import datetime, timezone from http import HTTPStatus +from auth_api.models import db +from auth_api.models.pubsub_message_processing import PubSubMessageProcessing from auth_api.services.gcp_queue import queue from auth_api.services.gcp_queue.gcp_auth import ensure_authorized_queue_user from auth_api.services.rest_service import RestService @@ -44,7 +46,10 @@ def worker(): return {}, HTTPStatus.OK try: - current_app.logger.info('Event Message Received: %s', json.dumps(dataclasses.asdict(event_message))) + current_app.logger.info('Event message received: %s', json.dumps(dataclasses.asdict(event_message))) + if is_message_processed(event_message): + current_app.logger.info('Event message already processed, skipping.') + return {}, HTTPStatus.OK message_type, email_msg = event_message.type, event_message.data email_msg['logo_url'] = minio_service.MinioService.get_minio_public_url('bc_logo_for_email.png') @@ -72,6 +77,18 @@ def worker(): return {}, HTTPStatus.OK +def is_message_processed(event_message): + """Check if the queue message is processed.""" + if PubSubMessageProcessing.find_by_cloud_event_id_and_type(event_message.id, event_message.type): + return True + pubsub_message_processing = PubSubMessageProcessing() + pubsub_message_processing.cloud_event_id = event_message.id + pubsub_message_processing.message_type = event_message.type + pubsub_message_processing.processed = datetime.now(timezone.utc) + db.session.add(pubsub_message_processing) + db.session.commit() + return False + def handle_drawdown_request(message_type, email_msg): """Handle the drawdown request message.""" if message_type != QueueMessageTypes.REFUND_DRAWDOWN_REQUEST.value: From 37091719662084ab4809eacdbb4153d063162679 Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Wed, 15 May 2024 15:01:03 -0700 Subject: [PATCH 13/29] Point at new auth-api --- queue_services/account-mailer/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queue_services/account-mailer/requirements.txt b/queue_services/account-mailer/requirements.txt index 7f295d4a80..0179b3c634 100644 --- a/queue_services/account-mailer/requirements.txt +++ b/queue_services/account-mailer/requirements.txt @@ -34,5 +34,5 @@ tornado==6.4 urllib3==1.26.18 zipp==3.18.1 -e git+https://github.com/bcgov/sbc-common-components.git#egg=sbc-common-components&subdirectory=python --e git+https://github.com/seeker25/sbc-auth.git@queue_upgrades#egg=auth-api&subdirectory=auth-api +-e git+https://github.com/bcgov/sbc-auth.git@queue_fixess#egg=auth-api&subdirectory=auth-api git+https://github.com/daxiom/simple-cloudevent.py.git From b3ec238938957e9135616cde42fd0e787d00e181 Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Wed, 15 May 2024 15:04:28 -0700 Subject: [PATCH 14/29] lint fix --- .../account-mailer/src/account_mailer/resources/worker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/queue_services/account-mailer/src/account_mailer/resources/worker.py b/queue_services/account-mailer/src/account_mailer/resources/worker.py index 8db0c3a774..545b178fca 100644 --- a/queue_services/account-mailer/src/account_mailer/resources/worker.py +++ b/queue_services/account-mailer/src/account_mailer/resources/worker.py @@ -89,6 +89,7 @@ def is_message_processed(event_message): db.session.commit() return False + def handle_drawdown_request(message_type, email_msg): """Handle the drawdown request message.""" if message_type != QueueMessageTypes.REFUND_DRAWDOWN_REQUEST.value: From 23d2250bb004cce21284cc19f86eb8bb31c9b9a8 Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Wed, 15 May 2024 15:08:28 -0700 Subject: [PATCH 15/29] lint fix --- queue_services/auth-queue/src/auth_queue/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/queue_services/auth-queue/src/auth_queue/__init__.py b/queue_services/auth-queue/src/auth_queue/__init__.py index 965ae0681d..597218c2ea 100644 --- a/queue_services/auth-queue/src/auth_queue/__init__.py +++ b/queue_services/auth-queue/src/auth_queue/__init__.py @@ -27,8 +27,10 @@ from auth_queue import config from auth_queue.resources.worker import bp as worker_endpoint + setup_logging(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'logging.conf')) # important to do this first + def register_endpoints(app: Flask): """Register endpoints with the flask application.""" # Allow base route to match with, and without a trailing slash From 99c8d57b3e91436e7534d521368b15b0eb4cd339 Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Wed, 15 May 2024 15:13:36 -0700 Subject: [PATCH 16/29] lint fixes --- queue_services/account-mailer/src/account_mailer/__init__.py | 2 ++ .../src/account_mailer/email_processors/pad_confirmation.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/queue_services/account-mailer/src/account_mailer/__init__.py b/queue_services/account-mailer/src/account_mailer/__init__.py index 76f7167279..9b5d148e24 100644 --- a/queue_services/account-mailer/src/account_mailer/__init__.py +++ b/queue_services/account-mailer/src/account_mailer/__init__.py @@ -32,8 +32,10 @@ from account_mailer import config from account_mailer.resources.worker import bp as worker_endpoint + setup_logging(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'logging.conf')) # important to do this first + def register_endpoints(app: Flask): """Register endpoints with the flask application.""" # Allow base route to match with, and without a trailing slash diff --git a/queue_services/account-mailer/src/account_mailer/email_processors/pad_confirmation.py b/queue_services/account-mailer/src/account_mailer/email_processors/pad_confirmation.py index 3379a41ef2..f1b5234b33 100644 --- a/queue_services/account-mailer/src/account_mailer/email_processors/pad_confirmation.py +++ b/queue_services/account-mailer/src/account_mailer/email_processors/pad_confirmation.py @@ -69,6 +69,8 @@ def _get_admin_emails(username): admin_emails = admin_user.contacts[0].contact.email else: admin_emails = admin_user.email + else: + raise ValueError('Admin user not found, cannot determine email address.') return admin_emails, admin_name From 4eefcf0b492e87989d39e9a47f5eb698ef80d493 Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Wed, 15 May 2024 15:27:46 -0700 Subject: [PATCH 17/29] Add in unit test for duplicate message ids --- .../tests/unit/test_worker_queue.py | 28 ++++++++++++++++++- .../account-mailer/tests/unit/utils.py | 8 +++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/queue_services/account-mailer/tests/unit/test_worker_queue.py b/queue_services/account-mailer/tests/unit/test_worker_queue.py index 5673837af3..c84fa0bd34 100644 --- a/queue_services/account-mailer/tests/unit/test_worker_queue.py +++ b/queue_services/account-mailer/tests/unit/test_worker_queue.py @@ -45,7 +45,33 @@ def test_refund_request(app, session, client): mock_send.assert_called -def test_lock_account_mailer_queue(app, session, client): +def test_duplicate_messages(app, session, client): + """Assert that duplicate messages are handled by the queue..""" + user = factory_user_model_with_contact() + org = factory_org_model() + factory_membership_model(user.id, org.id) + id = org.id + mail_details = { + 'accountId': id, + 'accountName': org.name + } + + with patch.object(notification_service, 'send_email', return_value=None) as mock_send: + helper_add_event_to_queue(client, message_type=QueueMessageTypes.NSF_LOCK_ACCOUNT.value, + mail_details=mail_details, message_id='f76e5ca9-93f3-44ee-a0f8-f47ee83b1971') + mock_send.assert_called + assert mock_send.call_args.args[0].get('recipients') == 'foo@bar.com' + assert mock_send.call_args.args[0].get('content').get('subject') == SubjectType.NSF_LOCK_ACCOUNT_SUBJECT.value + assert mock_send.call_args.args[0].get('attachments') is None + assert True + + with patch.object(notification_service, 'send_email', return_value=None) as mock_send: + helper_add_event_to_queue(client, message_type=QueueMessageTypes.NSF_LOCK_ACCOUNT.value, + mail_details=mail_details, message_id='f76e5ca9-93f3-44ee-a0f8-f47ee83b1971') + mock_send.assert_not_called + + +def test_duplicate_messages(app, session, client): """Assert that events can be retrieved and decoded from the Queue.""" user = factory_user_model_with_contact() org = factory_org_model() diff --git a/queue_services/account-mailer/tests/unit/utils.py b/queue_services/account-mailer/tests/unit/utils.py index fc555fbcaa..e470ef24d4 100644 --- a/queue_services/account-mailer/tests/unit/utils.py +++ b/queue_services/account-mailer/tests/unit/utils.py @@ -20,10 +20,10 @@ from simple_cloudevent import SimpleCloudEvent, to_queue_message -def build_request_for_queue_push(message_type, payload): +def build_request_for_queue_push(message_type, payload, message_id=None): """Build request for queue message.""" queue_message_bytes = to_queue_message(SimpleCloudEvent( - id=str(uuid.uuid4()), + id=str(message_id if message_id else uuid.uuid4()), source='account-mailer', subject=None, time=datetime.now(tz=timezone.utc).isoformat(), @@ -46,10 +46,10 @@ def post_to_queue(client, request_payload): assert response.status_code == 200 -def helper_add_event_to_queue(client, message_type: str, mail_details: dict): +def helper_add_event_to_queue(client, message_type: str, mail_details: dict, message_id=None): """Add event to the Queue.""" if not mail_details: mail_details = { } - payload = build_request_for_queue_push(message_type, mail_details) + payload = build_request_for_queue_push(message_type, mail_details, message_id) post_to_queue(client, payload) From 34f6f2969d633e00a794fef77ac153b721cf8f6c Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Wed, 15 May 2024 15:31:03 -0700 Subject: [PATCH 18/29] rename accident --- queue_services/account-mailer/tests/unit/test_worker_queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queue_services/account-mailer/tests/unit/test_worker_queue.py b/queue_services/account-mailer/tests/unit/test_worker_queue.py index c84fa0bd34..a61859e394 100644 --- a/queue_services/account-mailer/tests/unit/test_worker_queue.py +++ b/queue_services/account-mailer/tests/unit/test_worker_queue.py @@ -71,7 +71,7 @@ def test_duplicate_messages(app, session, client): mock_send.assert_not_called -def test_duplicate_messages(app, session, client): +def test_lock_account_mailer_queue(app, session, client): """Assert that events can be retrieved and decoded from the Queue.""" user = factory_user_model_with_contact() org = factory_org_model() From 7dd6a43e871132a1ba58e5344e4934abd411c3b1 Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Thu, 16 May 2024 06:48:08 -0700 Subject: [PATCH 19/29] Fix enums in account-mailer so they match sbc-common --- .../account-mailer/src/account_mailer/enums.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/queue_services/account-mailer/src/account_mailer/enums.py b/queue_services/account-mailer/src/account_mailer/enums.py index cb71f8e1fc..e908c36f14 100644 --- a/queue_services/account-mailer/src/account_mailer/enums.py +++ b/queue_services/account-mailer/src/account_mailer/enums.py @@ -43,8 +43,8 @@ class SubjectType(Enum): MEMBERSHIP_APPROVED_NOTIFICATION = '[BC Registries and Online Services] Welcome to the account {account_name}' MEMBERSHIP_APPROVED_NOTIFICATION_FOR_BCEID = '[BC Registries and Online Services] Welcome to the account ' \ '{account_name}' - NONBCSC_ORG_APPROVED_NOTIFICATION = '[BC Registries and Online Services] APPROVED Business Registry Account' - NONBCSC_ORG_REJECTED_NOTIFICATION = '[BC Registries and Online Services] YOUR ACTION REQUIRED: ' \ + NON_BCSC_ORG_APPROVED_NOTIFICATION = '[BC Registries and Online Services] APPROVED Business Registry Account' + NON_BCSC_ORG_REJECTED_NOTIFICATION = '[BC Registries and Online Services] YOUR ACTION REQUIRED: ' \ 'Business Registry Account cannot be approved' OTP_AUTHENTICATOR_RESET_NOTIFICATION = '[BC Registries and Online Services] Authenticator Has Been Reset' ROLE_CHANGED_NOTIFICATION = '[BC Registries and Online Services] Your Role Has Been Changed' @@ -86,8 +86,8 @@ class TitleType(Enum): GOVM_MEMBER_INVITATION = 'Invitation to Join an Account at Business Registry' MEMBERSHIP_APPROVED_NOTIFICATION = 'Your Membership Has Been Approved' MEMBERSHIP_APPROVED_NOTIFICATION_FOR_BCEID = 'Your Membership Has Been Approved' - NONBCSC_ORG_APPROVED_NOTIFICATION = 'Your Membership Has Been Approved' - NONBCSC_ORG_REJECTED_NOTIFICATION = 'Your Membership Has Been Rejected' + NON_BCSC_ORG_APPROVED_NOTIFICATION = 'Your Membership Has Been Approved' + NON_BCSC_ORG_REJECTED_NOTIFICATION = 'Your Membership Has Been Rejected' OTP_AUTHENTICATOR_RESET_NOTIFICATION = 'Your Authenticator Has Been Reset' ROLE_CHANGED_NOTIFICATION = 'Your Role Has Been Changed' STAFF_REVIEW_ACCOUNT = 'Notification from Business Registry' @@ -130,8 +130,8 @@ class TemplateType(Enum): GOVM_MEMBER_INVITATION_TEMPLATE_NAME = 'govm_member_invitation_email' MEMBERSHIP_APPROVED_NOTIFICATION_TEMPLATE_NAME = 'membership_approved_notification_email' MEMBERSHIP_APPROVED_NOTIFICATION_FOR_BCEID_TEMPLATE_NAME = 'membership_approved_notification_email_for_bceid' - NONBCSC_ORG_APPROVED_NOTIFICATION_TEMPLATE_NAME = 'nonbcsc_org_approved_notification_email' - NONBCSC_ORG_REJECTED_NOTIFICATION_TEMPLATE_NAME = 'nonbcsc_org_rejected_notification_email' + NON_BCSC_ORG_APPROVED_NOTIFICATION_TEMPLATE_NAME = 'nonbcsc_org_approved_notification_email' + NON_BCSC_ORG_REJECTED_NOTIFICATION_TEMPLATE_NAME = 'nonbcsc_org_rejected_notification_email' OTP_AUTHENTICATOR_RESET_NOTIFICATION_TEMPLATE_NAME = 'otp_authenticator_reset_notification_email' ROLE_CHANGED_NOTIFICATION_TEMPLATE_NAME = 'role_changed_notification_email' STAFF_REVIEW_ACCOUNT_TEMPLATE_NAME = 'staff_review_account_email' From c55e749b05667e58eb2ac375686df95ac889ce77 Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Thu, 16 May 2024 07:08:59 -0700 Subject: [PATCH 20/29] use correct enum --- auth-api/src/auth_api/services/org.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auth-api/src/auth_api/services/org.py b/auth-api/src/auth_api/services/org.py index f92b499a75..5d83fa20ad 100644 --- a/auth-api/src/auth_api/services/org.py +++ b/auth-api/src/auth_api/services/org.py @@ -878,9 +878,9 @@ def send_approved_rejected_notification(receipt_admin_emails, org_name, org_id, current_app.logger.debug(' Date: Thu, 16 May 2024 07:31:48 -0700 Subject: [PATCH 21/29] Update URL, other one causes an error --- .../src/account_mailer/email_templates/reset_passcode.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queue_services/account-mailer/src/account_mailer/email_templates/reset_passcode.html b/queue_services/account-mailer/src/account_mailer/email_templates/reset_passcode.html index 57b6bb7991..6b720a6fa2 100644 --- a/queue_services/account-mailer/src/account_mailer/email_templates/reset_passcode.html +++ b/queue_services/account-mailer/src/account_mailer/email_templates/reset_passcode.html @@ -6,7 +6,7 @@ ### To access the new application and get started on your online filing: -* Go to: [https://www.bcregistry.ca/business>](https://www.bcregistry.ca/business) +* Go to: [https://www.account.bcregistry.gov.bc.ca/](https://www.account.bcregistry.gov.bc.ca/) * If you have not yet accessed this website, and need to create a new account, select the "Create a BC Registries Account" button to start. Otherwise, please select the "Login" drop down in the upper right corner of the screen to login to your existing account. * Once logged into your new account, you can add your business on the manage businesses dashboard. Please select the "+ Add an Existing..." button on the right, and select "Business" from the dropdown selection. * Use your business incorporation number: {{ businessIdentifier }} and passcode, {{ passCode }} to link your business to your account. From cb0bf1489b3654af8c38b425f0fc5581bb52b85d Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Thu, 16 May 2024 08:03:02 -0700 Subject: [PATCH 22/29] Fix indent --- queue_services/account-mailer/src/account_mailer/enums.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queue_services/account-mailer/src/account_mailer/enums.py b/queue_services/account-mailer/src/account_mailer/enums.py index e908c36f14..5de5443768 100644 --- a/queue_services/account-mailer/src/account_mailer/enums.py +++ b/queue_services/account-mailer/src/account_mailer/enums.py @@ -45,7 +45,7 @@ class SubjectType(Enum): '{account_name}' NON_BCSC_ORG_APPROVED_NOTIFICATION = '[BC Registries and Online Services] APPROVED Business Registry Account' NON_BCSC_ORG_REJECTED_NOTIFICATION = '[BC Registries and Online Services] YOUR ACTION REQUIRED: ' \ - 'Business Registry Account cannot be approved' + 'Business Registry Account cannot be approved' OTP_AUTHENTICATOR_RESET_NOTIFICATION = '[BC Registries and Online Services] Authenticator Has Been Reset' ROLE_CHANGED_NOTIFICATION = '[BC Registries and Online Services] Your Role Has Been Changed' STAFF_REVIEW_ACCOUNT = '[BC Registries and Online Services] An out of province account needs to be approved.' From 814064d22a3d6d16029d35355445f0bd08d49e03 Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Thu, 16 May 2024 08:45:04 -0700 Subject: [PATCH 23/29] Add in duplicate message handling for auth-queue. --- .../models/pubsub_message_processing.py | 2 +- .../src/auth_queue/resources/worker.py | 21 +++++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/auth-api/src/auth_api/models/pubsub_message_processing.py b/auth-api/src/auth_api/models/pubsub_message_processing.py index 1e88e2c569..0f6b31abe2 100644 --- a/auth-api/src/auth_api/models/pubsub_message_processing.py +++ b/auth-api/src/auth_api/models/pubsub_message_processing.py @@ -1,7 +1,7 @@ """This model manages pubsub message processing. NOTE: Only use this when it's not possible to use other indicators to track message processing. - Currently used by the account-mailer. This prevents duplicates. + Currently used by the account-mailer / auth-queue. This prevents duplicates. """ import datetime as dt import pytz diff --git a/queue_services/auth-queue/src/auth_queue/resources/worker.py b/queue_services/auth-queue/src/auth_queue/resources/worker.py index ac3fd351fa..053bd6e5d4 100644 --- a/queue_services/auth-queue/src/auth_queue/resources/worker.py +++ b/queue_services/auth-queue/src/auth_queue/resources/worker.py @@ -14,7 +14,7 @@ """The unique worker functionality for this service is contained here.""" import dataclasses import json -from datetime import datetime +from datetime import datetime, timezone from http import HTTPStatus from auth_api.models import ActivityLog as ActivityLogModel @@ -22,6 +22,7 @@ from auth_api.models import Entity as EntityModel from auth_api.models import Org as OrgModel from auth_api.models import db +from auth_api.models.pubsub_message_processing import PubSubMessageProcessing from auth_api.services.gcp_queue import queue from auth_api.services.gcp_queue.gcp_auth import ensure_authorized_queue_user from auth_api.services.rest_service import RestService @@ -45,7 +46,10 @@ def worker(): return {}, HTTPStatus.OK try: - current_app.logger.info('Event Message Received: %s', json.dumps(dataclasses.asdict(event_message))) + current_app.logger.info('Event message received: %s', json.dumps(dataclasses.asdict(event_message))) + if is_message_processed(event_message): + current_app.logger.info('Event message already processed, skipping.') + return {}, HTTPStatus.OK if event_message.type == QueueMessageTypes.NAMES_EVENT.value: process_name_events(event_message) elif event_message.type == QueueMessageTypes.ACTIVITY_LOG.value: @@ -59,6 +63,19 @@ def worker(): return {}, HTTPStatus.OK +def is_message_processed(event_message): + """Check if the queue message is processed.""" + if PubSubMessageProcessing.find_by_cloud_event_id_and_type(event_message.id, event_message.type): + return True + pubsub_message_processing = PubSubMessageProcessing() + pubsub_message_processing.cloud_event_id = event_message.id + pubsub_message_processing.message_type = event_message.type + pubsub_message_processing.processed = datetime.now(timezone.utc) + db.session.add(pubsub_message_processing) + db.session.commit() + return False + + def process_activity_log(data): """Process activity log events.""" current_app.logger.debug('>>>>>>>process_activity_log>>>>>') From e929c89756c3b3bcb4649da9a607a0c4ab467488 Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Thu, 16 May 2024 09:02:23 -0700 Subject: [PATCH 24/29] fix up requirements --- queue_services/auth-queue/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queue_services/auth-queue/requirements.txt b/queue_services/auth-queue/requirements.txt index cb3c89c358..60772a8ea4 100644 --- a/queue_services/auth-queue/requirements.txt +++ b/queue_services/auth-queue/requirements.txt @@ -47,5 +47,5 @@ tornado==6.4 urllib3==1.26.18 zipp==3.18.1 -e git+https://github.com/bcgov/sbc-common-components.git#egg=sbc-common-components&subdirectory=python --e git+https://github.com/seeker25/sbc-auth.git@queue_upgrades#egg=auth-api&subdirectory=auth-api +-e git+https://github.com/bcgov/sbc-auth.git@queue_fixess#egg=auth-api&subdirectory=auth-api git+https://github.com/daxiom/simple-cloudevent.py.git From 5bfd00f690d3f14043af5459d25f49a7d426fc1f Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Thu, 16 May 2024 09:52:10 -0700 Subject: [PATCH 25/29] Fix linting issues --- auth-api/src/auth_api/models/membership.py | 2 +- auth-api/src/auth_api/models/pubsub_message_processing.py | 5 +++++ auth-api/src/auth_api/services/authorization.py | 1 + auth-api/src/auth_api/services/validators/payment_type.py | 1 + auth-api/src/auth_api/utils/custom_query.py | 2 +- 5 files changed, 9 insertions(+), 2 deletions(-) diff --git a/auth-api/src/auth_api/models/membership.py b/auth-api/src/auth_api/models/membership.py index a4bacd7406..ce049b0aaa 100644 --- a/auth-api/src/auth_api/models/membership.py +++ b/auth-api/src/auth_api/models/membership.py @@ -51,7 +51,7 @@ class Membership(VersionedModel): # pylint: disable=too-few-public-methods # Te user = relationship('User', foreign_keys=[user_id], lazy='select') org = relationship('Org', foreign_keys=[org_id], lazy='select') - def __init__(self, **kwargs): + def __init__(self, **kwargs): # pylint: disable=super-init-not-called """Initialize a new membership.""" self.org_id = kwargs.get('org_id') self.user_id = kwargs.get('user_id') diff --git a/auth-api/src/auth_api/models/pubsub_message_processing.py b/auth-api/src/auth_api/models/pubsub_message_processing.py index 0f6b31abe2..2e3dc2234f 100644 --- a/auth-api/src/auth_api/models/pubsub_message_processing.py +++ b/auth-api/src/auth_api/models/pubsub_message_processing.py @@ -21,6 +21,11 @@ class PubSubMessageProcessing(db.Model): message_type = Column(String(250), nullable=False) processed = Column(DateTime, nullable=True) + @classmethod + def find_by_id(cls, identifier): + """Find a pubsub message processing by id.""" + return cls.query.filter_by(id=identifier).one_or_none() + @classmethod def find_by_cloud_event_id_and_type(cls, cloud_event_id, message_type): """Find a pubsub message processing for cloud event id and type.""" diff --git a/auth-api/src/auth_api/services/authorization.py b/auth-api/src/auth_api/services/authorization.py index 991a25adb5..e7f39e51ea 100644 --- a/auth-api/src/auth_api/services/authorization.py +++ b/auth-api/src/auth_api/services/authorization.py @@ -226,6 +226,7 @@ def check_auth(**kwargs): if product_code_in_jwt == 'ALL': # Product code for super admin service account (sbc-auth-admin) return + auth = None if business_identifier: auth = Authorization.get_user_authorizations_for_entity(business_identifier) elif org_identifier: diff --git a/auth-api/src/auth_api/services/validators/payment_type.py b/auth-api/src/auth_api/services/validators/payment_type.py index 03180ee444..49695e3c8a 100644 --- a/auth-api/src/auth_api/services/validators/payment_type.py +++ b/auth-api/src/auth_api/services/validators/payment_type.py @@ -39,6 +39,7 @@ def validate(is_fatal=False, **kwargs) -> ValidatorResponse: OrgType.SBC_STAFF: non_ejv_payment_methods, OrgType.STAFF: non_ejv_payment_methods, } + payment_type = None if access_type == AccessType.GOVM.value: payment_type = PaymentMethod.EJV.value elif selected_payment_method: diff --git a/auth-api/src/auth_api/utils/custom_query.py b/auth-api/src/auth_api/utils/custom_query.py index fc6ed4f938..e4356e2b71 100644 --- a/auth-api/src/auth_api/utils/custom_query.py +++ b/auth-api/src/auth_api/utils/custom_query.py @@ -18,7 +18,7 @@ from sqlalchemy import String, func -class CustomQuery(BaseQuery): +class CustomQuery(BaseQuery): # pylint: disable=too-few-public-methods """Custom Query class to extend the base query class for helper functionality.""" def filter_conditionally(self, search_criteria, model_attribute, is_like: bool = False): From bf61a4c7d9c7eab47d1965de53db781ec7cdb359 Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Thu, 16 May 2024 14:18:47 -0700 Subject: [PATCH 26/29] See if unit test passes again --- auth-api/src/auth_api/services/authorization.py | 1 - 1 file changed, 1 deletion(-) diff --git a/auth-api/src/auth_api/services/authorization.py b/auth-api/src/auth_api/services/authorization.py index e7f39e51ea..991a25adb5 100644 --- a/auth-api/src/auth_api/services/authorization.py +++ b/auth-api/src/auth_api/services/authorization.py @@ -226,7 +226,6 @@ def check_auth(**kwargs): if product_code_in_jwt == 'ALL': # Product code for super admin service account (sbc-auth-admin) return - auth = None if business_identifier: auth = Authorization.get_user_authorizations_for_entity(business_identifier) elif org_identifier: From 488beaed2886e2309fdcf697614c64215efccdce Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Tue, 21 May 2024 09:53:33 -0700 Subject: [PATCH 27/29] restore update-env for makefiles --- auth-api/Makefile | 6 +++--- queue_services/account-mailer/Makefile | 6 +++--- queue_services/auth-queue/Makefile | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/auth-api/Makefile b/auth-api/Makefile index 699cc2455f..69644203e3 100644 --- a/auth-api/Makefile +++ b/auth-api/Makefile @@ -93,15 +93,15 @@ mac-cov: local-test ## Run the coverage report and display in a browser window ( ################################################################################# cd: ## CD flow ifeq ($(TAG_NAME), test) -cd: +cd: update-env oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):dev $(DOCKER_NAME):$(TAG_NAME) else ifeq ($(TAG_NAME), prod) -cd: +cd: update-env oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):$(TAG_NAME) $(DOCKER_NAME):$(TAG_NAME)-$(shell date +%F) oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):test $(DOCKER_NAME):$(TAG_NAME) else TAG_NAME=dev -cd: build tag +cd: build update-env tag endif build: ## Build the docker container diff --git a/queue_services/account-mailer/Makefile b/queue_services/account-mailer/Makefile index 28aeb959c3..2d7772dd2f 100644 --- a/queue_services/account-mailer/Makefile +++ b/queue_services/account-mailer/Makefile @@ -87,15 +87,15 @@ mac-cov: test ## Run the coverage report and display in a browser window (mac) ################################################################################# cd: ## CD flow ifeq ($(TAG_NAME), test) -cd: +cd: update-env oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):dev $(DOCKER_NAME):$(TAG_NAME) else ifeq ($(TAG_NAME), prod) -cd: +cd: update-env oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):$(TAG_NAME) $(DOCKER_NAME):$(TAG_NAME)-$(shell date +%F) oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):test $(DOCKER_NAME):$(TAG_NAME) else TAG_NAME=dev -cd: build tag +cd: build update-env tag endif build: ## Build the docker container diff --git a/queue_services/auth-queue/Makefile b/queue_services/auth-queue/Makefile index 2760fefe55..0c3fa31ca0 100644 --- a/queue_services/auth-queue/Makefile +++ b/queue_services/auth-queue/Makefile @@ -87,15 +87,15 @@ mac-cov: test ## Run the coverage report and display in a browser window (mac) ################################################################################# cd: ## CD flow ifeq ($(TAG_NAME), test) -cd: +cd: update-env oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):dev $(DOCKER_NAME):$(TAG_NAME) else ifeq ($(TAG_NAME), prod) -cd: +cd: update-env oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):$(TAG_NAME) $(DOCKER_NAME):$(TAG_NAME)-$(shell date +%F) oc -n "$(OPENSHIFT_REPOSITORY)-tools" tag $(DOCKER_NAME):test $(DOCKER_NAME):$(TAG_NAME) else TAG_NAME=dev -cd: build tag +cd: build update-env tag endif build: ## Build the docker container From 562bfe35a593bf0daf37d5f8c192024143bb81cd Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Tue, 21 May 2024 10:12:05 -0700 Subject: [PATCH 28/29] lint + test fix --- auth-api/src/auth_api/services/authorization.py | 1 + auth-api/tests/unit/services/test_authorization.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/auth-api/src/auth_api/services/authorization.py b/auth-api/src/auth_api/services/authorization.py index 991a25adb5..e7f39e51ea 100644 --- a/auth-api/src/auth_api/services/authorization.py +++ b/auth-api/src/auth_api/services/authorization.py @@ -226,6 +226,7 @@ def check_auth(**kwargs): if product_code_in_jwt == 'ALL': # Product code for super admin service account (sbc-auth-admin) return + auth = None if business_identifier: auth = Authorization.get_user_authorizations_for_entity(business_identifier) elif org_identifier: diff --git a/auth-api/tests/unit/services/test_authorization.py b/auth-api/tests/unit/services/test_authorization.py index 4952359fc2..02119ee999 100644 --- a/auth-api/tests/unit/services/test_authorization.py +++ b/auth-api/tests/unit/services/test_authorization.py @@ -284,8 +284,8 @@ def test_check_auth_staff_path(session, monkeypatch, test_desc, test_expect, add 'test_desc,test_expect,additional_kwargs,is_org_member,is_entity_affiliated,product_code_in_jwt', [ ( - 'Test UnboundLocalError when no role checks provided in kwargs, and no org_id or business_identifier.', - pytest.raises(UnboundLocalError), {}, False, False, ProductCode.BUSINESS.value + 'Test 403 when no role checks provided in kwargs, and no org_id or business_identifier.', + pytest.raises(Forbidden), {}, False, False, ProductCode.BUSINESS.value ), ( 'Test OK when no role checks provided in kwargs, but has ALL product in jwt. (bypass all checks).', @@ -359,8 +359,8 @@ def test_check_auth_system_path(session, monkeypatch, test_desc, test_expect, ad 'test_desc,test_expect,additional_kwargs,is_org_member,is_entity_affiliated', [ ( - 'Test UnboundLocalError when no role checks provided in kwargs.', - pytest.raises(UnboundLocalError), {}, False, False + 'Test HTTPException (403) when no role checks provided in kwargs.', + pytest.raises(HTTPException), {}, False, False ), ( 'Test 403 when org member, but no role checks provided in kwargs.', From dacc6d5957ad9d81ad8c94aa5c2ed6d531270eca Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Tue, 21 May 2024 10:25:33 -0700 Subject: [PATCH 29/29] Unit test touch up --- auth-api/tests/unit/services/test_authorization.py | 4 ++-- queue_services/account-mailer/requirements.txt | 2 +- queue_services/auth-queue/requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/auth-api/tests/unit/services/test_authorization.py b/auth-api/tests/unit/services/test_authorization.py index 02119ee999..5d1e6f57fe 100644 --- a/auth-api/tests/unit/services/test_authorization.py +++ b/auth-api/tests/unit/services/test_authorization.py @@ -359,8 +359,8 @@ def test_check_auth_system_path(session, monkeypatch, test_desc, test_expect, ad 'test_desc,test_expect,additional_kwargs,is_org_member,is_entity_affiliated', [ ( - 'Test HTTPException (403) when no role checks provided in kwargs.', - pytest.raises(HTTPException), {}, False, False + 'Test UnboundLocalError (403) when no role checks provided in kwargs.', + pytest.raises(UnboundLocalError), {}, False, False ), ( 'Test 403 when org member, but no role checks provided in kwargs.', diff --git a/queue_services/account-mailer/requirements.txt b/queue_services/account-mailer/requirements.txt index 0179b3c634..72554213e7 100644 --- a/queue_services/account-mailer/requirements.txt +++ b/queue_services/account-mailer/requirements.txt @@ -34,5 +34,5 @@ tornado==6.4 urllib3==1.26.18 zipp==3.18.1 -e git+https://github.com/bcgov/sbc-common-components.git#egg=sbc-common-components&subdirectory=python --e git+https://github.com/bcgov/sbc-auth.git@queue_fixess#egg=auth-api&subdirectory=auth-api +-e git+https://github.com/bcgov/sbc-auth.git#egg=auth-api&subdirectory=auth-api git+https://github.com/daxiom/simple-cloudevent.py.git diff --git a/queue_services/auth-queue/requirements.txt b/queue_services/auth-queue/requirements.txt index 60772a8ea4..da94942d3c 100644 --- a/queue_services/auth-queue/requirements.txt +++ b/queue_services/auth-queue/requirements.txt @@ -47,5 +47,5 @@ tornado==6.4 urllib3==1.26.18 zipp==3.18.1 -e git+https://github.com/bcgov/sbc-common-components.git#egg=sbc-common-components&subdirectory=python --e git+https://github.com/bcgov/sbc-auth.git@queue_fixess#egg=auth-api&subdirectory=auth-api +-e git+https://github.com/bcgov/sbc-auth.git#egg=auth-api&subdirectory=auth-api git+https://github.com/daxiom/simple-cloudevent.py.git