diff --git a/queue_services/payment-reconciliations/requirements.txt b/queue_services/payment-reconciliations/requirements.txt index 44e751bef..2cf9bd505 100644 --- a/queue_services/payment-reconciliations/requirements.txt +++ b/queue_services/payment-reconciliations/requirements.txt @@ -1,6 +1,6 @@ -e git+https://github.com/bcgov/lear.git@30dba30463c99aaedfdcfd463213e71ba0d35b51#egg=entity_queue_common&subdirectory=queue_services/common -e git+https://github.com/bcgov/sbc-common-components.git@b93585ea3ac273b9e51c4dd5ddbc8190fd95da6a#egg=sbc_common_components&subdirectory=python --e git+https://github.com/bcgov/sbc-pay.git@2e4373b8a9280d64737f7b6ed172831ba4e541d2#egg=pay_api&subdirectory=pay-api +-e git+https://github.com/bcgov/sbc-pay.git@1a1cb0c7b00978a36aed7c0e9d7bcd7148e0401a#egg=pay_api&subdirectory=pay-api Flask-Caching==2.0.2 Flask-Migrate==2.7.0 Flask-Moment==1.0.5 diff --git a/queue_services/payment-reconciliations/src/reconciliations/config.py b/queue_services/payment-reconciliations/src/reconciliations/config.py index b5cef8fba..3ce8d1bde 100644 --- a/queue_services/payment-reconciliations/src/reconciliations/config.py +++ b/queue_services/payment-reconciliations/src/reconciliations/config.py @@ -60,6 +60,7 @@ class _Config(): # pylint: disable=too-few-public-methods PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) PAY_LD_SDK_KEY = os.getenv('PAY_LD_SDK_KEY', None) + LEGISLATIVE_TIMEZONE = os.getenv('LEGISLATIVE_TIMEZONE', 'America/Vancouver') SENTRY_ENABLE = os.getenv('SENTRY_ENABLE', 'False') SENTRY_DSN = os.getenv('SENTRY_DSN', None) @@ -113,6 +114,9 @@ class _Config(): # pylint: disable=too-few-public-methods CFS_CLIENT_SECRET = os.getenv('CFS_CLIENT_SECRET') CONNECT_TIMEOUT = int(os.getenv('CONNECT_TIMEOUT', '10')) + # EFT Config + EFT_INVOICE_PREFIX = os.getenv('EFT_INVOICE_PREFIX', 'REG') + # Secret key for encrypting bank account ACCOUNT_SECRET_KEY = os.getenv('ACCOUNT_SECRET_KEY') diff --git a/queue_services/payment-reconciliations/src/reconciliations/eft/eft_errors.py b/queue_services/payment-reconciliations/src/reconciliations/eft/eft_errors.py index e61aab7d5..21b5906ad 100644 --- a/queue_services/payment-reconciliations/src/reconciliations/eft/eft_errors.py +++ b/queue_services/payment-reconciliations/src/reconciliations/eft/eft_errors.py @@ -31,4 +31,4 @@ class EFTError(Enum): INVALID_DEPOSIT_AMOUNT_CAD = 'Invalid transaction deposit amount CAD.' INVALID_TRANSACTION_DATE = 'Invalid transaction date.' INVALID_DEPOSIT_DATETIME = 'Invalid transaction deposit date time' - BCROS_ACCOUNT_NUMBER_REQUIRED = 'BCROS Account number is missing from the transaction description.' + ACCOUNT_SHORTNAME_REQUIRED = 'Account shortname is missing from the transaction description.' diff --git a/queue_services/payment-reconciliations/src/reconciliations/eft/eft_reconciliation.py b/queue_services/payment-reconciliations/src/reconciliations/eft/eft_reconciliation.py new file mode 100644 index 000000000..40029d239 --- /dev/null +++ b/queue_services/payment-reconciliations/src/reconciliations/eft/eft_reconciliation.py @@ -0,0 +1,506 @@ +# Copyright © 2023 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""EFT reconciliation file. + +The entry-point is the **cb_subscription_handler** + +The design and flow leverage a few constraints that are placed upon it +by NATS Streaming and using AWAIT on the default loop. +- NATS streaming queues require one message to be processed at a time. +- AWAIT on the default loop effectively runs synchronously + +If these constraints change, the use of Flask-SQLAlchemy would need to change. +Flask-SQLAlchemy currently allows the base model to be changed, or reworking +the model to a standalone SQLAlchemy usage with an async engine would need +to be pursued. +""" +from datetime import datetime +from operator import and_ +from typing import Dict, List + +from _decimal import Decimal +from entity_queue_common.service_utils import logger +from flask import current_app +from pay_api import db +from pay_api.factory.payment_system_factory import PaymentSystemFactory +from pay_api.models import EFTCredit as EFTCreditModel +from pay_api.models import EFTFile as EFTFileModel +from pay_api.models import EFTShortnames as EFTShortnameModel +from pay_api.models import EFTTransaction as EFTTransactionModel +from pay_api.models import Invoice as InvoiceModel +from pay_api.models import InvoiceReference as InvoiceReferenceModel +from pay_api.models import Payment as PaymentModel +from pay_api.models import PaymentAccount as PaymentAccountModel +from pay_api.models import Receipt as ReceiptModel +from pay_api.utils.enums import ( + EFTFileLineType, EFTProcessStatus, InvoiceReferenceStatus, InvoiceStatus, PaymentMethod, PaymentStatus) +from sentry_sdk import capture_message + +from reconciliations.eft import EFTHeader, EFTRecord, EFTTrailer +from reconciliations.minio import get_object + + +async def reconcile_eft_payments(msg: Dict[str, any]): # pylint: disable=too-many-locals + """Read the TDI17 file, create processing records and update payment details. + + 1: Check to see if file has been previously processed. + 2: Create / Update EFT File Model record + 3: Parse EFT header, transactions and trailer + 4: Validate header and trailer - persist any error messages + 4.1: If header and/or trailer is invalid, set FAIL state and return + 5: Validate and persist transaction details - persist any error messages + 5.1: If transaction details are invalid, set FAIL state and return + 6: Calculate total transaction balance per short name - dictionary + 7: Apply balance to outstanding EFT invoices - Update invoice paid amount and status, create payment, + invoice reference, and receipt + 8: Create EFT Credit records + 9: Finalize and complete + """ + # Fetch EFT File + file_name: str = msg.get('data').get('fileName') + minio_location: str = msg.get('data').get('location') + file = get_object(minio_location, file_name) + file_content = file.data.decode('utf-8-sig') + + # Split into lines + lines = file_content.splitlines() + + # Check if there is an existing EFT File record + eft_file_model: EFTFileModel = db.session.query(EFTFileModel).filter( + EFTFileModel.file_ref == file_name).one_or_none() + + if eft_file_model and eft_file_model.status_code == EFTProcessStatus.COMPLETED.value: + logger.info('File: %s already completed processing on %s.', file_name, eft_file_model.completed_on) + return + + # There is no existing EFT File record - instantiate one + if eft_file_model is None: + eft_file_model = EFTFileModel() + eft_file_model.file_ref = file_name + + # EFT File - In Progress + eft_file_model.status_code = EFTProcessStatus.IN_PROGRESS.value + eft_file_model.save() + + # EFT File parsed data holders + eft_header: EFTHeader = None + eft_trailer: EFTTrailer = None + eft_transactions: [EFTRecord] = [] + + # Read and parse EFT file header, trailer, transactions + for index, line in enumerate(lines): + if index == 0: + eft_header = EFTHeader(line, index) + elif index == len(lines) - 1: + eft_trailer = EFTTrailer(line, index) + else: + eft_transactions.append(EFTRecord(line, index)) + + eft_header_valid = _process_eft_header(eft_header, eft_file_model) + eft_trailer_valid = _process_eft_trailer(eft_trailer, eft_file_model) + + # If header and/or trailer has errors do not proceed + if not (eft_header_valid and eft_trailer_valid): + logger.error('Failed to process file %s with an invalid header or trailer.', file_name) + eft_file_model.status_code = EFTProcessStatus.FAILED.value + eft_file_model.save() + return + + has_eft_transaction_errors = False + + # Parse EFT Transactions + for eft_transaction in eft_transactions: + if eft_transaction.has_errors(): # Flag any instance of an error - will indicate file is partially processed + has_eft_transaction_errors = True + _save_eft_transaction(eft_record=eft_transaction, eft_file_model=eft_file_model, is_error=True) + else: + # Save TDI17 transaction record + _save_eft_transaction(eft_record=eft_transaction, eft_file_model=eft_file_model, is_error=False) + + # EFT Transactions have parsing errors - stop and FAIL transactions + # We want a full file to be parseable as we want to get a full accurate balance before applying them to invoices + if has_eft_transaction_errors: + logger.error('Failed to process file %s has transaction parsing errors.', file_name) + _update_transactions_to_fail(eft_file_model) + return + + # Generate dictionary with shortnames and total deposits + shortname_balance = _shortname_balance_as_dict(eft_transactions) + + # Process payments, update invoice and create receipt + has_eft_transaction_errors = _process_eft_payments(shortname_balance, eft_file_model) + + # Check and add credits + has_eft_credits_error = _process_eft_credits(shortname_balance, eft_file_model.id) + + # Mark EFT File partially processed due to transaction errors + # Rollback EFT transactions and update to FAIL status + if has_eft_transaction_errors or has_eft_credits_error: + db.session.rollback() + _update_transactions_to_fail(eft_file_model) + logger.error('Failed to process file %s due to transaction errors.', file_name) + return + + _finalize_process_state(eft_file_model) + + +def _finalize_process_state(eft_file_model: EFTFileModel): + """Set the final transaction and file statuses.""" + _update_transactions_to_complete(eft_file_model) + + status_code = EFTProcessStatus.COMPLETED.value + eft_file_model.status_code = status_code + eft_file_model.completed_on = datetime.now() + eft_file_model.save() + + +def _process_eft_header(eft_header: EFTHeader, eft_file_model: EFTFileModel) -> bool: + """Process the EFT Header.""" + if eft_header is None: + logger.error('Failed to process file %s with an invalid header.', eft_file_model.file_ref) + return False + + # Populate header and trailer data on EFT File record - values will return None if parsing failed + _set_eft_header_on_file(eft_header, eft_file_model) + + # Errors on parsing header - create EFT error records + if eft_header is not None and eft_header.has_errors(): + _save_eft_header_error(eft_header, eft_file_model) + return False + + return True + + +def _process_eft_trailer(eft_trailer: EFTTrailer, eft_file_model: EFTFileModel) -> bool: + """Process the EFT Trailer.""" + if eft_trailer is None: + logger.error('Failed to process file %s with an invalid trailer.', eft_file_model.file_ref) + return False + + # Populate header and trailer data on EFT File record - values will return None if parsing failed + _set_eft_trailer_on_file(eft_trailer, eft_file_model) + + # Errors on parsing trailer - create EFT error records + if eft_trailer is not None and eft_trailer.has_errors(): + _save_eft_trailer_error(eft_trailer, eft_file_model) + return False + + return True + + +def _process_eft_credits(shortname_balance, eft_file_id): + """Generate credits if there are any remaining balances.""" + has_credit_errors = False + for shortname in shortname_balance.keys(): + try: + # Skip if there is no balance + if not shortname_balance[shortname]['balance'] > 0: + continue + + eft_shortname = _get_shortname(shortname) + payment_account_id = None + + # Get payment account if short name is mapped to an auth account + if eft_shortname.auth_account_id is not None: + payment_account: PaymentAccountModel = PaymentAccountModel.\ + find_by_auth_account_id(eft_shortname.auth_account_id) + payment_account_id = payment_account.id + + # Check if there is an existing eft credit for this file + eft_credit_model = db.session.query(EFTCreditModel) \ + .filter(EFTCreditModel.eft_file_id == eft_file_id) \ + .filter(EFTCreditModel.short_name_id == eft_shortname.id) \ + .one_or_none() + + if eft_credit_model is None: + eft_credit_model = EFTCreditModel() + + balance = shortname_balance[shortname]['balance'] + + eft_credit_model.eft_file_id = eft_file_id + eft_credit_model.payment_account_id = payment_account_id + eft_credit_model.short_name_id = eft_shortname.id + eft_credit_model.amount = balance + eft_credit_model.remaining_amount = balance + db.session.add(eft_credit_model) + except Exception as e: # NOQA pylint: disable=broad-exception-caught + has_credit_errors = True + logger.error(e) + capture_message('EFT Failed to set EFT balance.', level='error') + return has_credit_errors + + +def _process_eft_payments(shortname_balance: Dict, eft_file: EFTFileModel) -> bool: + """Process payments by creating payment records and updating invoice state.""" + has_eft_transaction_errors = False + + for shortname in shortname_balance.keys(): + # Retrieve or Create shortname mapping + eft_shortname_model = _get_shortname(shortname) + + # No balance to apply - move to next shortname + if shortname_balance[shortname]['balance'] <= 0: + logger.warning('UNEXPECTED BALANCE: %s had zero or less balance on file: %s', shortname, eft_file.file_ref) + continue + + # check if short name is mapped to an auth account + if eft_shortname_model is not None and eft_shortname_model.auth_account_id is not None: + # We have a mapping and can continue processing + try: + auth_account_id = eft_shortname_model.auth_account_id + # Find invoices to be paid + invoices: List[InvoiceModel] = _get_invoices_owing(auth_account_id) + payment_account: PaymentAccountModel = PaymentAccountModel.find_by_auth_account_id(auth_account_id) + if invoices is not None: + for invoice in invoices: + _pay_invoice(payment_account=payment_account, + invoice=invoice, + shortname_balance=shortname_balance[shortname], + receipt_number=invoice.id) + + except Exception as e: # NOQA pylint: disable=broad-exception-caught + has_eft_transaction_errors = True + logger.error(e) + capture_message('EFT Failed to apply balance to invoice.', level='error') + + return has_eft_transaction_errors + + +def _set_eft_header_on_file(eft_header: EFTHeader, eft_file_model: EFTFileModel): + """Set EFT Header information on EFTFile model.""" + eft_file_model.file_creation_date = getattr(eft_header, 'creation_datetime', None) + eft_file_model.deposit_from_date = getattr(eft_header, 'starting_deposit_date', None) + eft_file_model.deposit_to_date = getattr(eft_header, 'ending_deposit_date', None) + + +def _set_eft_trailer_on_file(eft_trailer: EFTTrailer, eft_file_model: EFTFileModel): + """Set EFT Trailer information on EFTFile model.""" + eft_file_model.number_of_details = getattr(eft_trailer, 'number_of_details', None) + eft_file_model.total_deposit_cents = getattr(eft_trailer, 'total_deposit_amount', None) + + +def _set_eft_base_error(line_type: str, index: int, + eft_file_id: int, error_messages: [str]) -> EFTTransactionModel: + """Instantiate EFT Transaction model error record.""" + eft_transaction_model = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_id) \ + .filter(EFTTransactionModel.line_type == line_type).one_or_none() + + if eft_transaction_model is None: + eft_transaction_model = EFTTransactionModel() + + eft_transaction_model.line_type = line_type + eft_transaction_model.line_number = index + eft_transaction_model.file_id = eft_file_id + eft_transaction_model.status_code = EFTProcessStatus.FAILED.value + eft_transaction_model.error_messages = error_messages + + return eft_transaction_model + + +def _save_eft_header_error(eft_header: EFTHeader, eft_file_model: EFTFileModel): + """Save or update EFT Header error record.""" + eft_transaction_model = _set_eft_base_error(line_type=EFTFileLineType.HEADER.value, + index=eft_header.index, + eft_file_id=eft_file_model.id, + error_messages=eft_header.get_error_messages()) + eft_transaction_model.save() + + +def _save_eft_trailer_error(eft_trailer: EFTTrailer, eft_file_model: EFTFileModel): + """Save or update EFT Trailer error record.""" + eft_transaction_model = _set_eft_base_error(line_type=EFTFileLineType.TRAILER.value, + index=eft_trailer.index, + eft_file_id=eft_file_model.id, + error_messages=eft_trailer.get_error_messages()) + eft_transaction_model.save() + + +def _save_eft_transaction(eft_record: EFTRecord, eft_file_model: EFTFileModel, is_error: bool): + """Save or update EFT Transaction details record.""" + line_type = EFTFileLineType.TRANSACTION.value + status_code = EFTProcessStatus.FAILED.value if is_error else EFTProcessStatus.IN_PROGRESS.value + + eft_transaction_model = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_number == eft_record.index) \ + .filter(EFTTransactionModel.line_type == line_type).one_or_none() + + if eft_transaction_model is None: + eft_transaction_model = EFTTransactionModel() + + if eft_record.transaction_description: + eft_short_name: EFTShortnameModel = _get_shortname(eft_record.transaction_description) + eft_transaction_model.short_name_id = eft_short_name.id if eft_short_name else None + + eft_transaction_model.line_type = line_type + eft_transaction_model.line_number = eft_record.index + eft_transaction_model.file_id = eft_file_model.id + eft_transaction_model.status_code = status_code + eft_transaction_model.error_messages = eft_record.get_error_messages() + eft_transaction_model.batch_number = getattr(eft_record, 'batch_number', None) + eft_transaction_model.sequence_number = getattr(eft_record, 'transaction_sequence', None) + eft_transaction_model.jv_type = getattr(eft_record, 'jv_type', None) + eft_transaction_model.jv_number = getattr(eft_record, 'jv_number', None) + deposit_amount_cad = getattr(eft_record, 'deposit_amount_cad', None) + eft_transaction_model.deposit_amount_cents = deposit_amount_cad + eft_transaction_model.save() + + +def _update_transactions_to_fail(eft_file_model: EFTFileModel) -> int: + """Set EFT transactions to fail status.""" + result = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id, + EFTTransactionModel.line_type == EFTFileLineType.TRANSACTION.value) \ + .update({EFTTransactionModel.status_code: EFTProcessStatus.FAILED.value}, synchronize_session='fetch') + + eft_file_model.status_code = EFTProcessStatus.FAILED.value + eft_file_model.save() + + return result + + +def _update_transactions_to_complete(eft_file_model: EFTFileModel) -> int: + """Set EFT transactions to complete status if they are currently in progress.""" + result = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.status_code == EFTProcessStatus.IN_PROGRESS.value) \ + .update({EFTTransactionModel.status_code: EFTProcessStatus.COMPLETED.value}, synchronize_session='fetch') + db.session.commit() + + return result + + +def _get_shortname(short_name: str) -> EFTShortnameModel: + """Save short name if it doesn't exist.""" + eft_shortname = db.session.query(EFTShortnameModel) \ + .filter(EFTShortnameModel.short_name == short_name) \ + .one_or_none() + + if eft_shortname is None: + eft_shortname = EFTShortnameModel() + eft_shortname.short_name = short_name + eft_shortname.save() + + return eft_shortname + + +def _get_invoices_owing(auth_account_id: str) -> [InvoiceModel]: + """Return invoices that have not been fully paid.""" + unpaid_status = (InvoiceStatus.PARTIAL.value, + InvoiceStatus.CREATED.value, InvoiceStatus.OVERDUE.value) + query = db.session.query(InvoiceModel) \ + .join(PaymentAccountModel, and_(PaymentAccountModel.id == InvoiceModel.payment_account_id, + PaymentAccountModel.auth_account_id == auth_account_id)) \ + .filter(InvoiceModel.invoice_status_code.in_(unpaid_status)) \ + .order_by(InvoiceModel.created_on.asc()) + + return query.all() + + +def _shortname_balance_as_dict(eft_transactions: List[EFTRecord]) -> Dict: + """Create a dictionary mapping for shortname and total deposits from TDI17 file.""" + shortname_balance = {} + + for eft_transaction in eft_transactions: + # Skip any transactions with errors + if eft_transaction.has_errors(): + continue + + shortname = eft_transaction.transaction_description + transaction_date = eft_transaction.transaction_date + + if shortname in shortname_balance: + shortname_balance[shortname]['transaction_date'] = transaction_date + shortname_balance[shortname]['balance'] += eft_transaction.deposit_amount_cad / 100 + else: + shortname_balance[shortname] = {'transaction_date': transaction_date, + 'balance': eft_transaction.deposit_amount_cad / 100} + + return shortname_balance + + +def _pay_invoice(payment_account: PaymentAccountModel, invoice: InvoiceModel, shortname_balance: Dict, + receipt_number=None): + """Pay for an invoice and update invoice state.""" + payment_date = shortname_balance.get('transaction_date') or datetime.now() + balance = shortname_balance.get('balance') + + # Get the unpaid total - could be partial + unpaid_total = _get_invoice_unpaid_total(invoice) + + # Determine the payable amount based on what is available in the shortname balance + payable_amount = unpaid_total if balance >= unpaid_total else balance + + invoice.paid += payable_amount + invoice.payment_date = payment_date + + # Update the invoice state + if _get_invoice_unpaid_total(invoice) == 0: + invoice.invoice_status_code = InvoiceStatus.PAID.value + else: + invoice.invoice_status_code = InvoiceStatus.PARTIAL.value + + db.session.add(invoice) + + # Create the payment record + payment = _save_payment(payment_account=payment_account, + invoice=invoice, + payment_date=payment_date, + paid_amount=payable_amount) + + invoice_reference = InvoiceReferenceModel() + invoice_reference.invoice_id = invoice.id + invoice_reference.status_code = InvoiceReferenceStatus.ACTIVE.value + invoice_reference.invoice_number = payment.invoice_number + + db.session.add(invoice_reference) + + # Create Receipt records + receipt: ReceiptModel = ReceiptModel() + receipt.receipt_date = payment_date + receipt.receipt_amount = payable_amount + receipt.invoice_id = invoice.id + receipt.receipt_number = receipt_number + db.session.add(receipt) + + # Paid - update the shortname balance + shortname_balance['balance'] -= payable_amount + + +def _save_payment(payment_account: PaymentAccountModel, invoice: InvoiceModel, payment_date: datetime, + paid_amount, receipt_number=None) -> PaymentModel: + """Create a payment record for an invoice.""" + pay_service = PaymentSystemFactory.create_from_payment_method(PaymentMethod.EFT.value) + + payment = PaymentModel() + payment.payment_method_code = pay_service.get_payment_method_code() + payment.payment_status_code = PaymentStatus.COMPLETED.value + payment.payment_system_code = pay_service.get_payment_system_code() + payment.invoice_number = f'{current_app.config["EFT_INVOICE_PREFIX"]}{invoice.id}' + payment.invoice_amount = invoice.total + payment.payment_account_id = payment_account.id + payment.payment_date = payment_date + payment.paid_amount = paid_amount + payment.receipt_number = receipt_number + db.session.add(payment) + + return payment + + +def _get_invoice_unpaid_total(invoice: InvoiceModel) -> Decimal: + invoice_total = invoice.total or 0 + invoice_paid = invoice.paid or 0 + + return invoice_total - invoice_paid diff --git a/queue_services/payment-reconciliations/src/reconciliations/eft/eft_record.py b/queue_services/payment-reconciliations/src/reconciliations/eft/eft_record.py index 99a521be4..19acedc30 100644 --- a/queue_services/payment-reconciliations/src/reconciliations/eft/eft_record.py +++ b/queue_services/payment-reconciliations/src/reconciliations/eft/eft_record.py @@ -71,10 +71,10 @@ def _process(self): self.location_id = self.extract_value(15, 20) self.transaction_sequence = self.extract_value(24, 27) - # We are expecting a BCROS account number here, it is required + # We are expecting a SHORTNAME for matching here, it is required self.transaction_description = self.extract_value(27, 67) if len(self.transaction_description) == 0: - self.add_error(EFTParseError(EFTError.BCROS_ACCOUNT_NUMBER_REQUIRED)) + self.add_error(EFTParseError(EFTError.ACCOUNT_SHORTNAME_REQUIRED)) self.deposit_amount = self.parse_decimal(self.extract_value(67, 80), EFTError.INVALID_DEPOSIT_AMOUNT) self.currency = self.get_currency(self.extract_value(80, 82)) diff --git a/queue_services/payment-reconciliations/src/reconciliations/enums.py b/queue_services/payment-reconciliations/src/reconciliations/enums.py index a1aee70fa..12977a73c 100644 --- a/queue_services/payment-reconciliations/src/reconciliations/enums.py +++ b/queue_services/payment-reconciliations/src/reconciliations/enums.py @@ -75,3 +75,12 @@ class TargetTransaction(Enum): DEBIT_MEMO = 'DM' CREDIT_MEMO = 'CM' RECEIPT = 'RECEIPT' + + +class MessageType(Enum): + """Event message types.""" + + CAS_UPLOADED = 'bc.registry.payment.casSettlementUploaded' + CGI_ACK_RECEIVED = 'bc.registry.payment.cgi.ACKReceived' + CGI_FEEDBACK_RECEIVED = 'bc.registry.payment.cgi.FEEDBACKReceived' + EFT_FILE_UPLOADED = 'bc.registry.payment.eft.fileUploaded' diff --git a/queue_services/payment-reconciliations/src/reconciliations/version.py b/queue_services/payment-reconciliations/src/reconciliations/version.py index 60040d1f4..29391ed4d 100644 --- a/queue_services/payment-reconciliations/src/reconciliations/version.py +++ b/queue_services/payment-reconciliations/src/reconciliations/version.py @@ -22,4 +22,4 @@ Development release segment: .devN """ -__version__ = '1.1.1' # pylint: disable=invalid-name +__version__ = '1.1.2' # pylint: disable=invalid-name diff --git a/queue_services/payment-reconciliations/src/reconciliations/worker.py b/queue_services/payment-reconciliations/src/reconciliations/worker.py index 57b49c2df..bc47ad577 100644 --- a/queue_services/payment-reconciliations/src/reconciliations/worker.py +++ b/queue_services/payment-reconciliations/src/reconciliations/worker.py @@ -37,6 +37,8 @@ from reconciliations import config from reconciliations.cgi_reconciliations import reconcile_distributions +from reconciliations.eft.eft_reconciliation import reconcile_eft_payments +from reconciliations.enums import MessageType from reconciliations.payment_reconciliations import reconcile_payments @@ -54,12 +56,14 @@ async def process_event(event_message, flask_app): raise QueueException('Flask App not available.') with flask_app.app_context(): - if (message_type := event_message.get('type', None)) == 'bc.registry.payment.casSettlementUploaded': + if (message_type := event_message.get('type', None)) == MessageType.CAS_UPLOADED.value: await reconcile_payments(event_message) - elif message_type == 'bc.registry.payment.cgi.ACKReceived': + elif message_type == MessageType.CGI_ACK_RECEIVED.value: await reconcile_distributions(event_message) - elif message_type == 'bc.registry.payment.cgi.FEEDBACKReceived': + elif message_type == MessageType.CGI_FEEDBACK_RECEIVED.value: await reconcile_distributions(event_message, is_feedback=True) + elif message_type == MessageType.EFT_FILE_UPLOADED.value: + await reconcile_eft_payments(event_message) else: raise Exception('Invalid type') # pylint: disable=broad-exception-raised diff --git a/queue_services/payment-reconciliations/tests/integration/factory.py b/queue_services/payment-reconciliations/tests/integration/factory.py index f3a0f5106..e7ccc4bdd 100644 --- a/queue_services/payment-reconciliations/tests/integration/factory.py +++ b/queue_services/payment-reconciliations/tests/integration/factory.py @@ -61,6 +61,7 @@ def factory_invoice(payment_account: PaymentAccount, status_code: str = InvoiceS invoice_status_code=status_code, payment_account_id=payment_account.id, total=total, + paid=0, created_by='test', created_on=created_on, business_identifier=business_identifier, @@ -125,6 +126,16 @@ def factory_payment_transaction(payment_id: int): transaction_start_time=datetime.now()).save() +def factory_create_eft_account(auth_account_id='1234', status=CfsAccountStatus.ACTIVE.value, + cfs_account='1234'): + """Return Factory.""" + account = PaymentAccount(auth_account_id=auth_account_id, + payment_method=PaymentMethod.EFT.value, + name=f'Test EFT {auth_account_id}').save() + CfsAccount(status=status, account_id=account.id, cfs_account=cfs_account).save() + return account + + def factory_create_online_banking_account(auth_account_id='1234', status=CfsAccountStatus.PENDING.value, cfs_account='1234'): """Return Factory.""" diff --git a/queue_services/payment-reconciliations/tests/integration/test_eft_reconciliation.py b/queue_services/payment-reconciliations/tests/integration/test_eft_reconciliation.py new file mode 100644 index 000000000..38dead675 --- /dev/null +++ b/queue_services/payment-reconciliations/tests/integration/test_eft_reconciliation.py @@ -0,0 +1,669 @@ +# Copyright © 2023 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests to assure the EFT Reconciliation. + +Test-Suite to ensure that the EFT Reconciliation queue service and parser is working as expected. +""" +from datetime import datetime +from typing import List + +import pytest +from entity_queue_common.service_utils import subscribe_to_queue +from flask import current_app +from pay_api import db +from pay_api.models import EFTCredit as EFTCreditModel +from pay_api.models import EFTFile as EFTFileModel +from pay_api.models import EFTShortnames as EFTShortnameModel +from pay_api.models import EFTTransaction as EFTTransactionModel +from pay_api.models import Invoice as InvoiceModel +from pay_api.models import InvoiceReference as InvoiceReferenceModel +from pay_api.models import Payment as PaymentModel +from pay_api.models import PaymentAccount as PaymentAccountModel +from pay_api.models import Receipt as ReceiptModel +from pay_api.utils.enums import ( + EFTFileLineType, EFTProcessStatus, InvoiceReferenceStatus, InvoiceStatus, PaymentMethod, PaymentStatus) + +from reconciliations.eft.eft_enums import EFTConstants +from tests.integration.factory import factory_create_eft_account, factory_invoice +from tests.integration.utils import create_and_upload_eft_file, helper_add_eft_event_to_queue +from tests.utilities.factory_utils import factory_eft_header, factory_eft_record, factory_eft_trailer + + +@pytest.mark.asyncio +async def test_eft_tdi17_fail_header(session, app, stan_server, event_loop, client_id, events_stan, future, + mock_publish): + """Test EFT Reconciliations properly fails for a bad EFT header.""" + # Call back for the subscription + from reconciliations.worker import cb_subscription_handler + + await subscribe_to_queue(events_stan, + current_app.config.get('SUBSCRIPTION_OPTIONS').get('subject'), + current_app.config.get('SUBSCRIPTION_OPTIONS').get('queue'), + current_app.config.get('SUBSCRIPTION_OPTIONS').get('durable_name'), + cb_subscription_handler) + + # Generate file with invalid header + file_name: str = 'test_eft_tdi17.txt' + header = factory_eft_header(record_type=EFTConstants.HEADER_RECORD_TYPE.value, file_creation_date='20230814', + file_creation_time='FAIL', deposit_start_date='20230810', deposit_end_date='20230810') + + create_and_upload_eft_file(file_name, [header]) + + await helper_add_eft_event_to_queue(events_stan, file_name=file_name) + + # Assert EFT File record was created + eft_file_model: EFTFileModel = db.session.query(EFTFileModel).filter( + EFTFileModel.file_ref == file_name).one_or_none() + + assert eft_file_model is not None + assert eft_file_model.id is not None + assert eft_file_model.file_ref == file_name + assert eft_file_model.status_code == EFTProcessStatus.FAILED.value + assert eft_file_model.created_on is not None + assert eft_file_model.file_creation_date is None + assert eft_file_model.deposit_from_date == datetime(2023, 8, 10) + assert eft_file_model.deposit_to_date == datetime(2023, 8, 10) + assert eft_file_model.number_of_details is None + assert eft_file_model.total_deposit_cents is None + + eft_header_transaction: EFTTransactionModel = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.HEADER.value).one_or_none() + + assert eft_header_transaction is not None + assert eft_header_transaction.id is not None + assert eft_header_transaction.file_id == eft_file_model.id + assert eft_header_transaction.line_type == EFTFileLineType.HEADER.value + assert eft_header_transaction.status_code == EFTProcessStatus.FAILED.value + assert eft_header_transaction.line_number == 0 + assert len(eft_header_transaction.error_messages) == 1 + assert eft_header_transaction.error_messages[0] == 'Invalid header creation date time.' + + eft_trailer_transaction: EFTTransactionModel = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.TRAILER.value).one_or_none() + + assert eft_trailer_transaction is None + + eft_transactions: EFTTransactionModel = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.TRANSACTION.value).all() + + assert not bool(eft_transactions) + + +@pytest.mark.asyncio +async def test_eft_tdi17_fail_trailer(session, app, stan_server, event_loop, client_id, events_stan, future, + mock_publish): + """Test EFT Reconciliations properly fails for a bad EFT trailer.""" + # Call back for the subscription + from reconciliations.worker import cb_subscription_handler + + await subscribe_to_queue(events_stan, + current_app.config.get('SUBSCRIPTION_OPTIONS').get('subject'), + current_app.config.get('SUBSCRIPTION_OPTIONS').get('queue'), + current_app.config.get('SUBSCRIPTION_OPTIONS').get('durable_name'), + cb_subscription_handler) + + # Generate file with invalid trailer + file_name: str = 'test_eft_tdi17.txt' + header = factory_eft_header(record_type=EFTConstants.HEADER_RECORD_TYPE.value, file_creation_date='20230814', + file_creation_time='1601', deposit_start_date='20230810', deposit_end_date='20230810') + trailer = factory_eft_trailer(record_type=EFTConstants.TRAILER_RECORD_TYPE.value, number_of_details='A', + total_deposit_amount='3733750') + + create_and_upload_eft_file(file_name, [header, trailer]) + + await helper_add_eft_event_to_queue(events_stan, file_name=file_name) + + # Assert EFT File record was created + eft_file_model: EFTFileModel = db.session.query(EFTFileModel).filter( + EFTFileModel.file_ref == file_name).one_or_none() + + assert eft_file_model is not None + assert eft_file_model.id is not None + assert eft_file_model.file_ref == file_name + assert eft_file_model.status_code == EFTProcessStatus.FAILED.value + assert eft_file_model.created_on is not None + assert eft_file_model.file_creation_date == datetime(2023, 8, 14, 16, 1) + assert eft_file_model.deposit_from_date == datetime(2023, 8, 10) + assert eft_file_model.deposit_to_date == datetime(2023, 8, 10) + assert eft_file_model.number_of_details is None + assert eft_file_model.total_deposit_cents == 3733750 + + eft_trailer_transaction: EFTTransactionModel = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.TRAILER.value).one_or_none() + + assert eft_trailer_transaction is not None + assert eft_trailer_transaction.id is not None + assert eft_trailer_transaction.file_id == eft_file_model.id + assert eft_trailer_transaction.line_type == EFTFileLineType.TRAILER.value + assert eft_trailer_transaction.status_code == EFTProcessStatus.FAILED.value + assert eft_trailer_transaction.line_number == 1 + assert len(eft_trailer_transaction.error_messages) == 1 + assert eft_trailer_transaction.error_messages[0] == 'Invalid trailer number of details value.' + + eft_header_transaction: EFTTransactionModel = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.HEADER.value).one_or_none() + + assert eft_header_transaction is None + + eft_transactions: EFTTransactionModel = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.TRANSACTION.value).all() + + assert not bool(eft_transactions) + + +@pytest.mark.asyncio +async def test_eft_tdi17_fail_transactions(session, app, stan_server, event_loop, client_id, events_stan, future, + mock_publish): + """Test EFT Reconciliations properly fails for a bad EFT trailer.""" + # Call back for the subscription + from reconciliations.worker import cb_subscription_handler + + await subscribe_to_queue(events_stan, + current_app.config.get('SUBSCRIPTION_OPTIONS').get('subject'), + current_app.config.get('SUBSCRIPTION_OPTIONS').get('queue'), + current_app.config.get('SUBSCRIPTION_OPTIONS').get('durable_name'), + cb_subscription_handler) + + # Generate file with invalid trailer + file_name: str = 'test_eft_tdi17.txt' + header = factory_eft_header(record_type=EFTConstants.HEADER_RECORD_TYPE.value, file_creation_date='20230814', + file_creation_time='1601', deposit_start_date='20230810', deposit_end_date='20230810') + trailer = factory_eft_trailer(record_type=EFTConstants.TRAILER_RECORD_TYPE.value, number_of_details='1', + total_deposit_amount='3733750') + + transaction_1 = factory_eft_record(record_type=EFTConstants.TRANSACTION_RECORD_TYPE.value, ministry_code='AT', + program_code='0146', deposit_date='20230810', deposit_time='0000', + location_id='85004', transaction_sequence='001', + transaction_description='ABC123', deposit_amount='13500', + currency='', exchange_adj_amount='0', deposit_amount_cad='FAIL', + destination_bank_number='0003', batch_number='002400986', jv_type='I', + jv_number='002425669', transaction_date='') + + create_and_upload_eft_file(file_name, [header, transaction_1, trailer]) + + await helper_add_eft_event_to_queue(events_stan, file_name=file_name) + + # Assert EFT File record was created + eft_file_model: EFTFileModel = db.session.query(EFTFileModel).filter( + EFTFileModel.file_ref == file_name).one_or_none() + + assert eft_file_model is not None + assert eft_file_model.id is not None + assert eft_file_model.file_ref == file_name + assert eft_file_model.status_code == EFTProcessStatus.FAILED.value + assert eft_file_model.created_on is not None + assert eft_file_model.file_creation_date == datetime(2023, 8, 14, 16, 1) + assert eft_file_model.deposit_from_date == datetime(2023, 8, 10) + assert eft_file_model.deposit_to_date == datetime(2023, 8, 10) + assert eft_file_model.number_of_details == 1 + assert eft_file_model.total_deposit_cents == 3733750 + + eft_trailer_transaction: EFTTransactionModel = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.TRAILER.value).one_or_none() + + assert eft_trailer_transaction is None + + eft_header_transaction: EFTTransactionModel = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.HEADER.value).one_or_none() + + assert eft_header_transaction is None + + eft_transactions: EFTTransactionModel = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.TRANSACTION.value).all() + + assert eft_transactions is not None + assert len(eft_transactions) == 1 + assert eft_transactions[0].error_messages[0] == 'Invalid transaction deposit amount CAD.' + + +@pytest.mark.asyncio +async def test_eft_tdi17_basic_process(session, app, stan_server, event_loop, client_id, events_stan, future, + mock_publish): + """Test EFT Reconciliations worker is able to create basic EFT processing records.""" + # Call back for the subscription + from reconciliations.worker import cb_subscription_handler + + await subscribe_to_queue(events_stan, + current_app.config.get('SUBSCRIPTION_OPTIONS').get('subject'), + current_app.config.get('SUBSCRIPTION_OPTIONS').get('queue'), + current_app.config.get('SUBSCRIPTION_OPTIONS').get('durable_name'), + cb_subscription_handler) + + # Generate happy path file + file_name: str = 'test_eft_tdi17.txt' + generate_basic_tdi17_file(file_name) + + await helper_add_eft_event_to_queue(events_stan, file_name=file_name) + + # Assert EFT File record was created + eft_file_model: EFTFileModel = db.session.query(EFTFileModel).filter( + EFTFileModel.file_ref == file_name).one_or_none() + + assert eft_file_model is not None + assert eft_file_model.id is not None + assert eft_file_model.file_ref == file_name + assert eft_file_model.created_on is not None + assert eft_file_model.file_creation_date == datetime(2023, 8, 14, 16, 1) + assert eft_file_model.deposit_from_date == datetime(2023, 8, 10) + assert eft_file_model.deposit_to_date == datetime(2023, 8, 10) + assert eft_file_model.number_of_details == 5 + assert eft_file_model.total_deposit_cents == 3733750 + + # Partial as shortname is not mapped to an account + assert eft_file_model.status_code == EFTProcessStatus.COMPLETED.value + + # Stored as part of the EFT File record - expecting none when no errors + eft_header_transaction = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.HEADER.value).one_or_none() + + assert eft_header_transaction is None + + # Stored as part of the EFT File record - expecting none when no errors + eft_trailer_transaction = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.TRAILER.value).one_or_none() + + assert eft_trailer_transaction is None + + eft_transactions: List[EFTTransactionModel] = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.TRANSACTION.value).all() + + assert eft_transactions is not None + assert len(eft_transactions) == 2 + assert eft_transactions[0].short_name_id is not None + assert eft_transactions[1].short_name_id is not None + + eft_shortnames = db.session.query(EFTShortnameModel).all() + + assert eft_shortnames is not None + assert len(eft_shortnames) == 2 + assert eft_shortnames[0].auth_account_id is None + assert eft_shortnames[0].short_name == 'ABC123' + assert eft_shortnames[1].auth_account_id is None + assert eft_shortnames[1].short_name == 'DEF456' + + eft_credits: List[EFTCreditModel] = db.session.query(EFTCreditModel).all() + assert eft_credits is not None + assert len(eft_credits) == 2 + assert eft_credits[0].payment_account_id is None + assert eft_credits[0].short_name_id == eft_shortnames[0].id + assert eft_credits[0].eft_file_id == eft_file_model.id + assert eft_credits[0].amount == 135 + assert eft_credits[0].remaining_amount == 135 + assert eft_credits[1].payment_account_id is None + assert eft_credits[1].short_name_id == eft_shortnames[1].id + assert eft_credits[1].eft_file_id == eft_file_model.id + assert eft_credits[1].amount == 5250 + assert eft_credits[1].remaining_amount == 5250 + + +@pytest.mark.asyncio +async def test_eft_tdi17_process(session, app, stan_server, event_loop, client_id, events_stan, future, + mock_publish): + """Test EFT Reconciliations worker.""" + # Call back for the subscription + from reconciliations.worker import cb_subscription_handler + + payment_account, eft_shortname, invoice = create_test_data() + + assert payment_account is not None + assert eft_shortname is not None + assert invoice is not None + + await subscribe_to_queue(events_stan, + current_app.config.get('SUBSCRIPTION_OPTIONS').get('subject'), + current_app.config.get('SUBSCRIPTION_OPTIONS').get('queue'), + current_app.config.get('SUBSCRIPTION_OPTIONS').get('durable_name'), + cb_subscription_handler) + + # Generate happy path file + file_name: str = 'test_eft_tdi17.txt' + generate_tdi17_file(file_name) + + await helper_add_eft_event_to_queue(events_stan, file_name=file_name) + + # Assert EFT File record was created + eft_file_model: EFTFileModel = db.session.query(EFTFileModel).filter( + EFTFileModel.file_ref == file_name).one_or_none() + + assert eft_file_model is not None + assert eft_file_model.id is not None + assert eft_file_model.file_ref == file_name + assert eft_file_model.status_code == EFTProcessStatus.COMPLETED.value + assert eft_file_model.created_on is not None + assert eft_file_model.file_creation_date == datetime(2023, 8, 14, 16, 1) + assert eft_file_model.deposit_from_date == datetime(2023, 8, 10) + assert eft_file_model.deposit_to_date == datetime(2023, 8, 10) + assert eft_file_model.number_of_details == 5 + assert eft_file_model.total_deposit_cents == 3733750 + + # Stored as part of the EFT File record - expecting none when no errors + eft_header_transaction = db.session.query(EFTTransactionModel)\ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.HEADER.value).one_or_none() + + assert eft_header_transaction is None + + # Stored as part of the EFT File record - expecting none when no errors + eft_trailer_transaction = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.TRAILER.value).one_or_none() + + assert eft_trailer_transaction is None + + eft_transactions = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.TRANSACTION.value).all() + + assert eft_transactions is not None + assert len(eft_transactions) == 3 + assert eft_transactions[0].short_name_id is not None + assert eft_transactions[1].short_name_id is not None + assert eft_transactions[2].short_name_id is not None + + eft_shortnames = db.session.query(EFTShortnameModel).all() + + assert eft_shortnames is not None + assert len(eft_shortnames) == 2 + assert eft_shortnames[0].auth_account_id == eft_shortname.auth_account_id + assert eft_shortnames[0].short_name == 'TESTSHORTNAME' + assert eft_shortnames[1].auth_account_id is None + assert eft_shortnames[1].short_name == 'ABC123' + + today = datetime.now().date() + + # Assert Invoice is paid + invoice: InvoiceModel = InvoiceModel.find_by_id(invoice.id) + expected_amount = 100 + assert invoice is not None + assert invoice.payment_method_code == PaymentMethod.EFT.value + assert invoice.invoice_status_code == InvoiceStatus.PAID.value + assert invoice.payment_date is not None + assert invoice.payment_date.date() == today + assert invoice.paid == expected_amount + assert invoice.total == expected_amount + + receipt: ReceiptModel = ReceiptModel.find_by_invoice_id_and_receipt_number(invoice.id, invoice.id) + assert receipt is not None + assert receipt.receipt_number == str(invoice.id) + assert receipt.receipt_amount == expected_amount + + expected_invoice_number = f'{current_app.config["EFT_INVOICE_PREFIX"]}{invoice.id}' + payment: PaymentModel = PaymentModel.find_payment_for_invoice(invoice.id) + assert payment is not None + assert payment.payment_date.date() == today + assert payment.invoice_number == expected_invoice_number + assert payment.payment_account_id == payment_account.id + assert payment.payment_status_code == PaymentStatus.COMPLETED.value + assert payment.payment_method_code == PaymentMethod.EFT.value + assert payment.invoice_amount == expected_amount + + invoice_reference: InvoiceReferenceModel = InvoiceReferenceModel\ + .find_by_invoice_id_and_status(invoice.id, InvoiceReferenceStatus.ACTIVE.value) + + assert invoice_reference is not None + assert invoice_reference.invoice_id == invoice.id + assert invoice_reference.invoice_number == payment.invoice_number + assert invoice_reference.invoice_number == expected_invoice_number + assert invoice_reference.status_code == InvoiceReferenceStatus.ACTIVE.value + + eft_credits: List[EFTCreditModel] = db.session.query(EFTCreditModel).all() + assert eft_credits is not None + assert len(eft_credits) == 2 + assert eft_credits[0].payment_account_id == payment_account.id + assert eft_credits[0].short_name_id == eft_shortnames[0].id + assert eft_credits[0].eft_file_id == eft_file_model.id + assert eft_credits[0].amount == 50.5 + assert eft_credits[0].remaining_amount == 50.5 + assert eft_credits[1].payment_account_id is None + assert eft_credits[1].short_name_id == eft_shortnames[1].id + assert eft_credits[1].eft_file_id == eft_file_model.id + assert eft_credits[1].amount == 351.5 + assert eft_credits[1].remaining_amount == 351.5 + + +@pytest.mark.asyncio +async def test_eft_tdi17_rerun(session, app, stan_server, event_loop, client_id, events_stan, future, + mock_publish): + """Test EFT Reconciliations can be re-executed with a corrected file.""" + # Call back for the subscription + from reconciliations.worker import cb_subscription_handler + + payment_account, eft_shortname, invoice = create_test_data() + + await subscribe_to_queue(events_stan, + current_app.config.get('SUBSCRIPTION_OPTIONS').get('subject'), + current_app.config.get('SUBSCRIPTION_OPTIONS').get('queue'), + current_app.config.get('SUBSCRIPTION_OPTIONS').get('durable_name'), + cb_subscription_handler) + + # Generate file with invalid trailer + file_name: str = 'test_eft_tdi17.txt' + header = factory_eft_header(record_type=EFTConstants.HEADER_RECORD_TYPE.value, file_creation_date='20230814', + file_creation_time='1601', deposit_start_date='20230810', deposit_end_date='20230810') + trailer = factory_eft_trailer(record_type=EFTConstants.TRAILER_RECORD_TYPE.value, number_of_details='1', + total_deposit_amount='3733750') + + transaction_1 = factory_eft_record(record_type=EFTConstants.TRANSACTION_RECORD_TYPE.value, ministry_code='AT', + program_code='0146', deposit_date='20230810', deposit_time='0000', + location_id='85004', transaction_sequence='001', + transaction_description='TESTSHORTNAME', deposit_amount='13500', + currency='', exchange_adj_amount='0', deposit_amount_cad='FAIL', + destination_bank_number='0003', batch_number='002400986', jv_type='I', + jv_number='002425669', transaction_date='') + + create_and_upload_eft_file(file_name, [header, transaction_1, trailer]) + + await helper_add_eft_event_to_queue(events_stan, file_name=file_name) + + # Assert EFT File record was created + eft_file_model: EFTFileModel = db.session.query(EFTFileModel).filter( + EFTFileModel.file_ref == file_name).one_or_none() + + assert eft_file_model is not None + assert eft_file_model.status_code == EFTProcessStatus.FAILED.value + + eft_trailer_transaction: EFTTransactionModel = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.TRAILER.value).one_or_none() + + assert eft_trailer_transaction is None + + eft_header_transaction: EFTTransactionModel = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.HEADER.value).one_or_none() + + assert eft_header_transaction is None + + eft_transactions: EFTTransactionModel = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.TRANSACTION.value).all() + + assert eft_transactions is not None + assert len(eft_transactions) == 1 + assert eft_transactions[0].error_messages[0] == 'Invalid transaction deposit amount CAD.' + + # Correct transaction error and re-process + transaction_1 = factory_eft_record(record_type=EFTConstants.TRANSACTION_RECORD_TYPE.value, ministry_code='AT', + program_code='0146', deposit_date='20230810', deposit_time='0000', + location_id='85004', transaction_sequence='001', + transaction_description='TESTSHORTNAME', deposit_amount='13500', + currency='', exchange_adj_amount='0', deposit_amount_cad='13500', + destination_bank_number='0003', batch_number='002400986', jv_type='I', + jv_number='002425669', transaction_date='') + + create_and_upload_eft_file(file_name, [header, transaction_1, trailer]) + await helper_add_eft_event_to_queue(events_stan, file_name=file_name) + + # Check file is completed after correction + eft_file_model: EFTFileModel = db.session.query(EFTFileModel).filter( + EFTFileModel.file_ref == file_name).one_or_none() + + assert eft_file_model is not None + assert eft_file_model.status_code == EFTProcessStatus.COMPLETED.value + + eft_trailer_transaction: EFTTransactionModel = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.TRAILER.value).one_or_none() + + assert eft_trailer_transaction is None + + eft_header_transaction: EFTTransactionModel = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.HEADER.value).one_or_none() + + assert eft_header_transaction is None + + eft_transactions: List[EFTTransactionModel] = db.session.query(EFTTransactionModel) \ + .filter(EFTTransactionModel.file_id == eft_file_model.id) \ + .filter(EFTTransactionModel.line_type == EFTFileLineType.TRANSACTION.value).all() + + assert eft_transactions is not None + assert len(eft_transactions) == 1 + assert len(eft_transactions[0].error_messages) == 0 + assert eft_transactions[0].status_code == EFTProcessStatus.COMPLETED.value + assert eft_transactions[0].deposit_amount_cents == 13500 + + today = datetime.now().date() + # Assert Invoice is paid + invoice: InvoiceModel = InvoiceModel.find_by_id(invoice.id) + expected_amount = 100 + assert invoice is not None + assert invoice.payment_method_code == PaymentMethod.EFT.value + assert invoice.invoice_status_code == InvoiceStatus.PAID.value + assert invoice.payment_date is not None + assert invoice.payment_date.date() == today + assert invoice.paid == expected_amount + assert invoice.total == expected_amount + + receipt: ReceiptModel = ReceiptModel.find_by_invoice_id_and_receipt_number(invoice.id, invoice.id) + assert receipt is not None + assert receipt.receipt_number == str(invoice.id) + assert receipt.receipt_amount == expected_amount + + expected_invoice_number = f'{current_app.config["EFT_INVOICE_PREFIX"]}{invoice.id}' + payment: PaymentModel = PaymentModel.find_payment_for_invoice(invoice.id) + assert payment is not None + assert payment.payment_date.date() == today + assert payment.invoice_number == expected_invoice_number + assert payment.payment_account_id == payment_account.id + assert payment.payment_status_code == PaymentStatus.COMPLETED.value + assert payment.payment_method_code == PaymentMethod.EFT.value + assert payment.invoice_amount == expected_amount + + invoice_reference: InvoiceReferenceModel = InvoiceReferenceModel \ + .find_by_invoice_id_and_status(invoice.id, InvoiceReferenceStatus.ACTIVE.value) + + assert invoice_reference is not None + assert invoice_reference.invoice_id == invoice.id + assert invoice_reference.invoice_number == payment.invoice_number + assert invoice_reference.invoice_number == expected_invoice_number + assert invoice_reference.status_code == InvoiceReferenceStatus.ACTIVE.value + + eft_credits: List[EFTCreditModel] = db.session.query(EFTCreditModel).all() + assert eft_credits is not None + assert len(eft_credits) == 1 + assert eft_credits[0].payment_account_id == payment_account.id + assert eft_credits[0].short_name_id == eft_shortname.id + assert eft_credits[0].eft_file_id == eft_file_model.id + assert eft_credits[0].amount == 35 + assert eft_credits[0].remaining_amount == 35 + + +def create_test_data(): + """Create test seed data.""" + payment_account: PaymentAccountModel = factory_create_eft_account() + eft_shortname: EFTShortnameModel = EFTShortnameModel(short_name='TESTSHORTNAME', + auth_account_id=payment_account.auth_account_id).save() + + invoice: InvoiceModel = factory_invoice(payment_account=payment_account, total=100, service_fees=10.0, + payment_method_code=PaymentMethod.EFT.value) + + return payment_account, eft_shortname, invoice + + +def generate_basic_tdi17_file(file_name: str): + """Generate a complete TDI17 EFT file.""" + header = factory_eft_header(record_type=EFTConstants.HEADER_RECORD_TYPE.value, file_creation_date='20230814', + file_creation_time='1601', deposit_start_date='20230810', deposit_end_date='20230810') + + trailer = factory_eft_trailer(record_type=EFTConstants.TRAILER_RECORD_TYPE.value, number_of_details='5', + total_deposit_amount='3733750') + + transaction_1 = factory_eft_record(record_type=EFTConstants.TRANSACTION_RECORD_TYPE.value, ministry_code='AT', + program_code='0146', deposit_date='20230810', deposit_time='0000', + location_id='85004', transaction_sequence='001', + transaction_description='ABC123', deposit_amount='13500', + currency='', exchange_adj_amount='0', deposit_amount_cad='13500', + destination_bank_number='0003', batch_number='002400986', jv_type='I', + jv_number='002425669', transaction_date='') + + transaction_2 = factory_eft_record(record_type=EFTConstants.TRANSACTION_RECORD_TYPE.value, ministry_code='AT', + program_code='0146', deposit_date='20230810', deposit_time='0000', + location_id='85004', transaction_sequence='002', + transaction_description='DEF456', + deposit_amount='525000', currency='', exchange_adj_amount='0', + deposit_amount_cad='525000', destination_bank_number='0003', + batch_number='002400986', jv_type='I', jv_number='002425669', + transaction_date='') + + create_and_upload_eft_file(file_name, [header, transaction_1, transaction_2, trailer]) + + +def generate_tdi17_file(file_name: str): + """Generate a complete TDI17 EFT file.""" + header = factory_eft_header(record_type=EFTConstants.HEADER_RECORD_TYPE.value, file_creation_date='20230814', + file_creation_time='1601', deposit_start_date='20230810', deposit_end_date='20230810') + + trailer = factory_eft_trailer(record_type=EFTConstants.TRAILER_RECORD_TYPE.value, number_of_details='5', + total_deposit_amount='3733750') + + transaction_1 = factory_eft_record(record_type=EFTConstants.TRANSACTION_RECORD_TYPE.value, ministry_code='AT', + program_code='0146', deposit_date='20230810', deposit_time='0000', + location_id='85004', transaction_sequence='001', + transaction_description='TESTSHORTNAME', deposit_amount='10000', + currency='', exchange_adj_amount='0', deposit_amount_cad='10000', + destination_bank_number='0003', batch_number='002400986', jv_type='I', + jv_number='002425669', transaction_date='') + + transaction_2 = factory_eft_record(record_type=EFTConstants.TRANSACTION_RECORD_TYPE.value, ministry_code='AT', + program_code='0146', deposit_date='20230810', deposit_time='0000', + location_id='85004', transaction_sequence='002', + transaction_description='TESTSHORTNAME', + deposit_amount='5050', currency='', exchange_adj_amount='0', + deposit_amount_cad='5050', destination_bank_number='0003', + batch_number='002400986', jv_type='I', jv_number='002425669', + transaction_date='') + + transaction_3 = factory_eft_record(record_type=EFTConstants.TRANSACTION_RECORD_TYPE.value, ministry_code='AT', + program_code='0146', deposit_date='20230810', deposit_time='0000', + location_id='85004', transaction_sequence='003', + transaction_description='ABC123', deposit_amount='35150', + currency='', exchange_adj_amount='0', deposit_amount_cad='35150', + destination_bank_number='0003', batch_number='002400986', jv_type='I', + jv_number='002425669', transaction_date='') + + create_and_upload_eft_file(file_name, [header, transaction_1, transaction_2, transaction_3, trailer]) diff --git a/queue_services/payment-reconciliations/tests/integration/utils.py b/queue_services/payment-reconciliations/tests/integration/utils.py index 803341dea..098228b25 100644 --- a/queue_services/payment-reconciliations/tests/integration/utils.py +++ b/queue_services/payment-reconciliations/tests/integration/utils.py @@ -22,6 +22,8 @@ from flask import current_app from minio import Minio +from reconciliations.enums import MessageType + async def helper_add_event_to_queue(stan_client: stan.aio.client.Client, file_name: str): @@ -43,6 +45,26 @@ async def helper_add_event_to_queue(stan_client: stan.aio.client.Client, payload=json.dumps(payload).encode('utf-8')) +async def helper_add_eft_event_to_queue(stan_client: stan.aio.client.Client, file_name: str, + message_type: str = MessageType.EFT_FILE_UPLOADED.value): + """Add eft event to the Queue.""" + payload = { + 'specversion': '1.x-wip', + 'type': message_type, + 'source': 'https://api.business.bcregistry.gov.bc.ca/v1/accounts/1/', + 'id': 'C234-1234-1234', + 'time': '2020-08-28T17:37:34.651294+00:00', + 'datacontenttype': 'text/plain', + 'data': { + 'fileName': file_name, + 'location': current_app.config['MINIO_BUCKET_NAME'] + } + } + + await stan_client.publish(subject=current_app.config.get('SUBSCRIPTION_OPTIONS').get('subject'), + payload=json.dumps(payload).encode('utf-8')) + + async def helper_add_ejv_event_to_queue(stan_client: stan.aio.client.Client, file_name: str, message_type: str = 'ACKReceived'): """Add event to the Queue.""" @@ -81,6 +103,16 @@ def create_and_upload_settlement_file(file_name: str, rows: List[List]): upload_to_minio(f.read(), file_name) +def create_and_upload_eft_file(file_name: str, rows: List[List]): + """Create eft file, upload to minio and send event.""" + with open(file_name, mode='w') as eft_file: + for row in rows: + print(row, file=eft_file) + + with open(file_name, 'rb') as f: + upload_to_minio(f.read(), file_name) + + def upload_to_minio(value_as_bytes, file_name: str): """Return a pre-signed URL for new doc upload.""" minio_endpoint = current_app.config['MINIO_ENDPOINT'] diff --git a/queue_services/payment-reconciliations/tests/unit/test_eft_file_parser.py b/queue_services/payment-reconciliations/tests/unit/test_eft_file_parser.py index c2651e7f4..f6feb96f5 100644 --- a/queue_services/payment-reconciliations/tests/unit/test_eft_file_parser.py +++ b/queue_services/payment-reconciliations/tests/unit/test_eft_file_parser.py @@ -442,7 +442,7 @@ def test_eft_parse_record_invalid_dates(): assert record.transaction_date is None -def test_eft_parse_record_transaction_description_required(): +def test_eft_parse_record_invalid_numbers(): """Test EFT record parser for invalid numbers.""" content = factory_eft_record(record_type=EFTConstants.TRANSACTION_RECORD_TYPE.value, ministry_code='AT', @@ -478,7 +478,7 @@ def test_eft_parse_record_transaction_description_required(): assert record.errors[2].index == 0 -def test_eft_parse_record_invalid_numbers(): +def test_eft_parse_record_transaction_description_required(): """Test EFT record parser transaction description required.""" content = factory_eft_record(record_type=EFTConstants.TRANSACTION_RECORD_TYPE.value, ministry_code='AT', @@ -503,8 +503,8 @@ def test_eft_parse_record_invalid_numbers(): # We are expecting the transaction description as this is where we get the BCROS Account number assert record.errors assert len(record.errors) == 1 - assert record.errors[0].code == EFTError.BCROS_ACCOUNT_NUMBER_REQUIRED.name - assert record.errors[0].message == EFTError.BCROS_ACCOUNT_NUMBER_REQUIRED.value + assert record.errors[0].code == EFTError.ACCOUNT_SHORTNAME_REQUIRED.name + assert record.errors[0].message == EFTError.ACCOUNT_SHORTNAME_REQUIRED.value assert record.errors[0].index == 0