18681 - EFT GL Distribution supporting changes (#1378)

* 18681 - EFT GL Distribution supporting changes

- additional tables and fields to help with GL transfer processing and partner distribution
- updates to credit logic to allow linking to an invoice

* lint fix

* PR Feedback updates

* PR Feedback

pay-api/migrations/versions/2024_01_08_ff245db0cf76_eft_gl_transfer.py

@@ -0,0 +1,69 @@
+""" EFT GL Transfer tracking
+
+Revision ID: ff245db0cf76
+Revises: eec11500a81e
+Create Date: 2024-01-08 08:57:56.456585
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'ff245db0cf76'
+down_revision = 'eec11500a81e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    # EFT gl transfer tracking table
+    op.create_table('eft_gl_transfers',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('transfer_date', sa.DateTime(), nullable=False),
+        sa.Column('transfer_type', sa.String(length=50), nullable=False),
+        sa.Column('transfer_amount', sa.Numeric(precision=19, scale=2), nullable=False),
+        sa.Column('source_gl', sa.String(length=50), nullable=False),
+        sa.Column('target_gl', sa.String(length=50), nullable=False),
+        sa.Column('is_processed', sa.Boolean(), nullable=False),
+        sa.Column('processed_on', sa.DateTime(), nullable=True),
+        sa.Column('created_on', sa.DateTime(), nullable=False),
+        sa.Column('invoice_id', sa.Integer(), nullable=True),
+        sa.Column('short_name_id', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(['invoice_id'], ['invoices.id'], ),
+        sa.ForeignKeyConstraint(['short_name_id'], ['eft_short_names.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_eft_gl_transfers_transfer_date'), 'eft_gl_transfers', ['transfer_date'], unique=False)
+
+    # Add credit to invoice link to track what invoice are eft funds applied to
+    op.create_table('eft_credit_invoice_links',
+                    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+                    sa.Column('invoice_id', sa.Integer(), nullable=False),
+                    sa.Column('eft_credit_id', sa.Integer(), nullable=False),
+                    sa.Column('created_on', sa.DateTime(), nullable=False),
+                    sa.ForeignKeyConstraint(['eft_credit_id'], ['eft_credits.id'], ),
+                    sa.ForeignKeyConstraint(['invoice_id'], ['invoices.id'], ),
+                    sa.PrimaryKeyConstraint('id')
+                    )
+    op.create_index(op.f('ix_eft_credit_invoice_links_eft_credit_id'), 'eft_credit_invoice_links', ['eft_credit_id'], unique=False)
+    op.create_index(op.f('ix_eft_credit_invoice_links_invoice_id'), 'eft_credit_invoice_links', ['invoice_id'], unique=False)
+
+    # Add link between credits and the eft_transaction from TDI17 it came from for easier auditing
+    op.add_column('eft_credits', sa.Column('eft_transaction_id', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'eft_credits', 'eft_transactions', ['eft_transaction_id'], ['id'])
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_eft_gl_transfers_transfer_date'), table_name='eft_gl_transfers')
+    op.drop_table('eft_gl_transfers')
+    op.drop_index(op.f('ix_eft_credit_invoice_links_invoice_id'), table_name='eft_credit_invoice_links')
+    op.drop_index(op.f('ix_eft_credit_invoice_links_eft_credit_id'), table_name='eft_credit_invoice_links')
+    op.drop_table('eft_credit_invoice_links')
+    op.drop_column('eft_credits', 'eft_transaction_id')
+    # ### end Alembic commands ###

pay-api/src/pay_api/models/__init__.py

@@ -23,11 +23,14 @@
 from .cfs_account_status_code import CfsAccountStatusCode, CfsAccountStatusCodeSchema
 from .corp_type import CorpType, CorpTypeSchema  # noqa: I001
 from .credit import Credit
+from .custom_query import CustomQuery
 from .db import db, ma  # noqa: I001
 from .disbursement_status_code import DisbursementStatusCode
 from .distribution_code import DistributionCode, DistributionCodeLink
 from .eft_credit import EFTCredit
+from .eft_credit_invoice_link import EFTCreditInvoiceLink
 from .eft_file import EFTFile
+from .eft_gl_transfers import EFTGLTransfer
 from .eft_process_status_code import EFTProcessStatusCode
 from .eft_short_names import EFTShortnames, EFTShortnameSchema
 from .eft_transaction import EFTTransaction

pay-api/src/pay_api/models/base_model.py

@@ -38,6 +38,14 @@ def flush(self):
         self.create_activity(self)
         return self
 
+    def save_or_add(self, auto_save: bool):
+        """Run save if auto save is True."""
+        if auto_save:
+            self.save()
+        else:
+            db.session.add(self)
+        return self
+
     def save(self):
         """Save and commit."""
         db.session.add(self)

pay-api/src/pay_api/models/custom_query.py

@@ -0,0 +1,31 @@
+# Copyright © 2023 Province of British Columbia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Custom Query class to extend BaseQuery class functionality."""
+from datetime import datetime
+from flask_sqlalchemy import BaseQuery
+from sqlalchemy import func
+
+
+class CustomQuery(BaseQuery):
+    """Custom Query class to extend the base query class for helper functionality."""
+
+    def filter_conditionally(self, search_criteria, model_attribute):
+        """Add query filter if present."""
+        if search_criteria is None:
+            return self
+
+        if isinstance(search_criteria, datetime):
+            return self.filter(func.DATE(model_attribute) == search_criteria.date())
+
+        return self.filter(model_attribute == search_criteria)

pay-api/src/pay_api/models/db.py

@@ -19,12 +19,13 @@
 from flask_sqlalchemy import SQLAlchemy
 from sqlalchemy_continuum import make_versioned
 from sqlalchemy_continuum.plugins import ActivityPlugin
-
+from .custom_query import CustomQuery
 
 # by convention in the Flask community these are lower case,
 # whereas pylint wants them upper case
 ma = Marshmallow()  # pylint: disable=invalid-name
-db = SQLAlchemy()  # pylint: disable=invalid-name
+db = SQLAlchemy(query_class=CustomQuery)  # pylint: disable=invalid-name
+
 
 activity_plugin = ActivityPlugin()  # pylint: disable=invalid-name
 

pay-api/src/pay_api/models/eft_credit.py

@@ -39,6 +39,7 @@ class EFTCredit(BaseModel):  # pylint:disable=too-many-instance-attributes
             'amount',
             'created_on',
             'eft_file_id',
+            'eft_transaction_id',
             'short_name_id',
             'payment_account_id',
             'remaining_amount'
@@ -54,6 +55,7 @@ class EFTCredit(BaseModel):  # pylint:disable=too-many-instance-attributes
     eft_file_id = db.Column(db.Integer, ForeignKey('eft_files.id'), nullable=False)
     short_name_id = db.Column(db.Integer, ForeignKey('eft_short_names.id'), nullable=False)
     payment_account_id = db.Column(db.Integer, ForeignKey('payment_accounts.id'), nullable=True, index=True)
+    eft_transaction_id = db.Column(db.Integer, ForeignKey('eft_transactions.id'), nullable=True)
 
     @classmethod
     def find_by_payment_account_id(cls, payment_account_id: int):

pay-api/src/pay_api/models/eft_credit_invoice_link.py

@@ -0,0 +1,49 @@
+# Copyright © 2023 Province of British Columbia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Model to link invoices with EFT Credits."""
+from datetime import datetime
+
+from sqlalchemy import ForeignKey
+
+from .base_model import BaseModel
+from .db import db
+
+
+class EFTCreditInvoiceLink(BaseModel):  # pylint: disable=too-few-public-methods
+    """This class manages linkages between EFT Credits and invoices."""
+
+    __tablename__ = 'eft_credit_invoice_links'
+    # this mapper is used so that new and old versions of the service can be run simultaneously,
+    # making rolling upgrades easier
+    # This is used by SQLAlchemy to explicitly define which fields we're interested
+    # so it doesn't freak out and say it can't map the structure if other fields are present.
+    # This could occur from a failed deploy or during an upgrade.
+    # The other option is to tell SQLAlchemy to ignore differences, but that is ambiguous
+    # and can interfere with Alembic upgrades.
+    #
+    # NOTE: please keep mapper names in alpha-order, easier to track that way
+    #       Exception, id is always first, _fields first
+    __mapper_args__ = {
+        'include_properties': [
+            'id',
+            'created_on',
+            'eft_credit_id',
+            'invoice_id'
+        ]
+    }
+
+    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
+    invoice_id = db.Column(db.Integer, ForeignKey('invoices.id'), nullable=False, index=True)
+    eft_credit_id = db.Column(db.Integer, ForeignKey('eft_credits.id'), nullable=False, index=True)
+    created_on = db.Column('created_on', db.DateTime, nullable=True, default=datetime.now)

pay-api/src/pay_api/models/eft_gl_transfers.py

@@ -0,0 +1,65 @@
+# Copyright © 2023 Province of British Columbia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Model to track EFT GL transfers."""
+
+from datetime import datetime
+
+from sqlalchemy import ForeignKey
+
+from .base_model import BaseModel
+from .db import db
+
+
+class EFTGLTransfer(BaseModel):  # pylint: disable=too-many-instance-attributes
+    """This class manages the file data for EFT transactions."""
+
+    __tablename__ = 'eft_gl_transfers'
+    # this mapper is used so that new and old versions of the service can be run simultaneously,
+    # making rolling upgrades easier
+    # This is used by SQLAlchemy to explicitly define which fields we're interested
+    # so it doesn't freak out and say it can't map the structure if other fields are present.
+    # This could occur from a failed deploy or during an upgrade.
+    # The other option is to tell SQLAlchemy to ignore differences, but that is ambiguous
+    # and can interfere with Alembic upgrades.
+    #
+    # NOTE: please keep mapper names in alpha-order, easier to track that way
+    #       Exception, id is always first, _fields first
+    __mapper_args__ = {
+        'include_properties': [
+            'id',
+            'created_on',
+            'invoice_id',
+            'is_processed',
+            'processed_on',
+            'short_name_id',
+            'source_gl',
+            'target_gl',
+            'transfer_amount',
+            'transfer_type',
+            'transfer_date'
+        ]
+    }
+
+    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
+    # Intended to be populated based on TDI17 date for GL Transfers or payment date of an invoice for distributions
+    transfer_date = db.Column('transfer_date', db.DateTime, nullable=False, default=datetime.now, index=True)
+    transfer_type = db.Column('transfer_type', db.String(50), nullable=False)
+    transfer_amount = db.Column(db.Numeric(19, 2), nullable=False)
+    source_gl = db.Column('source_gl', db.String(50), nullable=False)
+    target_gl = db.Column('target_gl', db.String(50), nullable=False)
+    is_processed = db.Column('is_processed', db.Boolean(), nullable=False, default=False)
+    processed_on = db.Column('processed_on', db.DateTime, nullable=True)
+    created_on = db.Column('created_on', db.DateTime, nullable=False, default=datetime.now)
+    invoice_id = db.Column(db.Integer, ForeignKey('invoices.id'), nullable=True)
+    short_name_id = db.Column(db.Integer, ForeignKey('eft_short_names.id'), nullable=False)

pay-api/src/pay_api/services/__init__.py

@@ -15,6 +15,7 @@
 
 from .cfs_service import CFSService
 from .distribution_code import DistributionCode
+from .eft_gl_transfer import EFTGlTransfer
 from .fee_schedule import FeeSchedule
 from .hashing import HashingService
 from .internal_pay_service import InternalPayService

pay-api/src/pay_api/services/eft_gl_transfer.py

@@ -0,0 +1,89 @@
+# Copyright © 2023 Province of British Columbia
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Service to manage EFT GL Transfers."""
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Optional
+
+from pay_api.models import EFTGLTransfer as EFTGLTransferModel
+from pay_api.models import db
+
+
+@dataclass
+class EFTGlTransferSearch:  # pylint: disable=too-many-instance-attributes
+    """Used for searching EFT GL Transfer records."""
+
+    created_on: Optional[datetime] = None
+    invoice_id: Optional[int] = None
+    is_processed: Optional[bool] = None
+    processed_on: Optional[datetime] = None
+    short_name_id: Optional[int] = None
+    source_gl: Optional[str] = None
+    target_gl: Optional[str] = None
+    transfer_type: Optional[str] = None
+    transfer_date: Optional[datetime] = None
+
+
+class EFTGlTransfer:
+    """Service to manage EFT GL transfers."""
+
+    @staticmethod
+    def find_by_id(transfer_id: int) -> EFTGLTransferModel:
+        """Return EFT Transfers by id."""
+        return EFTGLTransferModel.find_by_id(transfer_id)
+
+    @staticmethod
+    def find_by_short_name_id(short_name_id: int, is_processed: bool = None) -> [EFTGLTransferModel]:
+        """Return EFT Transfers by short_name_id."""
+        query = db.session.query(EFTGLTransferModel) \
+            .filter(EFTGLTransferModel.short_name_id == short_name_id)
+
+        if is_processed is not None:
+            query = query.filter(EFTGLTransferModel.is_processed == is_processed) \
+
+        query = query.order_by(EFTGLTransferModel.created_on.asc())
+
+        return query.all()
+
+    @staticmethod
+    def find_by_invoice_id(invoice_id: int, is_processed: bool = None) -> [EFTGLTransferModel]:
+        """Return EFT Transfers by invoice_id."""
+        query = db.session.query(EFTGLTransferModel) \
+            .filter(EFTGLTransferModel.invoice_id == invoice_id)
+
+        if is_processed is not None:
+            query = query.filter(EFTGLTransferModel.is_processed == is_processed)
+
+        query = query.order_by(EFTGLTransferModel.created_on.asc())
+
+        return query.all()
+
+    @staticmethod
+    def search(search_criteria: EFTGlTransferSearch = EFTGlTransferSearch()) -> [EFTGLTransferModel]:
+        """Return EFT Transfers by search criteria."""
+        query = db.session.query(EFTGLTransferModel)
+
+        query = query.filter_conditionally(search_criteria.created_on, EFTGLTransferModel.created_on)
+        query = query.filter_conditionally(search_criteria.invoice_id, EFTGLTransferModel.invoice_id)
+        query = query.filter_conditionally(search_criteria.is_processed, EFTGLTransferModel.is_processed)
+        query = query.filter_conditionally(search_criteria.processed_on, EFTGLTransferModel.processed_on)
+        query = query.filter_conditionally(search_criteria.short_name_id, EFTGLTransferModel.short_name_id)
+        query = query.filter_conditionally(search_criteria.source_gl, EFTGLTransferModel.source_gl)
+        query = query.filter_conditionally(search_criteria.target_gl, EFTGLTransferModel.target_gl)
+        query = query.filter_conditionally(search_criteria.transfer_type, EFTGLTransferModel.transfer_type)
+        query = query.filter_conditionally(search_criteria.transfer_date, EFTGLTransferModel.transfer_date)
+
+        query = query.order_by(EFTGLTransferModel.created_on.asc())
+
+        return query.all()