22588 - Revert partner disbursements, fix migration processes, update versions, prod PREP

bcol-api/src/bcol_api/version.py

@@ -22,4 +22,4 @@
 Development release segment: .devN
 """
 
-__version__ = '0.1.0a0.dev'  # pylint: disable=invalid-name
+__version__ = '1.0.0'  # pylint: disable=invalid-name

jobs/payment-jobs/tasks/ejv_partner_distribution_task.py

@@ -23,14 +23,13 @@
 from pay_api.models import DistributionCodeLink as DistributionCodeLinkModel
 from pay_api.models import EjvFile as EjvFileModel
 from pay_api.models import EjvHeader as EjvHeaderModel
-from pay_api.models import EjvLink as EjvLinkModel
+from pay_api.models import EjvInvoiceLink as EjvInvoiceLinkModel
 from pay_api.models import FeeSchedule as FeeScheduleModel
 from pay_api.models import Invoice as InvoiceModel
 from pay_api.models import PaymentLineItem as PaymentLineItemModel
-from pay_api.models import RefundsPartial as RefundsPartialModel
 from pay_api.models import Receipt as ReceiptModel
 from pay_api.models import db
-from pay_api.utils.enums import DisbursementStatus, EjvFileType, EJVLinkType, InvoiceStatus, PaymentMethod
+from pay_api.utils.enums import DisbursementStatus, EjvFileType, InvoiceStatus, PaymentMethod
 from sqlalchemy import Date, cast
 
 from tasks.common.cgi_ejv import CgiEjv
@@ -53,17 +52,14 @@ def create_ejv_file(cls):
         cls._create_ejv_file_for_partner(batch_type='GI')  # Internal ministry
         cls._create_ejv_file_for_partner(batch_type='GA')  # External ministry
 
-    # TODO grab EFT invoices from PartnerDisbursements, eventually everything will run through partner disbursements.
     @staticmethod
     def get_invoices_for_disbursement(partner):
         """Return invoices for disbursement. Used by EJV and AP."""
         disbursement_date = datetime.today() - timedelta(days=current_app.config.get('DISBURSEMENT_DELAY_IN_DAYS'))
         invoices: List[InvoiceModel] = db.session.query(InvoiceModel) \
             .filter(InvoiceModel.invoice_status_code == InvoiceStatus.PAID.value) \
             .filter(
-            InvoiceModel.payment_method_code.notin_([PaymentMethod.INTERNAL.value,
-                                                     PaymentMethod.DRAWDOWN.value,
-                                                     PaymentMethod.EFT.value])) \
+            InvoiceModel.payment_method_code.notin_([PaymentMethod.INTERNAL.value, PaymentMethod.DRAWDOWN.value, PaymentMethod.EFT.value])) \
             .filter((InvoiceModel.disbursement_status_code.is_(None)) |
                     (InvoiceModel.disbursement_status_code == DisbursementStatus.ERRORED.value)) \
             .filter(~InvoiceModel.receipts.any(cast(ReceiptModel.receipt_date, Date) >= disbursement_date.date())) \
@@ -72,21 +68,6 @@ def get_invoices_for_disbursement(partner):
         current_app.logger.info(invoices)
         return invoices
 
-    @staticmethod
-    def get_refund_partial_payment_line_items_for_disbursement(partner) -> List[PaymentLineItemModel]:
-        """Return payment line items with partial refunds for disbursement."""
-        payment_line_items: List[PaymentLineItemModel] = db.session.query(PaymentLineItemModel) \
-            .join(InvoiceModel, PaymentLineItemModel.invoice_id == InvoiceModel.id) \
-            .join(RefundsPartialModel, PaymentLineItemModel.id == RefundsPartialModel.payment_line_item_id) \
-            .filter(InvoiceModel.invoice_status_code == InvoiceStatus.PAID.value) \
-            .filter(InvoiceModel.payment_method_code.in_([PaymentMethod.DIRECT_PAY.value])) \
-            .filter((RefundsPartialModel.disbursement_status_code.is_(None)) |
-                    (RefundsPartialModel.disbursement_status_code == DisbursementStatus.ERRORED.value)) \
-            .filter(InvoiceModel.corp_type_code == partner.code) \
-            .all()
-        current_app.logger.info(payment_line_items)
-        return payment_line_items
-
     @classmethod
     def get_invoices_for_refund_reversal(cls, partner):
         """Return invoices for refund reversal."""
@@ -97,9 +78,7 @@ def get_invoices_for_refund_reversal(cls, partner):
         invoices: List[InvoiceModel] = db.session.query(InvoiceModel) \
             .filter(InvoiceModel.invoice_status_code.in_(refund_inv_statuses)) \
             .filter(
-            InvoiceModel.payment_method_code.notin_([PaymentMethod.INTERNAL.value,
-                                                     PaymentMethod.DRAWDOWN.value,
-                                                     PaymentMethod.EFT.value])) \
+            InvoiceModel.payment_method_code.notin_([PaymentMethod.INTERNAL.value, PaymentMethod.DRAWDOWN.value, PaymentMethod.EFT.value])) \
             .filter(InvoiceModel.disbursement_status_code == DisbursementStatus.COMPLETED.value) \
             .filter(InvoiceModel.corp_type_code == partner.code) \
             .all()
@@ -134,16 +113,12 @@ def _create_ejv_file_for_partner(cls, batch_type: str):  # pylint:disable=too-ma
 
         for partner in partners:
             # Find all invoices for the partner to disburse.
-            # This includes invoices which are not PAID and invoices which are refunded and partial refunded.
+            # This includes invoices which are not PAID and invoices which are refunded.
             payment_invoices = cls.get_invoices_for_disbursement(partner)
             refund_reversals = cls.get_invoices_for_refund_reversal(partner)
             invoices = payment_invoices + refund_reversals
-
-            # Process partial refunds for each partner
-            refund_partial_items = cls.get_refund_partial_payment_line_items_for_disbursement(partner)
-
             # If no invoices continue.
-            if not invoices and not refund_partial_items:
+            if not invoices:
                 continue
 
             effective_date: str = cls.get_effective_date()
@@ -159,16 +134,11 @@ def _create_ejv_file_for_partner(cls, batch_type: str):  # pylint:disable=too-ma
             # and create one JV Header and detail for each.
             distribution_code_set = set()
             invoice_id_list = []
-            partial_line_item_id_list = []
             for inv in invoices:
                 invoice_id_list.append(inv.id)
                 for line_item in inv.payment_line_items:
                     distribution_code_set.add(line_item.fee_distribution_id)
 
-            for line_item in refund_partial_items:
-                partial_line_item_id_list.append(line_item.id)
-                distribution_code_set.add(line_item.fee_distribution_id)
-
             for distribution_code_id in list(distribution_code_set):
                 distribution_code: DistributionCodeModel = DistributionCodeModel.find_by_id(distribution_code_id)
                 credit_distribution_code: DistributionCodeModel = DistributionCodeModel.find_by_id(
@@ -177,22 +147,13 @@ def _create_ejv_file_for_partner(cls, batch_type: str):  # pylint:disable=too-ma
                 if credit_distribution_code.stop_ejv:
                     continue
 
-                line_items = cls._find_line_items_by_invoice_and_distribution(
-                        distribution_code_id, invoice_id_list)
-
-                refund_partial_items = cls._find_refund_partial_items_by_distribution(
-                        distribution_code_id, partial_line_item_id_list)
+                line_items = cls._find_line_items_by_invoice_and_distribution(distribution_code_id, invoice_id_list)
 
                 total: float = 0
                 for line in line_items:
                     total += line.total
 
-                partial_refund_total: float = 0
-                for refund_partial in refund_partial_items:
-                    partial_refund_total += refund_partial.refund_amount
-
                 batch_total += total
-                batch_total += partial_refund_total
 
                 debit_distribution = cls.get_distribution_string(distribution_code)  # Debit from BCREG GL
                 credit_distribution = cls.get_distribution_string(credit_distribution_code)  # Credit to partner GL
@@ -234,39 +195,20 @@ def _create_ejv_file_for_partner(cls, batch_type: str):  # pylint:disable=too-ma
 
                     control_total += 1
 
-                partial_refund_number: int = 0
-                for refund_partial in refund_partial_items:
-                    # JV Details for partial refunds
-                    partial_refund_number += 1
-                    # Flow Through add it as the refunds_partial id.
-                    flow_through = f'{refund_partial.id:<110}'
-                    refund_partial_number = f'#{refund_partial.id}'
-                    description = disbursement_desc[:-len(refund_partial_number)] + refund_partial_number
-                    description = f'{description[:100]:<100}'
-
-                    ejv_content = '{}{}'.format(ejv_content,  # pylint:disable=consider-using-f-string
-                                                cls.get_jv_line(batch_type, credit_distribution, description,
-                                                                effective_date, flow_through, journal_name,
-                                                                refund_partial.refund_amount,
-                                                                partial_refund_number, 'D'))
-                    partial_refund_number += 1
-                    control_total += 1
-
-                    # Add a line here for debit too
-                    ejv_content = '{}{}'.format(ejv_content,  # pylint:disable=consider-using-f-string
-                                                cls.get_jv_line(batch_type, debit_distribution, description,
-                                                                effective_date, flow_through, journal_name,
-                                                                refund_partial.refund_amount,
-                                                                partial_refund_number, 'C'))
-                    control_total += 1
-
-                    # Update partial refund status
-                    refund_partial.disbursement_status_code = DisbursementStatus.UPLOADED.value
-
-            # Create ejv invoice/partial_refund link records and set invoice status
             sequence = 1
-            sequence = cls._create_ejv_link(invoices, ejv_header_model, sequence, EJVLinkType.INVOICE.value)
-            cls._create_ejv_link(refund_partial_items, ejv_header_model, sequence, EJVLinkType.REFUND.value)
+            # Create ejv invoice link records and set invoice status
+            for inv in invoices:
+                # Create Ejv file link and flush
+                link_model = EjvInvoiceLinkModel(invoice_id=inv.id,
+                                                 ejv_header_id=ejv_header_model.id,
+                                                 disbursement_status_code=DisbursementStatus.UPLOADED.value,
+                                                 sequence=sequence)
+                # Set distribution status to invoice
+                db.session.add(link_model)
+                sequence += 1
+                inv.disbursement_status_code = DisbursementStatus.UPLOADED.value
+
+            db.session.flush()
 
         if not ejv_content:
             db.session.rollback()
@@ -298,18 +240,6 @@ def _find_line_items_by_invoice_and_distribution(cls, distribution_code_id, invo
             .filter(PaymentLineItemModel.fee_distribution_id == distribution_code_id)
         return line_items
 
-    @classmethod
-    def _find_refund_partial_items_by_distribution(cls, distribution_code_id, partial_line_item_id_list) \
-            -> List[RefundsPartialModel]:
-        """Find and return all payment line items for this distribution."""
-        line_items: List[RefundsPartialModel] = db.session.query(RefundsPartialModel) \
-            .join(PaymentLineItemModel, PaymentLineItemModel.id == RefundsPartialModel.payment_line_item_id) \
-            .filter(RefundsPartialModel.payment_line_item_id.in_(partial_line_item_id_list)) \
-            .filter(RefundsPartialModel.refund_amount > 0) \
-            .filter(PaymentLineItemModel.fee_distribution_id == distribution_code_id) \
-            .all()
-        return line_items
-
     @classmethod
     def _get_partners_by_batch_type(cls, batch_type) -> List[CorpTypeModel]:
         """Return partners by batch type."""
@@ -323,38 +253,22 @@ def _get_partners_by_batch_type(cls, batch_type) -> List[CorpTypeModel]:
 
         if batch_type == 'GA':
             # Rule for GA. Credit is 112 and debit is 112.
-            partner_distribution_code_ids: List[int] = db.session.scalars(query.filter(
+            partner_distribution_code_ids: List[int] = query.filter(
                 DistributionCodeModel.client == bc_reg_client_code
-            )).all()
+            ).all()
         else:
             # Rule for GI. Debit is 112 and credit is not 112.
-            partner_distribution_code_ids: List[int] = db.session.scalars(query.filter(
+            partner_distribution_code_ids: List[int] = query.filter(
                 DistributionCodeModel.client != bc_reg_client_code
-            )).all()
+            ).all()
 
         # Find all distribution codes who have these partner distribution codes as disbursement.
-        fee_query = db.session.query(DistributionCodeModel.distribution_code_id).filter(
-            DistributionCodeModel.disbursement_distribution_code_id.in_(partner_distribution_code_ids))
-        fee_distribution_codes: List[int] = db.session.scalars(fee_query).all()
+        fee_distribution_codes: List[int] = db.session.query(DistributionCodeModel.distribution_code_id).filter(
+            DistributionCodeModel.disbursement_distribution_code_id.in_(partner_distribution_code_ids)).all()
 
-        corp_type_query = db.session.query(FeeScheduleModel.corp_type_code). \
+        corp_type_codes: List[str] = db.session.query(FeeScheduleModel.corp_type_code). \
             join(DistributionCodeLinkModel,
-                 DistributionCodeLinkModel.fee_schedule_id == FeeScheduleModel.fee_schedule_id).\
-            filter(DistributionCodeLinkModel.distribution_code_id.in_(fee_distribution_codes))
-        corp_type_codes: List[str] = db.session.scalars(corp_type_query).all()
+                 DistributionCodeLinkModel.fee_schedule_id == FeeScheduleModel.fee_schedule_id). \
+            filter(DistributionCodeLinkModel.distribution_code_id.in_(fee_distribution_codes)).all()
 
         return db.session.query(CorpTypeModel).filter(CorpTypeModel.code.in_(corp_type_codes)).all()
-
-    @classmethod
-    def _create_ejv_link(cls, items, ejv_header_model, sequence, link_type):
-        for item in items:
-            link_model = EjvLinkModel(link_id=item.id,
-                                      link_type=link_type,
-                                      ejv_header_id=ejv_header_model.id,
-                                      disbursement_status_code=DisbursementStatus.UPLOADED.value,
-                                      sequence=sequence)
-            db.session.add(link_model)
-            sequence += 1
-            item.disbursement_status_code = DisbursementStatus.UPLOADED.value
-        db.session.flush()
-        return sequence

jobs/payment-jobs/tasks/ejv_payment_task.py

@@ -227,18 +227,16 @@ def _get_account_ids_for_payment(cls, batch_type) -> List[int]:
         """Return account IDs for payment."""
         # CREDIT : Distribution code against fee schedule
         # DEBIT : Distribution code against account.
+        # Rule for GA. Credit is 112 and debit is 112. For BCREG client code is 112
+        # Rule for GI. Credit is 112 and debit is not 112. For BCREG client code is 112
         bc_reg_client_code = current_app.config.get('CGI_BCREG_CLIENT_CODE')
-        query = db.session.query(DistributionCodeModel.account_id) \
+        account_ids = db.session.query(DistributionCodeModel.account_id) \
             .filter(DistributionCodeModel.stop_ejv.is_(False) | DistributionCodeModel.stop_ejv.is_(None)) \
-            .filter(DistributionCodeModel.account_id.isnot(None))
-
-        if batch_type == 'GA':
-            # Rule for GA. Credit is 112 and debit is 112. For BCREG client code is 112
-            account_ids: List[int] = query.filter(DistributionCodeModel.client == bc_reg_client_code)
-        else:
-            # Rule for GI. Credit is 112 and debit is not 112. For BCREG client code is 112
-            account_ids: List[int] = query.filter(DistributionCodeModel.client != bc_reg_client_code)
-        return db.session.scalars(account_ids).all()
+            .filter(DistributionCodeModel.account_id.isnot(None)) \
+            .filter_boolean(batch_type == 'GA', DistributionCodeModel.client == bc_reg_client_code) \
+            .filter_boolean(batch_type != 'GA', DistributionCodeModel.client != bc_reg_client_code) \
+            .all()
+        return [account_id_tuple[0] for account_id_tuple in account_ids]
 
     @classmethod
     def _get_invoices_for_payment(cls, account_id: int) -> List[InvoiceModel]:

pay-admin/admin/version.py

@@ -22,4 +22,4 @@
 Development release segment: .devN
 """
 
-__version__ = '0.1.0a0.dev'  # pylint: disable=invalid-name
+__version__ = '1.0.0'  # pylint: disable=invalid-name

pay-api/src/pay_api/models/custom_query.py

@@ -21,6 +21,14 @@
 class CustomQuery(Query):  # pylint: disable=too-many-ancestors
     """Custom Query class to extend the base query class for helper functionality."""
 
+    def filter_boolean(self, search_criteria, model_attribute):
+        """Add query filter for boolean value."""
+        if search_criteria is False:
+            return self
+        if search_criteria is None:
+            raise ValueError('Invalid search criteria None, not True or False')
+        return self.filter(model_attribute == search_criteria)
+
     def filter_conditionally(self, search_criteria, model_attribute, is_like: bool = False):
         """Add query filter if present."""
         if search_criteria is None:

pay-api/src/pay_api/services/base_payment_system.py

@@ -34,6 +34,7 @@
 from pay_api.models.refunds_partial import RefundPartialLine
 from pay_api.services import gcp_queue_publisher
 from pay_api.services.cfs_service import CFSService
+from pay_api.services.flags import flags
 from pay_api.services.invoice import Invoice
 from pay_api.services.invoice_reference import InvoiceReference
 from pay_api.services.payment import Payment
@@ -150,7 +151,8 @@ def ensure_no_payment_blockers(self, payment_account: PaymentAccount) -> None:
             # Note NSF (Account Unlocking) is paid using DIRECT_PAY - CC flow, not PAD.
             current_app.logger.warning(f'Account {payment_account.id} is frozen, rejecting invoice creation')
             raise BusinessException(Error.PAD_CURRENTLY_NSF)
-        if Invoice.has_overdue_invoices(payment_account.id):
+        if flags.is_on('enable-eft-payment-method', default=False) and \
+            Invoice.has_overdue_invoices(payment_account.id):
             raise BusinessException(Error.EFT_INVOICES_OVERDUE)
 
     @staticmethod

pay-api/src/pay_api/version.py

@@ -22,4 +22,4 @@
 Development release segment: .devN
 """
 
-__version__ = '1.21.3'  # pylint: disable=invalid-name
+__version__ = '1.21.4'  # pylint: disable=invalid-name