[CCFPCM-0426] PART 2 db normalization

apps/backend/fixtures/lambda/reconcile.json

@@ -3,7 +3,7 @@
     {
       "Sns": {
         "Message": {
-          "reconciliationMaxDate": "2023-09-16",
+          "reconciliationMaxDate": "2023-10-30",
           "program": "SBC",
           "reportEnabled": false,
           "byPassFileValidity": true

apps/backend/src/constants.ts

@@ -74,20 +74,6 @@ export interface AggregatedPosPayment {
   payments: PaymentEntity[];
 }
 
-export interface NormalizedLocation {
-  location_id: number;
-  source_id: string;
-  program_code: number;
-  ministry_client: number;
-  resp_code: string;
-  service_line_code: number;
-  stob_code: number;
-  project_code: number;
-  description: string;
-  merchant_ids: number[];
-  pt_location_id: number;
-}
-
 export const BankMerchantId = 999999999;
 
 export const SUPPORTED_FILE_EXTENSIONS: {

apps/backend/src/database/database.module.ts

@@ -7,6 +7,7 @@ import { join } from 'path';
 import { DatabaseLogger } from './database-logger';
 import { DatabaseService } from './database.service';
 import { entities } from './entity.config';
+import { DepositModule } from '../deposits/deposit.module';
 import { LocationModule } from '../location/location.module';
 import { NotificationModule } from '../notification/notification.module';
 import { S3ManagerModule } from '../s3-manager/s3-manager.module';
@@ -80,6 +81,7 @@ export const appOrmConfig: PostgresConnectionOptions = {
     S3ManagerModule,
     LocationModule,
     TransactionModule,
+    DepositModule,
     NotificationModule,
     TypeOrmModule.forRootAsync({
       useFactory: () => appOrmConfig,

apps/backend/src/database/database.service.ts

@@ -3,14 +3,22 @@ import { Inject, Injectable } from '@nestjs/common';
 import * as csv from 'csvtojson';
 import { masterData } from './const';
 import { FileTypes, Ministries } from '../constants';
-import { LocationEntity } from '../location/entities';
+import { CashDepositService } from '../deposits/cash-deposit.service';
+import { CashDepositEntity } from '../deposits/entities/cash-deposit.entity';
+import { POSDepositEntity } from '../deposits/entities/pos-deposit.entity';
+import { PosDepositService } from '../deposits/pos-deposit.service';
+import {
+  MinistryLocationEntity,
+  MasterLocationEntity,
+} from '../location/entities';
 import { LocationService } from '../location/location.service';
 import { FileIngestionRulesEntity } from '../notification/entities/file-ingestion-rules.entity';
 import { NotificationService } from '../notification/notification.service';
 import { ProgramRequiredFileEntity } from '../parse/entities/program-required-file.entity';
 import { S3ManagerService } from '../s3-manager/s3-manager.service';
 import { PaymentMethodEntity } from '../transaction/entities';
 import { PaymentMethodService } from '../transaction/payment-method.service';
+import { TransactionService } from '../transaction/transaction.service';
 
 @Injectable()
 export class DatabaseService {
@@ -20,20 +28,34 @@ export class DatabaseService {
     @Inject(NotificationService)
     private readonly notificationService: NotificationService,
     @Inject(PaymentMethodService)
-    private readonly paymentMethodService: PaymentMethodService
+    private readonly paymentMethodService: PaymentMethodService,
+    @Inject(TransactionService)
+    private readonly transactionService: TransactionService,
+    @Inject(PosDepositService)
+    private readonly posDepositService: PosDepositService,
+    @Inject(CashDepositService)
+    private readonly cashDepositService: CashDepositService
   ) {}
   /**
    * We rely on "master" data to join our txn/deposit table in order to match
    */
   async seedMasterData() {
-    const locations: LocationEntity[] = await this.locationService.findAll();
+    const locations: MasterLocationEntity[] =
+      await this.locationService.findAll();
 
     const paymentMethods: PaymentMethodEntity[] =
       await this.paymentMethodService.getPaymentMethods();
 
     const rules: FileIngestionRulesEntity[] =
       await this.notificationService.getAllRules();
 
+    const transactionsWithNullLocation =
+      await this.transactionService.findWithNullLocation();
+    const posDepositWithNullLocation =
+      await this.posDepositService.findWithNullLocation();
+    const cashDepositWithnullLocation =
+      await this.cashDepositService.findWithNullLocation();
+
     if (rules.length === 0) {
       await this.seedFileIngestionRules();
     }
@@ -60,6 +82,49 @@ export class DatabaseService {
           Ministries[rule.program as keyof typeof Ministries]
         );
       }
+      const locations: MinistryLocationEntity[] =
+        await this.locationService.findMinistryLocations(
+          Ministries[rule.program as keyof typeof Ministries]
+        );
+      if (transactionsWithNullLocation.length > 0) {
+        const txns = transactionsWithNullLocation.map((txn) => {
+          const location = locations.find(
+            (loc) =>
+              loc.source_id === txn.source_id &&
+              loc.location_id === txn.location_id
+          )!;
+          return { ...txn, location };
+        });
+        await this.transactionService.saveTransactions(txns);
+      }
+      if (posDepositWithNullLocation.length > 0) {
+        const merchants = locations.flatMap((itm) => itm.merchants);
+        const posDeposits = posDepositWithNullLocation.map(
+          (pos: POSDepositEntity) => {
+            const merchant = merchants.find(
+              (merch) => merch.merchant_id === pos.merchant_id
+            )!;
+            return {
+              ...pos,
+              timestamp: pos.timestamp,
+              merchant,
+            };
+          }
+        );
+        await this.posDepositService.savePOSDepositEntities(posDeposits);
+      }
+      if (cashDepositWithnullLocation.length > 0) {
+        const banks = locations.flatMap((itm) => itm.banks);
+        const cash = cashDepositWithnullLocation.map(
+          (cash: CashDepositEntity) => {
+            const bank = banks.find(
+              (bank) => bank.bank_id === cash.pt_location_id
+            )!;
+            return { ...cash, bank };
+          }
+        );
+        await this.cashDepositService.saveCashDepositEntities(cash);
+      }
     }
   }
 

apps/backend/src/database/entity.config.ts

@@ -3,7 +3,7 @@ import { POSDepositEntity } from '../deposits/entities/pos-deposit.entity';
 import {
   MinistryLocationEntity,
   BankLocationEntity,
-  LocationEntity,
+  MasterLocationEntity,
   MerchantEntity,
 } from '../location/entities';
 import { AlertDestinationEntity } from '../notification/entities/alert-destination.entity';
@@ -24,7 +24,7 @@ export const entities = [
   POSDepositEntity,
   CashDepositEntity,
   MinistryLocationEntity,
-  LocationEntity,
+  MasterLocationEntity,
   BankLocationEntity,
   MerchantEntity,
   FileUploadedEntity,

apps/backend/src/database/migrations/1698772487856-fk-migration.ts

@@ -0,0 +1,47 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class Migration1698772487856 implements MigrationInterface {
+  name = 'FKMigration1698772487856';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "location" DROP CONSTRAINT "UQ_62c907775331b5aa98ec8daf7dd"`
+    );
+    await queryRunner.query(`ALTER TABLE "transaction" ADD "location" uuid`);
+    await queryRunner.query(`ALTER TABLE "pos_deposit" ADD "merchant" uuid`);
+    await queryRunner.query(`ALTER TABLE "cash_deposit" ADD "bank" uuid`);
+    await queryRunner.query(
+      `ALTER TABLE "location" ADD CONSTRAINT "UQ_346127033e9c6ca7ffa2df9ecbf" UNIQUE ("location_id", "source_id")`
+    );
+    await queryRunner.query(
+      `ALTER TABLE "transaction" ADD CONSTRAINT "FK_dd423da8b6167163d79d3c06953" FOREIGN KEY ("location") REFERENCES "location"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`
+    );
+    await queryRunner.query(
+      `ALTER TABLE "pos_deposit" ADD CONSTRAINT "FK_0c2d1a0d19b5b5ea5f0af5f3bf2" FOREIGN KEY ("merchant") REFERENCES "location_merchant"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`
+    );
+    await queryRunner.query(
+      `ALTER TABLE "cash_deposit" ADD CONSTRAINT "FK_0889fcbcebfdf2845d242627936" FOREIGN KEY ("bank") REFERENCES "location_bank"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "cash_deposit" DROP CONSTRAINT "FK_0889fcbcebfdf2845d242627936"`
+    );
+    await queryRunner.query(
+      `ALTER TABLE "pos_deposit" DROP CONSTRAINT "FK_0c2d1a0d19b5b5ea5f0af5f3bf2"`
+    );
+    await queryRunner.query(
+      `ALTER TABLE "transaction" DROP CONSTRAINT "FK_dd423da8b6167163d79d3c06953"`
+    );
+    await queryRunner.query(
+      `ALTER TABLE "location" DROP CONSTRAINT "UQ_346127033e9c6ca7ffa2df9ecbf"`
+    );
+    await queryRunner.query(`ALTER TABLE "cash_deposit" DROP COLUMN "bank"`);
+    await queryRunner.query(`ALTER TABLE "pos_deposit" DROP COLUMN "merchant"`);
+    await queryRunner.query(`ALTER TABLE "transaction" DROP COLUMN "location"`);
+    await queryRunner.query(
+      `ALTER TABLE "location" ADD CONSTRAINT "UQ_62c907775331b5aa98ec8daf7dd" UNIQUE ("source_id", "location_id")`
+    );
+  }
+}

apps/backend/src/deposits/cash-deposit.service.ts

@@ -64,13 +64,13 @@ export class CashDepositService {
   async findCashDepositsByDate(
     program: Ministries,
     deposit_date: string,
-    pt_location_id: number,
+    pt_location_ids: number[],
     statuses?: MatchStatus[]
   ): Promise<CashDepositEntity[]> {
     const depositStatus = statuses ?? MatchStatusAll;
     return await this.cashDepositRepo.find({
       where: {
-        pt_location_id,
+        pt_location_id: In(pt_location_ids),
         metadata: { program: program },
         deposit_date,
         status: In(depositStatus),
@@ -88,13 +88,13 @@ export class CashDepositService {
    */
   async findAllCashDepositDatesPerLocation(
     program: Ministries,
-    pt_location_id: number,
+    pt_location_ids: number[],
     order: FindOptionsOrderValue
   ): Promise<string[]> {
     const deposits: CashDepositEntity[] = await this.cashDepositRepo.find({
       where: {
         metadata: { program },
-        pt_location_id,
+        pt_location_id: In(pt_location_ids),
       },
       order: {
         deposit_date: order,
@@ -133,11 +133,11 @@ export class CashDepositService {
   async findCashDepositExceptions(
     date: string,
     program: Ministries,
-    pt_location_id: number
+    pt_location_ids: number[]
   ): Promise<CashDepositEntity[]> {
     return await this.cashDepositRepo.find({
       where: {
-        pt_location_id,
+        pt_location_id: In(pt_location_ids),
         metadata: { program: program },
         deposit_date: LessThanOrEqual(date),
         status: MatchStatus.IN_PROGRESS,
@@ -251,4 +251,7 @@ export class CashDepositService {
     });
     return [...reconciled, ...in_progress, ...pending];
   }
+  async findWithNullLocation() {
+    return await this.cashDepositRepo.find({ where: { bank: undefined } });
+  }
 }

apps/backend/src/deposits/entities/cash-deposit.entity.ts

@@ -11,6 +11,7 @@ import { FileMetadata } from '../../common/columns/metadata';
 import { MatchStatus } from '../../common/const';
 import { FileTypes } from '../../constants';
 import { TDI17Details } from '../../flat-files';
+import { BankLocationEntity } from '../../location/entities';
 import { FileUploadedEntity } from '../../parse/entities/file-uploaded.entity';
 import { PaymentEntity } from '../../transaction/entities/payment.entity';
 
@@ -100,6 +101,10 @@ export class CashDepositEntity {
   @Column({ name: 'file_uploaded', nullable: true })
   fileUploadedEntityId?: string;
 
+  @ManyToOne(() => BankLocationEntity, { nullable: true })
+  @JoinColumn({ referencedColumnName: 'id', name: 'bank' })
+  bank: Relation<BankLocationEntity>;
+
   constructor(data?: TDI17Details) {
     Object.assign(this, data?.resource);
   }

apps/backend/src/deposits/entities/pos-deposit.entity.ts

@@ -13,6 +13,7 @@ import { FileMetadata } from '../../common/columns';
 import { MatchStatus } from '../../common/const';
 import { FileTypes } from '../../constants';
 import { TDI34Details } from '../../flat-files';
+import { MerchantEntity } from '../../location/entities';
 import { FileUploadedEntity } from '../../parse/entities/file-uploaded.entity';
 import { PosHeuristicRound } from '../../reconciliation/types/const';
 import { PaymentEntity, PaymentMethodEntity } from '../../transaction/entities';
@@ -94,6 +95,10 @@ export class POSDepositEntity {
   )
   payment_match?: Relation<PaymentEntity>;
 
+  @ManyToOne(() => MerchantEntity, { nullable: true })
+  @JoinColumn({ referencedColumnName: 'id', name: 'merchant' })
+  merchant: Relation<MerchantEntity>;
+
   constructor(data?: TDI34Details) {
     Object.assign(this, data?.resource);
   }

apps/backend/src/deposits/pos-deposit.service.ts

@@ -12,8 +12,11 @@
 import { POSDepositEntity } from './entities/pos-deposit.entity';
 import { MatchStatus, MatchStatusAll } from '../common/const';
 import { mapLimit } from '../common/promises';
-import { DateRange, Ministries, NormalizedLocation } from '../constants';
-import { LocationEntity } from '../location/entities';
+import { DateRange, Ministries } from '../constants';
+import {
+  MasterLocationEntity,
+  MinistryLocationEntity,
+} from '../location/entities';
 import { AppLogger } from '../logger/logger.service';
 
 @Injectable()
@@ -85,7 +88,7 @@
   }
 
   async findPOSBySettlementDate(
-    locations: NormalizedLocation[],
+    locations: MinistryLocationEntity[],
     program: Ministries,
     dateRange: DateRange
   ) {
@@ -98,13 +101,15 @@
     qb.addSelect('SUM(transaction_amt)::numeric(10,2)', 'transaction_amt');
     qb.addSelect('location_id');
     qb.leftJoin(
-      LocationEntity,
+      MasterLocationEntity,
       'master_location_data',
       'master_location_data.merchant_id = pos_deposit.merchant_id AND master_location_data.method = pos_deposit.payment_method'
     );
     qb.where({
       metadata: { program },
-      merchant_id: In(locations.flatMap((l) => l.merchant_ids)),
+      merchant_id: In(
+        locations.flatMap((l) => l.merchants.map((itm) => itm.merchant_id))
+      ),
       settlement_date: Raw(
         (alias) => `${alias} >= :minDate::date and ${alias} <= :maxDate::date`,
         { minDate, maxDate }
@@ -174,7 +179,7 @@
       await Promise.all(
         deposits.map((d) => {
           // eslint-disable-next-line @typescript-eslint/no-unused-vars
           const { timestamp, ...dep } = d;
           return manager.update(POSDepositEntity, { id: dep.id }, { ...dep });
         })
       );
@@ -250,4 +255,8 @@
     });
     return [...reconciled, ...in_progress, ...pending];
   }
+
+  async findWithNullLocation() {
+    return await this.posDepositRepo.find({ where: { merchant: undefined } });
+  }
 }