S3 document audit enhancements (#2849)

* add dedup func

* fetch associated contract or rate

* return the right thing

* return contract or rate

* simplify this

* fixes

* use existing

* store funcs

* find rate rev and contract

* fix parser

* debug

* debugs

* error return and log

* add revisions

* better parsing

* add missing data

* don't include contract

* no safe parse

* fix up types

* if we can't fetch, collect the errors

* naming

services/app-api/src/handlers/audit_s3.ts

@@ -1,10 +1,12 @@
 import type { APIGatewayProxyResultV2, Handler } from 'aws-lambda'
 import { initTracer, recordException } from '../../../uploads/src/lib/otel'
 import { configurePostgres } from './configuration'
-import { NewPostgresStore } from '../postgres'
+import { NewPostgresStore, NotFoundError } from '../postgres'
+import type { Store } from '../postgres'
 import { HeadObjectCommand, S3Client } from '@aws-sdk/client-s3'
 import type { S3ServiceException } from '@aws-sdk/client-s3'
 import type { AuditDocument } from '../domain-models'
+import type { ContractRevisionTable, RateRevisionTable } from '@prisma/client'
 
 const main: Handler = async (): Promise<APIGatewayProxyResultV2> => {
     // setup otel tracing
@@ -70,16 +72,32 @@ const main: Handler = async (): Promise<APIGatewayProxyResultV2> => {
             missingOrErrorDocuments.push(processResult)
         }
     }
+
+    // dedup
+    const uniqueDocuments = deduplicateDocuments(missingOrErrorDocuments)
     console.info(
-        `Missing ${missingOrErrorDocuments.length} of ${docResult.length} documents`
+        `Missing ${uniqueDocuments.length} of ${docResult.length} documents`
+    )
+
+    // get the contract or rate
+    const { results, errors } = await fetchAssociatedData(
+        store,
+        uniqueDocuments
     )
+    if (errors.length > 0) {
+        console.error(
+            'Errors encountered while fetching associated data:',
+            errors
+        )
+    }
+    console.info(`found ${results.length} documents with associations`)
     console.info(
-        `These documents could not be retreived from s3: ${JSON.stringify(missingOrErrorDocuments)}`
+        `These documents could not be retreived from s3: ${JSON.stringify(results)}`
     )
 
     const success: APIGatewayProxyResultV2 = {
         statusCode: 200,
-        body: JSON.stringify(missingOrErrorDocuments),
+        body: JSON.stringify(results),
         headers: {
             'Access-Control-Allow-Origin': '*',
             'Access-Control-Allow-Credentials': true,
@@ -165,4 +183,92 @@ async function processDocument(
     }
 }
 
+function deduplicateDocuments(documents: AuditDocument[]): AuditDocument[] {
+    const uniqueDocuments = new Map<string, AuditDocument>()
+
+    for (const doc of documents) {
+        if (!uniqueDocuments.has(doc.s3URL)) {
+            uniqueDocuments.set(doc.s3URL, doc)
+        }
+    }
+
+    return Array.from(uniqueDocuments.values())
+}
+
+type DocumentWithAssociation = AuditDocument & {
+    associatedContract?: ContractRevisionTable
+    associatedRate?: RateRevisionTable
+}
+
+type FetchError = {
+    documentId: string
+    type: 'contractDoc' | 'rateDoc'
+    error: Error | NotFoundError
+    revisionId: string | null
+}
+
+async function fetchAssociatedData(
+    store: Store,
+    documents: AuditDocument[]
+): Promise<{
+    results: DocumentWithAssociation[]
+    errors: FetchError[]
+}> {
+    const results: DocumentWithAssociation[] = []
+    const errors: FetchError[] = []
+
+    for (const doc of documents) {
+        let associatedData: ContractRevisionTable | RateRevisionTable | null =
+            null
+
+        if (doc.type === 'contractDoc' && doc.contractRevisionID) {
+            const contractResult = await store.findContractRevision(
+                doc.contractRevisionID
+            )
+            if (
+                contractResult instanceof Error ||
+                contractResult instanceof NotFoundError
+            ) {
+                errors.push({
+                    documentId: doc.id,
+                    type: doc.type,
+                    error: contractResult,
+                    revisionId: doc.contractRevisionID,
+                })
+            } else {
+                associatedData = contractResult
+            }
+        } else if (doc.type === 'rateDoc' && doc.rateRevisionID) {
+            const rateResult = await store.findRateRevision(doc.rateRevisionID)
+            if (
+                rateResult instanceof Error ||
+                rateResult instanceof NotFoundError
+            ) {
+                errors.push({
+                    documentId: doc.id,
+                    type: doc.type,
+                    error: rateResult,
+                    revisionId: doc.rateRevisionID,
+                })
+            } else {
+                associatedData = rateResult
+            }
+        }
+
+        results.push({
+            ...doc,
+            associatedContract:
+                doc.type === 'contractDoc'
+                    ? (associatedData as ContractRevisionTable)
+                    : undefined,
+            associatedRate:
+                doc.type === 'rateDoc'
+                    ? (associatedData as RateRevisionTable)
+                    : undefined,
+        })
+    }
+
+    return { results, errors }
+}
+
 module.exports = { main }

services/app-api/src/postgres/contractAndRates/findContractRevision.ts

@@ -0,0 +1,27 @@
+import type { PrismaTransactionType } from '../prismaTypes'
+import { NotFoundError } from '../postgresErrors'
+import type { ContractRevisionTable } from '@prisma/client'
+
+async function findContractRevision(
+    client: PrismaTransactionType,
+    contractRevID: string
+): Promise<ContractRevisionTable | Error> {
+    try {
+        const contractRev = await client.contractRevisionTable.findUnique({
+            where: {
+                id: contractRevID,
+            },
+        })
+
+        if (!contractRev) {
+            const err = `PRISMA ERROR: Cannot find contract revision with id: ${contractRevID}`
+            return new NotFoundError(err)
+        }
+        return contractRev
+    } catch (err) {
+        console.error('PRISMA ERROR', err)
+        return err
+    }
+}
+
+export { findContractRevision }

services/app-api/src/postgres/contractAndRates/findRateRevision.ts

@@ -0,0 +1,28 @@
+import type { PrismaTransactionType } from '../prismaTypes'
+import { NotFoundError } from '../postgresErrors'
+import type { RateRevisionTable } from '@prisma/client'
+
+async function findRateRevision(
+    client: PrismaTransactionType,
+    rateRevisionID: string
+): Promise<RateRevisionTable | Error> {
+    try {
+        const rateRevision = await client.rateRevisionTable.findUnique({
+            where: {
+                id: rateRevisionID,
+            },
+        })
+
+        if (!rateRevision) {
+            const err = `PRISMA ERROR: Cannot find rate revision with id: ${rateRevisionID}`
+            return new NotFoundError(err)
+        }
+
+        return rateRevision
+    } catch (err) {
+        console.error('PRISMA ERROR', err)
+        return err
+    }
+}
+
+export { findRateRevision }

services/app-api/src/postgres/contractAndRates/index.ts

@@ -24,3 +24,5 @@ export { replaceRateOnContract } from './replaceRateOnContract'
 export { withdrawRateInsideTransaction } from './withdrawRate'
 export { updateDraftContract } from './updateDraftContract'
 export type { FindAllRatesWithHistoryBySubmitType } from './findAllRatesWithHistoryBySubmitInfo'
+export { findContractRevision } from './findContractRevision'
+export { findRateRevision } from './findRateRevision'

services/app-api/src/postgres/documents/findAllDocuments.ts

@@ -188,7 +188,7 @@ async function getContractQuestionResponseDocument(
         const docs = await prisma.contractQuestionResponseDocument.findMany()
         return docs.map((doc) => ({
             ...doc,
-            questionID: doc.responseID,
+            responseID: doc.responseID,
         }))
     } catch (err) {
         return err instanceof Error

services/app-api/src/postgres/postgresStore.ts

@@ -1,4 +1,9 @@
-import type { PrismaClient, Division } from '@prisma/client'
+import type {
+    PrismaClient,
+    Division,
+    RateRevisionTable,
+    ContractRevisionTable,
+} from '@prisma/client'
 import type { StateCodeType } from '../../../app-web/src/common-code/healthPlanFormDataType'
 import type {
     ProgramType,
@@ -47,6 +52,8 @@ import {
     unlockContract,
     updateDraftContract,
     replaceRateOnContract,
+    findContractRevision,
+    findRateRevision,
 } from './contractAndRates'
 import type {
     SubmitContractArgsType,
@@ -190,6 +197,14 @@ type Store = {
     unlockRate: (args: UnlockRateArgsType) => Promise<RateType | Error>
 
     findAllDocuments: () => Promise<AuditDocument[] | Error>
+
+    findContractRevision: (
+        contractRevID: string
+    ) => Promise<ContractRevisionTable | Error>
+
+    findRateRevision: (
+        rateRevisionID: string
+    ) => Promise<RateRevisionTable | Error>
 }
 
 function NewPostgresStore(client: PrismaClient): Store {
@@ -266,6 +281,8 @@ function NewPostgresStore(client: PrismaClient): Store {
         unlockRate: (args) => unlockRate(client, args),
 
         findAllDocuments: () => findAllDocuments(client),
+        findContractRevision: (args) => findContractRevision(client, args),
+        findRateRevision: (args) => findRateRevision(client, args),
     }
 }