Update parsing and saving of multiple rule summaries for one condition (

#2923) * update parsing schemas to properly grab RR info * save loop for conditions and rule summaries working in postgres * update comments * update listEcrDataService and tests to retrieve conditions and summaries as arrays * wip * wip * concatenate string arrays with newlines for ecr library display * update snapshot test * update tests with new qeury * remove extra fields from BundleMetadata type * change parameter type * play with typing to make turbo build happy * play with typing to make turbo build happy * play with typing to make turbo build happy * fix scoping issue with variable * address comments * change conditions display to a UL update tests * cleanup random extras * updating saving for nested rule summaries * get sql server in working order * remove console logs * Updates to match Josh's sql server changes * remove unneeded localhost default
CDCgov · Nov 22, 2024 · dc2094d · dc2094d
1 parent c4b741e
commit dc2094d
Show file tree

Hide file tree

Showing 9 changed files with 118 additions and 44 deletions.
diff --git a/containers/ecr-viewer/docker-compose.yml b/containers/ecr-viewer/docker-compose.yml
@@ -10,6 +10,7 @@ services:
     environment:
       - APP_ENV=${APP_ENV:-test}
       - METADATA_DATABASE_TYPE=${METADATA_DATABASE_TYPE:-postgres}
+      - METADATA_DATABASE_SCHEMA=${METADATA_DATABASE_SCHEMA}
       - SOURCE=${SOURCE:-postgres}
       - DATABASE_URL=postgres://postgres:pw@postgres:5432/ecr_viewer_db
       - AWS_CUSTOM_ENDPOINT=http://aws-storage:4566
@@ -21,6 +22,9 @@ services:
       - AZURE_CONTAINER_NAME=${AZURE_CONTAINER_NAME:-ecr-viewer-files}
       - NEXT_PUBLIC_NON_INTEGRATED_VIEWER=${NEXT_PUBLIC_NON_INTEGRATED_VIEWER:-false}
       - NEXT_PUBLIC_BASEPATH=${NEXT_PUBLIC_BASEPATH:-/ecr-viewer}
+      - SQL_SERVER_USER=${SQL_SERVER_USER}
+      - SQL_SERVER_PASSWORD=${SQL_SERVER_PASSWORD}
+      - SQL_SERVER_HOST=${SQL_SERVER_HOST:-sqlserver}
     profiles:
       - ecr-viewer
 
@@ -54,8 +58,9 @@ services:
     ports:
       - "1433:1433"
     volumes:
-      - ./seed-scripts/sql:/var/opt/mssql/scripts
-    entrypoint: /var/opt/mssql/scripts/entrypoint.sh
+      - ./seed-scripts/sql/entrypoint.sh:/var/opt/mssql/scripts/entrypoint.sh
+      - ./sql/extended.sql:/var/opt/mssql/scripts/extended.sql
+    entrypoint: /bin/bash -c "/var/opt/mssql/scripts/entrypoint.sh"
     healthcheck:
       test:
         [

diff --git a/containers/ecr-viewer/seed-scripts/sql/entrypoint.sh b/containers/ecr-viewer/seed-scripts/sql/entrypoint.sh
@@ -2,6 +2,13 @@
 
 /opt/mssql/bin/sqlservr &
 
+# Wait for SQL Server to start up
+until /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P $MSSQL_SA_PASSWORD -Q "SELECT 1" -C &>/dev/null; do
+    echo "Waiting for SQL Server to start..."
+    sleep 1
+done
+
+# Run your SQL script
 /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P $MSSQL_SA_PASSWORD -d master -i /var/opt/mssql/scripts/extended.sql -C
 
 wait
diff --git a/containers/ecr-viewer/sql/extended.sql b/containers/ecr-viewer/sql/extended.sql
@@ -66,8 +66,8 @@ CREATE TABLE ecr_labs
     eICR_ID                                VARCHAR(200) REFERENCES ECR_DATA (eICR_ID),
     test_type                              VARCHAR(255),
     test_type_code                         VARCHAR(50),
-    test_type_system                       VARCHAR(50),
-    test_result_qualitative                VARCHAR(255),
+    test_type_system                       VARCHAR(255),
+    test_result_qualitative                VARCHAR(MAX),
     test_result_quantitative               FLOAT,
     test_result_units                      VARCHAR(50),
     test_result_code                       VARCHAR(50),

diff --git a/containers/ecr-viewer/src/app/api/save-fhir-data/save-fhir-data-service.ts b/containers/ecr-viewer/src/app/api/save-fhir-data/save-fhir-data-service.ts
@@ -175,14 +175,22 @@ export const saveMetadataToSqlServer = async (
   metadata: BundleExtendedMetadata,
   ecrId: string,
 ) => {
-  let pool = await sql.connect({
-    user: process.env.SQL_SERVER_USER,
-    password: process.env.SQL_SERVER_PASSWORD,
-    server: process.env.SQL_SERVER_HOST || "localhost",
-    options: {
-      trustServerCertificate: true,
-    },
-  });
+  let pool;
+  try {
+    pool = await sql.connect({
+      user: process.env.SQL_SERVER_USER,
+      password: process.env.SQL_SERVER_PASSWORD,
+      server: process.env.SQL_SERVER_HOST,
+      options: {
+        trustServerCertificate: true,
+      },
+    });
+  } catch (error: any) {
+    return NextResponse.json(
+      { message: "Failed to connect to SQL Server. " + error.message },
+      { status: 500 },
+    );
+  }
 
   const transaction = new sql.Transaction(pool);
   await transaction.begin();
@@ -191,7 +199,7 @@ export const saveMetadataToSqlServer = async (
     try {
       const ecrDataInsertRequest = new sql.Request(transaction);
       await ecrDataInsertRequest
-        .input("eICR_ID", sql.VarChar(200), metadata.eicr_id)
+        .input("eICR_ID", sql.VarChar(200), ecrId)
         .input("eicr_set_id", sql.VarChar(255), metadata.eicr_set_id)
         .input("fhir_reference_link", sql.VarChar(255), null) // Not implemented
         .input("last_name", sql.VarChar(255), metadata.last_name)
@@ -287,13 +295,13 @@ export const saveMetadataToSqlServer = async (
           const labInsertRequest = new sql.Request(transaction);
           await labInsertRequest
             .input("UUID", sql.VarChar(200), lab.uuid)
-            .input("eICR_ID", sql.VarChar(200), metadata.eicr_id)
+            .input("eICR_ID", sql.VarChar(200), ecrId)
             .input("test_type", sql.VarChar(200), lab.test_type)
             .input("test_type_code", sql.VarChar(50), lab.test_type_code)
-            .input("test_type_system", sql.VarChar(50), lab.test_type_system)
+            .input("test_type_system", sql.VarChar(255), lab.test_type_system)
             .input(
               "test_result_qualitative",
-              sql.VarChar(255),
+              sql.VarChar(sql.MAX),
               lab.test_result_qualitative,
             )
             .input(
@@ -354,25 +362,39 @@ export const saveMetadataToSqlServer = async (
       }
 
       if (metadata.rr) {
-        for (const rule of metadata.rr) {
+        // Loop through each condition/rule object in rr array
+        for (const rrItem of metadata.rr) {
           const rr_conditions_uuid = randomUUID();
           const rrConditionsInsertRequest = new sql.Request(transaction);
+
+          // Insert condition into ecr_rr_conditions
           await rrConditionsInsertRequest
             .input("UUID", sql.VarChar(200), rr_conditions_uuid)
-            .input("eICR_ID", sql.VarChar(200), metadata.eicr_id)
-            .input("condition", sql.VarChar(sql.MAX), rule.condition)
+            .input("eICR_ID", sql.VarChar(200), ecrId)
+            .input("condition", sql.VarChar(sql.MAX), rrItem.condition)
             .query(
               "INSERT INTO dbo.ecr_rr_conditions VALUES (@UUID, @eICR_ID, @condition)",
             );
 
-          const ruleSummaryInsertRequest = new sql.Request(transaction);
-          await ruleSummaryInsertRequest
-            .input("UUID", sql.VarChar(200), randomUUID())
-            .input("ECR_RR_CONDITIONS_ID", sql.VarChar(200), rr_conditions_uuid)
-            .input("rule_summary", sql.VarChar(sql.MAX), rule.rule_summaries)
-            .query(
-              "INSERT INTO dbo.ecr_rr_rule_summaries VALUES (@UUID, @ECR_RR_CONDITIONS_ID, @rule_summary)",
-            );
+          // Loop through the rule summaries array
+          if (rrItem.rule_summaries && rrItem.rule_summaries.length > 0) {
+            for (const summary of rrItem.rule_summaries) {
+              const ruleSummaryInsertRequest = new sql.Request(transaction);
+
+              // Insert each rule summary with reference to the condition
+              await ruleSummaryInsertRequest
+                .input("UUID", sql.VarChar(200), randomUUID())
+                .input(
+                  "ECR_RR_CONDITIONS_ID",
+                  sql.VarChar(200),
+                  rr_conditions_uuid,
+                )
+                .input("rule_summary", sql.VarChar(sql.MAX), summary.summary)
+                .query(
+                  "INSERT INTO dbo.ecr_rr_rule_summaries VALUES (@UUID, @ECR_RR_CONDITIONS_ID, @rule_summary)",
+                );
+            }
+          }
         }
       }
 
@@ -394,7 +416,9 @@ export const saveMetadataToSqlServer = async (
       );
     } finally {
       // Close the connection pool
-      pool.close();
+      if (pool) {
+        pool.close();
+      }
     }
   } else {
     return NextResponse.json(
@@ -451,13 +475,18 @@ export const saveMetadataToPostgres = async (
 
           const savedRRCondition = await t.one(saveRRConditions);
 
-          // Insert associated rule summary into ecr_rr_rule_summaries
-          const saveRRSummary = new PQ({
-            text: "INSERT INTO ecr_rr_rule_summaries (uuid, ecr_rr_conditions_id, rule_summary) VALUES (uuid_generate_v4(), $1, $2)",
-            values: [savedRRCondition.uuid, rrItem.rule_summaries],
-          });
-
-          await t.none(saveRRSummary);
+          // Loop through the rule summaries array
+          if (rrItem.rule_summaries && rrItem.rule_summaries.length > 0) {
+            for (const summaryObj of rrItem.rule_summaries) {
+              // Insert each associated summary into ecr_rr_rule_summaries
+              const saveRRSummary = new PQ({
+                text: "INSERT INTO ecr_rr_rule_summaries (uuid, ecr_rr_conditions_id, rule_summary) VALUES (uuid_generate_v4(), $1, $2)",
+                values: [savedRRCondition.uuid, summaryObj.summary],
+              });
+
+              await t.none(saveRRSummary);
+            }
+          }
         }
 
         return ecrData;
@@ -497,6 +526,7 @@ export const saveWithMetadata = async (
   let fhirDataResult;
   let metadataResult;
   const metadataSaveLocation = process.env.METADATA_DATABASE_TYPE;
+
   try {
     fhirDataResult = await saveFhirData(fhirBundle, ecrId, saveSource);
 

diff --git a/containers/ecr-viewer/src/app/api/save-fhir-data/types.ts b/containers/ecr-viewer/src/app/api/save-fhir-data/types.ts
@@ -19,9 +19,13 @@ interface Lab {
   specimen_collection_date: Date | undefined;
 }
 
+interface ruleSummary {
+  summary: string;
+}
+
 interface RR {
   condition: string;
-  rule_summaries: string;
+  rule_summaries: ruleSummary[];
 }
 
 export interface BundleExtendedMetadata {

diff --git a/containers/message-parser/app/default_schemas/ecr.json b/containers/message-parser/app/default_schemas/ecr.json
@@ -355,9 +355,16 @@
         }
       },
       "rule_summaries": {
-        "fhir_path": "Observation.extension.where(url = 'http://hl7.org/fhir/us/ecr/StructureDefinition/us-ph-determination-of-reportability-rule-extension').valueString",
-        "data_type": "string",
+        "fhir_path": "Observation.extension.where(url = 'http://hl7.org/fhir/us/ecr/StructureDefinition/us-ph-determination-of-reportability-rule-extension')",
+        "data_type": "array",
         "nullable": true,
+        "secondary_schema": {
+          "summary": {
+            "fhir_path": "valueString",
+            "data_type": "string",
+            "nullable": true
+          }
+        },
         "metadata": {
           "category": "RR Reportability Information"
         }

diff --git a/containers/message-parser/app/default_schemas/ecr_viewer_metadata.json b/containers/message-parser/app/default_schemas/ecr_viewer_metadata.json
@@ -50,9 +50,16 @@
         }
       },
       "rule_summaries": {
-        "fhir_path": "Observation.extension.where(url = 'http://hl7.org/fhir/us/ecr/StructureDefinition/us-ph-determination-of-reportability-rule-extension').valueString",
-        "data_type": "string",
+        "fhir_path": "Observation.extension.where(url = 'http://hl7.org/fhir/us/ecr/StructureDefinition/us-ph-determination-of-reportability-rule-extension')",
+        "data_type": "array",
         "nullable": true,
+        "secondary_schema": {
+          "summary": {
+            "fhir_path": "valueString",
+            "data_type": "string",
+            "nullable": true
+          }
+        },
         "metadata": {
           "category": "RR Reportability Information"
         }

diff --git a/containers/message-parser/app/default_schemas/ecr_with_metadata.json b/containers/message-parser/app/default_schemas/ecr_with_metadata.json
@@ -126,9 +126,16 @@
         }
       },
       "rule_summaries": {
-        "fhir_path": "Observation.extension.where(url = 'http://hl7.org/fhir/us/ecr/StructureDefinition/us-ph-determination-of-reportability-rule-extension').valueString",
-        "data_type": "string",
+        "fhir_path": "Observation.extension.where(url = 'http://hl7.org/fhir/us/ecr/StructureDefinition/us-ph-determination-of-reportability-rule-extension')",
+        "data_type": "array",
         "nullable": true,
+        "secondary_schema": {
+          "summary": {
+            "fhir_path": "valueString",
+            "data_type": "string",
+            "nullable": true
+          }
+        },
         "metadata": {
           "category": "RR Reportability Information"
         }

diff --git a/containers/message-parser/app/default_schemas/philly_ecr.json b/containers/message-parser/app/default_schemas/philly_ecr.json
@@ -357,9 +357,16 @@
         }
       },
       "rule_summaries": {
-        "fhir_path": "Observation.extension.where(url = 'http://hl7.org/fhir/us/ecr/StructureDefinition/us-ph-determination-of-reportability-rule-extension').valueString",
-        "data_type": "string",
+        "fhir_path": "Observation.extension.where(url = 'http://hl7.org/fhir/us/ecr/StructureDefinition/us-ph-determination-of-reportability-rule-extension')",
+        "data_type": "array",
         "nullable": true,
+        "secondary_schema": {
+          "summary": {
+            "fhir_path": "valueString",
+            "data_type": "string",
+            "nullable": true
+          }
+        },
         "metadata": {
           "category": "RR Reportability Information"
         }