Skip to content

Commit

Permalink
perf: send seed data requests asynchronously (#2825)
Browse files Browse the repository at this point in the history 
* perf: send seed data requests asynchronously

* refactor: favor early exits for less indents

* fix: cleanup printing

* fix: try to appease snyk

* fix: maybe appease snyk

* fix: wat
  • Loading branch information
@mcmcgrath13
mcmcgrath13 authored Nov 12, 2024
1 parent 04abd23 commit c4130e6
Show file tree
Hide file tree
Showing 2 changed files with 95 additions and 85 deletions.
176 changes: 92 additions & 84 deletions containers/ecr-viewer/seed-scripts/create-seed-data.py
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
import argparse
import json
import os
import traceback

import requests
import grequests

URL = "http://orchestration-service:8080"
BASEDIR = os.path.dirname(os.path.abspath(__file__))
Expand Down Expand Up @@ -32,92 +31,101 @@ def _process_files(args):
"""
print("Converting files...")
subfolders = ["LA"]
# Iterate over the subfolders

# Holds all of the rquests we are going to make
requests = []
folder_paths = []

def _process_eicrs(subfolder, folder, folder_path, payload):
r = grequests.post(f"{URL}/process-message", json=payload)
requests.append(r)
folder_paths.append(folder_path)

# Iterate over the subfolders to collect requests
for subfolder in subfolders:
subfolder_path = os.path.join(BASEDIR, "baseECR", subfolder)

# Check if the subfolder exists and is a directory
if os.path.isdir(subfolder_path):
# Now iterate through the folders inside each subfolder
for folder in os.listdir(subfolder_path):
folder_path = os.path.join(subfolder_path, folder)

# Check if it's a directory
if os.path.isdir(folder_path):
# If `bundle.json` exists and ski_convert is set just upload the bundle
if (
os.path.exists(os.path.join(folder_path, "bundle.json"))
and args.skip_convert
):
# Just upload the bundle
with open(
os.path.join(folder_path, "bundle.json")
) as fhir_file:
payload = {
"message_type": "fhir",
"data_type": "fhir",
"config_file_name": "save-bundle-to-ecr-viewer.json",
"message": json.load(fhir_file),
}
_process_eicrs(subfolder, folder, folder_path, payload)
# If we are not just inserting the bundle, check for the necessary files
elif os.path.exists(os.path.join(folder_path, "CDA_eICR.xml")):
# Open the necessary files in the folder
with (
open(
os.path.join(folder_path, "CDA_RR.xml"), "r"
) as rr_file,
open(
os.path.join(folder_path, "CDA_eICR.xml"), "r"
) as eicr_file,
):
payload = {
"message_type": "ecr",
"data_type": "ecr",
"config_file_name": "save-eicr-to-ecr-viewer-config.json",
"message": eicr_file.read(),
"rr_data": rr_file.read(),
}
_process_eicrs(subfolder, folder, folder_path, payload)
# If neither `bundle.json` nor `CDA_eICR.xml` exists, skip processing
else:
print(
f"Neither `bundle.json` nor `CDA_eICR.xml` found in {folder_path}. Skipping."
)
continue

# If the subfolder is not a directory, print a message
else:
print(f"{subfolder_path} is not a valid directory.")


def _process_eicrs(subfolder, folder, folder_path, payload):
try:
print(f"{URL}/process-message for {subfolder}/{folder}")
response = requests.post(f"{URL}/process-message", json=payload)
if response.status_code == 200:
responses_json = response.json()["processed_values"]["responses"]
for response in responses_json:
if "stamped_ecr" in response:
with open(
os.path.join(folder_path, "bundle.json"), "w"
) as fhir_file:
json.dump(
response["stamped_ecr"]["extended_bundle"],
fhir_file,
indent=4,
)
print(f"Converted {folder} in {subfolder} successfully.")
# Handle the case where the response fails
else:
print(f"Failed to convert {folder} in {subfolder}.")
# Handle file not found or other potential errors
except FileNotFoundError as e:
print(f"Required file not found in {folder_path}: {e}")
except Exception as e:
print(
f"An error occurred processing {folder} in {subfolder}: {e}\n\n{traceback.format_exc()}"
)
if not os.path.isdir(subfolder_path):
print(f"{subfolder_path} is not a valid directory.")
continue

# Now iterate through the folders inside each subfolder
for folder in os.listdir(subfolder_path):
folder_path = os.path.join(subfolder_path, folder)

# Check if it's a directory
if not os.path.isdir(folder_path):
continue

if (
os.path.exists(os.path.join(folder_path, "bundle.json"))
and args.skip_convert
):
# Just upload the bundle
with open(os.path.join(folder_path, "bundle.json")) as fhir_file:
payload = {
"message_type": "fhir",
"data_type": "fhir",
"config_file_name": "save-bundle-to-ecr-viewer.json",
"message": json.load(fhir_file),
}
_process_eicrs(subfolder, folder, folder_path, payload)

# If we are not just inserting the bundle, check for the necessary files
elif os.path.exists(os.path.join(folder_path, "CDA_eICR.xml")):
# Open the necessary files in the folder
with (
open(os.path.join(folder_path, "CDA_RR.xml"), "r") as rr_file,
open(os.path.join(folder_path, "CDA_eICR.xml"), "r") as eicr_file,
):
payload = {
"message_type": "ecr",
"data_type": "ecr",
"config_file_name": "save-eicr-to-ecr-viewer-config.json",
"message": eicr_file.read(),
"rr_data": rr_file.read(),
}

_process_eicrs(subfolder, folder, folder_path, payload)
# If neither `bundle.json` nor `CDA_eICR.xml` exists, skip processing
else:
print(
f"Neither `bundle.json` nor `CDA_eICR.xml` found in {folder_path}. Skipping."
)
continue

# Asynchronously send our collected requests
n = 0
failed = []
num_requests = len(requests)
print(f"Starting conversion and load of {num_requests} requests")
for index, response in grequests.imap_enumerated(requests, size=8):
n += 1
print(f"Received response {n} of {num_requests}")
folder_path = folder_paths[index]
if response.status_code != 200:
failed.append(folder_path)
print(f"Failed to convert {folder_path}.")
continue

responses_json = response.json()["processed_values"]["responses"]
for response in responses_json:
if "stamped_ecr" in response:
with open(
os.path.join(folder_path, "bundle.json"),
"w",
) as fhir_file:
json.dump(
response["stamped_ecr"]["extended_bundle"],
fhir_file,
indent=4,
)
print(f"Converted {folder_path} successfully.")

print(
f"Conversion complete: {n} records attempted with the following failues: {failed}"
)


if __name__ == "__main__":
Expand Down
4 changes: 3 additions & 1 deletion containers/ecr-viewer/seed-scripts/requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@
requests
grequests
requests
gevent>=23.9

0 comments on commit c4130e6

Please sign in to comment.