Update Prod to v0.3.4 - 11/7/2024

.gitignore

@@ -123,3 +123,6 @@ venv.bak/
 
 # copied as a result of local code mounted development
 collected_static/
+
+# vscode
+.vscode/

app/merge/merge_functions.py

@@ -24,28 +24,52 @@
 
 
 def process_MS2_data(ms1_data, ms2_data_list, mass_accuracy=10, rt_accuracy=0.2):
-    # NTAW-158: Adjust sheet names pulled from MS1 results
     matched_df = ms1_data if isinstance(ms1_data, pd.DataFrame) else ms1_data["chemical_results"]
-    # matched_df = ms1_data if isinstance(ms1_data, pd.DataFrame) else ms1_data["dsstox_search"]
-
     matched_df.rename(columns={"DTXCID_INDIVIDUAL_COMPONENT": "DTXCID"}, inplace=True)
 
     for ms2_data in ms2_data_list:
         filename = ms2_data["file_name"]
         cfmid_df = ms2_data["file_df"]
-        mass_col, rt_col, score_col = (f"MASS_MGF_{filename}", f"RT_{filename}", f"SUM_SCORE_{filename}")
+        # mass_col, rt_col, score_col, q_score_col, percentile_col = (f"MASS_MGF_{filename}", f"RT_{filename}", f"SUM_SCORE_{filename}", f"QUOTIENT_SCORE_{filename}", f"PERCENTILE_SCORE_{filename}")
+        mass_col, rt_col, score_col, q_score_col, percentile_col = (
+            f"MASS_MGF_{filename}",
+            f"RT_{filename}",
+            f"SUM_SCORE_{filename}",
+            f"QUOTIENT_SCORE_{filename}",
+            f"PERCENTILE_SCORE_{filename}",
+        )
         # logger.info('mass_col, rt_col, score_col')
         # logger.info(mass_col, rt_col, score_col)
 
-        # NTAW-158: Adjust columns renamed based on new MS2 column names
         # NTAW-158: Grab the neutral mass column from the MS2 data as this is going to be compared to the neutral mass from the MS1 data
-        cfmid_df.rename(columns={"MASS_NEUTRAL": mass_col, "RT": rt_col, "SUM_SCORE": score_col}, inplace=True)
-        # cfmid_df.rename(columns={"MASS_MGF": mass_col, "RT": rt_col, "SUM_SCORE": score_col}, inplace=True)
-        # cfmid_df.rename(columns = {'MASS_MGF': mass_col, 'RT': rt_col, 'SUM_SCORE' : score_col}, inplace = True)
-        # cfmid_df.rename(columns={"MASS_in_MGF": mass_col, "RT": rt_col, "energy_sum": score_col}, inplace=True)
+        cfmid_df.rename(
+            columns={
+                "MASS_NEUTRAL": mass_col,
+                "RT": rt_col,
+                "SUM_SCORE": score_col,
+                "Q-SCORE": q_score_col,
+                "PERCENTILE": percentile_col,
+            },
+            inplace=True,
+        )
+
+        # # NTAW-607: Convert retention time column units from seconds to minutes
+        # cfmid_df[rt_col] = cfmid_df[rt_col] / 60
+
+        # # NTAW-607: Add units to MS1 retention time column
+        # matched_df.rename(columns={"Retention_Time": "Retention_Time(min)"}, inplace=True)
 
         matched_df = matched_df.merge(
-            cfmid_df[["DTXCID", f"MASS_MGF_{filename}", f"RT_{filename}", f"SUM_SCORE_{filename}"]],
+            cfmid_df[
+                [
+                    "DTXCID",
+                    f"MASS_MGF_{filename}",
+                    f"RT_{filename}",
+                    f"SUM_SCORE_{filename}",
+                    f"QUOTIENT_SCORE_{filename}",
+                    f"PERCENTILE_SCORE_{filename}",
+                ]
+            ],
             how="left",
             on="DTXCID",
         )
@@ -57,10 +81,25 @@ def process_MS2_data(ms1_data, ms2_data_list, mass_accuracy=10, rt_accuracy=0.2)
             mass_diff + rt_diff if mass_diff <= mass_accuracy and rt_diff <= rt_accuracy else np.nan
             for mass_diff, rt_diff in zip(matched_df["mass_diff"], matched_df["rt_diff"])
         ]
-        matched_df[[mass_col, rt_col, score_col]] = matched_df[[mass_col, rt_col, score_col]].where(
-            (matched_df["mass_diff"] < mass_accuracy) & (matched_df["rt_diff"] < rt_accuracy), [np.nan, np.nan, np.nan]
+        matched_df[[mass_col, rt_col, score_col, q_score_col, percentile_col]] = matched_df[
+            [mass_col, rt_col, score_col, q_score_col, percentile_col]
+        ].where(
+            (matched_df["mass_diff"] < mass_accuracy) & (matched_df["rt_diff"] < rt_accuracy),
+            [np.nan, np.nan, np.nan, np.nan, np.nan],
         )
 
+        # NTAW-608: Quotient scores of 1 are showing up as empty cell. As a quick fix, fill in empty quotient cells with 1 (where the percentile cell has a value)
+        matched_df.loc[matched_df[q_score_col].isna() & matched_df[percentile_col].notna(), q_score_col] = 1
+
+    #     # NTAW-607: Round MS2 retention time, cfmid score columns to two decimal places
+    #     matched_df[f"RT_{filename}"] = matched_df[f"RT_{filename}"].round(2)
+    #     matched_df[score_col] = matched_df[score_col].round(2)
+    #     matched_df[q_score_col] = matched_df[q_score_col].round(2)
+    #     matched_df[percentile_col] = matched_df[percentile_col].round(2)
+
+    # # NTAW-607: Round MS1 retention time column to two decimal places
+    # matched_df["Retention_Time"] = matched_df["Retention_Time"].round(2)
+
     matched_df.drop(columns=["mass_diff", "rt_diff", "sum_diff"], inplace=True)
     matched_df["Median_MS2_Mass"] = matched_df[[col for col in matched_df.columns if "MASS_" in col]].apply(
         np.median, axis=1