Fix remaining relative paths on load

CHANGELOG.md

@@ -64,6 +64,7 @@ About changelog [here](https://keepachangelog.com/en/1.0.0/)
 - Default loglevel up to INFO, making logs with default start easier to read
 - Add XTR region to PAR region definition
 - Diagnoses can be searched on diagnoses page without waiting for load first
+- Explicitly store case file paths from load conf accessible on load as absolute paths for later access
 ### Fixed
 - Removed log info showing hgnc IDs used in variantS search
 - Maintain Matchmaker Exchange and Beacon submission status when a case is re-uploaded

scout/adapter/mongo/case.py

@@ -908,7 +908,9 @@ def load_case(self, config_data: dict, update: bool = False, keep_actions: bool
         try:
             for vcf_file in files:
                 # Check if file exists
-                if not case_obj["vcf_files"].get(vcf_file["file_name"]):
+                if not case_obj.get("vcf_files") or not case_obj["vcf_files"].get(
+                    vcf_file["file_name"]
+                ):
                     LOG.debug("didn't find {}, skipping".format(vcf_file["file_name"]))
                     continue
 

scout/build/case.py

@@ -1,6 +1,7 @@
 import logging
+import os
 from datetime import datetime
-from typing import Dict
+from typing import Dict, List, Optional, Union
 
 from scout.constants import CUSTOM_CASE_REPORTS, PHENOTYPE_GROUPS
 from scout.exceptions import ConfigError, IntegrityError
@@ -42,6 +43,59 @@ def _populate_pipeline_info(case_obj, case_data):
         case_obj["pipeline_version"] = case_data["exe_ver"]
 
 
+def set_abspath_case_file(case_obj: dict, case_data: dict, case_file: str):
+    """Abs path case file. E.g. demo files appear as relative path files, and storing absolute paths
+    ensures we do not have to load cases from the same working directory as we start the server process.
+    """
+    file_path = case_data.get(case_file)
+    if file_path and os.path.exists(file_path):
+        case_obj[case_file] = os.path.abspath(file_path)
+    else:
+        case_obj[case_file] = None
+
+
+def set_abspath_case_nested_files(case_obj: dict, case_data: dict, nested_file_key: str):
+    """Absolute path nested case files. Similar to the single abs path setter, but some paths reside nested
+    directly under a particular key.
+    """
+    case_obj[nested_file_key] = case_data.get(nested_file_key)
+    if case_obj.get(nested_file_key):
+        for file_type, nested_file_item in case_obj[nested_file_key].items():
+            if nested_file_item and os.path.exists(nested_file_item):
+                case_obj[nested_file_key][file_type] = os.path.abspath(nested_file_item)
+            else:
+                case_obj[nested_file_key][file_type] = None
+
+
+def set_abspath_case_nested_image_files(
+    case_obj: dict,
+    case_data: dict,
+    nested_file_key: Optional[str] = "custom_images",
+    path_key: Optional[str] = "path",
+):
+    """Absolute path for complexly nested custom image paths. Similar to the single abs path setter, but the custom image paths reside nested
+    in arrays, with the sub-key name "path" one or two levels into the dictionary.
+
+    E.g. case["custom_images"]["str_variants_images"][2]["path"] or
+    case["custom_images"]["case_images"]["section_one"][1]["path"].
+    """
+
+    def case_images_abspath(level: Union[Dict, List]):
+        """Recursively set path to abs_path for all path_key items in lists in dicts in level."""
+        for sub_level_key, sub_level in level.items():
+            if isinstance(sub_level, dict):
+                case_images_abspath(sub_level)
+            elif isinstance(sub_level, list):
+                for image in sub_level:
+                    image_path = image.get(path_key)
+                    if image_path and os.path.exists(image_path):
+                        image[path_key] = os.path.abspath(image_path)
+
+    if case_data.get(nested_file_key):
+        case_obj[nested_file_key] = case_data.get(nested_file_key)
+        case_images_abspath(case_obj[nested_file_key])
+
+
 def build_case(case_data, adapter):
     """Build a case object that is to be inserted to the database
 
@@ -278,24 +332,22 @@ def build_case(case_data, adapter):
     # Files
     case_obj["madeline_info"] = case_data.get("madeline_info")
 
-    case_obj["custom_images"] = case_data.get("custom_images")
+    set_abspath_case_nested_image_files(case_obj, case_data)
+
+    set_abspath_case_file(case_obj, case_data, "delivery_report")
 
     for report_key in [report.get("key_name") for report in CUSTOM_CASE_REPORTS.values()]:
         if report_key in case_data:
-            case_obj[report_key] = case_data.get(report_key)
+            set_abspath_case_file(case_obj, case_data, report_key)
 
-    case_obj["vcf_files"] = case_data.get("vcf_files", {})
-    case_obj["delivery_report"] = case_data.get("delivery_report")
+    set_abspath_case_nested_files(case_obj, case_data, "vcf_files")
 
     _populate_pipeline_info(case_obj, case_data)
 
-    case_obj["has_svvariants"] = bool(
-        case_obj["vcf_files"].get("vcf_sv") or case_obj["vcf_files"].get("vcf_sv_research")
-    )
-
-    case_obj["has_strvariants"] = bool(case_obj["vcf_files"].get("vcf_str"))
-
-    case_obj["has_meivariants"] = bool(case_obj["vcf_files"].get("vcf_mei"))
+    vcf_files = case_obj.get("vcf_files") or {}
+    case_obj["has_svvariants"] = bool(vcf_files.get("vcf_sv") or vcf_files.get("vcf_sv_research"))
+    case_obj["has_strvariants"] = bool(vcf_files.get("vcf_str"))
+    case_obj["has_meivariants"] = bool(vcf_files.get("vcf_mei"))
 
     case_obj["is_migrated"] = False
 

scout/build/individual.py

@@ -8,6 +8,7 @@
 BUILD_INDIVIDUAL_FILES = [
     "bam_file",
     "d4_file",
+    "mitodel_file",
     "mt_bam",
     "rhocall_bed",
     "rhocall_wig",
@@ -21,6 +22,43 @@
 ]
 
 
+def set_abspath_individual_file(ind_obj: dict, ind: dict, ind_file: str):
+    """Fix absolute path for individual files to be served from application.
+    This takes care of incomplete path for demo files. While most endpoints would attempt to make an
+    abs path when sending, storing them as absolute if we can access them on cli load ensures that
+    we can still find them from the web app later, even if that happens to be started in another working directory.
+
+    This may close a loophole for some very particular use cases, but in general should be safer.
+    """
+
+    file_path = ind.get(ind_file)
+    if file_path and os.path.exists(file_path):
+        ind_obj[ind_file] = os.path.abspath(file_path)
+    else:
+        ind_obj[ind_file] = None
+
+
+def set_abspath_nested_individual_files(ind_obj: dict, ind: dict, nested_file_key: str):
+    """Fix absolute path for nested files to be served from application.
+    For some of our more complicated nesting, e.g. Chromograph, the file endings are generalised lat (in js),
+    and only a template is stored on the individual object. We then still wish to update the dirname, treating the
+    basename lightly as a template without checking for file existence just yet. The endpoints will handle that later.
+    """
+    if ind.get(nested_file_key):
+        ind_obj[nested_file_key] = ind.get(nested_file_key)
+        for nested_file_item in ind_obj[nested_file_key]:
+            if nested_file_item:
+                if os.path.exists(nested_file_item):
+                    ind_obj[nested_file_key][nested_file_item] = os.path.abspath(nested_file_item)
+                    continue
+
+                nested_file_item_dirname = os.path.dirname(nested_file_item)
+                nested_file_item_basename = os.path.basename(nested_file_item)
+                ind_obj[nested_file_key][nested_file_item] = (
+                    os.path.abspath(nested_file_item_dirname) + "/" + nested_file_item_basename
+                )
+
+
 def build_individual(ind: dict) -> dict:
     """Build an Individual object
 
@@ -91,8 +129,9 @@ def build_individual(ind: dict) -> dict:
     except KeyError as err:
         raise (PedigreeError("Unknown phenotype: %s" % phenotype))
 
-    # Fix absolute path for individual bam files (takes care of incomplete path for demo files)
     for ind_file in BUILD_INDIVIDUAL_FILES:
+        set_abspath_individual_file(ind_obj, ind, ind_file)
+
         file_path = ind.get(ind_file)
         if file_path and os.path.exists(file_path):
             ind_obj[ind_file] = os.path.abspath(file_path)
@@ -105,10 +144,14 @@ def build_individual(ind: dict) -> dict:
     ind_obj["confirmed_sex"] = ind.get("confirmed_sex")
     ind_obj["confirmed_parent"] = ind.get("confirmed_parent")
     ind_obj["predicted_ancestry"] = ind.get("predicted_ancestry")
+
+    ind_obj["mitodel"] = ind.get("mitodel")
+
     ind_obj["chromograph_images"] = ind.get("chromograph_images")
     ind_obj["reviewer"] = ind.get("reviewer")
-    ind_obj["mitodel"] = ind.get("mitodel")
-    ind_obj["mitodel_file"] = ind.get("mitodel_file")
+
+    for nested_file_key in "chromograph_images", "reviewer":
+        set_abspath_nested_individual_files(ind_obj, ind, nested_file_key)
 
     # Check if the analysis type is ok
     analysis_type = ind.get("analysis_type", "unknown")

scripts/update_files_path.py

@@ -5,10 +5,10 @@
 from pymongo import MongoClient
 from tabulate import tabulate
 
+from scout.build.individual import BUILD_INDIVIDUAL_FILES as INDIVIDUAL_FILES
 from scout.constants import FILE_TYPE_MAP
 
 VCF_FILES = FILE_TYPE_MAP.keys()
-INDIVIDUAL_FILES = ["bam_file", "mt_bam", "vcf2cytosure"]
 
 
 @click.command()

tests/build/test_build_case.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
+
 import pytest
-from pprint import pprint as pp
-from scout.exceptions import PedigreeError, ConfigError, IntegrityError
 
 from scout.build import build_case
+from scout.exceptions import ConfigError, IntegrityError, PedigreeError
 
 
 def test_build_case(parsed_case, adapter, institute_obj, testpanel_obj):
@@ -54,7 +54,7 @@ def test_build_case(parsed_case, adapter, institute_obj, testpanel_obj):
 
     assert case_obj["madeline_info"] == parsed_case["madeline_info"]
 
-    assert case_obj["delivery_report"] == parsed_case["delivery_report"]
+    assert parsed_case["delivery_report"] in case_obj["delivery_report"]
 
     for vcf in case_obj["vcf_files"]:
         assert vcf in parsed_case["vcf_files"]