Fix issues with broken Exporter manifests (#206)

spine_items/exporter/executable_item.py

@@ -171,9 +171,7 @@ def execute(self, forward_resources, backward_resources, lock):
         result = self._process.run_until_complete()
         if len(result) > 1:
             self._result_files = result[1]
-            file_name = (
-                EXPORTER_EXECUTION_MANIFEST_FILE_PREFIX + (f"-{self.hash_filter_id()}" if self.filter_id else "")
-            ) + ".json"
+            file_name = self._manifest_file_name()
             with open(Path(self._data_dir, file_name), "w") as manifest:
                 dump(
                     {
@@ -187,14 +185,15 @@ def execute(self, forward_resources, backward_resources, lock):
 
     def exclude_execution(self, forward_resources, backward_resources, lock):
         """See base class."""
-        manifest_file_name = (
-            EXPORTER_EXECUTION_MANIFEST_FILE_PREFIX + (f"-{self.hash_filter_id()}" if self._filter_id else "")
-        ) + ".json"
+        manifest_file_name = self._manifest_file_name()
         manifest_file_path = Path(self._data_dir, manifest_file_name)
-        if not manifest_file_path.exists():
+        if not manifest_file_path.is_file():
             return
         with open(Path(self._data_dir, manifest_file_name)) as manifest_file:
-            manifest = json.load(manifest_file)
+            try:
+                manifest = json.load(manifest_file)
+            except json.decoder.JSONDecodeError:
+                return
         self._result_files = {label: set(files) for label, files in manifest.items()}
 
     def _output_resources_forward(self):
@@ -206,6 +205,16 @@ def _output_resources_forward(self):
             resources += output_database_resources(self.name, self._output_channels)
         return resources
 
+    def _manifest_file_name(self):
+        """Creates file name for manifest file.
+
+        Returns:
+            str: file name
+        """
+        return (
+                EXPORTER_EXECUTION_MANIFEST_FILE_PREFIX + (f"-{self.hash_filter_id()}" if self._filter_id else "")
+        ) + ".json"
+
     def _resolve_gams_system_directory(self):
         """Returns GAMS system path from Toolbox settings or None if GAMS default is to be used.
 

spine_items/exporter/export_manifest.py

@@ -68,9 +68,12 @@ def _collect_execution_manifests(data_dir):
     """
     manifests = None
     for path in Path(data_dir).iterdir():
-        if path.name.startswith(EXPORTER_EXECUTION_MANIFEST_FILE_PREFIX) and path.suffix == ".json":
+        if is_manifest_file(path):
             with open(path) as manifest_file:
-                manifest = json.load(manifest_file)
+                try:
+                    manifest = json.load(manifest_file)
+                except json.decoder.JSONDecodeError:
+                    continue
             for out_file_name, paths in manifest.items():
                 relative_paths = list()
                 for file_path in paths:
@@ -81,7 +84,7 @@ def _collect_execution_manifests(data_dir):
                             relative_paths.append(p.relative_to(data_dir))
                         except ValueError:
                             # Project may have been moved to another directory (or system)
-                            # so data_dir is differs from manifest file content.
+                            # so data_dir differs from manifest file content.
                             # Try resolving the relative path manually.
                             parts = tuple(dropwhile(lambda part: part != "output", p.parts))
                             relative_paths.append(str(Path(*parts)))
@@ -91,3 +94,15 @@ def _collect_execution_manifests(data_dir):
                     manifests = dict()
                 manifests.setdefault(out_file_name, list()).extend(paths)
     return manifests
+
+
+def is_manifest_file(path):
+    """Tests if given path is a manifest file.
+
+    Args:
+        path (Path): path to test
+
+    Returns:
+        bool: True if path looks like a manifest file, False otherwise
+    """
+    return path.is_file() and path.name.startswith(EXPORTER_EXECUTION_MANIFEST_FILE_PREFIX) and path.suffix == ".json"

spine_items/exporter/exporter.py

@@ -21,7 +21,7 @@
 from spine_engine.utils.serialization import deserialize_path
 from spine_engine.utils.helpers import ExecutionDirection
 from spinedb_api import clear_filter_configs
-from .export_manifest import exported_files_as_resources
+from .export_manifest import exported_files_as_resources, is_manifest_file
 from .specification import OutputFormat
 from ..commands import UpdateCancelOnErrorCommand
 from .mvcmodels.full_url_list_model import FullUrlListModel
@@ -30,7 +30,7 @@
 from .executable_item import ExecutableItem
 from .commands import CommandId, UpdateOutLabel, UpdateOutputTimeStampsFlag, UpdateOutUrl
 from .output_channel import OutputChannel
-from .utils import EXPORTER_EXECUTION_MANIFEST_FILE_PREFIX, output_database_resources
+from .utils import output_database_resources
 
 
 @dataclass
@@ -451,7 +451,7 @@ def rename(self, new_name, rename_data_dir_message):
         if not super().rename(new_name, rename_data_dir_message):
             return False
         for path in Path(self.data_dir).iterdir():
-            if path.name.startswith(EXPORTER_EXECUTION_MANIFEST_FILE_PREFIX) and path.suffix == ".json":
+            if is_manifest_file(path):
                 path.unlink()
         if self._exported_files is not None:
             data_dir_parts = Path(self.data_dir).parts