add re-attempts for saving netcdf

justfile

@@ -96,7 +96,7 @@ main-mb05 *more_args="":
                  --download-dir=NB_SPACE/DOWNLOADS \
                  --assume-downloaded-files \
                  --retain-downloaded-files \
-                 --max-segments=1 \
+                 --max-segments=5 \
                  {{more_args}}
 
 #                 --max-segments=5 \

src/__init__.py

@@ -1,4 +1,5 @@
 import os
+import time
 
 import xarray as xr
 
@@ -7,20 +8,33 @@
 
 def save_dataset_to_netcdf(logger: PbpLogger, ds: xr.Dataset, filename: str) -> bool:
     logger.info(f"  - saving dataset to: {filename}")
-    try:
-        ds.to_netcdf(
-            filename,
-            engine="h5netcdf",
-            encoding={
-                "effort": {"_FillValue": None},
-                "frequency": {"_FillValue": None},
-                "sensitivity": {"_FillValue": None},
-            },
-        )
-        return True
-    except Exception as e:  # pylint: disable=broad-exception-caught
-        logger.error(f"Unable to save {filename}: {e}")
-        return False
+
+    # retry a few times in case of failure (possibly due to concurrent access to parent directory)
+    max_attempts = 10
+    wait_secs = 3
+    # TODO similar re-attempt logic for the CSV (or other) output
+
+    for attempt in range(1, max_attempts):
+        try:
+            ds.to_netcdf(
+                filename,
+                engine="h5netcdf",
+                encoding={
+                    "effort": {"_FillValue": None},
+                    "frequency": {"_FillValue": None},
+                    "sensitivity": {"_FillValue": None},
+                },
+            )
+            return True
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            error = f"Unable to save {filename}: {e}  (attempt {attempt} of {max_attempts})"
+            logger.error(error)
+            print(error)
+            if attempt < max_attempts:
+                logger.info(f"  - retrying in {wait_secs} secs ...")
+                time.sleep(wait_secs)
+
+    return False
 
 
 def save_dataset_to_csv(logger: PbpLogger, ds: xr.Dataset, filename: str):