introduce a checkpoint which takes care of the local cache

plenoirf/production/__init__.py

@@ -29,6 +29,7 @@
 from . import simulate_loose_trigger
 from . import classify_cherenkov_photons
 from . import inspect_cherenkov_pool
+from . import checkpoint
 
 
 def make_example_job(
@@ -92,37 +93,16 @@ def run_job_in_dir(job, work_dir):
         )
 
     with jll.TimeDelta(logger, "inspect_cherenkov_pool"):
-        visible_cherenkov_photon_size = (
-            inspect_cherenkov_pool.inspect_cherenkov_pools(
-                cherenkov_pools_path=os.path.join(
-                    job["paths"]["work_dir"],
-                    "simulate_shower_and_collect_cherenkov_light_in_grid",
-                    "cherenkov_pools.tar",
-                ),
-                aperture_bin_edges=np.linspace(-50, 50, 51),
-                image_bin_edges_rad=np.linspace(
-                    np.deg2rad(-6.5), np.deg2rad(6.5), 51
-                ),
-                time_bin_edges=np.linspace(375e-6, 425e-6, 200),
-                out_dir=os.path.join(
-                    job["paths"]["work_dir"], "inspect_cherenkov_pool"
-                ),
-                field_of_view_center_rad=[0, 0],
-                field_of_view_half_angle_rad=np.deg2rad(6.5 / 2),
-                mirror_center=[0, 0],
-                mirror_radius=35.5,
-                threshold_num_photons=25,
-            )
-        )
-        with rnw.open(
-            os.path.join(
+        job = checkpoint.checkpoint(
+            job=job,
+            logger=logger,
+            func=inspect_cherenkov_pool.run_job,
+            cache_path=opj(
                 job["paths"]["work_dir"],
                 "inspect_cherenkov_pool",
-                "visible_cherenkov_photon_size.json",
+                "__job_cache__",
             ),
-            "wt",
-        ) as f:
-            f.write(json_utils.dumps(visible_cherenkov_photon_size))
+        )
 
     with jll.TimeDelta(logger, "inspect_particle_pool"):
         job = inspect_particle_pool.run_job(job=job, logger=logger)

plenoirf/production/checkpoint.py

@@ -0,0 +1,18 @@
+import os
+from . import job_io
+
+
+def checkpoint(job, logger, func, cache_path):
+    if os.path.exists(cache_path) and job["cache"]:
+        logger.info("{:s}, read cache".format(func.__name__))
+        return job_io.read(path=cache_path)
+    else:
+        logger.info("{:s}, run".format(func.__name__))
+
+        job = func(job=job, logger=logger)
+
+        if job["cache"]:
+            logger.info("{:s}, write cache".format(func.__name__))
+            job_io.write(path=cache_path, job=job)
+
+    return job

plenoirf/production/inspect_cherenkov_pool.py

@@ -5,10 +5,44 @@
 import binning_utils
 import sebastians_matplotlib_addons as sebplt
 import spherical_coordinates
+import json_utils
+import rename_after_writing as rnw
 
 from .. import bookkeeping
 
 
+def run_job(job, logger):
+    visible_cherenkov_photon_size = inspect_cherenkov_pools(
+        cherenkov_pools_path=os.path.join(
+            job["paths"]["work_dir"],
+            "simulate_shower_and_collect_cherenkov_light_in_grid",
+            "cherenkov_pools.tar",
+        ),
+        aperture_bin_edges=np.linspace(-50, 50, 51),
+        image_bin_edges_rad=np.linspace(np.deg2rad(-6.5), np.deg2rad(6.5), 51),
+        time_bin_edges=np.linspace(375e-6, 425e-6, 200),
+        out_dir=os.path.join(
+            job["paths"]["work_dir"], "inspect_cherenkov_pool"
+        ),
+        field_of_view_center_rad=[0, 0],
+        field_of_view_half_angle_rad=np.deg2rad(6.5 / 2),
+        mirror_center=[0, 0],
+        mirror_radius=35.5,
+        threshold_num_photons=25,
+    )
+    with rnw.open(
+        os.path.join(
+            job["paths"]["work_dir"],
+            "inspect_cherenkov_pool",
+            "visible_cherenkov_photon_size.json",
+        ),
+        "wt",
+    ) as f:
+        f.write(json_utils.dumps(visible_cherenkov_photon_size))
+
+    return job
+
+
 def inspect_cherenkov_pools(
     cherenkov_pools_path,
     aperture_bin_edges,