use checkpoint also for blocks

cherenkov-plenoscope · Feb 21, 2024 · 6957b78 · 6957b78
1 parent 42cb572
commit 6957b78
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 54 deletions.
diff --git a/plenoirf/production/__init__.py b/plenoirf/production/__init__.py
@@ -160,15 +160,37 @@ def _run_job_block(job, blk, block_id, logger):
     with jll.TimeDelta(
         logger, "simulate_hardware_block{:06d}".format(block_id)
     ):
-        job = simulate_hardware.run_job_block(
-            job=job, block_id=block_id, logger=logger
+        checkpoint.checkpoint(
+            job=job,
+            blk=blk,
+            logger=logger,
+            block_id=block_id,
+            func=simulate_hardware.run_job_block,
+            cache_path=opj(
+                job["paths"]["work_dir"],
+                "blocks",
+                "{block_id:06d}".format(block_id=block_id),
+                "inspect_cherenkov_pool",
+                "__job_cache__",
+            ),
         )
 
     with jll.TimeDelta(
         logger, "simulate_loose_trigger_block{:06d}".format(block_id)
     ):
-        job = simulate_loose_trigger.run_job_block(
-            job=job, blk=blk, block_id=block_id, logger=logger
+        checkpoint.checkpoint(
+            job=job,
+            blk=blk,
+            logger=logger,
+            block_id=block_id,
+            func=simulate_loose_trigger.run_job_block,
+            cache_path=opj(
+                job["paths"]["work_dir"],
+                "blocks",
+                "{block_id:06d}".format(block_id=block_id),
+                "simulate_loose_trigger",
+                "__job_cache__",
+            ),
         )
 
     """

diff --git a/plenoirf/production/checkpoint.py b/plenoirf/production/checkpoint.py
@@ -2,14 +2,17 @@
 from . import job_io
 
 
-def checkpoint(job, logger, func, cache_path):
+def checkpoint(job, logger, func, cache_path, block_id=None, blk=None):
     if os.path.exists(cache_path) and job["cache"]:
         logger.info("{:s}, read cache".format(func.__name__))
         return job_io.read(path=cache_path)
     else:
         logger.info("{:s}, run".format(func.__name__))
 
-        job = func(job=job, logger=logger)
+        if block_id is None:
+            job = func(job=job, logger=logger)
+        else:
+            job = func(job=job, logger=logger, block_id=block_id, blk=blk)
 
         if job["cache"]:
             logger.info("{:s}, write cache".format(func.__name__))

diff --git a/plenoirf/production/simulate_hardware.py b/plenoirf/production/simulate_hardware.py
@@ -8,31 +8,9 @@
 from . import job_io
 
 
-def run_job_block(job, block_id, logger):
-    block_dir = opj(
-        job["paths"]["work_dir"],
-        "blocks",
-        "{block_id:06d}".format(block_id=block_id),
-    )
-    work_dir = opj(block_dir, "simulate_hardware")
-    os.makedirs(work_dir, exist_ok=True)
-    cache_path = os.path.join(work_dir, "__job_cache__")
-
-    if os.path.exists(cache_path) and job["cache"]:
-        logger.info(
-            "simulate_hardware block{:06d}, read cache".format(block_id)
-        )
-        return job_io.read(path=cache_path)
-    else:
-        job = simulate_hardware(job=job, block_id=block_id)
-        make_debug_output(job=job, block_id=block_id)
-
-        if job["cache"]:
-            logger.info(
-                "simulate_hardware block{:06d}, write cache".format(block_id)
-            )
-            job_io.write(path=cache_path, job=job)
-
+def run_job_block(job, blk, block_id, logger):
+    job = simulate_hardware(job=job, block_id=block_id)
+    make_debug_output(job=job, block_id=block_id)
     return job
 
 

diff --git a/plenoirf/production/simulate_loose_trigger.py b/plenoirf/production/simulate_loose_trigger.py
@@ -13,30 +13,9 @@
 
 
 def run_job_block(job, blk, block_id, logger):
-    opj = os.path.join
-    block_dir = opj(
-        job["paths"]["work_dir"], "blocks", "{:06d}".format(block_id)
+    job = simulate_loose_trigger(
+        job=job, blk=blk, block_id=block_id, logger=logger
     )
-    work_dir = opj(block_dir, "simulate_loose_trigger")
-    os.makedirs(work_dir, exist_ok=True)
-    cache_path = os.path.join(work_dir, "__job_cache__")
-
-    if os.path.exists(cache_path) and job["cache"]:
-        logger.info(
-            "simulate_hardware block{:06d}, read cache".format(block_id)
-        )
-        return job_io.read(path=cache_path)
-    else:
-        job = simulate_loose_trigger(
-            job=job, blk=blk, block_id=block_id, logger=logger
-        )
-
-        if job["cache"]:
-            logger.info(
-                "simulate_hardware block{:06d}, write cache".format(block_id)
-            )
-            job_io.write(path=cache_path, job=job)
-
     return job