prepare provenance

plenoirf/__init__.py

@@ -16,6 +16,7 @@
 from . import unique
 from . import outer_telescope_array
 from . import configurating
+from . import tar_append
 
 import os
 from os import path as op
@@ -29,9 +30,17 @@
 
 
 def init(run_dir, build_dir="build"):
+    prov = provenance.make_provenance()
+
     run_dir = op.abspath(run_dir)
     makesuredirs(run_dir)
 
+    makesuredirs(opj(run_dir, "provenance"))
+    provenance.tar_open_append_close(
+        path=opj(run_dir, "provenance", "init.tar"),
+        provenance=prov,
+    )
+
     makesuredirs(opj(run_dir, "config"))
     with rnw.open(opj(run_dir, "config", "executables.json"), "wt") as f:
         f.write(

plenoirf/provenance.py

@@ -4,18 +4,23 @@
 import warnings
 import shutil
 import corsika_primary
-
-
-IMPORTANT_PROGRAMS = {
-    "git": {"version": "--version"},
-    "python": {"version": "--version"},
-    "cmake": {"version": "--version"},
-    "make": {"version": "--version"},
-    "gcc": {"version": "--version"},
-    "g++": {"version": "--version"},
-    "f77": {"version": "--version"},
-    "gfortran": {"version": "--version"},
-}
+import json_utils
+import gzip
+from . import tar_append
+
+
+def list_important_programs():
+    out = {
+        "git": {"version": "--version"},
+        "python": {"version": "--version"},
+        "cmake": {"version": "--version"},
+        "make": {"version": "--version"},
+        "gcc": {"version": "--version"},
+        "g++": {"version": "--version"},
+        "f77": {"version": "--version"},
+        "gfortran": {"version": "--version"},
+    }
+    return out
 
 
 def _get_ascii_stdout_stderr(command, cwd="."):
@@ -112,19 +117,21 @@ def get_current_working_directory():
 
 
 def make_provenance():
+    important_programs = list_important_programs()
+
     p = {}
     p["time"] = get_time_dict_now()
     p["hostname"] = get_hostname()
     p["username"] = get_username()
     p["current_working_directory"] = get_current_working_directory()
     p["which"] = {}
-    for prg in IMPORTANT_PROGRAMS:
+    for prg in important_programs:
         p["which"][prg] = which(prg)
 
     p["version"] = {}
-    for prg in IMPORTANT_PROGRAMS:
+    for prg in important_programs:
         _o, _ = get_ascii_stdout_stderr(
-            command=[prg, IMPORTANT_PROGRAMS[prg]["version"]]
+            command=[prg, important_programs[prg]["version"]]
         )
         p["version"][prg] = _o
 
@@ -180,3 +187,20 @@ def make_basic_version_str(
     )
     ver += "    hostname: {:s}\n".format(ap["hostname"])
     return ver
+
+
+def iso_time_str_to_filename(iso_time_str):
+    return str.replace(iso_time_str, ":", "-")
+
+
+def tar_open_append_close(path, provenance):
+    content = json_utils.dumps(provenance)
+    filename = (
+        iso_time_str_to_filename(iso_time_str=provenance["time"]["iso"])
+        + ".json.gz"
+    )
+    tar_append.tar_open_append_close(
+        path=path,
+        filename=filename,
+        filebytes=gzip.compress(data=content.encode()),
+    )

plenoirf/tar_append.py

@@ -0,0 +1,24 @@
+import tarfile
+import io
+import rename_after_writing as rnw
+import uuid
+import os
+
+
+def tar_open_append_close(path, filename, filebytes):
+    tmp_path = path + "." + uuid.uuid4().__str__()
+    if os.path.exists(path):
+        rnw.move(src=path, dst=tmp_path)
+    with tarfile.open(name=tmp_path, mode="a") as tarout:
+        filenames = [i.name for i in tarout.members]
+        assert filename not in filenames, "Filename already exists."
+        tar_append(tarout=tarout, filename=filename, filebytes=filebytes)
+    rnw.move(src=tmp_path, dst=path)
+
+
+def tar_append(tarout, filename, filebytes):
+    with io.BytesIO() as buff:
+        info = tarfile.TarInfo(filename)
+        info.size = buff.write(filebytes)
+        buff.seek(0)
+        tarout.addfile(info, buff)