Merge pull request #37 from VERITAS-Observatory/v1.1-dev1

v1.1-dev1 - Improvement in run scripts and container handling

README.md

@@ -46,7 +46,7 @@ cd scripts
    ../config/CORSIKA/input_template.dat
 # on DESY: log into the DAG submission node
 ./prepare_DAG_jobs.sh ../config/config_ATM61_template.dat
-./submit_DAG_jobs.sh <directory with DAG files> submit
+./submit_DAG_jobs.sh ../config/config_ATM61_template.dat <directory with DAG files>
 # otherwise: submit jobs to HT Condor - for each step (CORSIKA, GROPTICS, CARE)
 ./submit_jobs_to_htcondor.sh <directory with condor files / submission scripts> submit
 # now wait....for jobs to finish
@@ -190,6 +190,7 @@ Processing scripts are prepared for HT Condor systems.
 - note that the `MERGEVBF` step is not included in the DAG submission, as it combines all runs of a production. Run this as a final step at the ned of the production.
 
 Note that configuration and output directories are fine tuned for this setup.
+The preparation of the all temporary submission scripts is not very efficient in both time and number of files written (could be significantly improved).
 
 ## Using Apptainers
 
@@ -199,6 +200,7 @@ The simulation scripts are configured to use Apptainers with the correct paramet
 Note:
 
 - recommend to set `$APPTAINER_CACHEDIR` to a reasonable directory with sufficient disk space, as cluster jobs will use this directory to store the container images.
+- set `$VTSSIMPIPE_CONTAINER_DIR` to the directory where the container images are stored. This should be a "fast" disk, as each job will access the image files.
 
 ## Submitting jobs
 
@@ -226,7 +228,7 @@ This will generate the DAG files in the `VTSSIMPIPE_LOG_DIR/DAG` directory.
 To submit DAG jobs:
 
 ```bash
-./submit_DAG_jobs.sh <directory with DAG files> submit
+./submit_DAG_jobs.sh <config file> <directory with DAG files>
 ```
 
 Note that on DESY DAG jobs need to be submitted from a special node.
@@ -250,8 +252,7 @@ Configuration:
 
 ### CORSIKA parameters
 
-
-#### 2017 production:
+#### 2017 production
 
 ```Text
 Monte Carlo run header
@@ -286,7 +287,6 @@ SLANT 1
 
 - changed wavelength range from [200, 700] to [300, 600] nm
 
-
 ### corsikaIOreader parameters
 
 - apply pre-efficiency cut of 50%

config/GROPTICS/Upgrade_20120827_v420_3.cfg

@@ -86,10 +86,10 @@ from a file. There is a separate grid for each standard telescope.
      - nbinsy, number of bins in the y direction
      - fileName, name of grid file (for either reading or writing)
 Default (no GRIDF record) do full loop sequentially over all facets.
-* GRIDF 1 2 15 15 gridfileStdDC1.txt
-* GRIDF 2 2 15 15 gridfileStdDC2.txt
-* GRIDF 3 2 15 15 gridfileStdDC3.txt
-* GRIDF 4 2 15 15 gridfileStdDC4.txt
+* GRIDF 1 1 15 15 gridfileStdDC1.txt
+* GRIDF 2 1 15 15 gridfileStdDC2.txt
+* GRIDF 3 1 15 15 gridfileStdDC3.txt
+* GRIDF 4 1 15 15 gridfileStdDC4.txt
  GRIDF 2 1 15 15
 
 additional pilot files to combine with this file

config/config_ATM61_production.dat

@@ -1,5 +1,5 @@
 RUN_START=650000
-N_RUNS=10
+N_RUNS=1000
 N_SHOWER=2000
 ZENITH=20
 ATMOSPHERE=61

docker/Dockerfile-care

@@ -36,6 +36,8 @@ RUN mkdir CARE && tar -xzf v$CARE_VERSION.tar.gz -C CARE --strip-components=1 &&
     make && make clean && rm -f *.tar.gz
 
 FROM rootproject/root:6.24.06-centos7
+LABEL maintainer.name="VERITAS Collaboration"
+LABEL maintainer.email="gernot.maier@desy.de"
 WORKDIR /workdir
 COPY --from=build_image /workdir/VBF /workdir/VBF
 COPY --from=build_image /workdir/CARE /workdir/CARE

docker/Dockerfile-corsika

@@ -21,6 +21,8 @@ SHELL ["/bin/bash", "-c"]
 WORKDIR /workdir/
 
 FROM almalinux:9.3-minimal
+LABEL maintainer.name="VERITAS Collaboration"
+LABEL maintainer.email="gernot.maier@desy.de"
 WORKDIR /workdir
 ENV DEBIAN_FRONTEND=noninteractive
 ARG CORSIKA=corsika-77500

docker/Dockerfile-groptics

@@ -26,6 +26,8 @@ RUN mkdir GrOptics && tar -xzf v$GROPTICS_VERSION.tar.gz -C GrOptics --strip-com
     cd GrOptics && make
 
 FROM rootproject/root:6.24.06-centos7
+LABEL maintainer.name="VERITAS Collaboration"
+LABEL maintainer.email="gernot.maier@desy.de"
 WORKDIR /workdir
 COPY --from=build_image /workdir/corsikaIOreader/corsikaIOreader /workdir/corsikaIOreader/corsikaIOreader
 COPY --from=build_image /workdir/GrOptics /workdir/GrOptics

docker/Dockerfile-mergevbf

@@ -27,6 +27,8 @@ RUN mkdir EventDisplay_v4 && tar -xzf v490.5.tar.gz -C EventDisplay_v4 --strip-c
     cd EventDisplay_v4 && make mergeVBF && make clean
 
 FROM rootproject/root:6.24.06-centos7
+LABEL maintainer.name="VERITAS Collaboration"
+LABEL maintainer.email="gernot.maier@desy.de"
 WORKDIR /workdir
 COPY --from=build_image /workdir/VBF /workdir/VBF
 COPY --from=build_image /workdir/EventDisplay_v4 /workdir/EventDisplay_v4

env_setup_template.sh

@@ -8,11 +8,15 @@ export VTSSIMPIPE_DATA_DIR=vtssimpipe_corsika_dir
 # container type
 # export VTSSIMPIPE_CONTAINER="apptainer"
 export VTSSIMPIPE_CONTAINER="docker"
+# package url (docker style)
+export VTSSIMPIPE_CONTAINER_URL="ghcr.io/veritas-observatory/"
+# apptainers are stored in this directory
+export VTSSIMPIPE_CONTAINER_DIR=vtssimpipe_containe_dir
 # CORSIKA
-export VTSSIMPIPE_CORSIKA_IMAGE="ghcr.io/veritas-observatory/vtsimpipe-corsika:1.0.0"
+export VTSSIMPIPE_CORSIKA_IMAGE="vtsimpipe-corsika:1.0.0"
 # GROPTICS
-export VTSSIMPIPE_GROPTICS_IMAGE="ghcr.io/veritas-observatory/vtsimpipe-groptics:1.0.0"
+export VTSSIMPIPE_GROPTICS_IMAGE="vtsimpipe-groptics:1.0.0"
 # CARE
-export VTSSIMPIPE_CARE_IMAGE="ghcr.io/veritas-observatory/vtsimpipe-care:1.0.0"
+export VTSSIMPIPE_CARE_IMAGE="vtsimpipe-care:1.0.0"
 # MERGEVBF
-export VTSSIMPIPE_MERGEVBF_IMAGE="ghcr.io/veritas-observatory/vtsimpipe-mergevbf:1.0.0"
+export VTSSIMPIPE_MERGEVBF_IMAGE="vtsimpipe-mergevbf:1.0.0"

scripts/care.sh

@@ -66,9 +66,9 @@ generate_care_submission_script()
 
     echo "#!/bin/bash" > "$CAREFSCRIPT.sh"
     if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then
-        CARE_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR $VTSSIMPIPE_CARE_IMAGE"
+        CARE_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CARE_IMAGE}"
     elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then
-        CARE_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} --compat docker://$VTSSIMPIPE_CARE_IMAGE"
+        CARE_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_CARE_IMAGE/:/_}.sif"
     fi
     CARE_EXE="${CARE_EXE} bash -c \"cd /workdir/CARE && ${CARE}\""
     echo "$CARE_EXE > $CARE_DATA_DIR/$(basename "$OUTPUT_FILE").care.log 2>&1" >> "$CAREFSCRIPT.sh"

scripts/cleanup.sh

@@ -19,9 +19,7 @@
 generate_cleanup_submission_script()
 {
     CLEANUPSCRIPT="$1"
-    LOG_DIR=$(dirname "$CLEANUPSCRIPT")
     OUTPUT_FILE="$2"
-    RUN_NUMBER="$3"
     WOFF_LIST="$4"
     CLEANUP_DATA_DIR="${DATA_DIR}/CLEANUP/"
     mkdir -p "${CLEANUP_DATA_DIR}"
@@ -38,5 +36,6 @@ generate_cleanup_submission_script()
         echo "$CLEANUP_GROPTICS > $CLEANUP_DATA_DIR/$(basename "$OUTPUT_FILE").cleanup.log 2>&1" >> "$CLEANUPSCRIPT.sh"
     done
     echo "bzip2 -f -v ${CORSIKA_DATA_DIR}/$(basename "$OUTPUT_FILE").telescope >> $CLEANUP_DATA_DIR/$(basename "$OUTPUT_FILE").cleanup.log 2>&1" >> "$CLEANUPSCRIPT.sh"
+    echo "bzip2 -f -v ${CORSIKA_DATA_DIR}/$(basename "$OUTPUT_FILE").log >> $CLEANUP_DATA_DIR/$(basename "$OUTPUT_FILE").cleanup.log 2>&1" >> "$CLEANUPSCRIPT.sh"
     chmod u+x "$CLEANUPSCRIPT.sh"
 }

scripts/corsika.sh

@@ -6,14 +6,14 @@ prepare_corsika_containers()
 {
     DATA_DIR="$1"
     LOG_DIR="$2"
-    mkdir -p ${DATA_DIR}
+    mkdir -p "${DATA_DIR}"
 
     CONTAINER_EXTERNAL_DIR="-v \"${DATA_DIR}/CORSIKA:/workdir/external/data\" -v \"$LOG_DIR:/workdir/external/log\""
     CORSIKA_DATA_DIR="/workdir/external/data"
     if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then
-        COPY_COMMAND="docker run --rm $CONTAINER_EXTERNAL_DIR $VTSSIMPIPE_CORSIKA_IMAGE"
+        COPY_COMMAND="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE}"
     elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then
-        COPY_COMMAND="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} --compat docker://$VTSSIMPIPE_CORSIKA_IMAGE"
+        COPY_COMMAND="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_CORSIKA_IMAGE/:/_}.sif"
     fi
     # copy corsika directory to data dir (as apptainers are readonly)
     echo "Copy CORSIKA files to ${DATA_DIR}/CORSIKA/tmp_corsika_run_files"
@@ -35,11 +35,10 @@ generate_corsika_submission_script()
 
     echo "#!/bin/bash" > "$FSCRIPT.sh"
     mkdir -p "$(dirname $OUTPUT_FILE)"
-    rm -f "$OUTPUT_FILE".telescope
     if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then
-        CORSIKA_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR $VTSSIMPIPE_CORSIKA_IMAGE"
+        CORSIKA_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE}"
     elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then
-        CORSIKA_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} --compat docker://$VTSSIMPIPE_CORSIKA_IMAGE"
+        CORSIKA_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_CORSIKA_IMAGE/:/_}.sif"
     fi
     CORSIKA_EXE="${CORSIKA_EXE} bash -c \"cd /workdir/corsika-run && ./corsika77500Linux_QGSII_urqmd < $INPUT\""
     echo "$CORSIKA_EXE > $OUTPUT_FILE.log" >> "$FSCRIPT.sh"

scripts/groptics.sh

@@ -144,9 +144,9 @@ generate_groptics_submission_script()
 
     echo "#!/bin/bash" > "$GROPTICSFSCRIPT.sh"
     if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then
-        GROPTICS_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR $VTSSIMPIPE_GROPTICS_IMAGE"
+        GROPTICS_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_GROPTICS_IMAGE}"
     elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then
-        GROPTICS_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} --compat docker://$VTSSIMPIPE_GROPTICS_IMAGE"
+        GROPTICS_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_GROPTICS_IMAGE/:/_}.sif"
     fi
     GROPTICS_EXE="${GROPTICS_EXE} bash -c \"cd /workdir/GrOptics && ${CORSIKA_IO_READER} | ${GROPTICS}\""
     echo "$GROPTICS_EXE > $GROPTICS_DATA_DIR/$(basename "$OUTPUT_FILE").groptics.log 2>&1" >> "$GROPTICSFSCRIPT.sh"

scripts/mergevbf.sh

@@ -8,7 +8,7 @@ get_merge_file_name()
 
     # gamma_V6_CARE_std_Atmosphere61_zen20deg_1.0wob_160MHz_1.vbf.zst
     FNAME="gamma_V6_CARE"
-    if [[ $CARE_CONFIG == *"RHV"* ]]; then
+    if [[ $OBS_MODE == *"redHV"* ]]; then
         FNAME="${FNAME}_redHV"
     else
         FNAME="${FNAME}_std"
@@ -46,12 +46,12 @@ generate_mergevbf_submission_script()
     CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"$LOG_DIR:/workdir/external/log/\""
 
     if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then
-        CARE_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR $VTSSIMPIPE_MERGEVBF_IMAGE"
+        CARE_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_MERGEVBF_IMAGE}"
     elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then
-        CARE_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} --compat docker://$VTSSIMPIPE_MERGEVBF_IMAGE"
+        CARE_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_MERGEVBF_IMAGE/:/_}.sif"
     fi
 
-    batch_size=100
+    batch_size=250
     vbf_id="0"
     MERGEDFILE=$(get_merge_file_name "$WOBBLE" "$NSB" "$vbf_id")
     TMP_FL_LIST="$MERGEVBF_DATA_DIR"/tmp_file_list_${WOBBLE}_${NSB}.dat

scripts/prepare_DAG_jobs.sh

@@ -77,6 +77,7 @@ for ID in $(seq 0 "$N_RUNS"); do
     PARENT_CLEANUP="$PARENT_CLEANUP CHILD CLEANUP_${run_number}"
     echo "$PARENT_CLEANUP" >> "$DAG_FILE"
     echo "$PARENT_CORSIKA" >> "$DAG_FILE"
+    echo "DOT $DAG_DIR/run_${run_number}.dot" >> "$DAG_FILE"
 done
 
 echo "DAG directory: $DAG_DIR"

scripts/prepare_production.sh

@@ -129,8 +129,7 @@ else
     exit
 fi
 
-for ID in $(seq 0 "$N_RUNS");
-do
+for ((ID=0; ID<N_RUNS; ID++)); do
     run_number=$((ID + RUN_START))
     FSCRIPT="$LOG_DIR"/"run_${SIM_TYPE}_$run_number"
     INPUT="$LOG_DIR"/"input_$run_number.dat"

scripts/pull.sh

@@ -19,17 +19,18 @@ echo "VTSSIMPIPE_CARE_IMAGE: $VTSSIMPIPE_CARE_IMAGE"
 echo "VTSSIMPIPE_MERGEVBF_IMAGE: $VTSSIMPIPE_MERGEVBF_IMAGE"
 
 if [[ "$VTSSIMPIPE_CONTAINER" == "apptainer" ]]; then
-    echo "Pulling apptainer images."
-    apptainer pull --disable-cache --force docker://"$VTSSIMPIPE_CORSIKA_IMAGE"
-    apptainer pull --disable-cache --force docker://"$VTSSIMPIPE_GROPTICS_IMAGE"
-    apptainer pull --disable-cache --force docker://"$VTSSIMPIPE_CARE_IMAGE"
-    apptainer pull --disable-cache --force docker://"$VTSSIMPIPE_MERGEVBF_IMAGE"
+    echo "Pulling apptainer images to $VTSSIMPIPE_CONTAINER_DIR"
+    mkdir -p "${VTSSIMPIPE_CONTAINER_DIR}"
+    apptainer pull --force --dir "${VTSSIMPIPE_CONTAINER_DIR}/" docker://"${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE}"
+    apptainer pull --force --dir "${VTSSIMPIPE_CONTAINER_DIR}/" docker://"${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_GROPTICS_IMAGE}"
+    apptainer pull --force --dir "${VTSSIMPIPE_CONTAINER_DIR}/" docker://"${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CARE_IMAGE}"
+    apptainer pull --force --dir "${VTSSIMPIPE_CONTAINER_DIR}/" docker://"${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_MERGEVBF_IMAGE}"
 elif [[ "$VTSSIMPIPE_CONTAINER" == "docker" ]]; then
     echo "Pulling docker images."
-    docker pull "$VTSSIMPIPE_CORSIKA_IMAGE"
-    docker pull "$VTSSIMPIPE_GROPTICS_IMAGE"
-    docker pull "$VTSSIMPIPE_CARE_IMAGE"
-    docker pull "$VTSSIMPIPE_MERGEVBF_IMAGE"
+    docker pull "${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE}"
+    docker pull "${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_GROPTICS_IMAGE}"
+    docker pull "${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CARE_IMAGE}"
+    docker pull "${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_MERGEVBF_IMAGE}"
 else
     echo "Unknown container type $VTSSIMPIPE_CONTAINER."
     exit

scripts/submit_DAG_jobs.sh

@@ -7,20 +7,26 @@
 #
 set -e
 
-if [ $# -lt 1 ]
+if [ $# -lt 2 ]
 then
     echo "
-    ./submit_DAG_jobs.sh <dag directory> <submit/nosubmit>
+    ./submit_DAG_jobs.sh <config file> <dag directory>
 
     "
     exit
 fi
 
+CONFIG="$1"
+
 export _condor_SEC_TOKEN_DIRECTORY=$(mktemp -d)
-condor_token_fetch -lifetime $((7*24*60*60)) -token dag
+condor_token_fetch -lifetime $((30*24*60*60)) -token dag
+
+# shellcheck source=/dev/null
+. "$CONFIG"
 
-for dag_file in "${1}"/*.dag; do
-    condor_submit_dag "$dag_file"
+for ID in $(seq 0 "$N_RUNS"); do
+    run_number=$((ID + RUN_START))
+    condor_submit_dag "${2}/run_${run_number}.dag"
 done
 
 exit $?

scripts/submit_DAG_jobs_from_file_list.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# submit DAG jobs in DESY environment from a list of files
+# use htc-submit.zeuthen.desy.de
+# see https://dv-zeuthen.desy.de/services/batch/job_submission/ for details
+#
+# Submits all *.dag files in the given directory
+#
+set -e
+
+if [ $# -lt 1 ]
+then
+    echo "
+    ./submit_DAG_jobs_from_file_list.sh <file list>
+
+    "
+    exit
+fi
+
+FILELIST="$1"
+
+export _condor_SEC_TOKEN_DIRECTORY=$(mktemp -d)
+condor_token_fetch -lifetime $((30*24*60*60)) -token dag
+
+FLIST=$(cat $FILELIST)
+for F in ${FLIST}; do
+    echo $F
+    rm -f -v ${F}.*
+    condor_submit_dag ${F}
+done
+
+exit $?