Merge pull request #18 from GernotMaier/simplification

Simplification
VERITAS-Observatory · Jan 19, 2024 · 5d717d3 · 5d717d3
2 parents ff1835d + c7fbb91
commit 5d717d3
Show file tree

Hide file tree

Showing 12 changed files with 295 additions and 133 deletions.
diff --git a/README.md b/README.md
@@ -113,3 +113,34 @@ The simulation scripts are configured to use Apptainers with the correct paramet
 Note:
 
 - recommend to set `$APPTAINER_CACHEDIR` to a reasonable directory with sufficient disk space, as cluster jobs will use this directory to store the container images.
+
+## Submitting jobs
+
+The job submission scripts are written for HT Condor systems (as configured at DESY).
+The `prepare_production.sh` product scripts generates the condor files and submission scripts in the `VTSSIMPIPE_LOG_DIR` directory.
+
+To submit jobs, use the `submit_jobs_to_htcondor.sh` script:
+
+```bash
+./submit_jobs_to_htcondor.sh <directory with condor files / submission scripts> submit
+````
+
+This will submit all jobs in the directory to the HT Condor system.
+
+For efficiency reason, it is recommended to submit jobs using the DAG submission system (see [HTCondor DAGMan](https://htcondor.readthedocs.io/en/latest/automated-workflows/index.html)). This allows to run CORSIKA, followed by GrOptics, CARE, and the merging of the VBF files in a single job submission.
+
+To generate DAG files for the job submission:
+
+```bash
+./prepare_DAG_jobs.sh <config file>
+```
+
+This will generate the DAG files in the `VTSSIMPIPE_LOG_DIR/DAG` directory.
+
+To submit DAG jobs:
+
+```bash
+./submit_DAG_jobs.sh <directory with DAG files> submit
+```
+
+Note that on DESY DAG jobs need to be submitted from a special node.
diff --git a/config/config_ATM61_template.dat b/config/config_ATM61_template.dat
@@ -3,8 +3,8 @@ N_RUNS=10
 N_SHOWER=3
 ZENITH=25
 ATMOSPHERE=61
-WOBBLE="0.5"
-NSB="200"
+WOBBLE_LIST="0.5 1.0"
+NSB_LIST="200 300"
 
 # GrOptics
 EXTINCTION="Ext_results_VWinter_3_2_6.M5.txt"

diff --git a/config/config_ATM62_template.dat b/config/config_ATM62_template.dat
@@ -3,8 +3,8 @@ N_RUNS=10
 N_SHOWER=3
 ZENITH=25
 ATMOSPHERE=62
-WOBBLE="0.5"
-NSB="200"
+WOBBLE_LIST="0.5 1.0"
+NSB_LIST="200 300"
 
 # GrOptics
 EXTINCTION_FILE="Ext_results_VSummer_6_1_6.M5.txt"

diff --git a/env_setup_template.sh b/env_setup_template.sh
@@ -14,3 +14,5 @@ export VTSSIMPIPE_CORSIKA_IMAGE="ghcr.io/gernotmaier/vtsimpipe-corsika:latest"
 export VTSSIMPIPE_GROPTICS_IMAGE="ghcr.io/gernotmaier/vtsimpipe-groptics"
 # CARE
 export VTSSIMPIPE_CARE_IMAGE="ghcr.io/gernotmaier/vtsimpipe-care"
+# MERGEVBF (Eventdisplay)
+export VTSSIMPIPE_MERGEVBF_IMAGE="ghcr.io/veritas-observatory/eventdisplay_v4:latest"
diff --git a/scripts/care.sh b/scripts/care.sh
@@ -2,36 +2,23 @@
 # Helper functions for CARE; called from prepare_production.sh
 
 #####################################################################
-# generate GrOptics pilot file
-#
-# The pilot file is used to configure the GrOptics simulation.
-#
-# WOBBLE:     wobble offset and direction in degrees
-# ARRAYCONFIG: telescope model file
-# SEED:       seed for the random number generator (0=machine clock)
-generate_groptics_pilot_file()
+# preparation of CARE containers
+prepare_care_containers()
 {
-    LOG_DIR="$1"
-    RUN_NUMBER="$2"
+    DATA_DIR="$1"
+    WOBBLE="$2"
+    NSB="$3"
 
-    PILOT="${LOG_DIR}/pilot_${RUN_NUMBER}.dat"
-    rm -f "$PILOT"
-    touch "$PILOT"
+    CARE_DATA_DIR="${DATA_DIR}/CARE/W${WOBBLE}/NSB${NSB}"
+    TMP_CONFIG_DIR="${CARE_DATA_DIR}/model_files/"
+    mkdir -p "$TMP_CONFIG_DIR"
 
-    {
-        # (dummy file name; groptics is executed with the "-of" flag)
-        echo "* FILEOUT photonLocation.root allT T 0"
-        echo "* NSHOWER -1 -1"
-        echo "* WOBBLE $(get_wobble "${RUN_NUMBER}" "${WOBBLE}") 0.0 0.0 90."
-        echo "* ARRAYCONFIG ./data/${GROPTICS_CONFIG}"
-        echo "* SEED 0"
-        echo "* DEBUGBRANCHES 1"
-    } >> "$PILOT"
-    echo "/workdir/external/log/$(basename "$PILOT")"
+    # copy telescope model files for CARE (copy all files, although not all are needed)
+    cp -f "$(dirname "$0")"/../config/TELESCOPE_MODEL/* "${TMP_CONFIG_DIR}/"
 }
 
 #####################################################################
-# preparation of CARE containers
+# generate CARE input files and submission scripts
 #
 # Unfortunately quite fine tuned due to directory requirements of
 # corsikaIOreader and grOptics:
@@ -41,47 +28,24 @@ generate_groptics_pilot_file()
 # /workdir/CARE/Config:      directory into which model files
 #                            (configuration, pulse shapes) are copied.
 #
-prepare_care_containers()
+generate_care_submission_script()
 {
-    DATA_DIR="$1"
-    LOG_DIR="$2"
-    PULL="$3"
-    VTSSIMPIPE_CONTAINER="$4"
-    VTSSIMPIPE_CARE_IMAGE="$5"
-
-    TMP_CONFIG_DIR="${DATA_DIR}/CARE/model_files/"
-    mkdir -p "$TMP_CONFIG_DIR"
-
-    # copy telescope model files for CARE (copy all files, although not all are needed)
-    cp -f "$(dirname "$0")"/../config/TELESCOPE_MODEL/* "${TMP_CONFIG_DIR}/"
+    CAREFSCRIPT="$1"
+    LOG_DIR=$(dirname "$CAREFSCRIPT")
+    OUTPUT_FILE="$2"
+    rm -f "$OUTPUT_FILE.care.log"
+    WOBBLE="$3"
+    NSB="$4"
 
     # mount directories
-    GROPTICS_DATA_DIR="${DATA_DIR}/GROPTICS"
+    GROPTICS_DATA_DIR="${DATA_DIR}/GROPTICS/W${WOBBLE}"
+    CARE_DATA_DIR="${DATA_DIR}/CARE/W${WOBBLE}/NSB${NSB}"
+    TMP_CONFIG_DIR="${CARE_DATA_DIR}/model_files/"
     CONTAINER_EXTERNAL_DIR="-v \"${GROPTICS_DATA_DIR}:/workdir/external/groptics\""
-    CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"${DATA_DIR}/CARE:/workdir/external/care\""
+    CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"${CARE_DATA_DIR}:/workdir/external/care\""
     CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"${TMP_CONFIG_DIR}:/workdir/CARE/data\""
     CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"$LOG_DIR:/workdir/external/log/\""
 
-    if [[ $PULL == "TRUE" ]]; then
-        if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then
-            docker pull "$VTSSIMPIPE_CARE_IMAGE"
-        elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then
-            apptainer pull --disable-cache --force docker://"$VTSSIMPIPE_CARE_IMAGE"
-        fi
-    fi
-}
-
-#####################################################################
-# generate CARE input files and submission scripts
-generate_care_submission_script()
-{
-    FSCRIPT="$1"
-    LOG_DIR=$(dirname "$FSCRIPT")
-    OUTPUT_FILE="$2"
-    RUN_NUMBER="$3"
-    CONTAINER_EXTERNAL_DIR="$4"
-    rm -f "$OUTPUT_FILE.care.log"
-
     CARE="./CameraAndReadout \
      NSBRATEPERPIXEL \\\"0 0 ${NSB}\\\" \
      HIGHGAINPULSESHAPE \\\"0 /workdir/CARE/data/${CARE_HIGH_GAIN}\\\" \
@@ -94,13 +58,13 @@ generate_care_submission_script()
      --outputfile /workdir/external/care/$(basename "$OUTPUT_FILE") \
      --inputfile /workdir/external/groptics/$(basename "$OUTPUT_FILE").groptics.root"
 
-    echo "#!/bin/bash" > "$FSCRIPT.sh"
+    echo "#!/bin/bash" > "$CAREFSCRIPT.sh"
     if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then
         CARE_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR $VTSSIMPIPE_CARE_IMAGE"
     elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then
         CARE_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} --compat docker://$VTSSIMPIPE_CARE_IMAGE"
     fi
     CARE_EXE="${CARE_EXE} bash -c \"cd /workdir/CARE && ${CARE}\""
-    echo "$CARE_EXE > $OUTPUT_FILE.care.log 2>&1" >> "$FSCRIPT.sh"
-    chmod u+x "$FSCRIPT.sh"
+    echo "$CARE_EXE > $CARE_DATA_DIR/$(basename "$OUTPUT_FILE").care.log 2>&1" >> "$CAREFSCRIPT.sh"
+    chmod u+x "$CAREFSCRIPT.sh"
 }
diff --git a/scripts/corsika.sh b/scripts/corsika.sh
@@ -6,27 +6,18 @@ prepare_corsika_containers()
 {
     DATA_DIR="$1"
     LOG_DIR="$2"
-    PULL="$3"
-    VTSSIMPIPE_CONTAINER="$4"
-    VTSSIMPIPE_CORSIKA_IMAGE="$5"
 
     CONTAINER_EXTERNAL_DIR="-v \"${DATA_DIR}/CORSIKA:/workdir/external/data\" -v \"$LOG_DIR:/workdir/external/log\""
     CORSIKA_DATA_DIR="/workdir/external/data"
     if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then
         COPY_COMMAND="docker run --rm $CONTAINER_EXTERNAL_DIR $VTSSIMPIPE_CORSIKA_IMAGE"
-        PULL_COMMAND="docker pull $VTSSIMPIPE_CORSIKA_IMAGE"
     elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then
-        PULL_COMMAND="apptainer pull --disable-cache --force docker://$VTSSIMPIPE_CORSIKA_IMAGE"
         COPY_COMMAND="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} --compat docker://$VTSSIMPIPE_CORSIKA_IMAGE"
     fi
-    if [[ $PULL == "TRUE" ]]; then
-        eval "$PULL_COMMAND"
-    fi
     # copy corsika directory to data dir (as apptainers are readonly)
     echo "Copy CORSIKA files to ${DATA_DIR}/CORSIKA/tmp_corsika_run_files"
     mkdir -p "${DATA_DIR}/CORSIKA/tmp_corsika_run_files"
     COPY_COMMAND="$COPY_COMMAND bash -c \"cp /workdir/corsika-run/* /workdir/external/data/tmp_corsika_run_files\""
-    echo "$COPY_COMMAND"
     eval "$COPY_COMMAND"
 }
 
@@ -42,20 +33,18 @@ generate_corsika_submission_script()
     INPUT="/workdir/external/log/$(basename "$INPUT")"
 
     echo "#!/bin/bash" > "$FSCRIPT.sh"
-    mkdir -p $(dirname $OUTPUT_FILE)
-    rm -f $OUTPUT_FILE.telescope
+    mkdir -p "$(dirname $OUTPUT_FILE)"
+    rm -f "$OUTPUT_FILE".telescope
     if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then
         CORSIKA_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR $VTSSIMPIPE_CORSIKA_IMAGE"
     elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then
         CORSIKA_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} --compat docker://$VTSSIMPIPE_CORSIKA_IMAGE"
     fi
     CORSIKA_EXE="${CORSIKA_EXE} bash -c \"cd /workdir/corsika-run && ./corsika77500Linux_QGSII_urqmd < $INPUT\""
     echo "$CORSIKA_EXE > $OUTPUT_FILE.log" >> "$FSCRIPT.sh"
-#    echo "bzip2 -f -v $OUTPUT_FILE.telescope" >> "$FSCRIPT.sh"
     chmod u+x "$FSCRIPT.sh"
 }
 
-
 # core scatter area
 get_corsika_core_scatter()
 {
@@ -95,7 +84,6 @@ generate_corsika_input_card()
     CORE_SCATTER=$(get_corsika_core_scatter "$ZENITH")
     ATMOSPHERE="$7"
     CORSIKA_DATA_DIR="$8"
-    VTSSIMPIPE_CONTAINER="$9"
 
     INPUT="$LOG_DIR"/"input_$run_number.dat"
     rm -f "$INPUT"

diff --git a/scripts/groptics.sh b/scripts/groptics.sh
@@ -63,8 +63,9 @@ generate_groptics_pilot_file()
 {
     LOG_DIR="$1"
     RUN_NUMBER="$2"
+    WOBBLE="$3"
 
-    PILOT="${LOG_DIR}/pilot_${RUN_NUMBER}.dat"
+    PILOT="${LOG_DIR}/pilot_${RUN_NUMBER}_W${WOBBLE}.dat"
     rm -f "$PILOT"
     touch "$PILOT"
 
@@ -95,13 +96,12 @@ generate_groptics_pilot_file()
 prepare_groptics_containers()
 {
     DATA_DIR="$1"
-    LOG_DIR="$2"
-    ATMOSPHERE="$3"
-    PULL="$4"
-    VTSSIMPIPE_CONTAINER="$5"
-    VTSSIMPIPE_GROPTICS_IMAGE="$6"
+    ATMOSPHERE="$2"
+    WOBBLE="$3"
 
-    TMP_CONFIG_DIR="${DATA_DIR}/GROPTICS/model_files/"
+    mkdir -p "$DATA_DIR"
+    GROPTICS_DATA_DIR="${DATA_DIR}/GROPTICS/W${WOBBLE}"
+    TMP_CONFIG_DIR="${GROPTICS_DATA_DIR}/model_files/"
     mkdir -p "$TMP_CONFIG_DIR"
 
     # copy file for atmospheric extinction (corsikaIOreader required "M5" in file name)
@@ -112,49 +112,42 @@ prepare_groptics_containers()
     cp -f "$(dirname "$0")"/../config/TELESCOPE_MODEL/"$TELESCOPE_MODEL" "${TMP_CONFIG_DIR}/"
     # copy file for GrOptics configuration
     cp -f "$(dirname "$0")"/../config/TELESCOPE_MODEL/"$GROPTICS_CONFIG" "${TMP_CONFIG_DIR}/"
-
-    # mount directories
-    CORSIKA_DATA_DIR="${DATA_DIR}/CORSIKA"
-    CONTAINER_EXTERNAL_DIR="-v \"${CORSIKA_DATA_DIR}:/workdir/external/corsika\""
-    CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"${DATA_DIR}/GROPTICS:/workdir/external/groptics\""
-    # corsikaIOreader expects the atmprof file in the /workdir/external/groptics/data directory
-    CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"${TMP_CONFIG_DIR}:/workdir/GrOptics/data\""
-    CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"$LOG_DIR:/workdir/external/log/\""
-
-    if [[ $PULL == "TRUE" ]]; then
-        if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then
-            docker pull "$VTSSIMPIPE_GROPTICS_IMAGE"
-        elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then
-            apptainer pull --disable-cache --force docker://"$VTSSIMPIPE_GROPTICS_IMAGE"
-        fi
-    fi
 }
 
 #####################################################################
 # generate GrOptics input files and submission scripts
 generate_groptics_submission_script()
 {
-    FSCRIPT="$1"
-    LOG_DIR=$(dirname "$FSCRIPT")
+    GROPTICSFSCRIPT="$1"
+    LOG_DIR=$(dirname "$GROPTICSFSCRIPT")
     OUTPUT_FILE="$2"
     RUN_NUMBER="$3"
-    CONTAINER_EXTERNAL_DIR="$4"
+    WOBBLE="$4"
     rm -f "$OUTPUT_FILE.groptics.log"
 
+    GROPTICS_DATA_DIR="${DATA_DIR}/GROPTICS/W${WOBBLE}"
+    TMP_CONFIG_DIR="${GROPTICS_DATA_DIR}/model_files/"
+    # mount directories
+    CORSIKA_DATA_DIR="${DATA_DIR}/CORSIKA"
+    CONTAINER_EXTERNAL_DIR="-v \"${CORSIKA_DATA_DIR}:/workdir/external/corsika\""
+    CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"$GROPTICS_DATA_DIR:/workdir/external/groptics\""
     CORSIKA_FILE="${CORSIKA_DATA_DIR}/$(basename "$OUTPUT_FILE").telescope"
     CORSIKA_FILE="/workdir/external/corsika/$(basename "$CORSIKA_FILE")"
+    # corsikaIOreader expects the atmprof file in the /workdir/external/groptics/data directory
+    CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"${TMP_CONFIG_DIR}:/workdir/GrOptics/data\""
+    CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"$LOG_DIR:/workdir/external/log/\""
 
     CORSIKA_IO_READER=$(prepare_corsikaIOreader)
     GROPTICS="./grOptics -of /workdir/external/groptics/$(basename "$OUTPUT_FILE").groptics.root \
-     -p $(generate_groptics_pilot_file "$LOG_DIR" "$RUN_NUMBER")"
+     -p $(generate_groptics_pilot_file "$LOG_DIR" "$RUN_NUMBER" "$WOBBLE")"
 
-    echo "#!/bin/bash" > "$FSCRIPT.sh"
+    echo "#!/bin/bash" > "$GROPTICSFSCRIPT.sh"
     if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then
         GROPTICS_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR $VTSSIMPIPE_GROPTICS_IMAGE"
     elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then
         GROPTICS_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} --compat docker://$VTSSIMPIPE_GROPTICS_IMAGE"
     fi
     GROPTICS_EXE="${GROPTICS_EXE} bash -c \"cd /workdir/GrOptics && ${CORSIKA_IO_READER} | ${GROPTICS}\""
-    echo "$GROPTICS_EXE > $OUTPUT_FILE.groptics.log 2>&1" >> "$FSCRIPT.sh"
-    chmod u+x "$FSCRIPT.sh"
+    echo "$GROPTICS_EXE > $GROPTICS_DATA_DIR/$(basename "$OUTPUT_FILE").groptics.log 2>&1" >> "$GROPTICSFSCRIPT.sh"
+    chmod u+x "$GROPTICSFSCRIPT.sh"
 }
diff --git a/scripts/mergevbf.sh b/scripts/mergevbf.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+# Helper functions for mergevbf; called from prepare_production.sh
+
+#####################################################################
+# generate mergevbf input files and submission scripts
+#
+# /workdir/external/log:     directory for run scripts and pilot files
+# /workdir/external/care     :directory with CARE (input) files
+#
+generate_mergevbf_submission_script()
+{
+    MERGEVBFFSCRIPT="$1"
+    LOG_DIR=$(dirname "$MERGEVBFFSCRIPT")
+    OUTPUT_FILE="$2"
+    rm -f "$OUTPUT_FILE.mergevbf.log"
+    WOBBLE="$3"
+    NSB="$4"
+    RUNNUMBER="$5"
+
+    # mount directories
+    CARE_DATA_DIR="${DATA_DIR}/CARE/W${WOBBLE}/NSB${NSB}"
+    MERGEVBF_DATA_DIR="${DATA_DIR}/MERGEVBF/"
+    mkdir -p "$MERGEVBF_DATA_DIR"
+    CONTAINER_EXTERNAL_DIR="-v \"${MERGEVBF_DATA_DIR}:/workdir/external/mergevbf\""
+    CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"${CARE_DATA_DIR}:/workdir/external/care\""
+    CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"$LOG_DIR:/workdir/external/log/\""
+
+    MERGEVBF="./bin/mergeVBF \
+     /workdir/external/mergevbf/vbf_files.list \
+     /workdir/external/mergevbf/$(basename "$OUTPUT_FILE").vbf ${RUNNUMBER}"
+
+    echo "#!/bin/bash" > "$MERGEVBFFSCRIPT.sh"
+    if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then
+        CARE_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR $VTSSIMPIPE_MERGEVBF_IMAGE"
+    elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then
+        CARE_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} --compat docker://$VTSSIMPIPE_MERGEVBF_IMAGE"
+    fi
+    COLLECT_VBF="ls /workdir/external/care/*.vbf > /workdir/external/mergevbf/vbf_files.list"
+    MERGEVBF_EXE="${CARE_EXE} bash -c \"cd /opt/Eventdisplay && ${COLLECT_VBF} && ${MERGEVBF}\""
+    echo "$MERGEVBF_EXE > $MERGEVBF_DATA_DIR/$(basename "$OUTPUT_FILE").mergevbf.log 2>&1" >> "$MERGEVBFFSCRIPT.sh"
+    chmod u+x "$MERGEVBFFSCRIPT.sh"
+}