From 357acd2f7cea1fcaa3511e2b940e16c43056c4e4 Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Tue, 23 Jan 2024 07:51:24 +0100 Subject: [PATCH 01/19] loop stops at nrun-1 --- scripts/prepare_production.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/prepare_production.sh b/scripts/prepare_production.sh index d5ed916..248148c 100755 --- a/scripts/prepare_production.sh +++ b/scripts/prepare_production.sh @@ -129,8 +129,7 @@ else exit fi -for ID in $(seq 0 "$N_RUNS"); -do +for ((ID=0; ID Date: Tue, 23 Jan 2024 07:55:27 +0100 Subject: [PATCH 02/19] remove disable-cache for apptainer --- scripts/pull.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/pull.sh b/scripts/pull.sh index 6fb874a..2bb444a 100755 --- a/scripts/pull.sh +++ b/scripts/pull.sh @@ -20,10 +20,10 @@ echo "VTSSIMPIPE_MERGEVBF_IMAGE: $VTSSIMPIPE_MERGEVBF_IMAGE" if [[ "$VTSSIMPIPE_CONTAINER" == "apptainer" ]]; then echo "Pulling apptainer images." - apptainer pull --disable-cache --force docker://"$VTSSIMPIPE_CORSIKA_IMAGE" - apptainer pull --disable-cache --force docker://"$VTSSIMPIPE_GROPTICS_IMAGE" - apptainer pull --disable-cache --force docker://"$VTSSIMPIPE_CARE_IMAGE" - apptainer pull --disable-cache --force docker://"$VTSSIMPIPE_MERGEVBF_IMAGE" + apptainer pull docker://"$VTSSIMPIPE_CORSIKA_IMAGE" + apptainer pull docker://"$VTSSIMPIPE_GROPTICS_IMAGE" + apptainer pull docker://"$VTSSIMPIPE_CARE_IMAGE" + apptainer pull docker://"$VTSSIMPIPE_MERGEVBF_IMAGE" elif [[ "$VTSSIMPIPE_CONTAINER" == "docker" ]]; then echo "Pulling docker images." docker pull "$VTSSIMPIPE_CORSIKA_IMAGE" From 70a7f7973e97164a65f1020c113c84ade172490c Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Wed, 24 Jan 2024 22:00:59 +0100 Subject: [PATCH 03/19] add DOT command --- scripts/prepare_DAG_jobs.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/prepare_DAG_jobs.sh b/scripts/prepare_DAG_jobs.sh index 197761f..330189b 100755 --- a/scripts/prepare_DAG_jobs.sh +++ b/scripts/prepare_DAG_jobs.sh @@ -77,6 +77,7 @@ for ID in $(seq 0 "$N_RUNS"); do PARENT_CLEANUP="$PARENT_CLEANUP CHILD CLEANUP_${run_number}" echo "$PARENT_CLEANUP" >> "$DAG_FILE" echo "$PARENT_CORSIKA" >> "$DAG_FILE" + echo "DOT $DAG_DIR/run_${run_number}.dot" >> "$DAG_FILE" done echo "DAG directory: $DAG_DIR" From e77b70fd6b1b3d2069f95c85cc68bcf059873835 Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Wed, 24 Jan 2024 22:01:18 +0100 Subject: [PATCH 04/19] use config for DAG submit --- scripts/submit_DAG_jobs.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/submit_DAG_jobs.sh b/scripts/submit_DAG_jobs.sh index 1074032..ad6d808 100755 --- a/scripts/submit_DAG_jobs.sh +++ b/scripts/submit_DAG_jobs.sh @@ -7,20 +7,26 @@ # set -e -if [ $# -lt 1 ] +if [ $# -lt 2 ] then echo " - ./submit_DAG_jobs.sh + ./submit_DAG_jobs.sh " exit fi +CONFIG="$1" + export _condor_SEC_TOKEN_DIRECTORY=$(mktemp -d) condor_token_fetch -lifetime $((7*24*60*60)) -token dag -for dag_file in "${1}"/*.dag; do - condor_submit_dag "$dag_file" +# shellcheck source=/dev/null +. "$CONFIG" + +for ID in $(seq 0 "$N_RUNS"); do + run_number=$((ID + RUN_START)) + condor_submit_dag "${2}/run_${run_number}.dag" done exit $? From d4ca8cc7ce2e2fb072dbdb71a9703c2061e64d23 Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Wed, 24 Jan 2024 22:03:31 +0100 Subject: [PATCH 05/19] submit DAT improvement --- README.md | 4 ++-- config/config_ATM61_production.dat | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3e3a166..a47179d 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ cd scripts ../config/CORSIKA/input_template.dat # on DESY: log into the DAG submission node ./prepare_DAG_jobs.sh ../config/config_ATM61_template.dat -./submit_DAG_jobs.sh submit +./submit_DAG_jobs.sh ../config/config_ATM61_template.dat # otherwise: submit jobs to HT Condor - for each step (CORSIKA, GROPTICS, CARE) ./submit_jobs_to_htcondor.sh submit # now wait....for jobs to finish @@ -226,7 +226,7 @@ This will generate the DAG files in the `VTSSIMPIPE_LOG_DIR/DAG` directory. To submit DAG jobs: ```bash -./submit_DAG_jobs.sh submit +./submit_DAG_jobs.sh ``` Note that on DESY DAG jobs need to be submitted from a special node. diff --git a/config/config_ATM61_production.dat b/config/config_ATM61_production.dat index 59cea74..e7050d8 100644 --- a/config/config_ATM61_production.dat +++ b/config/config_ATM61_production.dat @@ -1,5 +1,5 @@ -RUN_START=650000 -N_RUNS=10 +RUN_START=650500 +N_RUNS=500 N_SHOWER=2000 ZENITH=20 ATMOSPHERE=61 From 38c0bb006ec9148211e7a5e521bf3a00a9294dbc Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Tue, 30 Jan 2024 10:55:52 +0100 Subject: [PATCH 06/19] longer DAG; zip of log files --- scripts/cleanup.sh | 1 + scripts/submit_DAG_jobs.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index 494e3a6..09a656f 100644 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -38,5 +38,6 @@ generate_cleanup_submission_script() echo "$CLEANUP_GROPTICS > $CLEANUP_DATA_DIR/$(basename "$OUTPUT_FILE").cleanup.log 2>&1" >> "$CLEANUPSCRIPT.sh" done echo "bzip2 -f -v ${CORSIKA_DATA_DIR}/$(basename "$OUTPUT_FILE").telescope >> $CLEANUP_DATA_DIR/$(basename "$OUTPUT_FILE").cleanup.log 2>&1" >> "$CLEANUPSCRIPT.sh" + echo "bzip2 -f -v ${CORSIKA_DATA_DIR}/$(basename "$OUTPUT_FILE").log >> $CLEANUP_DATA_DIR/$(basename "$OUTPUT_FILE").cleanup.log 2>&1" >> "$CLEANUPSCRIPT.sh" chmod u+x "$CLEANUPSCRIPT.sh" } diff --git a/scripts/submit_DAG_jobs.sh b/scripts/submit_DAG_jobs.sh index ad6d808..48020a2 100755 --- a/scripts/submit_DAG_jobs.sh +++ b/scripts/submit_DAG_jobs.sh @@ -19,7 +19,7 @@ fi CONFIG="$1" export _condor_SEC_TOKEN_DIRECTORY=$(mktemp -d) -condor_token_fetch -lifetime $((7*24*60*60)) -token dag +condor_token_fetch -lifetime $((30*24*60*60)) -token dag # shellcheck source=/dev/null . "$CONFIG" From bd6299cf20169e36e65eeec81e69b5083c572363 Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Fri, 2 Feb 2024 13:45:07 +0100 Subject: [PATCH 07/19] 1000 runs --- config/config_ATM61_production.dat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config_ATM61_production.dat b/config/config_ATM61_production.dat index e7050d8..3c054a5 100644 --- a/config/config_ATM61_production.dat +++ b/config/config_ATM61_production.dat @@ -1,5 +1,5 @@ -RUN_START=650500 -N_RUNS=500 +RUN_START=650000 +N_RUNS=1000 N_SHOWER=2000 ZENITH=20 ATMOSPHERE=61 From 220a2a8bca78d2c2fd5a43db2b7bf3c78b80cd09 Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Fri, 2 Feb 2024 13:45:19 +0100 Subject: [PATCH 08/19] improved batch size --- scripts/mergevbf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mergevbf.sh b/scripts/mergevbf.sh index 3351721..d79a344 100644 --- a/scripts/mergevbf.sh +++ b/scripts/mergevbf.sh @@ -51,7 +51,7 @@ generate_mergevbf_submission_script() CARE_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} --compat docker://$VTSSIMPIPE_MERGEVBF_IMAGE" fi - batch_size=100 + batch_size=250 vbf_id="0" MERGEDFILE=$(get_merge_file_name "$WOBBLE" "$NSB" "$vbf_id") TMP_FL_LIST="$MERGEVBF_DATA_DIR"/tmp_file_list_${WOBBLE}_${NSB}.dat From 93fb3ec3f114540b7041780f04ce4767578c59cc Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Sat, 3 Feb 2024 20:47:32 +0100 Subject: [PATCH 09/19] improved container handling --- env_setup_template.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/env_setup_template.sh b/env_setup_template.sh index f6e9e8a..2d9f0c1 100644 --- a/env_setup_template.sh +++ b/env_setup_template.sh @@ -8,11 +8,15 @@ export VTSSIMPIPE_DATA_DIR=vtssimpipe_corsika_dir # container type # export VTSSIMPIPE_CONTAINER="apptainer" export VTSSIMPIPE_CONTAINER="docker" +# package url (docker style) +export VTSSIMPIPE_CONTAINER_URL="ghcr.io/veritas-observatory/" +# apptainers are stored in this directory +export VTSSIMPIPE_CONTAINER_DIR=vtssimpipe_containe_dir # CORSIKA -export VTSSIMPIPE_CORSIKA_IMAGE="ghcr.io/veritas-observatory/vtsimpipe-corsika:1.0.0" +export VTSSIMPIPE_CORSIKA_IMAGE="vtsimpipe-corsika:1.0.0" # GROPTICS -export VTSSIMPIPE_GROPTICS_IMAGE="ghcr.io/veritas-observatory/vtsimpipe-groptics:1.0.0" +export VTSSIMPIPE_GROPTICS_IMAGE="vtsimpipe-groptics:1.0.0" # CARE -export VTSSIMPIPE_CARE_IMAGE="ghcr.io/veritas-observatory/vtsimpipe-care:1.0.0" +export VTSSIMPIPE_CARE_IMAGE="vtsimpipe-care:1.0.0" # MERGEVBF -export VTSSIMPIPE_MERGEVBF_IMAGE="ghcr.io/veritas-observatory/vtsimpipe-mergevbf:1.0.0" +export VTSSIMPIPE_MERGEVBF_IMAGE="vtsimpipe-mergevbf:1.0.0" From e4b37393e192fada09219133ed8850f2fb88d99c Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Sat, 3 Feb 2024 20:56:16 +0100 Subject: [PATCH 10/19] improved container handling --- scripts/care.sh | 4 ++-- scripts/corsika.sh | 21 +++++++++++---------- scripts/groptics.sh | 4 ++-- scripts/mergevbf.sh | 4 ++-- scripts/pull.sh | 19 ++++++++++--------- 5 files changed, 27 insertions(+), 25 deletions(-) diff --git a/scripts/care.sh b/scripts/care.sh index a11af95..bddd4fc 100644 --- a/scripts/care.sh +++ b/scripts/care.sh @@ -66,9 +66,9 @@ generate_care_submission_script() echo "#!/bin/bash" > "$CAREFSCRIPT.sh" if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then - CARE_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR $VTSSIMPIPE_CARE_IMAGE" + CARE_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CARE_IMAGE}" elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then - CARE_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} --compat docker://$VTSSIMPIPE_CARE_IMAGE" + CARE_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CARE_IMAGE/:/_}.sif" fi CARE_EXE="${CARE_EXE} bash -c \"cd /workdir/CARE && ${CARE}\"" echo "$CARE_EXE > $CARE_DATA_DIR/$(basename "$OUTPUT_FILE").care.log 2>&1" >> "$CAREFSCRIPT.sh" diff --git a/scripts/corsika.sh b/scripts/corsika.sh index a4092b3..ca4cf21 100644 --- a/scripts/corsika.sh +++ b/scripts/corsika.sh @@ -11,9 +11,9 @@ prepare_corsika_containers() CONTAINER_EXTERNAL_DIR="-v \"${DATA_DIR}/CORSIKA:/workdir/external/data\" -v \"$LOG_DIR:/workdir/external/log\"" CORSIKA_DATA_DIR="/workdir/external/data" if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then - COPY_COMMAND="docker run --rm $CONTAINER_EXTERNAL_DIR $VTSSIMPIPE_CORSIKA_IMAGE" + COPY_COMMAND="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE}" elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then - COPY_COMMAND="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} --compat docker://$VTSSIMPIPE_CORSIKA_IMAGE" + COPY_COMMAND="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE/:/_}.sif" fi # copy corsika directory to data dir (as apptainers are readonly) echo "Copy CORSIKA files to ${DATA_DIR}/CORSIKA/tmp_corsika_run_files" @@ -35,14 +35,15 @@ generate_corsika_submission_script() echo "#!/bin/bash" > "$FSCRIPT.sh" mkdir -p "$(dirname $OUTPUT_FILE)" - rm -f "$OUTPUT_FILE".telescope - if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then - CORSIKA_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR $VTSSIMPIPE_CORSIKA_IMAGE" - elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then - CORSIKA_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} --compat docker://$VTSSIMPIPE_CORSIKA_IMAGE" - fi - CORSIKA_EXE="${CORSIKA_EXE} bash -c \"cd /workdir/corsika-run && ./corsika77500Linux_QGSII_urqmd < $INPUT\"" - echo "$CORSIKA_EXE > $OUTPUT_FILE.log" >> "$FSCRIPT.sh" + # TMP rm -f "$OUTPUT_FILE".telescope + echo "cp -v /lustre/fs24/group/cta/tmp_corsika/CORSIKA/Data/$(basename $OUTPUT_FILE.telescope) $OUTPUT_FILE.telescope" >> "$FSCRIPT.sh" +# if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then +# CORSIKA_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE}" +# elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then +# CORSIKA_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE/:/_}.sif" +# fi +# CORSIKA_EXE="${CORSIKA_EXE} bash -c \"cd /workdir/corsika-run && ./corsika77500Linux_QGSII_urqmd < $INPUT\"" +# echo "$CORSIKA_EXE > $OUTPUT_FILE.log" >> "$FSCRIPT.sh" chmod u+x "$FSCRIPT.sh" } diff --git a/scripts/groptics.sh b/scripts/groptics.sh index 4526db4..3545239 100644 --- a/scripts/groptics.sh +++ b/scripts/groptics.sh @@ -144,9 +144,9 @@ generate_groptics_submission_script() echo "#!/bin/bash" > "$GROPTICSFSCRIPT.sh" if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then - GROPTICS_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR $VTSSIMPIPE_GROPTICS_IMAGE" + GROPTICS_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_GROPTICS_IMAGE}" elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then - GROPTICS_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} --compat docker://$VTSSIMPIPE_GROPTICS_IMAGE" + GROPTICS_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_GROPTICS_IMAGE/:/_}.sif" fi GROPTICS_EXE="${GROPTICS_EXE} bash -c \"cd /workdir/GrOptics && ${CORSIKA_IO_READER} | ${GROPTICS}\"" echo "$GROPTICS_EXE > $GROPTICS_DATA_DIR/$(basename "$OUTPUT_FILE").groptics.log 2>&1" >> "$GROPTICSFSCRIPT.sh" diff --git a/scripts/mergevbf.sh b/scripts/mergevbf.sh index d79a344..d48a89e 100644 --- a/scripts/mergevbf.sh +++ b/scripts/mergevbf.sh @@ -46,9 +46,9 @@ generate_mergevbf_submission_script() CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"$LOG_DIR:/workdir/external/log/\"" if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then - CARE_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR $VTSSIMPIPE_MERGEVBF_IMAGE" + CARE_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_MERGEVBF_IMAGE}" elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then - CARE_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} --compat docker://$VTSSIMPIPE_MERGEVBF_IMAGE" + CARE_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_MERGEVBF_IMAGE/:/_}.sif" fi batch_size=250 diff --git a/scripts/pull.sh b/scripts/pull.sh index 2bb444a..fa4f94d 100755 --- a/scripts/pull.sh +++ b/scripts/pull.sh @@ -19,17 +19,18 @@ echo "VTSSIMPIPE_CARE_IMAGE: $VTSSIMPIPE_CARE_IMAGE" echo "VTSSIMPIPE_MERGEVBF_IMAGE: $VTSSIMPIPE_MERGEVBF_IMAGE" if [[ "$VTSSIMPIPE_CONTAINER" == "apptainer" ]]; then - echo "Pulling apptainer images." - apptainer pull docker://"$VTSSIMPIPE_CORSIKA_IMAGE" - apptainer pull docker://"$VTSSIMPIPE_GROPTICS_IMAGE" - apptainer pull docker://"$VTSSIMPIPE_CARE_IMAGE" - apptainer pull docker://"$VTSSIMPIPE_MERGEVBF_IMAGE" + echo "Pulling apptainer images to $VTSSIMPIPE_CONTAINER_DIR" + mkdir -p "${VTSSIMPIPE_CONTAINER_DIR}" + apptainer pull --force --dir "${VTSSIMPIPE_CONTAINER_DIR}/" docker://"${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE}" + apptainer pull --force --dir "${VTSSIMPIPE_CONTAINER_DIR}/" docker://"${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_GROPTICS_IMAGE}" + apptainer pull --force --dir "${VTSSIMPIPE_CONTAINER_DIR}/" docker://"${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CARE_IMAGE}" + apptainer pull --force --dir "${VTSSIMPIPE_CONTAINER_DIR}/" docker://"${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_MERGEVBF_IMAGE}" elif [[ "$VTSSIMPIPE_CONTAINER" == "docker" ]]; then echo "Pulling docker images." - docker pull "$VTSSIMPIPE_CORSIKA_IMAGE" - docker pull "$VTSSIMPIPE_GROPTICS_IMAGE" - docker pull "$VTSSIMPIPE_CARE_IMAGE" - docker pull "$VTSSIMPIPE_MERGEVBF_IMAGE" + docker pull "${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE}" + docker pull "${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_GROPTICS_IMAGE}" + docker pull "${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CARE_IMAGE}" + docker pull "${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_MERGEVBF_IMAGE}" else echo "Unknown container type $VTSSIMPIPE_CONTAINER." exit From 0688449151b0f88db28ba737190c777680855875 Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Sat, 3 Feb 2024 21:37:07 +0100 Subject: [PATCH 11/19] do not write grid face file --- config/GROPTICS/Upgrade_20120827_v420_3.cfg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/GROPTICS/Upgrade_20120827_v420_3.cfg b/config/GROPTICS/Upgrade_20120827_v420_3.cfg index 55e2b1a..a38b0f4 100644 --- a/config/GROPTICS/Upgrade_20120827_v420_3.cfg +++ b/config/GROPTICS/Upgrade_20120827_v420_3.cfg @@ -86,10 +86,10 @@ from a file. There is a separate grid for each standard telescope. - nbinsy, number of bins in the y direction - fileName, name of grid file (for either reading or writing) Default (no GRIDF record) do full loop sequentially over all facets. -* GRIDF 1 2 15 15 gridfileStdDC1.txt -* GRIDF 2 2 15 15 gridfileStdDC2.txt -* GRIDF 3 2 15 15 gridfileStdDC3.txt -* GRIDF 4 2 15 15 gridfileStdDC4.txt +* GRIDF 1 1 15 15 gridfileStdDC1.txt +* GRIDF 2 1 15 15 gridfileStdDC2.txt +* GRIDF 3 1 15 15 gridfileStdDC3.txt +* GRIDF 4 1 15 15 gridfileStdDC4.txt GRIDF 2 1 15 15 additional pilot files to combine with this file From 0ed03d476f5ee954d627f535bc5a7f858cdbf022 Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Sun, 4 Feb 2024 10:43:02 +0100 Subject: [PATCH 12/19] fix apptainer command --- scripts/care.sh | 2 +- scripts/corsika.sh | 4 ++-- scripts/groptics.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/care.sh b/scripts/care.sh index bddd4fc..45faecc 100644 --- a/scripts/care.sh +++ b/scripts/care.sh @@ -68,7 +68,7 @@ generate_care_submission_script() if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then CARE_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CARE_IMAGE}" elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then - CARE_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CARE_IMAGE/:/_}.sif" + CARE_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_CARE_IMAGE/:/_}.sif" fi CARE_EXE="${CARE_EXE} bash -c \"cd /workdir/CARE && ${CARE}\"" echo "$CARE_EXE > $CARE_DATA_DIR/$(basename "$OUTPUT_FILE").care.log 2>&1" >> "$CAREFSCRIPT.sh" diff --git a/scripts/corsika.sh b/scripts/corsika.sh index ca4cf21..3928e2e 100644 --- a/scripts/corsika.sh +++ b/scripts/corsika.sh @@ -13,7 +13,7 @@ prepare_corsika_containers() if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then COPY_COMMAND="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE}" elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then - COPY_COMMAND="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE/:/_}.sif" + COPY_COMMAND="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_CORSIKA_IMAGE/:/_}.sif" fi # copy corsika directory to data dir (as apptainers are readonly) echo "Copy CORSIKA files to ${DATA_DIR}/CORSIKA/tmp_corsika_run_files" @@ -40,7 +40,7 @@ generate_corsika_submission_script() # if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then # CORSIKA_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE}" # elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then -# CORSIKA_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE/:/_}.sif" +# CORSIKA_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_CORSIKA_IMAGE/:/_}.sif" # fi # CORSIKA_EXE="${CORSIKA_EXE} bash -c \"cd /workdir/corsika-run && ./corsika77500Linux_QGSII_urqmd < $INPUT\"" # echo "$CORSIKA_EXE > $OUTPUT_FILE.log" >> "$FSCRIPT.sh" diff --git a/scripts/groptics.sh b/scripts/groptics.sh index 3545239..a4616d4 100644 --- a/scripts/groptics.sh +++ b/scripts/groptics.sh @@ -146,7 +146,7 @@ generate_groptics_submission_script() if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then GROPTICS_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_GROPTICS_IMAGE}" elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then - GROPTICS_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_GROPTICS_IMAGE/:/_}.sif" + GROPTICS_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_GROPTICS_IMAGE/:/_}.sif" fi GROPTICS_EXE="${GROPTICS_EXE} bash -c \"cd /workdir/GrOptics && ${CORSIKA_IO_READER} | ${GROPTICS}\"" echo "$GROPTICS_EXE > $GROPTICS_DATA_DIR/$(basename "$OUTPUT_FILE").groptics.log 2>&1" >> "$GROPTICSFSCRIPT.sh" From 133a61a13a2e0a21fbdb1d27f837607fecb45910 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sun, 4 Feb 2024 11:23:01 +0100 Subject: [PATCH 13/19] Improve readme; remove temporary corsika entries. --- README.md | 2 ++ scripts/corsika.sh | 16 +++++++--------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a47179d..3272cb5 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,7 @@ Processing scripts are prepared for HT Condor systems. - note that the `MERGEVBF` step is not included in the DAG submission, as it combines all runs of a production. Run this as a final step at the ned of the production. Note that configuration and output directories are fine tuned for this setup. +The preparation of the all temporary submission scripts is not very efficient in both time and number of files written (could be significantly improved). ## Using Apptainers @@ -199,6 +200,7 @@ The simulation scripts are configured to use Apptainers with the correct paramet Note: - recommend to set `$APPTAINER_CACHEDIR` to a reasonable directory with sufficient disk space, as cluster jobs will use this directory to store the container images. +- set `$VTSSIMPIPE_CONTAINER_DIR` to the directory where the container images are stored. This should be a "fast" disk, as each job will access the image files. ## Submitting jobs diff --git a/scripts/corsika.sh b/scripts/corsika.sh index 3928e2e..ecbc9d5 100644 --- a/scripts/corsika.sh +++ b/scripts/corsika.sh @@ -35,15 +35,13 @@ generate_corsika_submission_script() echo "#!/bin/bash" > "$FSCRIPT.sh" mkdir -p "$(dirname $OUTPUT_FILE)" - # TMP rm -f "$OUTPUT_FILE".telescope - echo "cp -v /lustre/fs24/group/cta/tmp_corsika/CORSIKA/Data/$(basename $OUTPUT_FILE.telescope) $OUTPUT_FILE.telescope" >> "$FSCRIPT.sh" -# if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then -# CORSIKA_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE}" -# elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then -# CORSIKA_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_CORSIKA_IMAGE/:/_}.sif" -# fi -# CORSIKA_EXE="${CORSIKA_EXE} bash -c \"cd /workdir/corsika-run && ./corsika77500Linux_QGSII_urqmd < $INPUT\"" -# echo "$CORSIKA_EXE > $OUTPUT_FILE.log" >> "$FSCRIPT.sh" + if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then + CORSIKA_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE}" + elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then + CORSIKA_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_CORSIKA_IMAGE/:/_}.sif" + fi + CORSIKA_EXE="${CORSIKA_EXE} bash -c \"cd /workdir/corsika-run && ./corsika77500Linux_QGSII_urqmd < $INPUT\"" + echo "$CORSIKA_EXE > $OUTPUT_FILE.log" >> "$FSCRIPT.sh" chmod u+x "$FSCRIPT.sh" } From 8f085f96203fe282bbaceb9ad4c483d7e3e03300 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sun, 4 Feb 2024 11:37:18 +0100 Subject: [PATCH 14/19] Schellcheck --- README.md | 4 +--- scripts/cleanup.sh | 2 -- scripts/corsika.sh | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3272cb5..ec06987 100644 --- a/README.md +++ b/README.md @@ -252,8 +252,7 @@ Configuration: ### CORSIKA parameters - -#### 2017 production: +#### 2017 production ```Text Monte Carlo run header @@ -288,7 +287,6 @@ SLANT 1 - changed wavelength range from [200, 700] to [300, 600] nm - ### corsikaIOreader parameters - apply pre-efficiency cut of 50% diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index 09a656f..8ec3874 100644 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -19,9 +19,7 @@ generate_cleanup_submission_script() { CLEANUPSCRIPT="$1" - LOG_DIR=$(dirname "$CLEANUPSCRIPT") OUTPUT_FILE="$2" - RUN_NUMBER="$3" WOFF_LIST="$4" CLEANUP_DATA_DIR="${DATA_DIR}/CLEANUP/" mkdir -p "${CLEANUP_DATA_DIR}" diff --git a/scripts/corsika.sh b/scripts/corsika.sh index ecbc9d5..52875ce 100644 --- a/scripts/corsika.sh +++ b/scripts/corsika.sh @@ -6,7 +6,7 @@ prepare_corsika_containers() { DATA_DIR="$1" LOG_DIR="$2" - mkdir -p ${DATA_DIR} + mkdir -p "${DATA_DIR}" CONTAINER_EXTERNAL_DIR="-v \"${DATA_DIR}/CORSIKA:/workdir/external/data\" -v \"$LOG_DIR:/workdir/external/log\"" CORSIKA_DATA_DIR="/workdir/external/data" From f5b1905768c6a70cca2aca8c0e0a4a67d80e1dbb Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sun, 4 Feb 2024 11:44:03 +0100 Subject: [PATCH 15/19] Add maintainer labels to Dockerfiles --- docker/Dockerfile-care | 2 ++ docker/Dockerfile-corsika | 2 ++ docker/Dockerfile-groptics | 2 ++ docker/Dockerfile-mergevbf | 2 ++ 4 files changed, 8 insertions(+) diff --git a/docker/Dockerfile-care b/docker/Dockerfile-care index ea3f290..e9cc139 100644 --- a/docker/Dockerfile-care +++ b/docker/Dockerfile-care @@ -36,6 +36,8 @@ RUN mkdir CARE && tar -xzf v$CARE_VERSION.tar.gz -C CARE --strip-components=1 && make && make clean && rm -f *.tar.gz FROM rootproject/root:6.24.06-centos7 +LABEL maintainer.name="VERITAS Collaboration" +LABEL maintainer.email="gernot.maier@desy.de" WORKDIR /workdir COPY --from=build_image /workdir/VBF /workdir/VBF COPY --from=build_image /workdir/CARE /workdir/CARE diff --git a/docker/Dockerfile-corsika b/docker/Dockerfile-corsika index f506682..7acd3e9 100644 --- a/docker/Dockerfile-corsika +++ b/docker/Dockerfile-corsika @@ -21,6 +21,8 @@ SHELL ["/bin/bash", "-c"] WORKDIR /workdir/ FROM almalinux:9.3-minimal +LABEL maintainer.name="VERITAS Collaboration" +LABEL maintainer.email="gernot.maier@desy.de" WORKDIR /workdir ENV DEBIAN_FRONTEND=noninteractive ARG CORSIKA=corsika-77500 diff --git a/docker/Dockerfile-groptics b/docker/Dockerfile-groptics index 3f48c7f..8ab53ee 100644 --- a/docker/Dockerfile-groptics +++ b/docker/Dockerfile-groptics @@ -26,6 +26,8 @@ RUN mkdir GrOptics && tar -xzf v$GROPTICS_VERSION.tar.gz -C GrOptics --strip-com cd GrOptics && make FROM rootproject/root:6.24.06-centos7 +LABEL maintainer.name="VERITAS Collaboration" +LABEL maintainer.email="gernot.maier@desy.de" WORKDIR /workdir COPY --from=build_image /workdir/corsikaIOreader/corsikaIOreader /workdir/corsikaIOreader/corsikaIOreader COPY --from=build_image /workdir/GrOptics /workdir/GrOptics diff --git a/docker/Dockerfile-mergevbf b/docker/Dockerfile-mergevbf index 121814e..f065a8d 100644 --- a/docker/Dockerfile-mergevbf +++ b/docker/Dockerfile-mergevbf @@ -27,6 +27,8 @@ RUN mkdir EventDisplay_v4 && tar -xzf v490.5.tar.gz -C EventDisplay_v4 --strip-c cd EventDisplay_v4 && make mergeVBF && make clean FROM rootproject/root:6.24.06-centos7 +LABEL maintainer.name="VERITAS Collaboration" +LABEL maintainer.email="gernot.maier@desy.de" WORKDIR /workdir COPY --from=build_image /workdir/VBF /workdir/VBF COPY --from=build_image /workdir/EventDisplay_v4 /workdir/EventDisplay_v4 From 2499ff44b13949f4dd2fd40e56fc2365a7ba739a Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Wed, 7 Feb 2024 10:09:26 +0100 Subject: [PATCH 16/19] submit jobs from file list (mostly to rescue) --- scripts/submit_DAG_jobs_from_file_list.sh | 30 +++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 scripts/submit_DAG_jobs_from_file_list.sh diff --git a/scripts/submit_DAG_jobs_from_file_list.sh b/scripts/submit_DAG_jobs_from_file_list.sh new file mode 100755 index 0000000..2a73934 --- /dev/null +++ b/scripts/submit_DAG_jobs_from_file_list.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# submit DAG jobs in DESY environment from a list of files +# use htc-submit.zeuthen.desy.de +# see https://dv-zeuthen.desy.de/services/batch/job_submission/ for details +# +# Submits all *.dag files in the given directory +# +set -e + +if [ $# -lt 1 ] +then + echo " + ./submit_DAG_jobs_from_file_list.sh + + " + exit +fi + +FILELIST="$1" + +export _condor_SEC_TOKEN_DIRECTORY=$(mktemp -d) +condor_token_fetch -lifetime $((30*24*60*60)) -token dag + +FLIST=$(cat $FILELIST) +for F in ${FLIST}; do + echo $F + condor_submit_dag ${F} +done + +exit $? From f3fe5a9d2af4ef9332a91e5e9a78682ffcd2d4bf Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Fri, 9 Feb 2024 13:28:20 +0100 Subject: [PATCH 17/19] bug fix in apptainer dir --- scripts/mergevbf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mergevbf.sh b/scripts/mergevbf.sh index d48a89e..6e04506 100644 --- a/scripts/mergevbf.sh +++ b/scripts/mergevbf.sh @@ -48,7 +48,7 @@ generate_mergevbf_submission_script() if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then CARE_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_MERGEVBF_IMAGE}" elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then - CARE_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_MERGEVBF_IMAGE/:/_}.sif" + CARE_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_MERGEVBF_IMAGE/:/_}.sif" fi batch_size=250 From 75ee86197f1a952ac8735c951a8707d461c6eab0 Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Sat, 10 Feb 2024 11:42:13 +0100 Subject: [PATCH 18/19] remove existing dag files --- scripts/submit_DAG_jobs_from_file_list.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/submit_DAG_jobs_from_file_list.sh b/scripts/submit_DAG_jobs_from_file_list.sh index 2a73934..cfd25c7 100755 --- a/scripts/submit_DAG_jobs_from_file_list.sh +++ b/scripts/submit_DAG_jobs_from_file_list.sh @@ -24,6 +24,7 @@ condor_token_fetch -lifetime $((30*24*60*60)) -token dag FLIST=$(cat $FILELIST) for F in ${FLIST}; do echo $F + rm -f -v ${F}.* condor_submit_dag ${F} done From c0edab7fa5424561acf870bb10aa5b2b9f96980e Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Sat, 10 Feb 2024 11:48:52 +0100 Subject: [PATCH 19/19] fix name of redHV file name --- scripts/mergevbf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mergevbf.sh b/scripts/mergevbf.sh index 6e04506..76af1f0 100644 --- a/scripts/mergevbf.sh +++ b/scripts/mergevbf.sh @@ -8,7 +8,7 @@ get_merge_file_name() # gamma_V6_CARE_std_Atmosphere61_zen20deg_1.0wob_160MHz_1.vbf.zst FNAME="gamma_V6_CARE" - if [[ $CARE_CONFIG == *"RHV"* ]]; then + if [[ $OBS_MODE == *"redHV"* ]]; then FNAME="${FNAME}_redHV" else FNAME="${FNAME}_std"