From 539ad3f82af46d6e89a092dfc0459c4d65c95475 Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Mon, 12 Feb 2024 08:45:43 +0100 Subject: [PATCH 01/10] pack log files scripts --- scripts/pack_log_files.sh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 scripts/pack_log_files.sh diff --git a/scripts/pack_log_files.sh b/scripts/pack_log_files.sh new file mode 100755 index 0000000..017ebbb --- /dev/null +++ b/scripts/pack_log_files.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Pack log files in production directories into +# individual tar ball depending on the data level +# + +if [ ! -n "$1" ] || [ "$1" = "-h" ]; then +echo " +Pack Simulation log files for each level. + +./pack_log_files.sh + +Production directory is e.g., $VTSSIMPIPE_DATA_DIR/ATM62/Zd65/ + +" +exit +fi + + +PDIR="${1}" +DLIST="CORSIKA CARE_redHV CARE_std CLEANUP GROPTICS MERGEVBF_redHV MERGEVBF_std" + +cd "${PDIR}" + +for DIR in $DLIST; do + echo $DIR + if [ -e $DIR ]; then + rm -f logs_${DIR}.tar.gz + find ${DIR} -type f -name "*.log" -exec tar -rf logs_${DIR}.tar.gz {} + + fi +done From a2527990951249c64a7eef952db627c6b194849e Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Mon, 12 Feb 2024 15:18:38 +0100 Subject: [PATCH 02/10] simplified submission --- scripts/corsika.sh | 149 ++++++++++++++++++++++------------ scripts/prepare_DAG_jobs.sh | 4 +- scripts/prepare_production.sh | 39 ++++----- 3 files changed, 116 insertions(+), 76 deletions(-) diff --git a/scripts/corsika.sh b/scripts/corsika.sh index 52875ce..c62fce1 100644 --- a/scripts/corsika.sh +++ b/scripts/corsika.sh @@ -13,7 +13,7 @@ prepare_corsika_containers() if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then COPY_COMMAND="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE}" elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then - COPY_COMMAND="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_CORSIKA_IMAGE/:/_}.sif" + COPY_COMMAND="apptainer exec --cleanenv --no-mount bind-paths ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_CORSIKA_IMAGE/:/_}.sif" fi # copy corsika directory to data dir (as apptainers are readonly) echo "Copy CORSIKA files to ${DATA_DIR}/CORSIKA/tmp_corsika_run_files" @@ -22,28 +22,6 @@ prepare_corsika_containers() eval "$COPY_COMMAND" } -generate_corsika_submission_script() -{ - FSCRIPT="$1" - INPUT="$2" - OUTPUT_FILE="$3" - CONTAINER_EXTERNAL_DIR="$4" - rm -f "$OUTPUT_FILE.log" - rm -f "$OUTPUT_FILE.telescope" - - INPUT="/workdir/external/log/$(basename "$INPUT")" - - echo "#!/bin/bash" > "$FSCRIPT.sh" - mkdir -p "$(dirname $OUTPUT_FILE)" - if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then - CORSIKA_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE}" - elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then - CORSIKA_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_CORSIKA_IMAGE/:/_}.sif" - fi - CORSIKA_EXE="${CORSIKA_EXE} bash -c \"cd /workdir/corsika-run && ./corsika77500Linux_QGSII_urqmd < $INPUT\"" - echo "$CORSIKA_EXE > $OUTPUT_FILE.log" >> "$FSCRIPT.sh" - chmod u+x "$FSCRIPT.sh" -} # core scatter area (in m) get_corsika_core_scatter() @@ -74,38 +52,107 @@ get_corsika_energy_min() # CORSIKA input card generate_corsika_input_card() { - LOG_DIR="$1" - run_number="$2" - S1="$3" - INPUT_TEMPLATE="$4" - N_SHOWER="$5" - ZENITH="$6" + OFILE="$1" + N_SHOWER="$2" + ZENITH="$3" ENERGY_MIN=$(get_corsika_energy_min "$ZENITH") CORE_SCATTER=$(get_corsika_core_scatter "$ZENITH") - ATMOSPHERE="$7" - CORSIKA_DATA_DIR="$8" + ATMOSPHERE="$4" + CORSIKA_DATA_DIR="$5" + + echo "S1=\$((RANDOM % 900000000 - 1))" >> "$OFILE" + echo "S2=\$((S1 + 2))" >> "$OFILE" + echo "S3=\$((S2 + 2))" >> "$OFILE" + echo "S4=\$((S3 + 2))" >> "$OFILE" - INPUT="$LOG_DIR"/"input_$run_number.dat" - rm -f "$INPUT" + { + cat << EOF +# CORSIKACONFIG DATDIR $CORSIKA_DATA_DIR/tmp_corsika_run_files +# CORSIKACONFIG RUNNR RUN_NUMBER +# CORSIKACONFIG EVTNR 1 +# CORSIKACONFIG NSHOW $N_SHOWER +# CORSIKACONFIG CSCAT 5 $CORE_SCATTER 0. +# CORSIKACONFIG PRMPAR 1 +# CORSIKACONFIG ERANGE $ENERGY_MIN 200.E3 +# CORSIKACONFIG ESLOPE -1.5 +# CORSIKACONFIG THETAP $ZENITH $ZENITH +# CORSIKACONFIG PHIP 0. 360. +# CORSIKACONFIG SEED S1 0 0 +# CORSIKACONFIG SEED S2 0 0 +# CORSIKACONFIG SEED S3 0 0 +# CORSIKACONFIG SEED S4 0 0 +# CORSIKACONFIG ATMOD 1 +# CORSIKACONFIG MAGNET 25.2 40.88 +# CORSIKACONFIG ARRANG 10.4 +# CORSIKACONFIG ELMFLG F T +# CORSIKACONFIG RADNKG 200.E2 +# CORSIKACONFIG FIXCHI 0. +# CORSIKACONFIG HADFLG 0 0 0 0 0 2 +# CORSIKACONFIG QGSJET T 0 +# CORSIKACONFIG QGSSIG T +# CORSIKACONFIG HILOW 100. +# CORSIKACONFIG ECUTS 0.30 0.05 0.02 0.02 +# CORSIKACONFIG MUADDI F +# CORSIKACONFIG MUMULT T +# CORSIKACONFIG LONGI T 20. F F +# CORSIKACONFIG MAXPRT 50 +# CORSIKACONFIG PAROUT F F +# CORSIKACONFIG ECTMAP 1.E5 +# CORSIKACONFIG DEBUG F 6 F 1000000 +# CORSIKACONFIG DIRECT $CORSIKA_DATA_DIR +# CORSIKACONFIG USER user +# CORSIKACONFIG ATMOSPHERE $ATMOSPHERE T +# CORSIKACONFIG OBSLEV 1270.E2 +# CORSIKACONFIG TELESCOPE -23.7E2 37.6E2 0.E2 7.E2 +# CORSIKACONFIG TELESCOPE -47.7E2 -44.1E2 4.4E2 7.E2 +# CORSIKACONFIG TELESCOPE 60.1E2 -29.4E2 9.8E2 7.E2 +# CORSIKACONFIG TELESCOPE 11.3E2 35.9E2 7.E2 7.E2 +# CORSIKACONFIG TELESCOPE -8.61E2 -135.48E2 12.23E2 7.E2 +# CORSIKACONFIG TELFIL $CORSIKA_DATA_DIR/DATRUN_NUMBER.telescope +# CORSIKACONFIG CERFIL 0 +# CORSIKACONFIG CERSIZ 5. +# CORSIKACONFIG CWAVLG 250. 700. +# CORSIKACONFIG EXIT +EOF +} >> ${OFILE} - S1=$((S1 + 2)) - S2=$((S1 + 2)) - S3=$((S2 + 2)) - S4=$((S3 + 2)) +echo "input_file=$(dirname "$OFILE")/input_\${RUN_NUMBER}.dat" >> $OFILE - echo "DATDIR $CORSIKA_DATA_DIR/tmp_corsika_run_files" > "$INPUT" - sed -e "s|run_number|$run_number|" \ - -e "s|number_of_showers|$N_SHOWER|" \ - -e "s|core_scatter_area|$CORE_SCATTER|" \ - -e "s|energy_min|$ENERGY_MIN|" \ - -e "s|atmosphere_id|$ATMOSPHERE|" \ - -e "s|zenith_angle|$ZENITH|g" \ - -e "s|output_directory|$CORSIKA_DATA_DIR|" \ - -e "s|seed_1|$S1|" \ - -e "s|seed_2|$S2|" \ - -e "s|seed_3|$S3|" \ - -e "s|seed_4|$S4|" \ - "$INPUT_TEMPLATE" >> "$INPUT" +echo "rm -f \$input_file" >> $OFILE +echo "sed -n '/DATDIR/,/EXIT/{/DATDIR/!{/EXIT/!s/# CORSIKACONFIG //p}}' "\$0" > \$input_file" >> $OFILE +echo "sed -i \"s/RUN_NUMBER/\$RUN_NUMBER/\" \$input_file" >> $OFILE +echo "sed -i \"s/S1/\$S1/\" \$input_file" >> $OFILE +echo "sed -i \"s/S2/\$S2/\" \$input_file" >> $OFILE +echo "sed -i \"s/S3/\$S3/\" \$input_file" >> $OFILE +echo "sed -i \"s/S4/\$S4/\" \$input_file" >> $OFILE - echo "$S4" +} + +generate_corsika_submission_script() +{ + FSCRIPT="$1" + OUTPUT_DIR="$2" + CONTAINER_EXTERNAL_DIR="$3" + N_SHOWER="$4" + ZENITH="$5" + ATMOSPHERE="$6" + CORSIKA_DATA_DIR="${7}" + + INPUT="/workdir/external/log/input_\${RUN_NUMBER}.dat" + OUTPUT="/workdir/external/data/DAT\${RUN_NUMBER}" + + echo "#!/bin/bash" > "$FSCRIPT.sh" + echo "RUN_NUMBER=\$1" >> "$FSCRIPT.sh" + + generate_corsika_input_card $FSCRIPT.sh "$N_SHOWER" "$ZENITH" "$ATMOSPHERE" "$CORSIKA_DATA_DIR" + echo "rm -f ${OUTPUT_DIR}/DAT\${RUN_NUMBER}.log" >> "$FSCRIPT.sh" + echo "rm -f ${OUTPUT_DIR}/DAT\${RUN_NUMBER}.telescope" >> "$FSCRIPT.sh" + if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then + CORSIKA_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CORSIKA_IMAGE}" + elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then + CORSIKA_EXE="apptainer exec --cleanenv --no-mount bind-paths ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_CORSIKA_IMAGE/:/_}.sif" + fi + CORSIKA_EXE="${CORSIKA_EXE} bash -c \"cd /workdir/external/data/tmp_corsika_run_files && /workdir/corsika-run/corsika77500Linux_QGSII_urqmd < $INPUT\"" + echo "$CORSIKA_EXE > ${OUTPUT_DIR}/DAT\${RUN_NUMBER}.log" >> "$FSCRIPT.sh" + chmod u+x "$FSCRIPT.sh" } diff --git a/scripts/prepare_DAG_jobs.sh b/scripts/prepare_DAG_jobs.sh index 330189b..868cdcc 100755 --- a/scripts/prepare_DAG_jobs.sh +++ b/scripts/prepare_DAG_jobs.sh @@ -45,12 +45,14 @@ for ID in $(seq 0 "$N_RUNS"); do touch "$DAG_FILE" # CORSIKA - job_corsika="$VTSSIMPIPE_LOG_DIR"/"$DIRSUFF"/CORSIKA/run_CORSIKA_${run_number}.sh.condor + job_corsika="$VTSSIMPIPE_LOG_DIR"/"$DIRSUFF"/CORSIKA/run_CORSIKA.sh.condor { echo "# DAG file for run $run_number" echo "JOB CORSIKA_${run_number} $job_corsika" + echo "VARS CORSIKA_${run_number} run_number=\"${run_number}\"" } >> "$DAG_FILE" PARENT_CORSIKA="PARENT CORSIKA_${run_number} CHILD" + continue # CLEANUP PARENT_CLEANUP="PARENT" diff --git a/scripts/prepare_production.sh b/scripts/prepare_production.sh index 248148c..fb9ce59 100755 --- a/scripts/prepare_production.sh +++ b/scripts/prepare_production.sh @@ -1,26 +1,24 @@ #!/bin/bash -# Generate CORSIKA input files and submission scripts +# Generate simulation submission scripts # set -e -echo "Generate simulation input files and submission scripts." +echo "Generate simulation submission scripts." echo if [ $# -lt 2 ]; then -echo "./prepare_production.sh [input file template] +echo "./prepare_production.sh Allowed simulation steps: CORSIKA, GROPTICS, CARE, MERGEVBF, CLEANUP CORSIKA: - template configuration file, see ./config/config_ATM61_template.dat -- input file template, see ./config/CORSIKA/input_template.dat " exit fi SIM_TYPE="$1" CONFIG="$2" -[[ "$3" ]] && INPUT_TEMPLATE=$3 || INPUT_TEMPLATE="" if [[ ! -e "$CONFIG" ]]; then echo "Configuration file $CONFIG does not exist." @@ -52,16 +50,12 @@ echo "VTSSIMPIPE_CORSIKA_IMAGE: $VTSSIMPIPE_CORSIKA_IMAGE" echo "VTSSIMPIPE_GROPTICS_IMAGE: $VTSSIMPIPE_GROPTICS_IMAGE" echo "VTSSIMPIPE_CARE_IMAGE: $VTSSIMPIPE_CARE_IMAGE" -echo "Generating for $SIM_TYPE $N_RUNS input files and submission scripts (starting from run number $RUN_START)." +echo "Generating for $SIM_TYPE $N_RUNS submission scripts (starting from run number $RUN_START)." echo "Number of showers per run: $N_SHOWER" echo "Atmosphere: $ATMOSPHERE" echo "Zenith angle: $ZENITH deg" echo "Wobble angle: $WOBBLE_LIST deg" echo "NSB rate: $NSB_LIST MHz" -if [[ $SIM_TYPE == "CORSIKA" ]]; then - S1=$((RANDOM % 900000000 - 1)) - echo "First CORSIKA seed: $S1" -fi # directories DIRSUFF="ATM${ATMOSPHERE}/Zd${ZENITH}" @@ -79,6 +73,7 @@ generate_htcondor_file() cat > "${SUBFIL}" < Date: Mon, 12 Feb 2024 18:48:24 +0100 Subject: [PATCH 03/10] improved job submission --- scripts/care.sh | 50 +++++++------ scripts/cleanup.sh | 21 +++--- scripts/groptics.sh | 93 +++++++++++++++++-------- scripts/prepare_DAG_jobs.sh | 8 ++- scripts/prepare_all_production_steps.sh | 9 ++- scripts/prepare_production.sh | 55 +++++---------- 6 files changed, 125 insertions(+), 111 deletions(-) diff --git a/scripts/care.sh b/scripts/care.sh index 45faecc..7b75ba7 100644 --- a/scripts/care.sh +++ b/scripts/care.sh @@ -6,11 +6,9 @@ prepare_care_containers() { DATA_DIR="$1" - WOBBLE="$2" - NSB="$3" - OBS_MODE="$4" + OBS_MODE="$2" - CARE_DATA_DIR="${DATA_DIR}/CARE_${OBS_MODE}/W${WOBBLE}/NSB${NSB}" + CARE_DATA_DIR="${DATA_DIR}/CARE_${OBS_MODE}" TMP_CONFIG_DIR="${CARE_DATA_DIR}/model_files/" mkdir -p "$TMP_CONFIG_DIR" @@ -35,22 +33,11 @@ generate_care_submission_script() LOG_DIR=$(dirname "$CAREFSCRIPT") OUTPUT_FILE="$2" rm -f "$OUTPUT_FILE.care.log" - WOBBLE="$3" - NSB="$4" - CARE_CONFIG="$5" - OBS_MODE="$6" - - # mount directories - GROPTICS_DATA_DIR="${DATA_DIR}/GROPTICS/W${WOBBLE}" - CARE_DATA_DIR="${DATA_DIR}/CARE_${OBS_MODE}/W${WOBBLE}/NSB${NSB}" - TMP_CONFIG_DIR="${CARE_DATA_DIR}/model_files/" - CONTAINER_EXTERNAL_DIR="-v \"${GROPTICS_DATA_DIR}:/workdir/external/groptics\"" - CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"${CARE_DATA_DIR}:/workdir/external/care\"" - CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"${TMP_CONFIG_DIR}:/workdir/CARE/data\"" - CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"$LOG_DIR:/workdir/external/log/\"" + CARE_CONFIG="$3" + OBS_MODE="$4" CARE="./CameraAndReadout \ - NSBRATEPERPIXEL \\\"0 0 ${NSB}\\\" \ + NSBRATEPERPIXEL \\\"0 0 \${NSB_LEVEL}\\\" \ HIGHGAINPULSESHAPE \\\"0 /workdir/CARE/data/${CARE_HIGH_GAIN}\\\" \ LOWGAINPULSESHAPE \\\"0 /workdir/CARE/data/${CARE_LOW_GAIN}\\\" \ SIMULATORNAME \\\"$(echo $LOGNAME)\\\" \ @@ -61,16 +48,35 @@ generate_care_submission_script() --vbfrunnumber 10000 \ --writepedestals 1 \ --configfile /workdir/CARE/data/${CARE_CONFIG} \ - --outputfile /workdir/external/care/$(basename "$OUTPUT_FILE") \ - --inputfile /workdir/external/groptics/$(basename "$OUTPUT_FILE").groptics.root" + --outputfile /workdir/external/care/DAT\${RUN_NUMBER} \ + --inputfile /workdir/external/groptics/DAT\${RUN_NUMBER}.groptics.root" echo "#!/bin/bash" > "$CAREFSCRIPT.sh" + echo "RUN_NUMBER=\$1" >> "$CAREFSCRIPT.sh" + echo "WOBBLE_OFFSET=\$2" >> "$CAREFSCRIPT.sh" + echo "NSB_LEVEL=\$3" >> "$CAREFSCRIPT.sh" + echo "" >> "$CAREFSCRIPT.sh" + echo "mkdir -p ${DATA_DIR}/CARE_${OBS_MODE}/W\${WOBBLE_OFFSET}/NSB\${NSB_LEVEL}" >> "$CAREFSCRIPT.sh" + + # mount directories +{ +cat << EOF +GROPTICS_DATA_DIR="${DATA_DIR}/GROPTICS/W\${WOBBLE_OFFSET}" +CARE_DATA_DIR="${DATA_DIR}/CARE_${OBS_MODE}/W\${WOBBLE_OFFSET}/NSB\${NSB_LEVEL}" +TMP_CONFIG_DIR="\${CARE_DATA_DIR}/model_files/" +CONTAINER_EXTERNAL_DIR="-v \${GROPTICS_DATA_DIR}:/workdir/external/groptics" +CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \${CARE_DATA_DIR}:/workdir/external/care" +CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v ${TMP_CONFIG_DIR}:/workdir/CARE/data" +CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v $LOG_DIR:/workdir/external/log/" +EOF +} >> "$CAREFSCRIPT.sh" + if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then CARE_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CARE_IMAGE}" elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then - CARE_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_CARE_IMAGE/:/_}.sif" + CARE_EXE="apptainer exec --cleanenv --no-mount bind-paths ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_CARE_IMAGE/:/_}.sif" fi CARE_EXE="${CARE_EXE} bash -c \"cd /workdir/CARE && ${CARE}\"" - echo "$CARE_EXE > $CARE_DATA_DIR/$(basename "$OUTPUT_FILE").care.log 2>&1" >> "$CAREFSCRIPT.sh" + echo "$CARE_EXE > \${CARE_DATA_DIR}/DAT\${RUN_NUMBER}.care.log 2>&1" >> "$CAREFSCRIPT.sh" chmod u+x "$CAREFSCRIPT.sh" } diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index 8ec3874..fc96e4a 100644 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -19,23 +19,20 @@ generate_cleanup_submission_script() { CLEANUPSCRIPT="$1" - OUTPUT_FILE="$2" - WOFF_LIST="$4" + OUTPUT_DIR="$2" CLEANUP_DATA_DIR="${DATA_DIR}/CLEANUP/" mkdir -p "${CLEANUP_DATA_DIR}" - rm -f "$CLEANUP_DATA_DIR/$(basename "$OUTPUT_FILE").cleanup.log" - # CORSIKA - files are bzipped2 CORSIKA_DATA_DIR="${DATA_DIR}/CORSIKA" echo "#!/bin/bash" > "$CLEANUPSCRIPT.sh" - for WOBBLE in ${WOFF_LIST}; do - # GROPTICS - files are removed - GROPTICS_DATA_DIR="${DATA_DIR}/GROPTICS/W${WOBBLE}" - CLEANUP_GROPTICS="rm -f -v ${GROPTICS_DATA_DIR}/$(basename "$OUTPUT_FILE").groptics.root" - echo "$CLEANUP_GROPTICS > $CLEANUP_DATA_DIR/$(basename "$OUTPUT_FILE").cleanup.log 2>&1" >> "$CLEANUPSCRIPT.sh" - done - echo "bzip2 -f -v ${CORSIKA_DATA_DIR}/$(basename "$OUTPUT_FILE").telescope >> $CLEANUP_DATA_DIR/$(basename "$OUTPUT_FILE").cleanup.log 2>&1" >> "$CLEANUPSCRIPT.sh" - echo "bzip2 -f -v ${CORSIKA_DATA_DIR}/$(basename "$OUTPUT_FILE").log >> $CLEANUP_DATA_DIR/$(basename "$OUTPUT_FILE").cleanup.log 2>&1" >> "$CLEANUPSCRIPT.sh" + echo "RUN_NUMBER=\$1" >> "$CLEANUPSCRIPT.sh" + echo "rm -f \"$CLEANUP_DATA_DIR/DAT\${RUN_NUMBER}.cleanup.log\"" >> "$$CLEANUPSCRIPT.sh" + # GROPTICS - files are removed + # TODO - not removed yet + CLEANUP_GROPTICS="find ${DATA_DIR}/GROPTICS/ -name \"DAT\${RUN_NUMBER}*.groptics.root\" -print" >> "$CLEANUPSCRIPT.sh" + echo "$CLEANUP_GROPTICS > $CLEANUP_DATA_DIR/DAT\${RUN_NUMBER}.cleanup.log 2>&1" >> "$CLEANUPSCRIPT.sh" + echo "bzip2 -f -v ${CORSIKA_DATA_DIR}/DAT\${RUN_NUMBER}.telescope >> $CLEANUP_DATA_DIR/DAT\${RUN_NUMBER}.cleanup.log 2>&1" >> "$CLEANUPSCRIPT.sh" + echo "bzip2 -f -v ${CORSIKA_DATA_DIR}/DAT\${RUN_NUMBER}.log >> $CLEANUP_DATA_DIR/DAT\${RUN_NUMBER}.cleanup.log 2>&1" >> "$CLEANUPSCRIPT.sh" chmod u+x "$CLEANUPSCRIPT.sh" } diff --git a/scripts/groptics.sh b/scripts/groptics.sh index a4616d4..7269318 100644 --- a/scripts/groptics.sh +++ b/scripts/groptics.sh @@ -61,24 +61,41 @@ get_wobble() # SEED: seed for the random number generator (0=machine clock) generate_groptics_pilot_file() { - LOG_DIR="$1" - RUN_NUMBER="$2" - WOBBLE="$3" - - PILOT="${LOG_DIR}/pilot_${RUN_NUMBER}_W${WOBBLE}.dat" - rm -f "$PILOT" - touch "$PILOT" + OFILE="$1" { - # (dummy file name; groptics is executed with the "-of" flag) - echo "* FILEOUT photonLocation.root allT T 0" - echo "* NSHOWER -1 -1" - echo "* WOBBLE $(get_wobble "${RUN_NUMBER}" "${WOBBLE}") 0.0 0.0 90." - echo "* ARRAYCONFIG ./data/${GROPTICS_CONFIG}" - echo "* SEED 0" - echo "* DEBUGBRANCHES 1" - } >> "$PILOT" - echo "/workdir/external/log/$(basename "$PILOT")" + cat << EOF +get_wobble() +{ + RUN_NUMBER="\$1" + WOBBLE="\$2" + if [[ \$((RUN_NUMBER % 4)) == 0 ]]; then + # North + echo "0.0 \${WOBBLE}" + elif [[ \$((RUN_NUMBER % 4)) == 1 ]]; then + # East + echo "\${WOBBLE} 0.0" + elif [[ \$((RUN_NUMBER % 4)) == 2 ]]; then + # South + echo "0.0 -\${WOBBLE}" + else + # West + echo "-\${WOBBLE} 0.0" + fi +} + +WOBBLE_STRING="\$(get_wobble "\${RUN_NUMBER}" "\${WOBBLE_OFFSET}") 0.0 0.0 90." + +# (dummy file name; groptics is executed with the "-of" flag) +# GROPTICSCONFIG * FILEOUT photonLocation.root allT T 0 +# GROPTICSCONFIG * NSHOWER -1 -1 +# GROPTICSCONFIG * WOBBLE WOBBLE_STRING +# GROPTICSCONFIG * ARRAYCONFIG ./data/${GROPTICS_CONFIG} +# GROPTICSCONFIG * SEED 0 +# GROPTICSCONFIG * DEBUGBRANCHES 1 +EOF +} >> "$OFILE" + } ##################################################################### @@ -125,30 +142,46 @@ generate_groptics_submission_script() WOBBLE="$4" rm -f "$OUTPUT_FILE.groptics.log" - GROPTICS_DATA_DIR="${DATA_DIR}/GROPTICS/W${WOBBLE}" - TMP_CONFIG_DIR="${GROPTICS_DATA_DIR}/model_files/" - # mount directories + OUTPUT="/workdir/external/data/DAT\${RUN_NUMBER}" + CORSIKA_DATA_DIR="${DATA_DIR}/CORSIKA" - CONTAINER_EXTERNAL_DIR="-v \"${CORSIKA_DATA_DIR}:/workdir/external/corsika\"" - CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"$GROPTICS_DATA_DIR:/workdir/external/groptics\"" CORSIKA_FILE="${CORSIKA_DATA_DIR}/$(basename "$OUTPUT_FILE").telescope" CORSIKA_FILE="/workdir/external/corsika/$(basename "$CORSIKA_FILE")" - # corsikaIOreader expects the atmprof file in the /workdir/external/groptics/data directory - # groptics expect cfg files in the /workdir/external/groptics/data directory - CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"${TMP_CONFIG_DIR}:/workdir/GrOptics/data\"" - CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"$LOG_DIR:/workdir/external/log/\"" - CORSIKA_IO_READER=$(prepare_corsikaIOreader) - GROPTICS="./grOptics -of /workdir/external/groptics/$(basename "$OUTPUT_FILE").groptics.root \ - -p $(generate_groptics_pilot_file "$LOG_DIR" "$RUN_NUMBER" "$WOBBLE")" +# GROPTICS="./grOptics -of /workdir/external/groptics/$(basename "$OUTPUT_FILE").groptics.root \ +# -p $(generate_groptics_pilot_file "$LOG_DIR" "$RUN_NUMBER" "$WOBBLE")" echo "#!/bin/bash" > "$GROPTICSFSCRIPT.sh" + echo "RUN_NUMBER=\$1" >> "$GROPTICSFSCRIPT.sh" + echo "WOBBLE_OFFSET=\$2" >> "$GROPTICSFSCRIPT.sh" + +{ +cat << EOF +GROPTICS_DATA_DIR="${DATA_DIR}/GROPTICS/W\${WOBBLE_OFFSET}" +TMP_CONFIG_DIR="\${GROPTICS_DATA_DIR}/model_files/" +# mount directories +CONTAINER_EXTERNAL_DIR="-v \"${CORSIKA_DATA_DIR}:/workdir/external/corsika\"" +CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"\$GROPTICS_DATA_DIR:/workdir/external/groptics\"" +# corsikaIOreader expects the atmprof file in the /workdir/external/groptics/data directory +# groptics expect cfg files in the /workdir/external/groptics/data directory +CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"\${TMP_CONFIG_DIR}:/workdir/GrOptics/data\"" +CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"$LOG_DIR:/workdir/external/log/\"" +EOF +} >> "$GROPTICSFSCRIPT.sh" + + echo "PILOTFILE=\"/workdir/external/log/pilot_\${RUN_NUMBER}_W\${WOBBLE_OFFSET}.dat\"" >> "$GROPTICSFSCRIPT.sh" + echo "rm -f \"\$(dirname \$0)/pilot_\${RUN_NUMBER}_W\${WOBBLE_OFFSET}.dat\"" >> "$GROPTICSFSCRIPT.sh" + generate_groptics_pilot_file "$GROPTICSFSCRIPT.sh" "$PILOTFILE" + echo "PILOT=\" \$(dirname \$0)/pilot_\${RUN_NUMBER}_W\${WOBBLE_OFFSET}.dat\"" >> "$GROPTICSFSCRIPT.sh" + echo "sed -n '/* FILEOUT/,/* DEBUGBRANCHES/{/*FILEOUT/!{/*DEBUGBRANCHES /!s/# GROPTICSCONFIG//p}}' "\$0" > \$PILOT" >> "$GROPTICSFSCRIPT.sh" + echo "sed -i \"s/WOBBLE_STRING/\$WOBBLE_STRING/\" \$PILOT" >> "$GROPTICSFSCRIPT.sh" + if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then GROPTICS_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_GROPTICS_IMAGE}" elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then - GROPTICS_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_GROPTICS_IMAGE/:/_}.sif" + GROPTICS_EXE="apptainer exec --cleanenv --no-mount bind-paths ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_GROPTICS_IMAGE/:/_}.sif" fi GROPTICS_EXE="${GROPTICS_EXE} bash -c \"cd /workdir/GrOptics && ${CORSIKA_IO_READER} | ${GROPTICS}\"" - echo "$GROPTICS_EXE > $GROPTICS_DATA_DIR/$(basename "$OUTPUT_FILE").groptics.log 2>&1" >> "$GROPTICSFSCRIPT.sh" + echo "$GROPTICS_EXE > \$GROPTICS_DATA_DIR/DAT\${RUN_NUMBER}.groptics.log 2>&1" >> "$GROPTICSFSCRIPT.sh" chmod u+x "$GROPTICSFSCRIPT.sh" } diff --git a/scripts/prepare_DAG_jobs.sh b/scripts/prepare_DAG_jobs.sh index 868cdcc..1da9d8e 100755 --- a/scripts/prepare_DAG_jobs.sh +++ b/scripts/prepare_DAG_jobs.sh @@ -52,17 +52,18 @@ for ID in $(seq 0 "$N_RUNS"); do echo "VARS CORSIKA_${run_number} run_number=\"${run_number}\"" } >> "$DAG_FILE" PARENT_CORSIKA="PARENT CORSIKA_${run_number} CHILD" - continue # CLEANUP PARENT_CLEANUP="PARENT" - job_cleanup="$VTSSIMPIPE_LOG_DIR"/"$DIRSUFF"/CLEANUP/run_CLEANUP_${run_number}.sh.condor + job_cleanup="$VTSSIMPIPE_LOG_DIR"/"$DIRSUFF"/CLEANUP/run_CLEANUP.sh.condor echo "JOB CLEANUP_${run_number} $job_cleanup" >> "$DAG_FILE" + echo "VARS CLEANUP_${run_number} run_number=\"${run_number}\"" >> "$DAG_FILE" # GROPTICS and CARE for WOBBLE in ${WOBBLE_LIST}; do - job_groptics="$VTSSIMPIPE_LOG_DIR"/"$DIRSUFF"/GROPTICS/run_GROPTICS_${run_number}_${WOBBLE}.sh.condor + job_groptics="$VTSSIMPIPE_LOG_DIR"/"$DIRSUFF"/GROPTICS/run_GROPTICS.sh.condor echo "JOB GROPTICS_${run_number}_${WOBBLE} $job_groptics" >> "$DAG_FILE" + echo "VARS GROPTICS_${run_number}_${WOBBLE} run_number=\"${run_number}\" wobble_offset=\"${WOBBLE}\"" >> "$DAG_FILE" PARENT_CORSIKA="$PARENT_CORSIKA GROPTICS_${run_number}_${WOBBLE}" PARENT_GROPTICS="PARENT GROPTICS_${run_number}_${WOBBLE} CHILD" for config in $(get_care_configs); do @@ -70,6 +71,7 @@ for ID in $(seq 0 "$N_RUNS"); do for NSB in ${!care_nsb_list}; do job_care="$VTSSIMPIPE_LOG_DIR"/"$DIRSUFF"/CARE/run_CARE_${run_number}_${config}_${WOBBLE}_${NSB}.sh.condor echo "JOB CARE_${run_number}_${config}_${WOBBLE}_${NSB} $job_care" >> "$DAG_FILE" + echo "VARS CARE_${run_number}_${config}_${WOBBLE}_${NSB} run_number=\"${run_number}\" wobble_offset=\"${WOBBLE}\" nsb_level=\"${NSB}\"" >> "$DAG_FILE" PARENT_GROPTICS="$PARENT_GROPTICS CARE_${run_number}_${config}_${WOBBLE}_${NSB}" PARENT_CLEANUP="$PARENT_CLEANUP CARE_${run_number}_${config}_${WOBBLE}_${NSB}" done diff --git a/scripts/prepare_all_production_steps.sh b/scripts/prepare_all_production_steps.sh index 5f022bc..746878d 100755 --- a/scripts/prepare_all_production_steps.sh +++ b/scripts/prepare_all_production_steps.sh @@ -1,14 +1,13 @@ #!/bin/bash # Prepare all simulation production steps -if [ $# -lt 2 ]; then - echo "./prepare_all_production_steps.sh [remove previous file (Dangerous!)" +if [ $# -lt 1 ]; then + echo "./prepare_all_production_steps.sh [remove previous file (Dangerous!)" exit fi CONFIG="$1" -INPUT="$2" -CLEAN_ALL_FILES="$3" +CLEAN_ALL_FILES="$2" # shellcheck source=/dev/null . "$(dirname "$0")"/../env_setup.sh @@ -25,5 +24,5 @@ for step in CORSIKA GROPTICS CARE MERGEVBF CLEANUP; do rm -rf "${VTSSIMPIPE_DATA_DIR:?}"/"$DIRSUFF"/"${step}_redHV" fi echo "prepare $step" - ./prepare_production.sh $step "${CONFIG}" "${INPUT}" + ./prepare_production.sh $step "${CONFIG}" done diff --git a/scripts/prepare_production.sh b/scripts/prepare_production.sh index fb9ce59..9be2a56 100755 --- a/scripts/prepare_production.sh +++ b/scripts/prepare_production.sh @@ -73,7 +73,7 @@ generate_htcondor_file() cat > "${SUBFIL}" < Date: Mon, 12 Feb 2024 19:06:34 +0100 Subject: [PATCH 04/10] fixes --- scripts/cleanup.sh | 2 +- scripts/groptics.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index fc96e4a..60d45a6 100644 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -27,7 +27,7 @@ generate_cleanup_submission_script() echo "#!/bin/bash" > "$CLEANUPSCRIPT.sh" echo "RUN_NUMBER=\$1" >> "$CLEANUPSCRIPT.sh" - echo "rm -f \"$CLEANUP_DATA_DIR/DAT\${RUN_NUMBER}.cleanup.log\"" >> "$$CLEANUPSCRIPT.sh" + echo "rm -f \"$CLEANUP_DATA_DIR/DAT\${RUN_NUMBER}.cleanup.log\"" >> "$CLEANUPSCRIPT.sh" # GROPTICS - files are removed # TODO - not removed yet CLEANUP_GROPTICS="find ${DATA_DIR}/GROPTICS/ -name \"DAT\${RUN_NUMBER}*.groptics.root\" -print" >> "$CLEANUPSCRIPT.sh" diff --git a/scripts/groptics.sh b/scripts/groptics.sh index 7269318..0cfe821 100644 --- a/scripts/groptics.sh +++ b/scripts/groptics.sh @@ -173,7 +173,7 @@ EOF echo "rm -f \"\$(dirname \$0)/pilot_\${RUN_NUMBER}_W\${WOBBLE_OFFSET}.dat\"" >> "$GROPTICSFSCRIPT.sh" generate_groptics_pilot_file "$GROPTICSFSCRIPT.sh" "$PILOTFILE" echo "PILOT=\" \$(dirname \$0)/pilot_\${RUN_NUMBER}_W\${WOBBLE_OFFSET}.dat\"" >> "$GROPTICSFSCRIPT.sh" - echo "sed -n '/* FILEOUT/,/* DEBUGBRANCHES/{/*FILEOUT/!{/*DEBUGBRANCHES /!s/# GROPTICSCONFIG//p}}' "\$0" > \$PILOT" >> "$GROPTICSFSCRIPT.sh" + echo "sed -n '/* FILEOUT/,/* DEBUGBRANCHES/{/*FILEOUT/!{/*DEBUGBRANCHES /!s/# GROPTICSCONFIG //p}}' "\$0" > \$PILOT" >> "$GROPTICSFSCRIPT.sh" echo "sed -i \"s/WOBBLE_STRING/\$WOBBLE_STRING/\" \$PILOT" >> "$GROPTICSFSCRIPT.sh" if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then @@ -181,6 +181,7 @@ EOF elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then GROPTICS_EXE="apptainer exec --cleanenv --no-mount bind-paths ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_GROPTICS_IMAGE/:/_}.sif" fi + GROPTICS="./grOptics -of /workdir/external/groptics/DAT\${RUN_NUMBER}.groptics.root -p /workdir/external/log/pilot_\${RUN_NUMBER}_W\${WOBBLE_OFFSET}.dat" GROPTICS_EXE="${GROPTICS_EXE} bash -c \"cd /workdir/GrOptics && ${CORSIKA_IO_READER} | ${GROPTICS}\"" echo "$GROPTICS_EXE > \$GROPTICS_DATA_DIR/DAT\${RUN_NUMBER}.groptics.log 2>&1" >> "$GROPTICSFSCRIPT.sh" chmod u+x "$GROPTICSFSCRIPT.sh" From 896c530800365042d5606f1cb27c8d7de4dca6a8 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 12 Feb 2024 21:40:11 +0100 Subject: [PATCH 05/10] Remove duplicated code --- scripts/groptics.sh | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/scripts/groptics.sh b/scripts/groptics.sh index 0cfe821..48c1ef5 100644 --- a/scripts/groptics.sh +++ b/scripts/groptics.sh @@ -28,29 +28,6 @@ prepare_corsikaIOreader() echo "$CORSIKA_IO_READER" } - -##################################################################### -# get wobble direction (use run number) -get_wobble() -{ - RUN_NUMBER="$1" - WOBBLE="$2" - if [[ $((RUN_NUMBER % 4)) == 0 ]]; then - # North - echo "0.0 ${WOBBLE}" - elif [[ $((RUN_NUMBER % 4)) == 1 ]]; then - # East - echo "${WOBBLE} 0.0" - elif [[ $((RUN_NUMBER % 4)) == 2 ]]; then - # South - echo "0.0 -${WOBBLE}" - else - # West - echo "-${WOBBLE} 0.0" - fi -} - - ##################################################################### # generate GrOptics pilot file # From 5161e58edfe4eba3830d52f64f8717210ebbe07d Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Tue, 13 Feb 2024 09:26:44 +0100 Subject: [PATCH 06/10] path fixes --- scripts/care.sh | 2 +- scripts/groptics.sh | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/care.sh b/scripts/care.sh index 7b75ba7..a445710 100644 --- a/scripts/care.sh +++ b/scripts/care.sh @@ -74,7 +74,7 @@ EOF if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then CARE_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_CARE_IMAGE}" elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then - CARE_EXE="apptainer exec --cleanenv --no-mount bind-paths ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_CARE_IMAGE/:/_}.sif" + CARE_EXE="apptainer exec --cleanenv --no-mount bind-paths \${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_CARE_IMAGE/:/_}.sif" fi CARE_EXE="${CARE_EXE} bash -c \"cd /workdir/CARE && ${CARE}\"" echo "$CARE_EXE > \${CARE_DATA_DIR}/DAT\${RUN_NUMBER}.care.log 2>&1" >> "$CAREFSCRIPT.sh" diff --git a/scripts/groptics.sh b/scripts/groptics.sh index 48c1ef5..548ac52 100644 --- a/scripts/groptics.sh +++ b/scripts/groptics.sh @@ -137,12 +137,12 @@ cat << EOF GROPTICS_DATA_DIR="${DATA_DIR}/GROPTICS/W\${WOBBLE_OFFSET}" TMP_CONFIG_DIR="\${GROPTICS_DATA_DIR}/model_files/" # mount directories -CONTAINER_EXTERNAL_DIR="-v \"${CORSIKA_DATA_DIR}:/workdir/external/corsika\"" -CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"\$GROPTICS_DATA_DIR:/workdir/external/groptics\"" +CONTAINER_EXTERNAL_DIR="-v ${CORSIKA_DATA_DIR}:/workdir/external/corsika" +CONTAINER_EXTERNAL_DIR="\$CONTAINER_EXTERNAL_DIR -v \$GROPTICS_DATA_DIR:/workdir/external/groptics" # corsikaIOreader expects the atmprof file in the /workdir/external/groptics/data directory # groptics expect cfg files in the /workdir/external/groptics/data directory -CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"\${TMP_CONFIG_DIR}:/workdir/GrOptics/data\"" -CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"$LOG_DIR:/workdir/external/log/\"" +CONTAINER_EXTERNAL_DIR="\$CONTAINER_EXTERNAL_DIR -v \${TMP_CONFIG_DIR}:/workdir/GrOptics/data" +CONTAINER_EXTERNAL_DIR="\$CONTAINER_EXTERNAL_DIR -v $LOG_DIR:/workdir/external/log/" EOF } >> "$GROPTICSFSCRIPT.sh" @@ -156,7 +156,7 @@ EOF if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then GROPTICS_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_GROPTICS_IMAGE}" elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then - GROPTICS_EXE="apptainer exec --cleanenv --no-mount bind-paths ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_GROPTICS_IMAGE/:/_}.sif" + GROPTICS_EXE="apptainer exec --cleanenv --no-mount bind-paths \${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_GROPTICS_IMAGE/:/_}.sif" fi GROPTICS="./grOptics -of /workdir/external/groptics/DAT\${RUN_NUMBER}.groptics.root -p /workdir/external/log/pilot_\${RUN_NUMBER}_W\${WOBBLE_OFFSET}.dat" GROPTICS_EXE="${GROPTICS_EXE} bash -c \"cd /workdir/GrOptics && ${CORSIKA_IO_READER} | ${GROPTICS}\"" From 9999d261f1943498902c55648d19abd3098a0e94 Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Tue, 13 Feb 2024 10:34:14 +0100 Subject: [PATCH 07/10] mounts --- scripts/care.sh | 6 +++--- scripts/groptics.sh | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/care.sh b/scripts/care.sh index a445710..aa00a68 100644 --- a/scripts/care.sh +++ b/scripts/care.sh @@ -65,9 +65,9 @@ GROPTICS_DATA_DIR="${DATA_DIR}/GROPTICS/W\${WOBBLE_OFFSET}" CARE_DATA_DIR="${DATA_DIR}/CARE_${OBS_MODE}/W\${WOBBLE_OFFSET}/NSB\${NSB_LEVEL}" TMP_CONFIG_DIR="\${CARE_DATA_DIR}/model_files/" CONTAINER_EXTERNAL_DIR="-v \${GROPTICS_DATA_DIR}:/workdir/external/groptics" -CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \${CARE_DATA_DIR}:/workdir/external/care" -CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v ${TMP_CONFIG_DIR}:/workdir/CARE/data" -CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v $LOG_DIR:/workdir/external/log/" +CONTAINER_EXTERNAL_DIR="\$CONTAINER_EXTERNAL_DIR -v \${CARE_DATA_DIR}:/workdir/external/care" +CONTAINER_EXTERNAL_DIR="\$CONTAINER_EXTERNAL_DIR -v ${TMP_CONFIG_DIR}:/workdir/CARE/data" +CONTAINER_EXTERNAL_DIR="\$CONTAINER_EXTERNAL_DIR -v $LOG_DIR:/workdir/external/log/" EOF } >> "$CAREFSCRIPT.sh" diff --git a/scripts/groptics.sh b/scripts/groptics.sh index 548ac52..4b9efb5 100644 --- a/scripts/groptics.sh +++ b/scripts/groptics.sh @@ -122,8 +122,7 @@ generate_groptics_submission_script() OUTPUT="/workdir/external/data/DAT\${RUN_NUMBER}" CORSIKA_DATA_DIR="${DATA_DIR}/CORSIKA" - CORSIKA_FILE="${CORSIKA_DATA_DIR}/$(basename "$OUTPUT_FILE").telescope" - CORSIKA_FILE="/workdir/external/corsika/$(basename "$CORSIKA_FILE")" + CORSIKA_FILE="/workdir/external/corsika/DAT\${RUN_NUMBER}.telescope" CORSIKA_IO_READER=$(prepare_corsikaIOreader) # GROPTICS="./grOptics -of /workdir/external/groptics/$(basename "$OUTPUT_FILE").groptics.root \ # -p $(generate_groptics_pilot_file "$LOG_DIR" "$RUN_NUMBER" "$WOBBLE")" From f1c6b8be64fbccf960eff7cfe9e19520168ecb49 Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Tue, 13 Feb 2024 15:36:28 +0100 Subject: [PATCH 08/10] correct DAT config --- scripts/prepare_DAG_jobs.sh | 7 ++++++- scripts/prepare_production.sh | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/prepare_DAG_jobs.sh b/scripts/prepare_DAG_jobs.sh index 1da9d8e..6bd4775 100755 --- a/scripts/prepare_DAG_jobs.sh +++ b/scripts/prepare_DAG_jobs.sh @@ -36,6 +36,10 @@ get_care_configs() DAG_DIR="$VTSSIMPIPE_LOG_DIR"/"$DIRSUFF"/DAG mkdir -p "$DAG_DIR" +rm -f "${DAG_DIR}/config.dag.config" +echo "DAGMAN_MAX_JOBS_SUBMITTED 200" > "${DAG_DIR}/config.dag.config" + + for ID in $(seq 0 "$N_RUNS"); do run_number=$((ID + RUN_START)) @@ -69,7 +73,7 @@ for ID in $(seq 0 "$N_RUNS"); do for config in $(get_care_configs); do care_nsb_list="NSB_LIST_$config" for NSB in ${!care_nsb_list}; do - job_care="$VTSSIMPIPE_LOG_DIR"/"$DIRSUFF"/CARE/run_CARE_${run_number}_${config}_${WOBBLE}_${NSB}.sh.condor + job_care="$VTSSIMPIPE_LOG_DIR"/"$DIRSUFF"/CARE/run_CARE_${config}.sh.condor echo "JOB CARE_${run_number}_${config}_${WOBBLE}_${NSB} $job_care" >> "$DAG_FILE" echo "VARS CARE_${run_number}_${config}_${WOBBLE}_${NSB} run_number=\"${run_number}\" wobble_offset=\"${WOBBLE}\" nsb_level=\"${NSB}\"" >> "$DAG_FILE" PARENT_GROPTICS="$PARENT_GROPTICS CARE_${run_number}_${config}_${WOBBLE}_${NSB}" @@ -82,6 +86,7 @@ for ID in $(seq 0 "$N_RUNS"); do echo "$PARENT_CLEANUP" >> "$DAG_FILE" echo "$PARENT_CORSIKA" >> "$DAG_FILE" echo "DOT $DAG_DIR/run_${run_number}.dot" >> "$DAG_FILE" + echo "CONFIG ${DAG_DIR}/config.dag.config" >> "$DAG_FILE" done echo "DAG directory: $DAG_DIR" diff --git a/scripts/prepare_production.sh b/scripts/prepare_production.sh index 9be2a56..eec4dfc 100755 --- a/scripts/prepare_production.sh +++ b/scripts/prepare_production.sh @@ -78,7 +78,7 @@ Log = ${SUBSCRIPT}.\$(Cluster)_\$(Process).log Output = ${SUBSCRIPT}.\$(Cluster)_\$(Process).output Error = ${SUBSCRIPT}.\$(Cluster)_\$(Process).error Log = ${SUBSCRIPT}.\$(Cluster)_\$(Process).log -request_memory = 2000M +request_memory = 4000M getenv = True max_materialize = 250 queue 1 From f45d5b7781a2bb7115ebd86453f49d540d35bca2 Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Tue, 13 Feb 2024 16:06:11 +0100 Subject: [PATCH 09/10] remove configuration --- scripts/prepare_DAG_jobs.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/prepare_DAG_jobs.sh b/scripts/prepare_DAG_jobs.sh index 6bd4775..a8aa379 100755 --- a/scripts/prepare_DAG_jobs.sh +++ b/scripts/prepare_DAG_jobs.sh @@ -36,10 +36,6 @@ get_care_configs() DAG_DIR="$VTSSIMPIPE_LOG_DIR"/"$DIRSUFF"/DAG mkdir -p "$DAG_DIR" -rm -f "${DAG_DIR}/config.dag.config" -echo "DAGMAN_MAX_JOBS_SUBMITTED 200" > "${DAG_DIR}/config.dag.config" - - for ID in $(seq 0 "$N_RUNS"); do run_number=$((ID + RUN_START)) @@ -86,7 +82,6 @@ for ID in $(seq 0 "$N_RUNS"); do echo "$PARENT_CLEANUP" >> "$DAG_FILE" echo "$PARENT_CORSIKA" >> "$DAG_FILE" echo "DOT $DAG_DIR/run_${run_number}.dot" >> "$DAG_FILE" - echo "CONFIG ${DAG_DIR}/config.dag.config" >> "$DAG_FILE" done echo "DAG directory: $DAG_DIR" From e71cf952bc75131568e92eb658e345cdf51ed428 Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Tue, 13 Feb 2024 16:21:46 +0100 Subject: [PATCH 10/10] remove groptics --- scripts/cleanup.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index 60d45a6..bb73956 100644 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -29,8 +29,7 @@ generate_cleanup_submission_script() echo "RUN_NUMBER=\$1" >> "$CLEANUPSCRIPT.sh" echo "rm -f \"$CLEANUP_DATA_DIR/DAT\${RUN_NUMBER}.cleanup.log\"" >> "$CLEANUPSCRIPT.sh" # GROPTICS - files are removed - # TODO - not removed yet - CLEANUP_GROPTICS="find ${DATA_DIR}/GROPTICS/ -name \"DAT\${RUN_NUMBER}*.groptics.root\" -print" >> "$CLEANUPSCRIPT.sh" + CLEANUP_GROPTICS="find ${DATA_DIR}/GROPTICS/ -name \"DAT\${RUN_NUMBER}*.groptics.root\" -print -exec rm -f -v {} \\;" >> "$CLEANUPSCRIPT.sh" echo "$CLEANUP_GROPTICS > $CLEANUP_DATA_DIR/DAT\${RUN_NUMBER}.cleanup.log 2>&1" >> "$CLEANUPSCRIPT.sh" echo "bzip2 -f -v ${CORSIKA_DATA_DIR}/DAT\${RUN_NUMBER}.telescope >> $CLEANUP_DATA_DIR/DAT\${RUN_NUMBER}.cleanup.log 2>&1" >> "$CLEANUPSCRIPT.sh" echo "bzip2 -f -v ${CORSIKA_DATA_DIR}/DAT\${RUN_NUMBER}.log >> $CLEANUP_DATA_DIR/DAT\${RUN_NUMBER}.cleanup.log 2>&1" >> "$CLEANUPSCRIPT.sh"