Skip to content

Commit

Permalink
Adding a (somewhat) generic PBS cluster (using mpirun).
Browse files Browse the repository at this point in the history
  • Loading branch information
anaegel committed Feb 23, 2024
1 parent 029c5c1 commit 89e1e6a
Show file tree
Hide file tree
Showing 2 changed files with 115 additions and 0 deletions.
4 changes: 4 additions & 0 deletions scripts/shell/clusters
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@ elif [ $UGSUBMIT_TYPE == "DoraCSCS" ]; then

elif [ $UGSUBMIT_TYPE == "moab" ]; then
source $scriptpath/schedulers/moab

elif [ $UGSUBMIT_TYPE == "PBSGeneric" ]; then
source $scriptpath/schedulers/pbs-generic

############################################################
#
# you can add your own cluster like this:
Expand Down
111 changes: 111 additions & 0 deletions scripts/shell/schedulers/pbs-generic
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
#!/bin/bash
######################################################################
# qsub (w/ mpirun) on GENERIC cluster.
#-------------------------------------------------------------------
######################################################################

function UJS_Submit
{
UJS_GetOptions

if [ $walltime == "unlimited" ]; then
walltime=01:00:00
fi

if [ $((npe%nppn)) -ne 0 ]; then
echo "NOTE: npe=$npe is not dividable by nppn=$nppn. rounding up"
fi


#omplaceOption="omplace -c 0-127:st=4"
omplaceOption=""

echo "Cluster: GENERIC. Scheduler: qsub"
if [ $nppn -gt $nppnmaxSingleThread ]; then
qsubrun="mpirun -np $npe $executable $args"
#qsub doesn't know about hyperthreading. Only specify used procs. Round up.
# nppnRaw=$(((nppn+1)/2))

else
qsubrun="mpirun -np $npe $omplaceOption $executable $args"
# nppnRaw=$((nppn))

fi


# PBSnodes="-l select=$nnodes:node_type=rome:mpiprocs=$nppn"
PBSnodes=""


if [ -z "$queue" ]; then
qsubqueue=""
else
qsubqueue="-queue $queue"
fi

if [ $exclusive == true ]; then
echo "WARNING: Exclusive not yet supported on this machine."
fi


PBSMail=""
if [ $mail == true ]; then
if [ -z "$UGSUBMIT_EMAIL" ]; then
echo "please set UGSUBMIT_EMAIL or specify email with -email".
exit
fi
PBSMail="-M $UGSUBMIT_EMAIL -m $PBSMailtype"
fi

# interactive: qsub -IX -l walltime=00:50:00 -l mppwidth=4 -l mppnppn=4

qsubargs="$qsubqueue -v UG4_ROOT -N $jobname -o job.output -e job.error -j oe -l walltime=$walltime $PBSnodes $PBSMail"

echo "echo \"cd $outdir; $qsubrun\" | qsub $qsubargs" >> info.txt

if [ $test == true ]; then
echo "ONLY testing - NOT executing."
return
fi





if [ $interactive == true ]; then
echo "Interactive mode currently not supported on hermit. Aborting."
else
jobid=`echo "cd $outdir; $qsubrun" | qsub $qsubargs`
return=$?
if [ ! $return == 0 ]; then
echo "ERROR: qsub returned $return. Job has NOT been scheduled."
exit
fi
jobid=`echo $jobid | sed 's/\([0-9]*\).*/\1/'`
fi
}



function UJS_GetOptions
{
echo "WARNING: Unknown platform. guessed default values for MaxNumProcsPerNode (nppnmax) and MaxProcs (pemax) may be wrong."
nppnmax=48
pemax=1344
nppnmaxSingleThread=48
}

function UJS_Info
{
echo "Using $UGSUBMIT_TYPE"
echo "qstat -u $USER -r"
qstat -u $USER
}


function UJS_Cancel
{
echo "Using $UGSUBMIT_TYPE"
echo "qdel $1"
qdel $1
}

0 comments on commit 89e1e6a

Please sign in to comment.