-
Notifications
You must be signed in to change notification settings - Fork 23
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Adding a (somewhat) generic PBS cluster (using mpirun).
- Loading branch information
Showing
2 changed files
with
115 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,111 @@ | ||
#!/bin/bash | ||
###################################################################### | ||
# qsub (w/ mpirun) on GENERIC cluster. | ||
#------------------------------------------------------------------- | ||
###################################################################### | ||
|
||
function UJS_Submit | ||
{ | ||
UJS_GetOptions | ||
|
||
if [ $walltime == "unlimited" ]; then | ||
walltime=01:00:00 | ||
fi | ||
|
||
if [ $((npe%nppn)) -ne 0 ]; then | ||
echo "NOTE: npe=$npe is not dividable by nppn=$nppn. rounding up" | ||
fi | ||
|
||
|
||
#omplaceOption="omplace -c 0-127:st=4" | ||
omplaceOption="" | ||
|
||
echo "Cluster: GENERIC. Scheduler: qsub" | ||
if [ $nppn -gt $nppnmaxSingleThread ]; then | ||
qsubrun="mpirun -np $npe $executable $args" | ||
#qsub doesn't know about hyperthreading. Only specify used procs. Round up. | ||
# nppnRaw=$(((nppn+1)/2)) | ||
|
||
else | ||
qsubrun="mpirun -np $npe $omplaceOption $executable $args" | ||
# nppnRaw=$((nppn)) | ||
|
||
fi | ||
|
||
|
||
# PBSnodes="-l select=$nnodes:node_type=rome:mpiprocs=$nppn" | ||
PBSnodes="" | ||
|
||
|
||
if [ -z "$queue" ]; then | ||
qsubqueue="" | ||
else | ||
qsubqueue="-queue $queue" | ||
fi | ||
|
||
if [ $exclusive == true ]; then | ||
echo "WARNING: Exclusive not yet supported on this machine." | ||
fi | ||
|
||
|
||
PBSMail="" | ||
if [ $mail == true ]; then | ||
if [ -z "$UGSUBMIT_EMAIL" ]; then | ||
echo "please set UGSUBMIT_EMAIL or specify email with -email". | ||
exit | ||
fi | ||
PBSMail="-M $UGSUBMIT_EMAIL -m $PBSMailtype" | ||
fi | ||
|
||
# interactive: qsub -IX -l walltime=00:50:00 -l mppwidth=4 -l mppnppn=4 | ||
|
||
qsubargs="$qsubqueue -v UG4_ROOT -N $jobname -o job.output -e job.error -j oe -l walltime=$walltime $PBSnodes $PBSMail" | ||
|
||
echo "echo \"cd $outdir; $qsubrun\" | qsub $qsubargs" >> info.txt | ||
|
||
if [ $test == true ]; then | ||
echo "ONLY testing - NOT executing." | ||
return | ||
fi | ||
|
||
|
||
|
||
|
||
|
||
if [ $interactive == true ]; then | ||
echo "Interactive mode currently not supported on hermit. Aborting." | ||
else | ||
jobid=`echo "cd $outdir; $qsubrun" | qsub $qsubargs` | ||
return=$? | ||
if [ ! $return == 0 ]; then | ||
echo "ERROR: qsub returned $return. Job has NOT been scheduled." | ||
exit | ||
fi | ||
jobid=`echo $jobid | sed 's/\([0-9]*\).*/\1/'` | ||
fi | ||
} | ||
|
||
|
||
|
||
function UJS_GetOptions | ||
{ | ||
echo "WARNING: Unknown platform. guessed default values for MaxNumProcsPerNode (nppnmax) and MaxProcs (pemax) may be wrong." | ||
nppnmax=48 | ||
pemax=1344 | ||
nppnmaxSingleThread=48 | ||
} | ||
|
||
function UJS_Info | ||
{ | ||
echo "Using $UGSUBMIT_TYPE" | ||
echo "qstat -u $USER -r" | ||
qstat -u $USER | ||
} | ||
|
||
|
||
function UJS_Cancel | ||
{ | ||
echo "Using $UGSUBMIT_TYPE" | ||
echo "qdel $1" | ||
qdel $1 | ||
} |