Skip to content

Commit

Permalink
machines: eccc: update our machine files (CICE-Consortium#725)
Browse files Browse the repository at this point in the history
A few notes:
- We do need to request memory even on machines where we have excluseive
node access. 20 GB was chosen rather arbitrarily.
- We set umask to 022 to make the <jobname>.o and <jobname>.e files
readable by group and others.
- We use the minimal SSM package for the compiler and Intel MPI, but we
keep the setup using environment modules commented if ever we need to
weak things (i.e. I_MPI_LIBRARY_KIND)
- We set OMP_STACKSIZE. Since d1e972a (Update OMP (CICE-Consortium#680),
2022-02-18), OpenMP threading is active in 'ice_transport_remap.F90',
and the default OpenMP stack size needs to be adjusted to avoid stack
overflows. We set it to a 64 Mb size as used for other machines.

Also, remove dead code setting 'CICE_ACCT'. This variable was last used
in 98e0307 (Update scripts, rename variables from CICE_ to ICE_ to be
more reusable in icepack., 2017-09-15), and so did not do anything for
any of the machines that were using it after that commit. Remove code in
machines env files that was setting it based on '~/.cice_proj'.

A few notes specific to 'gpsc3':

- Since we use an '--export' directive to choose which environment
variables are exported to the job environment by SLURM, SSMUSE_BASE and
SSMUSE_PATH are not present in the environnement and loading domains
without their full paths fails on csh, so use a full path.
- We use the compiler package from main/opt instead of eccc/all/opt
since we do not need the EC-specific variables to be set (and it also
leads to job failures since BASE_ARCH is not defined).

(cherry picked from commit c334aee)
  • Loading branch information
phil-blain committed Dec 12, 2022
1 parent ab08728 commit 7ec729f
Show file tree
Hide file tree
Showing 23 changed files with 761 additions and 99 deletions.
37 changes: 37 additions & 0 deletions configuration/scripts/cice.batch.csh
Original file line number Diff line number Diff line change
Expand Up @@ -216,8 +216,45 @@ cat >> ${jobfile} << EOFB
#PBS -j oe
#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds}
#PBS -l walltime=${batchtime}
#PBS -W umask=022
EOFB

else if (${ICE_MACHINE} =~ robert* || ${ICE_MACHINE} =~ underhill* || ${ICE_MACHINE} =~ ppp6* || ${ICE_MACHINE} =~ ppp5*) then
cat >> ${jobfile} << EOFB
#PBS -N ${ICE_CASENAME}
#PBS -j oe
#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds}:mem=20gb
#PBS -l walltime=${batchtime}
#PBS -W umask=022
EOFB

else if (${ICE_MACHINE} =~ ppp3*) then
cat >> ${jobfile} << EOFB
#PBS -N ${ICE_CASENAME}
#PBS -j oe
#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds}:mem=20gb:res_tmpfs=1000:res_image=eccc/eccc_all_ppp_ubuntu-18.04-amd64_latest
#PBS -l walltime=${batchtime}
#PBS -W umask=022
#PBS -q development
#PBS -o ${ICE_CASEDIR}
#PBS -S /bin/csh
EOFB

else if (${ICE_MACHINE} =~ gpsc3*) then
cat >> ${jobfile} << EOFB
#SBATCH --export=USER,LOGNAME,HOME,MAIL,PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
#SBATCH -J ${ICE_CASENAME}
#SBATCH -A ${acct}
#SBATCH --partition ${queue}
#SBATCH --time ${batchtime}
#SBATCH --nodes ${nnodes}
#SBATCH --ntasks ${ntasks}
#SBATCH --cpus-per-task ${nthrds}
#SBATCH --mem-per-cpu=5G
#SBATCH --comment="image=eccc/eccc_all_default_ubuntu-18.04-amd64_latest"
EOFB


else if (${ICE_MACHINE} =~ freya* ) then
cat >> ${jobfile} << EOFB
#PBS -N ${ICE_CASENAME}
Expand Down
37 changes: 37 additions & 0 deletions configuration/scripts/cice.launch.csh
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,43 @@ aprun -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_F
EOFR
endif

#=======
else if (${ICE_MACHINE} =~ ppp5* || ${ICE_MACHINE} =~ ppp6* || ${ICE_MACHINE} =~ robert* || ${ICE_MACHINE} =~ underhill*) then
if (${ICE_COMMDIR} =~ serial*) then
cat >> ${jobfile} << EOFR
./cice >&! \$ICE_RUNLOG_FILE
EOFR
else
cat >> ${jobfile} << EOFR
mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE
EOFR
endif

#=======
else if (${ICE_MACHINE} =~ ppp3*) then
if (${ICE_COMMDIR} =~ serial*) then
cat >> ${jobfile} << EOFR
./cice >&! \$ICE_RUNLOG_FILE
EOFR
else
cat >> ${jobfile} << EOFR
rumpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE
EOFR
endif

#=======
else if (${ICE_MACHINE} =~ gpsc3*) then
if (${ICE_COMMDIR} =~ serial*) then
cat >> ${jobfile} << EOFR
./cice >&! \$ICE_RUNLOG_FILE
EOFR
else
cat >> ${jobfile} << EOFR
mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE
EOFR
endif


#=======
else if (${ICE_MACHINE} =~ freya*) then
if (${ICE_COMMDIR} =~ serial*) then
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#==============================================================================
# Makefile macros for "millikan"
# Makefile macros for "gpsc3"
#==============================================================================
# For use with intel compiler
#==============================================================================
Expand All @@ -11,12 +11,12 @@ CFLAGS := -c -O2 -fp-model precise

FIXEDFLAGS := -132
FREEFLAGS := -FR
FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -no-wrap-margin
FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -no-wrap-margin
#-xHost
FFLAGS_NOOPT:= -O0

ifeq ($(ICE_BLDDEBUG), true)
FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays
FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created
#-init=snan,arrays
# -heap-arrays 1024
else
FFLAGS += -O2
Expand All @@ -36,25 +36,14 @@ else
endif
LD:= $(FC)

NETCDF_PATH := /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150/ubuntu-14.04-amd64-64/

PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs

#PNETCDF_PATH := $(PNETCDF)
#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib

INCLDIR := $(INCLDIR)

LIB_NETCDF := $(NETCDF_PATH)/lib
LIB_PNETCDF := $(PNETCDF_PATH)/lib
LIB_MPI := $(IMPILIBDIR)

#SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf -L$(LIB_PNETCDF) -lpnetcdf -lgptl
SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf
ifeq ($(ICE_IOTYPE), netcdf)
INCLDIR += $(shell nf-config --fflags)
SLIBS := $(shell nf-config --flibs)
endif

ifeq ($(ICE_THREADED), true)
LDFLAGS += -openmp
CFLAGS += -openmp
FFLAGS += -openmp
LDFLAGS += -qopenmp
CFLAGS += -qopenmp
FFLAGS += -qopenmp
endif

Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#==============================================================================
# Makefile macros for "cesium"
# Makefile macros for "ppp3"
#==============================================================================
# For use with intel compiler
#==============================================================================
Expand All @@ -11,12 +11,12 @@ CFLAGS := -c -O2 -fp-model precise

FIXEDFLAGS := -132
FREEFLAGS := -FR
FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -no-wrap-margin
FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -no-wrap-margin
#-xHost
FFLAGS_NOOPT:= -O0

ifeq ($(ICE_BLDDEBUG), true)
FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays
FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created
#-init=snan,arrays
# -heap-arrays 1024
else
FFLAGS += -O2
Expand All @@ -36,25 +36,14 @@ else
endif
LD:= $(FC)

NETCDF_PATH := /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150/ubuntu-14.04-amd64-64/

PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs

#PNETCDF_PATH := $(PNETCDF)
#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib

INCLDIR := $(INCLDIR)

LIB_NETCDF := $(NETCDF_PATH)/lib
LIB_PNETCDF := $(PNETCDF_PATH)/lib
LIB_MPI := $(IMPILIBDIR)

#SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf -L$(LIB_PNETCDF) -lpnetcdf -lgptl
SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf -llapack -lblas
ifeq ($(ICE_IOTYPE), netcdf)
INCLDIR += $(shell nf-config --fflags)
SLIBS := $(shell nf-config --flibs)
endif

ifeq ($(ICE_THREADED), true)
LDFLAGS += -openmp
CFLAGS += -openmp
FFLAGS += -openmp
LDFLAGS += -qopenmp
CFLAGS += -qopenmp
FFLAGS += -qopenmp
endif

49 changes: 49 additions & 0 deletions configuration/scripts/machines/Macros.ppp5_intel
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
#==============================================================================
# Makefile macros for "ppp5"
#==============================================================================
# For use with intel compiler
#==============================================================================

CPP := fpp
CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
CFLAGS := -c -O2 -fp-model precise
#-xHost

FIXEDFLAGS := -132
FREEFLAGS := -FR
FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -no-wrap-margin
#-xHost

ifeq ($(ICE_BLDDEBUG), true)
FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -check nooutput_conversion -debug-parameters all
#-init=snan,arrays
# -heap-arrays 1024
else
FFLAGS += -O2
endif

SCC := icc
SFC := ifort
MPICC := mpiicc
MPIFC := mpiifort

ifeq ($(ICE_COMMDIR), mpi)
FC := $(MPIFC)
CC := $(MPICC)
else
FC := $(SFC)
CC := $(SCC)
endif
LD:= $(FC)

ifeq ($(ICE_IOTYPE), netcdf)
INCLDIR += $(shell nf-config --fflags)
SLIBS := $(shell nf-config --flibs)
endif

ifeq ($(ICE_THREADED), true)
LDFLAGS += -qopenmp
CFLAGS += -qopenmp
FFLAGS += -qopenmp
endif

44 changes: 44 additions & 0 deletions configuration/scripts/machines/Macros.ppp6_gnu
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
#==============================================================================
# Makefile macros for ECCC ppp6
#==============================================================================
# For use with GNU compiler
#==============================================================================

CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
CFLAGS := -c -O2
#-xHost

FREEFLAGS := -ffree-form
FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none
#-xHost

ifeq ($(ICE_BLDDEBUG), true)
FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow
else
FFLAGS += -O2
endif

SCC := gcc
SFC := gfortran
MPICC := mpicc
MPIFC := mpifort

ifeq ($(ICE_COMMDIR), mpi)
FC := $(MPIFC)
CC := $(MPICC)
else
FC := $(SFC)
CC := $(SCC)
endif
LD:= $(FC)

ifeq ($(ICE_IOTYPE), netcdf)
INCLDIR += $(shell nf-config --fflags)
SLIBS := $(shell nf-config --flibs)
endif

ifeq ($(ICE_THREADED), true)
LDFLAGS += -fopenmp
CFLAGS += -fopenmp
FFLAGS += -fopenmp
endif
44 changes: 44 additions & 0 deletions configuration/scripts/machines/Macros.ppp6_gnu-impi
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
#==============================================================================
# Makefile macros for ECCC ppp6
#==============================================================================
# For use with GNU compiler
#==============================================================================

CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
CFLAGS := -c -O2
#-xHost

FREEFLAGS := -ffree-form
FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none
#-xHost

ifeq ($(ICE_BLDDEBUG), true)
FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow
else
FFLAGS += -O2
endif

SCC := gcc
SFC := gfortran
MPICC := mpicc
MPIFC := mpif90

ifeq ($(ICE_COMMDIR), mpi)
FC := $(MPIFC)
CC := $(MPICC)
else
FC := $(SFC)
CC := $(SCC)
endif
LD:= $(FC)

ifeq ($(ICE_IOTYPE), netcdf)
INCLDIR += $(shell nf-config --fflags)
SLIBS := $(shell nf-config --flibs)
endif

ifeq ($(ICE_THREADED), true)
LDFLAGS += -fopenmp
CFLAGS += -fopenmp
FFLAGS += -fopenmp
endif
49 changes: 49 additions & 0 deletions configuration/scripts/machines/Macros.ppp6_intel
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
#==============================================================================
# Makefile macros for "ppp6"
#==============================================================================
# For use with intel compiler
#==============================================================================

CPP := fpp
CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
CFLAGS := -c -O2 -fp-model precise
#-xHost

FIXEDFLAGS := -132
FREEFLAGS := -FR
FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -no-wrap-margin
#-xHost

ifeq ($(ICE_BLDDEBUG), true)
FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -check nooutput_conversion -debug-parameters all
#-init=snan,arrays
# -heap-arrays 1024
else
FFLAGS += -O2
endif

SCC := icc
SFC := ifort
MPICC := mpiicc
MPIFC := mpiifort

ifeq ($(ICE_COMMDIR), mpi)
FC := $(MPIFC)
CC := $(MPICC)
else
FC := $(SFC)
CC := $(SCC)
endif
LD:= $(FC)

ifeq ($(ICE_IOTYPE), netcdf)
INCLDIR += $(shell nf-config --fflags)
SLIBS := $(shell nf-config --flibs)
endif

ifeq ($(ICE_THREADED), true)
LDFLAGS += -qopenmp
CFLAGS += -qopenmp
FFLAGS += -qopenmp
endif

Loading

0 comments on commit 7ec729f

Please sign in to comment.