Merge branch 'azamat/machines/theta' (PR #1482)

Machine update for ANL Theta: This is a port of ACME to ANL Theta. Supplements #730. [BFB]
ESMCI · May 5, 2017 · 8c5879b · 8c5879b
2 parents b5797b4 + 1f62785
commit 8c5879b
Show file tree

Hide file tree

Showing 3 changed files with 120 additions and 1 deletion.
diff --git a/config/acme/machines/config_batch.xml b/config/acme/machines/config_batch.xml
@@ -224,7 +224,13 @@
         <queue walltimemax="01:00:00" jobmin="1" jobmax="65536" default="true">default</queue>
       </queues>
     </batch_system>
-
+
+    <batch_system MACH="theta" type="cobalt_theta">
+      <queues>
+        <queue walltimemax="00:60:00" jobmin="1" jobmax="3200" default="true">default</queue>
+      </queues>
+    </batch_system>
+
     <batch_system MACH="cascade" type="slurm">
       <directives>
 	<directive>--mail-type=END</directive>

diff --git a/config/acme/machines/config_compilers.xml b/config/acme/machines/config_compilers.xml
@@ -958,6 +958,24 @@ for mct, etc.
   <ALBANY_PATH>/projects/ccsm/libs/AlbanyTrilinos/Albany/build/install</ALBANY_PATH>
 </compiler>
 
+<compiler COMPILER="intel" MACH="theta">
+  <MPIFC> ftn </MPIFC>
+  <MPICC> cc </MPICC>
+  <MPICXX> CC </MPICXX>
+  <ADD_FFLAGS DEBUG="FALSE"> -O2 -xMIC-AVX512 </ADD_FFLAGS>
+  <ADD_CFLAGS DEBUG="FALSE"> -O2 -xMIC-AVX512 </ADD_CFLAGS>
+  <ADD_FFLAGS DEBUG="TRUE"> -O0 -g -xMIC-AVX512 </ADD_FFLAGS>
+  <ADD_CFLAGS DEBUG="TRUE"> -O0 -g -xMIC-AVX512 </ADD_CFLAGS>
+  <ADD_CFLAGS compile_threaded="true"> -qopenmp </ADD_CFLAGS>
+  <ADD_FFLAGS compile_threaded="true"> -qopenmp </ADD_FFLAGS>
+  <ADD_FFLAGS_NOOPT compile_threaded="true"> -qopenmp </ADD_FFLAGS_NOOPT>
+  <ADD_LDFLAGS compile_threaded="true"> -qopenmp </ADD_LDFLAGS>
+  <CONFIG_ARGS> --host=Linux </CONFIG_ARGS>
+  <ADD_SLIBS> -L$(NETCDF_DIR)/lib -lnetcdff -L$(NETCDF_DIR)/lib -lnetcdf -Wl,-rpath -Wl,$(NETCDF_DIR)/lib </ADD_SLIBS>
+  <ADD_SLIBS> -mkl -lpthread -lm </ADD_SLIBS>
+  <ADD_CPPFLAGS> -DHAVE_COMM_F2C </ADD_CPPFLAGS>
+</compiler>
+
 <compiler COMPILER="pgi" MACH="blues">
   <PNETCDF_PATH>$(PNETCDFROOT)</PNETCDF_PATH>
   <NETCDF_PATH>$(NETCDFROOT)</NETCDF_PATH>

diff --git a/config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml
@@ -1175,6 +1175,101 @@
     </environment_variables>
 </machine>
 
+<machine MACH="theta">
+    <DESC>ALCF Cray XC* KNL, os is CNL, 64 pes/node, batch system is cobalt</DESC>
+    <NODENAME_REGEX>theta.*</NODENAME_REGEX>
+    <TESTS>acme_developer</TESTS>
+    <COMPILERS>intel,gnu,cray</COMPILERS>
+    <MPILIBS>mpt</MPILIBS>
+    <CIME_OUTPUT_ROOT>/projects/EarlyPerf_theta/acme/scratch/$USER</CIME_OUTPUT_ROOT>
+    <DIN_LOC_ROOT>/projects/EarlyPerf_theta/acme/inputdata</DIN_LOC_ROOT>
+    <DIN_LOC_ROOT_CLMFORC>/projects/EarlyPerf_theta/acme/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
+    <DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
+    <DOUT_L_MSROOT>/projects/EarlyPerf_theta/acme/csm/$CASE</DOUT_L_MSROOT>
+    <CCSM_BASELINE>/projects/EarlyPerf_theta/acme/baselines</CCSM_BASELINE>
+    <CCSM_CPRNC>/projects/EarlyPerf_theta/acme/tools/cprnc</CCSM_CPRNC>
+    <SAVE_TIMING_DIR>/projects/EarlyPerf_theta/acme</SAVE_TIMING_DIR>
+    <OS>CNL</OS>
+    <BATCH_SYSTEM>cobalt_theta</BATCH_SYSTEM>
+    <SUPPORTED_BY>acme</SUPPORTED_BY>
+    <GMAKE_J>8</GMAKE_J>
+    <MAX_TASKS_PER_NODE>64</MAX_TASKS_PER_NODE>
+    <PES_PER_NODE>64</PES_PER_NODE>
+    <PROJECT>Theta_ESP</PROJECT>
+    <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
+    <PIO_CONFIG_OPTS> -D PIO_BUILD_TIMING:BOOL=ON </PIO_CONFIG_OPTS>
+    <mpirun mpilib="default">
+      <executable>aprun</executable>
+      <arguments>
+        <arg name="num_tasks" >-n $TOTALPES</arg>
+        <arg name="tasks_per_node" >-N $PES_PER_NODE</arg>
+        <arg name="thread_count">--cc depth -d $OMP_NUM_THREADS</arg>
+        <arg name="env_omp_stacksize">-e OMP_STACKSIZE=128M</arg>
+        <arg name="env_thread_count">-e OMP_NUM_THREADS=$OMP_NUM_THREADS</arg>
+      </arguments>
+    </mpirun>
+    <module_system type="module">
+      <init_path lang="perl">/opt/modules/default/init/perl.pm</init_path>
+      <init_path lang="python">/opt/modules/default/init/python.py</init_path>
+      <init_path lang="sh">/opt/modules/default/init/sh</init_path>
+      <init_path lang="csh">/opt/modules/default/init/csh</init_path>
+      <cmd_path lang="perl">/opt/modules/default/bin/modulecmd perl</cmd_path>
+      <cmd_path lang="python">/opt/modules/default/bin/modulecmd python</cmd_path>
+      <cmd_path lang="sh">module</cmd_path>
+      <cmd_path lang="csh">module</cmd_path>
+      <modules>
+        <command name="rm">PrgEnv-intel</command>
+        <command name="rm">PrgEnv-cray</command>
+        <command name="rm">PrgEnv-gnu</command>
+        <command name="rm">intel</command>
+        <command name="rm">cce</command>
+        <command name="rm">cray-parallel-netcdf</command>
+        <command name="rm">cray-hdf5-parallel</command>
+        <command name="rm">pmi</command>
+        <command name="rm">cray-libsci</command>
+        <command name="rm">cray-mpich</command>
+        <command name="rm">cray-netcdf</command>
+        <command name="rm">cray-hdf5</command>
+        <command name="rm">cray-netcdf-hdf5parallel</command>
+        <command name="rm">craype</command>
+      </modules>
+      <modules compiler="intel">
+        <command name="load">PrgEnv-intel/6.0.3</command>
+        <command name="switch">intel intel/17.0.2.174</command>
+        <command name="rm">cray-libsci</command>
+      </modules>
+      <modules compiler="cray">
+        <command name="load">PrgEnv-cray/6.0.3</command>
+        <command name="switch">cce cce/8.5.4</command>
+      </modules>
+      <modules compiler="gnu">
+        <command name="load">PrgEnv-gnu/6.0.3</command>
+        <command name="switch">gcc gcc/6.2.0</command>
+      </modules>
+      <modules>
+        <command name="swap">craype craype/2.5.9</command>
+      </modules>
+      <modules compiler="!intel">
+        <command name="switch">cray-libsci/16.09.1</command>
+      </modules>
+      <modules>
+        <command name="load">cray-mpich/7.5.3</command>
+      </modules>
+      <modules mpilib="mpt">
+        <command name="load">cray-netcdf-hdf5parallel/4.4.1.1</command>
+        <command name="load">cray-hdf5-parallel/1.10.0.1</command>
+        <command name="load">cray-parallel-netcdf/1.8.0</command>
+      </modules>
+    </module_system>
+    <environment_variables>
+      <env name="MPICH_ENV_DISPLAY">1</env>
+      <env name="MPICH_VERSION_DISPLAY">1</env>
+      <env name="OMP_STACKSIZE">128M</env>
+      <env name="OMP_PROC_BIND">spread</env>
+      <env name="OMP_PLACES">threads</env>
+    </environment_variables>
+</machine>
+
 <machine MACH="sooty">
     <DESC>PNL cluster, OS is Linux, batch system is SLURM</DESC>
     <NODENAME_REGEX>sooty</NODENAME_REGEX>