Add Preliminary MHD Integrator

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "cholla-tests-data"]
+	path = cholla-tests-data
+	url = https://github.com/cholla-hydro/cholla-tests-data.git

Makefile

@@ -10,7 +10,7 @@ CUDA_ARCH ?= sm_70
 
 DIRS     := src src/analysis src/chemistry_gpu src/cooling src/cooling_grackle src/cosmology \
             src/cpu src/global src/gravity src/gravity/paris src/grid src/hydro \
-            src/integrators src/io src/main.cpp src/main_tests.cpp \
+            src/integrators src/io src/main.cpp src/main_tests.cpp src/mhd\
             src/model src/mpi src/old_cholla src/particles src/reconstruction \
             src/riemann_solvers src/system_tests src/utils src/dust
 
@@ -134,6 +134,7 @@ ifdef HIPCONFIG
   DFLAGS    += -DO_HIP
   CXXFLAGS  += $(HIPCONFIG)
   GPUCXX    ?= hipcc
+  #GPUFLAGS  += -Wall
   LD        := $(CXX)
   LDFLAGS   := $(CXXFLAGS) -L$(ROCM_PATH)/lib
   LIBS      += -lamdhip64

builds/make.host.c3po

@@ -3,13 +3,9 @@ CC                = mpicc
 CXX               = mpicxx
 CFLAGS_DEBUG      = -g -O0
 CFLAGS_OPTIMIZE   = -g -O2
-CXXFLAGS_DEBUG    = -g -O0 -std=c++11 ${F_OFFLOAD}
-CXXFLAGS_OPTIMIZE = -g -Ofast -std=c++11 ${F_OFFLOAD}
-GPUFLAGS_DEBUG    = -g -O0 -std=c++11 -ccbin=mpicxx
-GPUFLAGS_OPTIMIZE = -g -O3 -std=c++11 -ccbin=mpicxx
 CXXFLAGS_DEBUG    = -g -O0 -std=c++17 ${F_OFFLOAD}
 CXXFLAGS_OPTIMIZE = -g -Ofast -std=c++17 ${F_OFFLOAD}
-GPUFLAGS_DEBUG    = -g -G -cudart shared -O0 -std=c++17 -ccbin=mpicxx
+GPUFLAGS_DEBUG    = -g -G -cudart shared -O0 -std=c++17 -ccbin=mpicxx -Xcompiler -rdynamic
 GPUFLAGS_OPTIMIZE = -g -O3 -std=c++17 -ccbin=mpicxx
 
 OMP_NUM_THREADS   = 7

builds/make.host.frontier

@@ -11,8 +11,8 @@ CFLAGS_OPTIMIZE   = -g -O2
 CXXFLAGS_DEBUG    = -g -O0 -std=c++17
 CXXFLAGS_OPTIMIZE = -g -Ofast -std=c++17 -Wno-unused-result
 
-GPUFLAGS_OPTIMIZE = -std=c++17 --offload-arch=gfx90a -Wno-unused-result
-GPUFLAGS_DEBUG    = -g -O0 -std=c++17 --offload-arch=gfx90a -Wno-unused-result
+GPUFLAGS_OPTIMIZE = -std=c++17 --offload-arch=gfx90a -Wall -Wno-unused-result
+GPUFLAGS_DEBUG    = -g -O0 -std=c++17 --offload-arch=gfx90a -Wall -Wno-unused-result
 HIPCONFIG	  = -I$(ROCM_PATH)/include $(shell hipconfig -C) # workaround for Rocm 5.2 warnings
 #HIPCONFIG	  = $(shell hipconfig -C)
 

builds/make.host.spock

@@ -6,6 +6,7 @@ CXX               = CC
 
 CFLAGS_DEBUG      = -g -O0
 CFLAGS_OPTIMIZE   = -g -O2
+
 CXXFLAGS_DEBUG    = -g -O0 -std=c++17
 CXXFLAGS_OPTIMIZE = -g -Ofast -std=c++17
 

builds/make.type.mhd

@@ -9,15 +9,14 @@ MPI_GPU   ?=
 DFLAGS    += -DCUDA
 DFLAGS    += -DMPI_CHOLLA
 DFLAGS    += -DPRECISION=2
-DFLAGS    += -DPPMP
+# Currently MHD only supports PCM reconstruction. Higher order reconstruction
+# methods will be added later
+DFLAGS    += -DPCM
 DFLAGS    += -DHLLD
 DFLAGS    += -DMHD
 
-ifeq ($(findstring cosmology,$(TYPE)),cosmology)
-DFLAGS    += -DSIMPLE
-else
+# MHD only supports the Van Leer integrator
 DFLAGS    += -DVL
-endif
 
 # need this if using Disk_3D
 # DFLAGS += -DDISK_ICS
@@ -45,3 +44,15 @@ DFLAGS    += $(OUTPUT)
 #If not specified, MPI_GPU is off by default
 #This is set in the system make.host file
 DFLAGS    += $(MPI_GPU)
+
+# NOTE: The following macros are to help facilitate debugging and should not be
+# used on scientific runs
+
+# Do CUDA error checking
+DFLAGS += -DCUDA_ERROR_CHECK
+
+# Limit the number of steps to evolve.
+# DFLAGS += -DN_STEPS_LIMIT=1000
+
+# Output on every time step
+# DFLAGS += -DOUTPUT_ALWAYS

builds/run_tests.sh

@@ -110,7 +110,7 @@ buildCholla ()
 {
   echo -e "\nBuilding Cholla...\n"
   builtin cd $CHOLLA_ROOT
-  make -j TYPE=${CHOLLA_MAKE_TYPE} BUILD=${1}
+  make --jobs=$(nproc) TYPE=${CHOLLA_MAKE_TYPE} BUILD=${1}
 }
 # ==============================================================================
 
@@ -121,7 +121,7 @@ buildChollaTests ()
 {
   echo
   builtin cd $CHOLLA_ROOT
-  make -j TYPE=${CHOLLA_MAKE_TYPE} TEST=true
+  make --jobs=$(nproc) TYPE=${CHOLLA_MAKE_TYPE} TEST=true
 }
 # ==============================================================================
 

examples/1D/sound_wave.txt

@@ -30,18 +30,17 @@ outdir=./
 
 #################################################
 # Parameters for linear wave problems
-# initial density 
+# initial density
 rho=1.0
-# velocity in the x direction 
+# velocity in the x direction
 vx=0
 # velocity in the y direction
 vy=0
 # velocity in the z direction
 vz=0
-# initial pressure 
+# initial pressure
 P=0.6
 # amplitude of perturbing oscillations
 A=1e-4
 # value of gamma
 gamma=1.666666666666667
-

examples/2D/sound_wave.txt

@@ -33,18 +33,17 @@ outdir=./
 
 #################################################
 # Parameters for linear wave problems
-# initial density 
+# initial density
 rho=1.0
-# velocity in the x direction 
+# velocity in the x direction
 vx=0
 # velocity in the y direction
 vy=0
 # velocity in the z direction
 vz=0
-# initial pressure 
+# initial pressure
 P=0.6
 # amplitude of perturbing oscillations
 A=1e-4
 # value of gamma
 gamma=1.666666666666667
-

examples/3D/Brio_and_Wu.txt

@@ -6,11 +6,11 @@
 
 ################################################
 # number of grid cells in the x dimension
-nx=32
+nx=256
 # number of grid cells in the y dimension
-ny=32
+ny=256
 # number of grid cells in the z dimension
-nz=32
+nz=256
 # final output time
 tout=0.1
 # time interval for output

examples/3D/Ryu_and_Jones_1a.txt

@@ -0,0 +1,74 @@
+#
+# Parameter File for 3D Ryu & Jones MHD shock tube 4d.
+# Citation: Ryu & Jones 1995 "Numerical Magnetohydrodynamics in Astrophysics:
+# Algorithms and Tests for One-Dimensional Flow"
+#
+# Note: There are many shock tubes in this paper. This settings file is
+# specifically for shock tube 4d
+#
+
+################################################
+# number of grid cells in the x dimension
+nx=32
+# number of grid cells in the y dimension
+ny=32
+# number of grid cells in the z dimension
+nz=32
+# final output time
+tout=0.08
+# time interval for output
+outstep=0.08
+# name of initial conditions
+init=Riemann
+
+# domain properties
+xmin=0.0
+ymin=0.0
+zmin=0.0
+xlen=1.0
+ylen=1.0
+zlen=1.0
+
+# type of boundary conditions
+xl_bcnd=3
+xu_bcnd=3
+yl_bcnd=3
+yu_bcnd=3
+zl_bcnd=3
+zu_bcnd=3
+
+# path to output directory
+outdir=./
+
+#################################################
+# Parameters for 1D Riemann problems
+# density of left state
+rho_l=1.0
+# velocity of left state
+vx_l=10.0
+vy_l=0.0
+vz_l=0.0
+# pressure of left state
+P_l=20.0
+# Magnetic field of the left state
+Bx_l=1.4104739588693909
+By_l=1.4104739588693909
+Bz_l=0.0
+
+# density of right state
+rho_r=1.0
+# velocity of right state
+vx_r=-10.0
+vy_r=0.0
+vz_r=0.0
+# pressure of right state
+P_r=1.0
+# Magnetic field of the right state
+Bx_r=1.4104739588693909
+By_r=1.4104739588693909
+Bz_r=0.0
+
+# location of initial discontinuity
+diaph=0.5
+# value of gamma
+gamma=1.6666666666666667

examples/3D/Ryu_and_Jones_2a.txt

@@ -0,0 +1,74 @@
+#
+# Parameter File for 3D Ryu & Jones MHD shock tube 4d.
+# Citation: Ryu & Jones 1995 "Numerical Magnetohydrodynamics in Astrophysics:
+# Algorithms and Tests for One-Dimensional Flow"
+#
+# Note: There are many shock tubes in this paper. This settings file is
+# specifically for shock tube 4d
+#
+
+################################################
+# number of grid cells in the x dimension
+nx=32
+# number of grid cells in the y dimension
+ny=32
+# number of grid cells in the z dimension
+nz=32
+# final output time
+tout=0.2
+# time interval for output
+outstep=0.2
+# name of initial conditions
+init=Riemann
+
+# domain properties
+xmin=0.0
+ymin=0.0
+zmin=0.0
+xlen=1.0
+ylen=1.0
+zlen=1.0
+
+# type of boundary conditions
+xl_bcnd=3
+xu_bcnd=3
+yl_bcnd=3
+yu_bcnd=3
+zl_bcnd=3
+zu_bcnd=3
+
+# path to output directory
+outdir=./
+
+#################################################
+# Parameters for 1D Riemann problems
+# density of left state
+rho_l=1.08
+# velocity of left state
+vx_l=1.2
+vy_l=0.01
+vz_l=0.5
+# pressure of left state
+P_l=0.95
+# Magnetic field of the left state
+Bx_l=7.0898154036220635
+By_l=1.0155412503859613
+Bz_l=0.5641895835477563
+
+# density of right state
+rho_r=1.0
+# velocity of right state
+vx_r=0.0
+vy_r=0.0
+vz_r=0.0
+# pressure of right state
+P_r=1.0
+# Magnetic field of the right state
+Bx_r=7.0898154036220635
+By_r=1.1283791670955126
+Bz_r=0.5641895835477563
+
+# location of initial discontinuity
+diaph=0.5
+# value of gamma
+gamma=1.6666666666666667

examples/3D/alfven_wave.txt

@@ -0,0 +1,71 @@
+#
+# Parameter File for MHD Alfven Wave
+# See [this blog post](https://robertcaddy.com/posts/Classes-and-bugfixing-6/)
+# for details on each wave
+# The right eigenvector for this wave is:
+# (1/3) * [0, 0, +/-1, -/+2*sqrt(2), 0, -1, 2*sqrt(2), 0]
+# The terms with two sign options: use the left one for right moving waves and
+# the right one for left moving waves
+#
+
+################################################
+# number of grid cells in the x dimension
+nx=256
+# number of grid cells in the y dimension
+ny=256
+# number of grid cells in the z dimension
+nz=256
+# final output time
+tout=1.0
+# time interval for output
+outstep=1.0
+# name of initial conditions
+init=Linear_Wave
+# domain properties
+xmin=0.0
+ymin=0.0
+zmin=0.0
+xlen=1.0
+ylen=1.0
+zlen=1.0
+# type of boundary conditions
+xl_bcnd=1
+xu_bcnd=1
+yl_bcnd=1
+yu_bcnd=1
+zl_bcnd=1
+zu_bcnd=1
+# path to output directory
+outdir=./
+
+#################################################
+# Parameters for linear wave problems
+# initial density
+rho=1.0
+# velocity in the x direction
+vx=0
+# velocity in the y direction
+vy=0
+# velocity in the z direction
+vz=0
+# initial pressure
+P=0.6
+# magnetic field in the x direction
+Bx=1
+# magnetic field in the y direction
+By=1.5
+# magnetic field in the z direction
+Bz=0
+# amplitude of perturbing oscillations
+A=1e-6
+# value of gamma
+gamma=1.666666666666667
+# The right eigenvectors to set the wave properly
+rEigenVec_rho=0
+rEigenVec_MomentumX=0
+rEigenVec_MomentumY=0
+rEigenVec_MomentumZ=-1
+rEigenVec_Bx=0
+rEigenVec_By=0
+rEigenVec_Bz=1
+rEigenVec_E=0