Swapnil/eos table

Jenkinsfile

@@ -1,6 +1,6 @@
 pipeline {
     triggers { pollSCM('') }  // Run tests whenever a new commit is detected.
-    agent { dockerfile {args '--gpus all'}} // Use the Dockerfile defined in the root Flash-X directory
+    agent { dockerfile {args '--gpus all -v /mnt/scratch/tables:/tables:ro'}} // Use the Dockerfile defined in the root Flash-X directory
     environment {
 		// Get rid of Read -1, expected <someNumber>, errno =1 error
     	// See https://github.com/open-mpi/ompi/issues/4948
@@ -16,7 +16,7 @@ pipeline {
 	    sh 'nvidia-smi'
 	    sh 'nvcc -V'
 	    sh 'git submodule update --init'
-	    sh 'cp makefiles/GNUmakefile_jenkins Exec/GNUmakefile'
+	    sh 'cp makefiles/GNUmakefile_jenkins_HDF5_CUDA Exec/GNUmakefile'
 	    dir('Exec'){
 	        sh 'make generate; make -j'
 	    }
@@ -66,8 +66,8 @@ pipeline {
 		dir('Exec'){
 			sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi.py'
 			sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_1d_fiducial'
-			sh 'python ../Scripts/data_reduction/reduce_data.py'
 			sh 'python ../Scripts/data_reduction/reduce_data_fft.py'
+			sh 'python ../Scripts/data_reduction/reduce_data.py'
 			sh 'python ../Scripts/data_reduction/combine_files.py plt _reduced_data.h5'
 			sh 'python ../Scripts/data_reduction/combine_files.py plt _reduced_data_fft_power.h5'
 			sh 'python ../Scripts/babysitting/avgfee.py'
@@ -85,14 +85,14 @@ pipeline {
 	        sh 'make realclean; make generate; make -j'
 			sh 'python ../Scripts/initial_conditions/st4_linear_moment_ffi_3F.py'
 			sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.ex ../sample_inputs/inputs_1d_fiducial'
-			sh 'python3 ../Scripts/babysitting/avgfee_HDF5.py'
+			/*sh 'python3 ../Scripts/babysitting/avgfee_HDF5.py'*/
 			sh 'rm -rf plt*'
 		}
 	}}
 
 	stage('Collisions flavor instability'){ steps{
 		dir('Exec'){
-			sh 'cp ../makefiles/GNUmakefile_jenkins GNUmakefile'
+			sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile'
 	        sh 'make realclean; make generate NUM_FLAVORS=2; make -j NUM_FLAVORS=2'
 			sh 'python ../Scripts/initial_conditions/st8_coll_inst_test.py'
 			sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_collisional_instability_test'
@@ -104,7 +104,7 @@ pipeline {
 
 	stage('Collisions to equilibrium'){ steps{
 		dir('Exec'){
-			sh 'cp ../makefiles/GNUmakefile_jenkins GNUmakefile'
+			sh 'cp ../makefiles/GNUmakefile_jenkins_HDF5_CUDA GNUmakefile'
 	        sh 'make realclean; make generate NUM_FLAVORS=3; make -j NUM_FLAVORS=3'
 			sh 'python ../Scripts/initial_conditions/st7_empty_particles.py'
 			sh 'mpirun -np 4 ./main3d.gnu.TPROF.MPI.CUDA.ex ../sample_inputs/inputs_coll_equi_test'

Scripts/tests/coll_inst_test.py

@@ -273,4 +273,4 @@ def QKE(t,y):
     plt.yscale('log')
     plt.legend()
     plt.savefig('EMU_Julien_LucasLSA_Neu.pdf')
-    plt.close()
+    plt.close()

Source/DataReducer.H

@@ -7,6 +7,11 @@
 #include <AMReX_MultiFab.H>
 #include <AMReX_ParticleMesh.H>
 #include <FlavoredNeutrinoContainer.H>
+
+//We use the AMReX binary format to write data for now
+//That's because the HDF5 write format gives errors when running with CUDA.
+#undef AMREX_USE_HDF5
+
 #ifdef AMREX_USE_HDF5
 #include <../submodules/HighFive/include/highfive/H5File.hpp>
 #include <../submodules/HighFive/include/highfive/H5DataSpace.hpp>

Source/DataReducer.cpp

@@ -4,6 +4,11 @@
 #include "ArithmeticArray.H"
 #include <cmath>
 #include <string>
+
+//We use the AMReX binary format to write data for now
+//That's because the HDF5 write format gives errors when running with CUDA.
+#undef AMREX_USE_HDF5
+
 #ifdef AMREX_USE_HDF5
 #include <../submodules/HighFive/include/highfive/H5File.hpp>
 #include <../submodules/HighFive/include/highfive/H5DataSpace.hpp>
@@ -107,6 +112,8 @@ void DataReducer::InitializeFiles()
     outfile << j << ":sumTrN\t";
     j++;
     outfile << j << ":sumTrHN\t";
+    j++;
+    outfile << j << ":Vphase\t";
     outfile << std::endl;
     outfile.close();
 
@@ -127,18 +134,23 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom,
   // Do reductions over the particles //
   //==================================//
   using PType = typename FlavoredNeutrinoContainer::ParticleType;
-  amrex::ReduceOps<ReduceOpSum,ReduceOpSum> reduce_ops;
-  auto particleResult = amrex::ParticleReduce< ReduceData<amrex::Real, amrex::Real> >(neutrinos,
-      [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple<amrex::Real, amrex::Real> {
+  amrex::ReduceOps<ReduceOpSum,ReduceOpSum,ReduceOpSum> reduce_ops;
+  auto particleResult = amrex::ParticleReduce< ReduceData<amrex::Real, amrex::Real, amrex::Real> >(neutrinos,
+      [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple<amrex::Real, amrex::Real, amrex::Real> {
           Real TrHN = p.rdata(PIdx::TrHN);
-	  Real TrN = 0;
+	        Real TrN = 0;
+          Real Vphase = p.rdata(PIdx::Vphase);
 #include "generated_files/DataReducer.cpp_fill_particles"
-	  return GpuTuple{TrN,TrHN};
+	        return GpuTuple{TrN,TrHN, Vphase};
       }, reduce_ops);
   Real TrN  = amrex::get<0>(particleResult);
   Real TrHN = amrex::get<1>(particleResult);
+  Real Vphase = amrex::get<2>(particleResult);
   ParallelDescriptor::ReduceRealSum(TrN);
   ParallelDescriptor::ReduceRealSum(TrHN);
+  ParallelDescriptor::ReduceRealSum(Vphase);
+
+  printf("TrN=%g, TrHN=%g, Vphase=%g\n", TrN, TrHN, Vphase);
 
   //=============================//
   // Do reductions over the grid //
@@ -326,6 +338,7 @@ DataReducer::WriteReducedData0D(const amrex::Geometry& geom,
     outfile << N_offdiag_mag << "\t";
     outfile << TrN << "\t";
     outfile << TrHN << "\t";
+    outfile << Vphase << "\t";
     outfile << std::endl;
     outfile.close();
 #endif

Source/EosTable.H

@@ -0,0 +1,71 @@
+#ifndef EOS_TABLE_H
+#define EOS_TABLE_H
+
+void ReadEosTable(const std::string nuceos_table_name);
+
+// OLD TODO: remove hard coded constants
+// OLD TODO: introduce defines for table index of variables
+#define HAVEGR 1
+#define MAX2(x, y) (((x) > (y)) ? (x) : (y))
+#define MIN2(x, y) (((x) < (y)) ? (x) : (y))
+#define NTABLES 19
+#define LENGTHGF 6.77269222552442e-06
+#define TIMEGF 2.03040204956746e05
+#define RHOGF 1.61887093132742e-18
+#define PRESSGF 1.80123683248503e-39
+#define EPSGF 1.11265005605362e-21
+#define INVRHOGF 6.17714470405638e17
+#define INVEPSGF 8.98755178736818e20
+#define INVPRESSGF 5.55174079257738e38
+#define CLIGHT 1
+
+// table key
+// 0 logpress 
+// 1 logenergy
+// 2 entropy
+// 3 munu
+// 4 cs2
+// 5 dedt
+// 6 dpdrhoe
+// 7 dpderho
+// 8 muhat
+// 9 mu_e
+// 10 mu_p
+// 11 mu_n
+// 12 Xa
+// 13 Xh
+// 14 Xn
+// 15 Xp
+// 16 Abar
+// 17 Zbar
+// 18 Gamma
+//  enum eos_var {i_logpress=0, i_logenergy, i_entropy, i_munu, i_cs2, i_dedt,
+                //i_dpdrhoe, i_dpderho, i_muhat, i_mu_e, i_mu_p, i_mu_n, i_Xa,
+                //i_Xh, i_Xn, i_Xp, i_Abar, i_Zbar, i_Gamma};
+//}
+
+namespace nuc_eos_private {
+
+// table data
+  extern double *alltables;
+  extern double *epstable;
+  extern double *logrho;
+  extern double *logtemp;
+  extern double *yes;
+
+#define EOSVAR(VAR) helperVarsReal[helperVarsEnumReal::VAR] 
+//WARNING: If the number of variables is increased here, then memory allocation for helperVarsReal/helperVarsInt pointer in GRHydroX_ReadEOSTable.cxx will also need to be increased.
+  enum helperVarsEnumReal { eos_rhomin, eos_rhomax, eos_tempmin, eos_tempmax, eos_yemin, eos_yemax, 
+                            energy_shift, dlintemp, dlintempi, drholintempi, dlintempyei, drholintempyei, 
+                            dtemp, dtempi, drho, drhoi, dye, dyei, drhotempi, drhoyei, dtempyei, 
+                            drhotempyei, eos_epsmin, eos_epsmax };
+  extern double *helperVarsReal;
+
+#define EOSVAR_INT(VAR) helperVarsInt[helperVarsEnumInt::VAR]
+  enum helperVarsEnumInt { nrho, ntemp, nye } ;
+  extern int *helperVarsInt;
+
+
+}
+
+#endif // EOS_TABLE_H

Source/EosTableFunctions.H

@@ -0,0 +1,42 @@
+#ifndef EOSTABLEFUCNTIONS_HXX
+#define EOSTABLEFUCNTIONS_HXX
+
+
+#include "EosTableHelpers.H"
+
+struct EOS_tabulated {
+    double *alltables, *epstable, *logrho, *logtemp, *yes, *helperVarsReal;
+    int *helperVarsInt;
+
+    //constructor for Tabulated EOS
+    AMREX_GPU_DEVICE AMREX_GPU_HOST EOS_tabulated() = default;//Need to keep it
+    AMREX_GPU_DEVICE AMREX_GPU_HOST EOS_tabulated(double *alltables, double *epstable, double *logrho, 
+    double *logtemp, double *yes, double *helperVarsReal, int *helperVarsInt): 
+    alltables(alltables), epstable(epstable), logrho(logrho), logtemp(logtemp), 
+    yes(yes), helperVarsReal(helperVarsReal), helperVarsInt(helperVarsInt) {}
+
+    //--------------- get helperVarsReal pointer ---------------------------------------------
+    AMREX_GPU_DEVICE AMREX_GPU_HOST double *get_helperVarsReal() const {
+        return helperVarsReal;
+    };//get_helperVarsReal
+
+
+    //--------------- get mu_e and muhat for tabulated EOS ---------------------------------------------  
+    AMREX_GPU_DEVICE AMREX_GPU_HOST void get_mue_muhat(double rho, double temperature, double Ye,
+                                             double &mue, double &muhat, 
+                                             int &keyerr, int &anyerr) const {
+
+        nuc_eos_mue_muhat(rho, temperature, Ye, mue, muhat, keyerr, anyerr, 
+                                alltables, logrho, logtemp, yes, helperVarsReal, helperVarsInt);
+        return;
+
+        /*Actual steps:
+        (1) check bounds
+        (2) get interp spots
+        (3) do interpolation to get mu_e and muhat
+        */
+    };//get_mue_muhat
+
+}; //struct EOS_tabulated
+
+#endif // EOSTABLEFUCNTIONS_HXX