Merge pull request #2131 from mmorale3/add_dense_hamiltonian

Add dense hamiltonian

CMakeLists.txt

@@ -244,9 +244,6 @@ SET(BUILD_AFQMC_WITH_NCCL 0 CACHE BOOL "Build AFQMC with NCCL library.")
 If (BUILD_AFQMC AND NOT QMC_MPI)
   MESSAGE(FATAL_ERROR "AFQMC requires building with MPI (QMC_MPI=1). Set BUILD_AFQMC=0 or configure MPI.")
 ENDIF()
-If (BUILD_AFQMC AND ENABLE_CUDA AND NOT QMC_COMPLEX)
-  MESSAGE(FATAL_ERROR "AFQMC only works on GPUs if QMC_COMPLEX=1.")
-ENDIF()
 SET(BUILD_FCIQMC 0 CACHE BOOL "Build with FCIQMC")
 #SET(BUILD_QMCTOOLS 1 CACHE BOOL "Build tools for QMCPACK")
 #SET(MPIP_PROFILE 0 CACHE BOOL "Build with mpip for mpi profile")

external_codes/mpi_wrapper/mpi3/shared_window.hpp

@@ -9,9 +9,9 @@
 #include "../mpi3/dynamic_window.hpp"
 #include "../mpi3/group.hpp"
 
-#include <boost/interprocess/containers/vector.hpp>
-#include <boost/interprocess/allocators/allocator.hpp>
-#include <boost/interprocess/managed_shared_memory.hpp>
+//#include <boost/interprocess/containers/vector.hpp>
+//#include <boost/interprocess/allocators/allocator.hpp>
+//#include <boost/interprocess/managed_shared_memory.hpp>
 
 #include<mpi.h>
 
@@ -20,9 +20,9 @@ namespace mpi3{
 
 template<class T>
 struct shared_window : window<T>{
-//	shared_communicator& comm_;
-	shared_window(shared_communicator& comm, mpi3::size_t n, int disp_unit = alignof(T)) //: //sizeof(T)) : // here we assume that disp_unit is used for align
-		//window<T>()//, comm_{comm}
+	shared_communicator& comm_;
+	shared_window(shared_communicator& comm, mpi3::size_t n, int disp_unit = alignof(T)) : //sizeof(T)) : // here we assume that disp_unit is used for align
+		window<T>(), comm_{comm}
 	{
 		void* base_ptr = nullptr;
 		auto e = static_cast<enum error>(
@@ -37,12 +37,12 @@ struct shared_window : window<T>{
 		shared_window(comm, 0, disp_unit)
 	{}
 	shared_window(shared_window const&) = default;
-	shared_window(shared_window&& other) noexcept : window<T>{std::move(other)}//, comm_{other.comm_}
+	shared_window(shared_window&& other) noexcept : window<T>{std::move(other)}, comm_{other.comm_}
 	{}
 	group get_group() const{
 		group r; MPI3_CALL(MPI_Win_get_group)(this->impl_, &(&r)); return r;
 	}
-//	shared_communicator& get_communicator() const{return comm_;}
+	shared_communicator& get_communicator() const{return comm_;}
 	struct query_t{
 		mpi3::size_t size;
 		int disp_unit;

src/AFQMC/CMakeLists.txt

@@ -16,6 +16,8 @@ SET (AFQMC_SRCS
   Hamiltonians/HSPotential_Helpers.cpp
   Hamiltonians/FactorizedSparseHamiltonian.cpp
   Hamiltonians/KPFactorizedHamiltonian.cpp
+  Hamiltonians/RealDenseHamiltonian.cpp
+  Hamiltonians/RealDenseHamiltonian_v2.cpp
 #  Hamiltonians/KPTHCHamiltonian.cpp
   Hamiltonians/HamiltonianFactory_Helper.cpp
   Hamiltonians/HamiltonianFactory.cpp
@@ -47,6 +49,8 @@ IF(ENABLE_CUDA)
     Numerics/detail/CUDA/Kernels/KaKjw_to_KKwaj.cu
     Numerics/detail/CUDA/Kernels/batched_dot_wabn_wban.cu
     Numerics/detail/CUDA/Kernels/batched_Tab_to_Klr.cu
+    Numerics/detail/CUDA/Kernels/dot_wabn.cu
+    Numerics/detail/CUDA/Kernels/Tab_to_Kl.cu
     Numerics/detail/CUDA/Kernels/sampleGaussianRNG.cu
     Numerics/detail/CUDA/Kernels/construct_X.cu
     Numerics/detail/CUDA/Kernels/reference_operations.cu

src/AFQMC/Estimators/Observables/n2r.hpp

@@ -82,14 +82,16 @@ class n2r: public AFQMCInfo
 
   n2r(afqmc::TaskGroup_& tg_, AFQMCInfo& info, xmlNodePtr cur, WALKER_TYPES wlk, 
            bool host_mem, aux_Allocator alloc_, aux_Allocator orb_alloc_, int nave_=1, int bsize=1):
-                AFQMCInfo(info),aux_alloc(alloc_),TG(tg_),walker_type(wlk),writer(false),
-                block_size(bsize),nave(nave_),counter(0),use_host_memory(host_mem),
+                AFQMCInfo(info),aux_alloc(alloc_),block_size(bsize),nave(nave_),counter(0),TG(tg_),
+                walker_type(wlk),dm_size(0),writer(false),
+                use_host_memory(host_mem),
                 hdf_walker_output(""),
-                denom(iextensions<1u>{0},shared_allocator<ComplexType>{TG.TG_local()}),
-                DMWork({0,0},shared_allocator<ComplexType>{TG.TG_local()}),
                 Orbitals({0,0},orb_alloc_), 
-                Buff(iextensions<1u>{0},alloc_), 
-                DMAverage({0,0},shared_allocator<ComplexType>{TG.TG_local()})
+                DMAverage({0,0},shared_allocator<ComplexType>{TG.TG_local()}),
+                DMWork({0,0},shared_allocator<ComplexType>{TG.TG_local()}),
+                denom(iextensions<1u>{0},shared_allocator<ComplexType>{TG.TG_local()}),
+                Buff(iextensions<1u>{0},alloc_),
+                Buff2(iextensions<1u>{0})
   {
 
     app_log()<<"  --  Adding Back Propagated on-top pair density (N2R) estimator. -- \n ";

src/AFQMC/HamiltonianOperations/HamiltonianOperations.hpp

@@ -28,6 +28,10 @@
 #include "AFQMC/HamiltonianOperations/KP3IndexFactorization_batched.hpp"
 #include "AFQMC/HamiltonianOperations/KP3IndexFactorization_batched_ooc.hpp"
 //#include "AFQMC/HamiltonianOperations/KPTHCOps.hpp"
+#else
+#include "AFQMC/HamiltonianOperations/Real3IndexFactorization.hpp"
+#include "AFQMC/HamiltonianOperations/Real3IndexFactorization_batched.hpp"
+#include "AFQMC/HamiltonianOperations/Real3IndexFactorization_batched_v2.hpp"
 #endif
 
 namespace qmcplusplus
@@ -164,7 +168,10 @@ class HamiltonianOperations:
                                   SparseTensor<RealType,RealType>,
                                   SparseTensor<RealType,ComplexType>,
                                   SparseTensor<ComplexType,RealType>,
-                                  SparseTensor<ComplexType,ComplexType>
+                                  SparseTensor<ComplexType,ComplexType>,
+                                  Real3IndexFactorization,
+                                  Real3IndexFactorization_batched,
+                                  Real3IndexFactorization_batched_v2
                              >
 #endif
 {
@@ -182,6 +189,9 @@ class HamiltonianOperations:
     explicit HamiltonianOperations(STRR&& other) : variant(std::move(other)) {}
     explicit HamiltonianOperations(STRC&& other) : variant(std::move(other)) {}
     explicit HamiltonianOperations(STCR&& other) : variant(std::move(other)) {}
+    explicit HamiltonianOperations(Real3IndexFactorization&& other) : variant(std::move(other)) {}
+    explicit HamiltonianOperations(Real3IndexFactorization_batched&& other) : variant(std::move(other)) {}
+    explicit HamiltonianOperations(Real3IndexFactorization_batched_v2&& other) : variant(std::move(other)) {}
 #else
     explicit HamiltonianOperations(KP3IndexFactorization&& other) : variant(std::move(other)) {}
     explicit HamiltonianOperations(KP3IndexFactorization_batched&& other) : variant(std::move(other)) {}
@@ -195,6 +205,9 @@ class HamiltonianOperations:
     explicit HamiltonianOperations(STRR const& other) = delete;
     explicit HamiltonianOperations(STRC const& other) = delete;
     explicit HamiltonianOperations(STCR const& other) = delete;
+    explicit HamiltonianOperations(Real3IndexFactorization const& other) = delete;
+    explicit HamiltonianOperations(Real3IndexFactorization_batched const& other) = delete;
+    explicit HamiltonianOperations(Real3IndexFactorization_batched_v2 const& other) = delete;
 #else
     explicit HamiltonianOperations(KP3IndexFactorization const& other) = delete;
     explicit HamiltonianOperations(KP3IndexFactorization_batched const& other) = delete;