Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Named SoA Support #4163

Open
wants to merge 10 commits into
base: development
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 34 additions & 7 deletions Src/Particle/AMReX_ParticleContainer.H
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
#include <memory>
#include <numeric>
#include <random>
#include <string>
#include <tuple>
#include <type_traits>
#include <utility>
Expand Down Expand Up @@ -1144,7 +1145,8 @@ public:
*/
ParticleTileType& DefineAndReturnParticleTile (int lev, int grid, int tile)
{
m_particles[lev][std::make_pair(grid, tile)].define(NumRuntimeRealComps(), NumRuntimeIntComps());
m_particles[lev][std::make_pair(grid, tile)].define(NumRuntimeRealComps(), NumRuntimeIntComps(), &m_soa_rdata_names, &m_soa_idata_names);

return ParticlesAt(lev, grid, tile);
}

Expand Down Expand Up @@ -1247,10 +1249,10 @@ public:

Long superParticleSize() const { return superparticle_size; }

template <typename T,
std::enable_if_t<std::is_same_v<T,bool>,int> = 0>
void AddRealComp (T communicate=true)
void AddRealComp (std::string const & name, bool communicate=true)
{
m_soa_rdata_names.push_back(name);

m_runtime_comps_defined = true;
m_num_runtime_real++;
h_redistribute_real_comp.push_back(communicate);
Expand All @@ -1270,10 +1272,16 @@ public:
}
}

template <typename T,
std::enable_if_t<std::is_same_v<T,bool>,int> = 0>
void AddIntComp (T communicate=true)
void AddRealComp (bool communicate=true)
{
std::string default_name = "runtime_real" + std::to_string(m_num_runtime_real);
AddRealComp(default_name, communicate);
}

void AddIntComp (std::string const & name, bool communicate=true)
{
m_soa_idata_names.push_back(name);

m_runtime_comps_defined = true;
m_num_runtime_int++;
h_redistribute_int_comp.push_back(communicate);
Expand All @@ -1293,6 +1301,12 @@ public:
}
}

void AddIntComp (bool communicate=true)
{
std::string default_name = "runtime_int" + std::to_string(m_num_runtime_int);
AddIntComp(default_name, communicate);
}

int NumRuntimeRealComps () const { return m_num_runtime_real; }
int NumRuntimeIntComps () const { return m_num_runtime_int; }

Expand Down Expand Up @@ -1403,6 +1417,15 @@ public:
#include "AMReX_ParticlesHDF5.H"
#endif

/** Overwrite the default names for the compile-time SoA components */
void SetSoACompileTimeNames (std::vector<std::string> const & rdata_name, std::vector<std::string> const & idata_name);

/** Get the names for the real SoA components **/
std::vector<std::string> GetRealSoANames () const {return m_soa_rdata_names;}

/** Get the names for the int SoA components **/
std::vector<std::string> GetIntSoANames () const {return m_soa_idata_names;}

protected:

template <class RTYPE>
Expand Down Expand Up @@ -1435,6 +1458,10 @@ private:
size_t particle_size, superparticle_size;
int num_real_comm_comps, num_int_comm_comps;
Vector<ParticleLevel> m_particles;

// names of both compile-time and runtime Real and Int SoA data
std::vector<std::string> m_soa_rdata_names;
std::vector<std::string> m_soa_idata_names;
};

template <int T_NStructReal, int T_NStructInt, int T_NArrayReal, int T_NArrayInt, template<class> class Allocator, class CellAssignor>
Expand Down
51 changes: 48 additions & 3 deletions Src/Particle/AMReX_ParticleContainerI.H
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
#include <type_traits>
#include <AMReX_MakeParticle.H>

#include <string>
#include <type_traits>
#include <vector>


template <typename ParticleType, int NArrayReal, int NArrayInt,
template<class> class Allocator, class CellAssignor>
void
Expand Down Expand Up @@ -60,10 +64,51 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
pp.query("do_unlink", doUnlink);
pp.queryAdd("do_mem_efficient_sort", memEfficientSort);

// add default names for SoA Real and Int compile-time arguments
int first_r_name = 0;
// push back x,y,z
if constexpr (ParticleType::is_soa_particle) {
constexpr int x_in_ascii = 120;
for (int i=0; i<AMREX_SPACEDIM; ++i)
{
std::string const name{char(x_in_ascii+i)};
m_soa_rdata_names.push_back(name);
}
first_r_name = AMREX_SPACEDIM;
}
for (int i=first_r_name; i<NArrayReal; ++i)
{
m_soa_rdata_names.push_back("real_comp" + std::to_string(i-first_r_name));
}
for (int i=0; i<NArrayInt; ++i)
{
m_soa_idata_names.push_back("int_comp" + std::to_string(i));
}

initialized = true;
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We discussed that we want to make this two functions (one for real, one for int) that take the ParticleType and index to return a default name on the current index.

This way, we can reuse this in the:

  • current I/O logic (existing naming scheme we took over here)
  • here
  • in AddRealComp/AddIntComp without name argument

}
}

template <typename ParticleType, int NArrayReal, int NArrayInt,
template<class> class Allocator, class CellAssignor>
void
ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor> :: SetSoACompileTimeNames (
std::vector<std::string> const & rdata_name, std::vector<std::string> const & idata_name
)
{
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(rdata_name.size() == NArrayReal, "rdata_name must be equal to NArrayReal");
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(idata_name.size() == NArrayInt, "idata_name must be equal to NArrayInt");

for (int i=0; i<NArrayReal; ++i)
{
m_soa_rdata_names.at(i) = rdata_name.at(i);
}
for (int i=0; i<NArrayInt; ++i)
{
m_soa_idata_names.at(i) = idata_name.at(i);
}
}

template <typename ParticleType, int NArrayReal, int NArrayInt,
template<class> class Allocator, class CellAssignor>
template <typename P, typename Assignor>
Expand Down Expand Up @@ -1161,7 +1206,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
}
} else {
ParticleTileType ptile_tmp;
ptile_tmp.define(m_num_runtime_real, m_num_runtime_int);
ptile_tmp.define(m_num_runtime_real, m_num_runtime_int, &m_soa_rdata_names, &m_soa_idata_names);
ptile_tmp.resize(np_total);
// copy re-ordered particles
gatherParticles(ptile_tmp, ptile, np, permutations);
Expand Down Expand Up @@ -1498,7 +1543,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
tmp_local[lev][index].resize(num_threads);
soa_local[lev][index].resize(num_threads);
for (int t = 0; t < num_threads; ++t) {
soa_local[lev][index][t].define(m_num_runtime_real, m_num_runtime_int);
soa_local[lev][index][t].define(m_num_runtime_real, m_num_runtime_int, &m_soa_rdata_names, &m_soa_idata_names);
}
}
}
Expand Down
12 changes: 10 additions & 2 deletions Src/Particle/AMReX_ParticleTile.H
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,10 @@
#include <AMReX_RealVect.H>

#include <array>
#include <string>
#include <type_traits>
#include <vector>


namespace amrex {

Expand Down Expand Up @@ -730,10 +733,15 @@ struct ParticleTile
ParticleTile& operator= (ParticleTile &&) noexcept = default;
#endif

void define (int a_num_runtime_real, int a_num_runtime_int)
void define (
int a_num_runtime_real,
int a_num_runtime_int,
std::vector<std::string>* soa_rdata_names=nullptr,
std::vector<std::string>* soa_idata_names=nullptr
)
{
m_defined = true;
GetStructOfArrays().define(a_num_runtime_real, a_num_runtime_int);
GetStructOfArrays().define(a_num_runtime_real, a_num_runtime_int, soa_rdata_names, soa_idata_names);
m_runtime_r_ptrs.resize(a_num_runtime_real);
m_runtime_i_ptrs.resize(a_num_runtime_int);
m_runtime_r_cptrs.resize(a_num_runtime_real);
Expand Down
74 changes: 73 additions & 1 deletion Src/Particle/AMReX_StructOfArrays.H
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@
#include <AMReX_Vector.H>
#include <AMReX_GpuContainers.H>

#include <algorithm>
#include <array>
#include <string>
#include <vector>


namespace amrex {

Expand All @@ -19,11 +23,18 @@
using RealVector = amrex::PODVector<ParticleReal, Allocator<ParticleReal> >;
using IntVector = amrex::PODVector<int, Allocator<int> >;

void define (int a_num_runtime_real, int a_num_runtime_int)
void define (
int a_num_runtime_real,
int a_num_runtime_int,
std::vector<std::string>* soa_rdata_names=nullptr,
std::vector<std::string>* soa_idata_names=nullptr
Comment on lines +29 to +30
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, this is indeed not super save... Should we store a reference to a PC instead?

Or store the names in StructOfArrays and look them up when we need them from PC and ParticleTile?

)
{
m_defined = true;
m_runtime_rdata.resize(a_num_runtime_real);
m_runtime_idata.resize(a_num_runtime_int );
m_rdata_names = soa_rdata_names;

Check warning

Code scanning / CodeQL

Local variable address stored in non-local memory Warning

A stack address which arrived via a
parameter
may be assigned to a non-local variable.
m_idata_names = soa_idata_names;

Check warning

Code scanning / CodeQL

Local variable address stored in non-local memory Warning

A stack address which arrived via a
parameter
may be assigned to a non-local variable.
}

[[nodiscard]] int NumRealComps () const noexcept { return NReal + m_runtime_rdata.size(); }
Expand Down Expand Up @@ -79,6 +90,32 @@
}
}

/** Get access to a particle Real component Array (compile-time and runtime component)
*
* @param name named component component with 0...NReal-1 compile-time and NReal... runtime arguments
*/
[[nodiscard]] RealVector& GetRealData (std::string const & name) {
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_rdata_names != nullptr, "SoA Real names were not defined.");
auto const pos = std::find(m_rdata_names->begin(), m_rdata_names->end(), name);
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(pos != m_rdata_names->end(), "Soa Real name='" + name + "' was not found components");

int const index = std::distance(m_rdata_names->begin(), pos);
return GetRealData(index);
}

/** Get access to a particle Real component Array (compile-time and runtime component)
*
* @param name named component component with 0...NReal-1 compile-time and NReal... runtime arguments
*/
[[nodiscard]] const RealVector& GetRealData (std::string const & name) const {
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_rdata_names != nullptr, "SoA Real names were not defined.");
auto const pos = std::find(m_rdata_names->begin(), m_rdata_names->end(), name);
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(pos != m_rdata_names->end(), "Soa Real name='" + name + "' was not found components");

int const index = std::distance(m_rdata_names->begin(), pos);
return GetRealData(index);
}

/** Get access to a particle Int component Array (compile-time and runtime component)
*
* @param index component with 0...NInt-1 compile-time and NInt... runtime arguments
Expand Down Expand Up @@ -118,6 +155,34 @@
}
}

/** Get access to a particle Int component Array (compile-time and runtime component)
*
* @param index component with 0...NInt-1 compile-time and NInt... runtime arguments
* @return
*/
[[nodiscard]] IntVector& GetIntData (std::string const & name) {
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_idata_names != nullptr, "SoA Int names were not defined.");
auto const pos = std::find(m_idata_names->begin(), m_idata_names->end(), name);
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(pos != m_idata_names->end(), "Soa Int name='" + name + "' was not found components");

int const index = std::distance(m_idata_names->begin(), pos);
return GetIntData(index);
}

/** Get access to a particle Int component Array (compile-time and runtime component)
*
* @param index component with 0...NInt-1 compile-time and NInt... runtime arguments
* @return
*/
[[nodiscard]] const IntVector& GetIntData (std::string const & name) const {
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_idata_names != nullptr, "SoA Int names were not defined.");
auto const pos = std::find(m_idata_names->begin(), m_idata_names->end(), name);
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(pos != m_idata_names->end(), "Soa Int name='" + name + "' was not found components");

int const index = std::distance(m_idata_names->begin(), pos);
return GetIntData(index);
}

/**
* \brief Returns the total number of particles (real and neighbor)
*
Expand Down Expand Up @@ -226,13 +291,20 @@
int m_num_neighbor_particles{0};

private:
// compile-time data
IdCPU m_idcpu;
std::array<RealVector, NReal> m_rdata;
std::array< IntVector, NInt> m_idata;

// runtime data
std::vector<RealVector> m_runtime_rdata;
std::vector<IntVector > m_runtime_idata;

// names of both compile-time and runtime Real and Int data
std::vector<std::string>* m_rdata_names = nullptr;
std::vector<std::string>* m_idata_names = nullptr;
Comment on lines +303 to +305
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could own the names here and let PC and ParticleTile reference them...


//! whether the runtime components are sized correctly
bool m_defined{false};
};

Expand Down
10 changes: 10 additions & 0 deletions Tests/Particles/NamedSoAComponents/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
foreach(D IN LISTS AMReX_SPACEDIM)
set(_sources main.cpp)
#set(_input_files)
#set(_input_files inputs)

setup_test(${D} _sources _input_files NTHREADS 2)

unset(_sources)
unset(_input_files)
endforeach()
22 changes: 22 additions & 0 deletions Tests/Particles/NamedSoAComponents/GNUmakefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
AMREX_HOME = ../../../

DEBUG = FALSE

DIM = 3

COMP = gcc

USE_MPI = TRUE
USE_OMP = FALSE
USE_CUDA = FALSE

#TINY_PROFILE = TRUE
USE_PARTICLES = TRUE

include $(AMREX_HOME)/Tools/GNUMake/Make.defs

include ./Make.package
include $(AMREX_HOME)/Src/Base/Make.package
include $(AMREX_HOME)/Src/Particle/Make.package

include $(AMREX_HOME)/Tools/GNUMake/Make.rules
1 change: 1 addition & 0 deletions Tests/Particles/NamedSoAComponents/Make.package
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
CEXE_sources += main.cpp
Loading
Loading