Reduce use of SuperParticle

.github/workflows/gcc.yml

@@ -376,7 +376,7 @@ jobs:
       # /home/runner/work/amrex/amrex/Src/Base/AMReX_IntVect.H:194:92: error: array subscript -1 is below array bounds of ‘int [3]’ [-Werror=array-bounds]
       # int& operator[] (int i) noexcept { BL_ASSERT(i>=0 && i < AMREX_SPACEDIM); return vect[i]; }
       #
-      env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wunreachable-code -Wnon-virtual-dtor -Wlogical-op -Wmisleading-indentation -Wduplicated-cond -Wduplicated-branches -Wmissing-include-dirs -Wno-array-bounds"}
+      env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wunreachable-code -Wnon-virtual-dtor -Wlogical-op -Wmisleading-indentation -Wduplicated-cond -Wduplicated-branches -Wmissing-include-dirs -Wno-array-bounds"}
       run: |
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10

Src/Extern/HDF5/AMReX_ParticleHDF5.H

@@ -66,9 +66,9 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
     WriteHDF5ParticleData(dir, name, write_real_comp, write_int_comp,
                           tmp_real_comp_names, tmp_int_comp_names,
                           compression,
-                          [=] AMREX_GPU_HOST_DEVICE (const SuperParticleType& p) -> int
+                          [=] AMREX_GPU_HOST_DEVICE (const ConstPTDType& ptd, int i) -> int
                           {
-                              return p.id() > 0;
+                              return ptd.id(i) > 0;
                           }, true);
 }
 
@@ -102,9 +102,9 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
     WriteHDF5ParticleData(dir, name, write_real_comp, write_int_comp,
                           real_comp_names, int_comp_names,
                           compression,
-                          [=] AMREX_GPU_HOST_DEVICE (const SuperParticleType& p) -> int
+                          [=] AMREX_GPU_HOST_DEVICE (const ConstPTDType& ptd, int i) -> int
                           {
-                              return p.id() > 0;
+                              return ptd.id(i) > 0;
                           });
 }
 
@@ -138,9 +138,9 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
     WriteHDF5ParticleData(dir, name, write_real_comp, write_int_comp,
                           real_comp_names, int_comp_names,
                           compression,
-                          [=] AMREX_GPU_HOST_DEVICE (const SuperParticleType& p)
+                          [=] AMREX_GPU_HOST_DEVICE (const ConstPTDType& ptd, int i)
                           {
-                              return p.id() > 0;
+                              return ptd.id(i) > 0;
                           });
 }
 
@@ -166,9 +166,9 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
                           write_real_comp, write_int_comp,
                           real_comp_names, int_comp_names,
                           compression,
-                          [=] AMREX_GPU_HOST_DEVICE (const SuperParticleType& p)
+                          [=] AMREX_GPU_HOST_DEVICE (const ConstPTDType& ptd, int i)
                           {
-                              return p.id() > 0;
+                              return ptd.id(i) > 0;
                           });
 }
 
@@ -200,9 +200,9 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
                           write_real_comp, write_int_comp,
                           real_comp_names, int_comp_names,
                           compression,
-                          [=] AMREX_GPU_HOST_DEVICE (const ParticleType& p)
+                          [=] AMREX_GPU_HOST_DEVICE (const ConstPTDType& ptd, int i)
                           {
-                              return p.id() > 0;
+                              return ptd.id(i) > 0;
                           });
 }
 
@@ -238,9 +238,9 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
     WriteHDF5ParticleData(dir, name, write_real_comp, write_int_comp,
                           real_comp_names, int_comp_names,
                           compression,
-                          [=] AMREX_GPU_HOST_DEVICE (const SuperParticleType& p)
+                          [=] AMREX_GPU_HOST_DEVICE (const ConstPTDType& ptd, int i)
                           {
-                              return p.id() > 0;
+                              return ptd.id(i) > 0;
                           });
 }
 
@@ -261,9 +261,9 @@ WritePlotFileHDF5 (const std::string& dir, const std::string& name,
                           write_real_comp, write_int_comp,
                           real_comp_names, int_comp_names,
                           compression,
-                          [=] AMREX_GPU_HOST_DEVICE (const SuperParticleType& p)
+                          [=] AMREX_GPU_HOST_DEVICE (const ConstPTDType& ptd, int i)
                           {
-                              return p.id() > 0;
+                              return ptd.id(i) > 0;
                           });
 }
 

Src/Particle/AMReX_Particle.H

@@ -337,6 +337,7 @@ struct alignas(sizeof(double)) Particle
 
     //! The floating point type used for the particles.
     using RealType = ParticleReal;
+    using IntType = int;
 
     static Long the_next_id;
 
@@ -366,6 +367,12 @@ struct alignas(sizeof(double)) Particle
 #endif
     }
 
+    [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
+    uint64_t& idcpu () & { return this->m_idcpu; }
+
+    [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
+    const uint64_t& idcpu () const & { return this->m_idcpu; }
+
     [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
     RealVect pos () const & {return RealVect(AMREX_D_DECL(this->m_pos[0], this->m_pos[1], this->m_pos[2]));}
 

Src/Particle/AMReX_ParticleContainer.H

@@ -185,6 +185,8 @@ public:
     //! A single level worth of particles is indexed (grid id, tile id)
     //! for both SoA and AoS data.
     using ParticleLevel = std::map<std::pair<int, int>, ParticleTileType>;
+    using PTDType = typename ParticleTileType::ParticleTileDataType;
+    using ConstPTDType = typename ParticleTileType::ConstParticleTileDataType;
     using AoS = typename ParticleTileType::AoS;
     using SoA = typename ParticleTileType::SoA;
 

Src/Particle/AMReX_ParticleContainerI.H

@@ -954,42 +954,47 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
         auto dst = virts.getParticleTileData();
         ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k)
         {
-             if(imf_arr(i,j,k,0)!=0)
-             {
-                 SuperParticleType p;
-                 p.cpu() = 0;
-                 p.id() = LongParticleIds::VirtualParticleID;
-
-                 //Set rdata(0) first so we can normalize the weighted fields
-                 p.rdata(0) = static_cast<ParticleReal>(partData(i,j,k,AMREX_SPACEDIM));
-                 //Set pos with the normalized weighted field
-                 for (int n = 0; n < AMREX_SPACEDIM; ++n)
-                 {
-                     p.pos(n) = static_cast<ParticleReal>(partData(i,j,k,n) / p.rdata(0));
-                 }
-                 //Set rdata(n>0) with the normalized weighted field for NStructReal
-                 for (int n = 1; n < NStructReal; ++n)
-                 {
-                     p.rdata(n) = static_cast<ParticleReal>(partData(i,j,k,AMREX_SPACEDIM+n) / p.rdata(0));
-                 }
-                 //Set rdata(n>0) with the normalized weighted field for NArrayReal
-                 for (int n = 0; n < NArrayReal; ++n)
-                 {
-                     p.rdata(NStructReal+n) = static_cast<ParticleReal>(partData(i,j,k,AMREX_SPACEDIM+NStructReal+n));
-                 }
-                 //Set idata with the "first" particles idata field for NStructInt
-                 for (int n = 0; n < NStructInt; ++n)
-                 {
-                     p.idata(n) = imf_arr(i,j,k,1+n);
-                 }
-                 //Set idata with the "first" particles idata field for NArrayInt
-                 for (int n = 0; n < NArrayInt; ++n)
-                 {
-                     p.idata(NStructInt+n) = imf_arr(i,j,k,1+NStructInt+n);
-                 }
-                 //Set the new particle in dst tile
-                 dst.setSuperParticle(p, last_offset+offsets_ptr[((i-imf_arr.begin.x)+(j-imf_arr.begin.y)*imf_arr.jstride+(k-imf_arr.begin.z)*imf_arr.kstride)]);
-             }
+            if(imf_arr(i,j,k,0)!=0)
+            {
+                const auto idx = last_offset + offsets_ptr[
+                    ((i-imf_arr.begin.x)
+                    +(j-imf_arr.begin.y)*imf_arr.jstride
+                    +(k-imf_arr.begin.z)*imf_arr.kstride)
+                ];
+
+                dst.cpu(idx) = 0;
+                dst.id(idx) = LongParticleIds::VirtualParticleID;
+
+                auto& p = dst[idx];
+                //Set rdata(0) first so we can normalize the weighted fields
+                //Note that this does not work for soa PC
+                p.rdata(0) = static_cast<ParticleReal>(partData(i,j,k,AMREX_SPACEDIM));;
+                //Set pos with the normalized weighted field
+                for (int n = 0; n < AMREX_SPACEDIM; ++n)
+                {
+                    p.pos(n) = static_cast<ParticleReal>(partData(i,j,k,n) / p.rdata(0));
+                }
+                //Set rdata(n>0) with the normalized weighted field for NStructReal
+                for (int n = 1; n < NStructReal; ++n)
+                {
+                    p.rdata(n) = static_cast<ParticleReal>(partData(i,j,k,AMREX_SPACEDIM+n) / p.rdata(0));
+                }
+                //Set rdata(n>0) with the normalized weighted field for NArrayReal
+                for (int n = 0; n < NArrayReal; ++n)
+                {
+                    dst.rdata(n)[idx] = static_cast<ParticleReal>(partData(i,j,k,AMREX_SPACEDIM+NStructReal+n));
+                }
+                //Set idata with the "first" particles idata field for NStructInt
+                for (int n = 0; n < NStructInt; ++n)
+                {
+                    p.idata(n) = imf_arr(i,j,k,1+n);
+                }
+                //Set idata with the "first" particles idata field for NArrayInt
+                for (int n = 0; n < NArrayInt; ++n)
+                {
+                    dst.idata(n)[idx] = imf_arr(i,j,k,1+NStructInt+n);
+                }
+            }
 
         });
         last_offset+=next_offset;