Skip to content

Commit 676e82a

Browse files
deboggardner48
andauthored
Adding Particles Attribute and CI Test (#2178)
* added a particle advection without terrain test * added temperature attribute that is interpolated from the flow temperature * removed unnecessary vars * added particle tests to cmake and github workflows * disabling random initial placement of particles for regtests * syntax correction in windows.yml * removed unused var in ERF_Prob.cpp * Fixed another unused var warning in ERF_LagrangianMicrophysics.H Fixed another unused var warning * corrected exec path * test with an without particles * update AMReX submodule * update test names --------- Co-authored-by: David J. Gardner <[email protected]>
1 parent 05dd7a4 commit 676e82a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+21596
-14
lines changed

.github/workflows/ci.yml

+7-3
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,12 @@ concurrency:
1818
jobs:
1919

2020
Build-And-Test:
21-
2221
runs-on: ${{matrix.os}}
22+
name: OS ${{matrix.os}} - Particles ${{matrix.particles}}
2323
strategy:
2424
matrix:
2525
os: [ubuntu-22.04]
26+
particles: [OFF, ON]
2627
include:
2728
- os: ubuntu-22.04
2829
install_deps: sudo apt-get install mpich libmpich-dev
@@ -66,6 +67,7 @@ jobs:
6667
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
6768
-DERF_DIM:STRING=3 \
6869
-DERF_ENABLE_MPI:BOOL=ON \
70+
-DERF_ENABLE_PARTICLES:BOOL=${{matrix.particles}} \
6971
-DERF_ENABLE_TESTS:BOOL=ON \
7072
-DERF_ENABLE_ALL_WARNINGS:BOOL=ON \
7173
-DERF_ENABLE_FCOMPARE:BOOL=ON \
@@ -80,8 +82,10 @@ jobs:
8082
export CCACHE_MAXSIZE=600M
8183
ccache -z
8284
83-
cmake --build ${{runner.workspace}}/ERF/build-${{matrix.os}} --parallel ${{env.NPROCS}} \
84-
2>&1 | tee -a ${{runner.workspace}}/build-output.txt;
85+
cmake \
86+
--build ${{runner.workspace}}/ERF/build-${{matrix.os}} \
87+
--parallel ${{env.NPROCS}} \
88+
2>&1 | tee -a ${{runner.workspace}}/build-output.txt;
8589
8690
ccache -s
8791
du -hs ~/.cache/ccache

.github/workflows/cuda-ci.yml

+3-1
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,11 @@ concurrency:
1818
jobs:
1919
cuda-build:
2020
runs-on: ubuntu-22.04
21-
name: CUDA v${{matrix.cuda_ver}}
21+
name: CUDA v${{matrix.cuda_ver}} - Particles ${{matrix.particles}}
2222
strategy:
2323
matrix:
2424
cuda_pkg: [12-0]
25+
particles: [OFF, ON]
2526
include:
2627
- cuda_ver: "12.0"
2728
cuda_pkg: 12-0
@@ -70,6 +71,7 @@ jobs:
7071
-DERF_DIM:STRING=3 \
7172
-DERF_ENABLE_MPI:BOOL=ON \
7273
-DERF_ENABLE_CUDA:BOOL=ON \
74+
-DERF_ENABLE_PARTICLES:BOOL=${{matrix.particles}} \
7375
-DERF_ENABLE_REGRESSION_TESTS_ONLY:BOOL=ON .
7476
echo "Building with $(nproc) processes"
7577
cmake --build build-${{matrix.cuda_pkg}} --parallel $(nproc) \

.github/workflows/gcc.yml

+7-1
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,15 @@ concurrency:
88

99
jobs:
1010
library:
11-
name: [email protected] C++17 Release
11+
name: [email protected] C++17 Release - Particles ${{matrix.particles}}
1212
runs-on: ubuntu-22.04
1313
# env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual"}
14+
strategy:
15+
matrix:
16+
particles: [OFF, ON]
17+
1418
steps:
19+
1520
- uses: actions/checkout@v4
1621
with:
1722
submodules: true
@@ -39,6 +44,7 @@ jobs:
3944
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
4045
-DERF_DIM:STRING=3 \
4146
-DERF_ENABLE_MPI:BOOL=ON \
47+
-DERF_ENABLE_PARTICLES:BOOL=${{matrix.particles}} \
4248
-DERF_ENABLE_TESTS:BOOL=ON \
4349
-DERF_ENABLE_ALL_WARNINGS:BOOL=ON \
4450
-DERF_ENABLE_FCOMPARE:BOOL=ON \

.github/workflows/hip.yml

+7-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ jobs:
2020
# https://gitlab.kitware.com/cmake/cmake/-/issues/21968
2121
# https://github.com/ROCm-Developer-Tools/HIP/issues/2246
2222
Build-And-Test-HIP:
23-
name: HIP ROCm [email protected] C++17 [tests]
23+
name: HIP ROCm [email protected] C++17 - Particles ${{matrix.particles}}
2424
runs-on: ubuntu-22.04
2525
# Have to have -Wno-deprecated-declarations due to deprecated atomicAddNoRet
2626
# Have to have -Wno-gnu-zero-variadic-macro-arguments to avoid
@@ -34,7 +34,12 @@ jobs:
3434
# #define select_impl_(_1, _2, impl_, ...) impl_
3535
# NOTE: -Werror was removed because ERF specifically had a lot of warnings. It will be a separate task to go through and fix them all...
3636
env: {CXXFLAGS: "-Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wno-deprecated-declarations -Wno-gnu-zero-variadic-macro-arguments -Wno-pass-failed"}
37+
strategy:
38+
matrix:
39+
particles: [OFF, ON]
40+
3741
steps:
42+
3843
- uses: actions/checkout@v4
3944
with:
4045
submodules: true
@@ -75,6 +80,7 @@ jobs:
7580
-DERF_ENABLE_MPI:BOOL=ON \
7681
-DERF_ENABLE_HIP:BOOL=ON \
7782
-DAMReX_AMD_ARCH=gfx908 \
83+
-DERF_ENABLE_PARTICLES:BOOL=${{matrix.particles}} \
7884
-DERF_ENABLE_TESTS:BOOL=ON \
7985
-DERF_ENABLE_ALL_WARNINGS:BOOL=ON \
8086
-DERF_ENABLE_FCOMPARE:BOOL=ON \

.github/workflows/macos.yml

+7-1
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,15 @@ concurrency:
88

99
jobs:
1010
MacOS-Clang:
11-
name: Apple [email protected]
11+
name: Apple [email protected] - Particles ${{matrix.particles}}
1212
runs-on: macos-latest
1313
# env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual"}
14+
strategy:
15+
matrix:
16+
particles: [OFF, ON]
17+
1418
steps:
19+
1520
- uses: actions/checkout@v4
1621
with:
1722
submodules: true
@@ -37,6 +42,7 @@ jobs:
3742
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
3843
-DERF_DIM:STRING=3 \
3944
-DERF_ENABLE_MPI:BOOL=TRUE \
45+
-DERF_ENABLE_PARTICLES:BOOL=${{matrix.particles}} \
4046
-DERF_ENABLE_TESTS:BOOL=TRUE \
4147
-DERF_ENABLE_ALL_WARNINGS:BOOL=TRUE \
4248
-DERF_ENABLE_FCOMPARE:BOOL=TRUE \

.github/workflows/sycl.yml

+7-1
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,14 @@ concurrency:
1717

1818
jobs:
1919
Build-And-Test-SYCL:
20-
name: oneAPI SYCL
20+
name: oneAPI SYCL - Particles ${{matrix.particles}}
2121
runs-on: ubuntu-22.04
22+
strategy:
23+
matrix:
24+
particles: [OFF, ON]
25+
2226
steps:
27+
2328
- uses: actions/checkout@v4
2429
with:
2530
submodules: true
@@ -62,6 +67,7 @@ jobs:
6267
-DERF_ENABLE_TESTS:BOOL=ON \
6368
-DERF_ENABLE_ALL_WARNINGS:BOOL=ON \
6469
-DERF_ENABLE_FCOMPARE:BOOL=ON \
70+
-DERF_ENABLE_PARTICLES:BOOL=${{matrix.particles}} \
6571
-DCMAKE_C_COMPILER=$(which icx) \
6672
-DCMAKE_CXX_COMPILER=$(which icpx) \
6773
-DCMAKE_CXX_STANDARD=17

.github/workflows/windows.yml

+7-1
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,14 @@ concurrency:
88

99
jobs:
1010
WIN32-MSVC10:
11-
name: WIN32 [email protected]
11+
name: WIN32 [email protected] - Particles ${{matrix.particles}}
1212
runs-on: windows-latest
13+
strategy:
14+
matrix:
15+
particles: [OFF, ON]
16+
1317
steps:
18+
1419
- uses: actions/checkout@v4
1520
with:
1621
submodules: true
@@ -22,6 +27,7 @@ jobs:
2227
-DCMAKE_BUILD_TYPE:STRING=Debug `
2328
-DERF_DIM:STRING=3 `
2429
-DERF_ENABLE_MPI:BOOL=OFF `
30+
-DERF_ENABLE_PARTICLES:BOOL=${{matrix.particles}} `
2531
-DERF_ENABLE_TESTS:BOOL=ON `
2632
-DERF_ENABLE_ALL_WARNINGS:BOOL=OFF `
2733
-DERF_ENABLE_FCOMPARE:BOOL=ON `

Exec/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ else ()
3232
add_subdirectory(DryRegTests/DensityCurrent)
3333
add_subdirectory(DryRegTests/EkmanSpiral)
3434
add_subdirectory(DryRegTests/IsentropicVortex)
35+
add_subdirectory(DryRegTests/ParticleAdvection)
3536
add_subdirectory(DryRegTests/ParticlesOverWoA)
3637
add_subdirectory(DryRegTests/ScalarAdvDiff)
3738
add_subdirectory(DryRegTests/TaylorGreenVortex)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
if(ERF_ENABLE_PARTICLES)
2+
set(erf_exe_name erf_particles_advect)
3+
4+
add_executable(${erf_exe_name} "")
5+
target_sources(${erf_exe_name}
6+
PRIVATE
7+
ERF_Prob.cpp
8+
)
9+
10+
target_include_directories(${erf_exe_name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
11+
12+
include(${CMAKE_SOURCE_DIR}/CMake/BuildERFExe.cmake)
13+
build_erf_exe(${erf_exe_name})
14+
endif()
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#ifndef ERF_PROB_H_
2+
#define ERF_PROB_H_
3+
4+
#include <string>
5+
6+
#include "AMReX_REAL.H"
7+
8+
#include "ERF_ProbCommon.H"
9+
10+
struct ProbParm : ProbParmDefaults {
11+
amrex::Real T_0 = 300.0; // surface temperature == mean potential temperature
12+
amrex::Real U_0 = 0.0;
13+
amrex::Real V_0 = 0.0;
14+
amrex::Real x_c = 0.0; // center of thermal perturbation
15+
amrex::Real z_c = 3000.0;
16+
amrex::Real x_r = 4000.0;
17+
amrex::Real z_r = 2000.0;
18+
amrex::Real T_pert = -15.0; // perturbation temperature
19+
// overridden physical constants
20+
amrex::Real C_p = 1004.0;
21+
}; // namespace ProbParm
22+
23+
class Problem : public ProblemBase
24+
{
25+
public:
26+
Problem();
27+
28+
#include "Prob/ERF_InitDensityHSEDry.H"
29+
30+
void init_custom_pert (
31+
const amrex::Box& bx,
32+
const amrex::Box& xbx,
33+
const amrex::Box& ybx,
34+
const amrex::Box& zbx,
35+
amrex::Array4<amrex::Real const> const& state,
36+
amrex::Array4<amrex::Real > const& state_pert,
37+
amrex::Array4<amrex::Real > const& x_vel_pert,
38+
amrex::Array4<amrex::Real > const& y_vel_pert,
39+
amrex::Array4<amrex::Real > const& z_vel_pert,
40+
amrex::Array4<amrex::Real > const& r_hse,
41+
amrex::Array4<amrex::Real > const& p_hse,
42+
amrex::Array4<amrex::Real const> const& z_nd,
43+
amrex::Array4<amrex::Real const> const& z_cc,
44+
amrex::GeometryData const& geomdata,
45+
amrex::Array4<amrex::Real const> const& mf_m,
46+
amrex::Array4<amrex::Real const> const& mf_u,
47+
amrex::Array4<amrex::Real const> const& mf_v,
48+
const SolverChoice& sc) override;
49+
50+
protected:
51+
std::string name() override { return "Particles over Witch of Agnesi"; }
52+
53+
private:
54+
ProbParm parms;
55+
};
56+
57+
#endif
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
#include "ERF_Prob.H"
2+
#include "ERF_EOS.H"
3+
#include "ERF_TerrainMetrics.H"
4+
5+
using namespace amrex;
6+
7+
std::unique_ptr<ProblemBase>
8+
amrex_probinit(
9+
const amrex_real* /*problo*/,
10+
const amrex_real* /*probhi*/)
11+
{
12+
return std::make_unique<Problem>();
13+
}
14+
15+
Problem::Problem()
16+
{
17+
// Parse params
18+
ParmParse pp("prob");
19+
pp.query("T_0", parms.T_0);
20+
pp.query("U_0", parms.U_0);
21+
pp.query("x_c", parms.x_c);
22+
pp.query("z_c", parms.z_c);
23+
pp.query("x_r", parms.x_r);
24+
pp.query("z_r", parms.z_r);
25+
pp.query("T_pert", parms.T_pert);
26+
27+
init_base_parms(parms.rho_0, parms.T_0);
28+
}
29+
30+
void
31+
Problem::init_custom_pert(
32+
const Box& bx,
33+
const Box& xbx,
34+
const Box& ybx,
35+
const Box& zbx,
36+
Array4<Real const> const& /*state*/,
37+
Array4<Real > const& state_pert,
38+
Array4<Real > const& x_vel_pert,
39+
Array4<Real > const& y_vel_pert,
40+
Array4<Real > const& z_vel_pert,
41+
Array4<Real > const& r_hse,
42+
Array4<Real > const& p_hse,
43+
Array4<Real const> const& z_nd,
44+
Array4<Real const> const& z_cc,
45+
GeometryData const& geomdata,
46+
Array4<Real const> const& /*mf_m*/,
47+
Array4<Real const> const& /*mf_u*/,
48+
Array4<Real const> const& /*mf_v*/,
49+
const SolverChoice& sc)
50+
{
51+
const int klo = geomdata.Domain().smallEnd()[2];
52+
const int khi = geomdata.Domain().bigEnd()[2];
53+
54+
const bool use_moisture = (sc.moisture_type != MoistureType::None);
55+
56+
AMREX_ALWAYS_ASSERT(bx.length()[2] == khi+1);
57+
58+
ParallelFor(bx, [=, parms_d=parms] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
59+
{
60+
// Geometry (note we must include these here to get the data on device)
61+
const auto prob_lo = geomdata.ProbLo();
62+
const auto dx = geomdata.CellSize();
63+
const Real x = prob_lo[0] + (i + 0.5) * dx[0];
64+
const Real z = z_cc(i,j,k);
65+
66+
// Temperature that satisfies the EOS given the hydrostatically balanced (r,p)
67+
const Real Tbar_hse = p_hse(i,j,k) / (R_d * r_hse(i,j,k));
68+
69+
Real L = std::sqrt(
70+
std::pow((x - parms_d.x_c)/parms_d.x_r, 2) +
71+
std::pow((z - parms_d.z_c)/parms_d.z_r, 2)
72+
);
73+
if (L <= 1.0) {
74+
Real dT = parms_d.T_pert * (std::cos(PI*L) + 1.0)/2.0;
75+
76+
// Note: dT is a temperature perturbation, theta_perturbed is base state + perturbation in theta
77+
Real theta_perturbed = (Tbar_hse+dT)*std::pow(p_0/p_hse(i,j,k), R_d/parms_d.C_p);
78+
79+
// This version perturbs rho but not p
80+
state_pert(i, j, k, Rho_comp) = getRhoThetagivenP(p_hse(i,j,k)) / theta_perturbed - r_hse(i,j,k);
81+
}
82+
83+
// Set scalar = 0 everywhere
84+
state_pert(i, j, k, RhoScalar_comp) = 0.0;
85+
86+
if (use_moisture) {
87+
state_pert(i, j, k, RhoQ1_comp) = 0.0;
88+
state_pert(i, j, k, RhoQ2_comp) = 0.0;
89+
}
90+
});
91+
92+
// Set the x-velocity
93+
ParallelFor(xbx, [=, parms_d=parms] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
94+
{
95+
Real ztop = z_nd(i,j,khi+1);
96+
Real zht = z_nd(i,j,klo);
97+
x_vel_pert(i, j, k) = parms_d.U_0 * ztop / (ztop - zht);
98+
});
99+
100+
// Set the y-velocity
101+
ParallelFor(ybx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
102+
{
103+
y_vel_pert(i, j, k) = 0.0;
104+
});
105+
106+
const auto dx = geomdata.CellSize();
107+
amrex::GpuArray<Real, AMREX_SPACEDIM> dxInv;
108+
dxInv[0] = 1. / dx[0];
109+
dxInv[1] = 1. / dx[1];
110+
dxInv[2] = 1. / dx[2];
111+
112+
// Set the z-velocity from impenetrable condition
113+
ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
114+
{
115+
z_vel_pert(i, j, k) = WFromOmega(i, j, k, 0.0, x_vel_pert, y_vel_pert, z_nd, dxInv);
116+
});
117+
118+
amrex::Gpu::streamSynchronize();
119+
120+
}

0 commit comments

Comments
 (0)