Skip to content

Commit 7ed2e8f

Browse files
AMLattanziAMLattanzi
authored
Explicit MOST runtime (#1605)
* Make explicit most a runtime option. * Fix function signature mismatch. * Remove commented flag for windfarm. * Add docs update. * Fix SmnSmn since it was a ifndef.
1 parent 1a5597f commit 7ed2e8f

22 files changed

+304
-485
lines changed

‎CMake/BuildERFExe.cmake

-4
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,6 @@ function(build_erf_lib erf_lib_name)
1717

1818
target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_MOISTURE)
1919

20-
if(ERF_ENABLE_EXPLICIT_MOST_STRESS)
21-
target_compile_definitions(${erf_lib_name} PUBLIC ERF_EXPLICIT_MOST_STRESS)
22-
endif()
23-
2420
if(ERF_ENABLE_MULTIBLOCK)
2521
target_sources(${erf_lib_name} PRIVATE
2622
${SRC_DIR}/MultiBlock/MultiBlockContainer.cpp)

‎CMakeLists.txt

-2
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@ option(ERF_ENABLE_PARTICLES "Enable Lagrangian particles" OFF)
1919
option(ERF_ENABLE_FCOMPARE "Enable building fcompare when not testing" OFF)
2020
set(ERF_PRECISION "DOUBLE" CACHE STRING "Floating point precision SINGLE or DOUBLE")
2121

22-
option(ERF_ENABLE_EXPLICIT_MOST_STRESS "Explicitly include MOST stresses in momentum diffusion" OFF)
23-
2422
option(ERF_ENABLE_MOISTURE "Enable Full Moisture" ON)
2523
option(ERF_ENABLE_WARM_NO_PRECIP "Enable Warm Moisture" OFF)
2624
option(ERF_ENABLE_RRTMGP "Enable RTE-RRTMGP Radiation" OFF)

‎Docs/sphinx_doc/MOST.rst

+8-1
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,14 @@ In ERF, when the MOST boundary condition is applied, velocity and temperature in
134134

135135
.. math::
136136
137-
(\rho \theta)_{i,j,-n} = \rho_{i,j,-n} \left[ \frac{(\rho\theta)_{i,j,0}}{\rho_{i,j,0}} - \left. \frac{\tau_{\theta z}}{\rho} \right|_{i,j,0} \frac{\rho_{i,j,0}}{K_{\theta,v,(i,j,0)}} n \Delta z \right]
137+
(\rho \theta)_{i,j,-n} = \rho_{i,j,-n} \left[ \frac{(\rho\theta)_{i,j,0}}{\rho_{i,j,0}} - \left. \frac{\tau_{\theta z}}{\rho} \right|_{i,j,0} \frac{\rho_{i,j,0}}{K_{\theta,v,(i,j,0)}} n \Delta z \right].
138+
139+
The above implementation explicitly sets the ghost cells so that the local stresses in (6) are recovered. This formulation will depend upon the eddy diffusivity :math:`K_{\phi,v}` in the near-wall region. Since :math:`K_{\phi,v}` may be a function of near-wall gradients, circular dependencies may occur. An **explicit MOST** formulation has also been implemented where the stress tensors are directly populated with the values computed for :math:`\tau_{\phi z}` and the ghost cells are filled according the recommendation made in `Moeng, Journal of the Atmospheric Sciences, 1984 <https://journals.ametsoc.org/view/journals/atsc/41/13/1520-0469_1984_041_2052_alesmf_2_0_co_2.xml>`_; see below. To enable the **explicit MOST** formulation, users may add the line ``erf.use_explicit_most = true``.
140+
141+
.. math::
142+
143+
(\rho \theta)_{z} = \frac{(\rho \theta)_{i,j,1} - (\rho \theta)_{i,j,0}}{\Delta z}
144+
(\rho \theta)_{i,j,-n} = (\rho \theta)_{i,j,0} - (\rho \theta)_{z} n \Delta z .
138145
139146
Finally, it must be noted that complex terrain will modify the surface normal and tangent vectors. Consequently, the MOST implentation with terrain will require local vector rotations. While the ERF dycore accounts for
140147
terrain metrics when computing fluxes (e.g. for advection, diffusion, etc.), the impact of terrain metrics on MOST is still a work in progress. Therefore, running with terrain (``erf.use_terrain = true``) and with MOST

‎Exec/ABL/GNUmakefile

-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ TEST = TRUE
2525
USE_ASSERTION = TRUE
2626

2727
#USE_POISSON_SOLVE = TRUE
28-
USE_EXPLICIT_MOST_STRESS = FALSE
2928

3029
# GNU Make
3130
Bpack := ./Make.package

‎Exec/Make.ERF

-4
Original file line numberDiff line numberDiff line change
@@ -170,10 +170,6 @@ ERF_LSM_MM5_DIR = $(ERF_SOURCE_DIR)/LandSurfaceModel/MM5
170170
include $(ERF_LSM_MM5_DIR)/Make.package
171171
VPATH_LOCATIONS += $(ERF_LSM_MM5_DIR)
172172
INCLUDE_LOCATIONS += $(ERF_LSM_MM5_DIR)
173-
174-
ifeq ($(USE_EXPLICIT_MOST_STRESS), TRUE)
175-
DEFINES += -DERF_EXPLICIT_MOST_STRESS
176-
endif
177173

178174
ifeq ($(COMPUTE_ERROR), TRUE)
179175
DEFINES += -DERF_COMPUTE_ERROR

‎Source/BoundaryConditions/ABLMost.H

+6-9
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ public:
3030

3131
// Constructor
3232
explicit ABLMost (const amrex::Vector<amrex::Geometry>& geom,
33+
bool& use_exp_most,
3334
amrex::Vector<amrex::Vector<amrex::MultiFab>>& vars_old,
3435
amrex::Vector<std::unique_ptr<amrex::MultiFab>>& Theta_prim,
3536
amrex::Vector<std::unique_ptr<amrex::MultiFab>>& Qv_prim,
@@ -40,7 +41,8 @@ public:
4041
amrex::Vector<amrex::Vector<amrex::MultiFab*>> lsm_flux,
4142
amrex::Real start_bdy_time = 0.0,
4243
amrex::Real bdy_time_interval = 0.0)
43-
: m_start_bdy_time(start_bdy_time),
44+
: m_exp_most(use_exp_most),
45+
m_start_bdy_time(start_bdy_time),
4446
m_bdy_time_interval(bdy_time_interval),
4547
m_geom(geom),
4648
m_ma(geom,vars_old,Theta_prim,Qv_prim,z_phys_nd)
@@ -220,26 +222,20 @@ public:
220222
void
221223
impose_most_bcs (const int& lev,
222224
const amrex::Vector<amrex::MultiFab*>& mfs,
223-
#ifdef ERF_EXPLICIT_MOST_STRESS
224225
amrex::MultiFab* xzmom_flux, amrex::MultiFab* zxmom_flux,
225226
amrex::MultiFab* yzmom_flux, amrex::MultiFab* zymom_flux,
226227
amrex::MultiFab* heat_flux,
227-
#else
228228
amrex::MultiFab* eddyDiffs,
229-
#endif
230229
amrex::MultiFab* z_phys);
231230

232231
template<typename FluxCalc>
233232
void
234233
compute_most_bcs (const int& lev,
235234
const amrex::Vector<amrex::MultiFab*>& mfs,
236-
#ifdef ERF_EXPLICIT_MOST_STRESS
237-
amrex::MultiFab* xzmom_flux, amrex::MultiFab* zxmom_flux,
238-
amrex::MultiFab* yzmom_flux, amrex::MultiFab* zymom_flux,
235+
amrex::MultiFab* xzmom_flux, amrex::MultiFab* zxmom_flux,
236+
amrex::MultiFab* yzmom_flux, amrex::MultiFab* zymom_flux,
239237
amrex::MultiFab* heat_flux,
240-
#else
241238
amrex::MultiFab* eddyDiffs,
242-
#endif
243239
amrex::MultiFab* z_phys,
244240
const amrex::Real& dz_no_terrain,
245241
const FluxCalc& flux_comp);
@@ -319,6 +315,7 @@ public:
319315

320316
private:
321317
bool use_moisture;
318+
bool m_exp_most = false;
322319
amrex::Real z0_const;
323320
amrex::Real surf_temp;
324321
amrex::Real surf_heating_rate{0};

‎Source/BoundaryConditions/ABLMost.cpp

+29-77
Original file line numberDiff line numberDiff line change
@@ -121,48 +121,36 @@ ABLMost::compute_fluxes (const int& lev,
121121
void
122122
ABLMost::impose_most_bcs (const int& lev,
123123
const Vector<MultiFab*>& mfs,
124-
#ifdef ERF_EXPLICIT_MOST_STRESS
125124
MultiFab* xzmom_flux, MultiFab* zxmom_flux,
126125
MultiFab* yzmom_flux, MultiFab* zymom_flux,
127126
MultiFab* heat_flux,
128-
#else
129127
MultiFab* eddyDiffs,
130-
#endif
131128
MultiFab* z_phys)
132129
{
133130
const int klo = 0;
134131
if (flux_type == FluxCalcType::MOENG) {
135132
moeng_flux flux_comp(klo);
136133
compute_most_bcs(lev, mfs,
137-
#ifdef ERF_EXPLICIT_MOST_STRESS
138134
xzmom_flux, xzmom_flux,
139135
yzmom_flux, yzmom_flux,
140136
heat_flux,
141-
#else
142137
eddyDiffs,
143-
#endif
144138
z_phys, m_geom[lev].CellSize(2), flux_comp);
145139
} else if (flux_type == FluxCalcType::DONELAN) {
146140
donelan_flux flux_comp(klo);
147141
compute_most_bcs(lev, mfs,
148-
#ifdef ERF_EXPLICIT_MOST_STRESS
149142
xzmom_flux, xzmom_flux,
150143
yzmom_flux, yzmom_flux,
151144
heat_flux,
152-
#else
153145
eddyDiffs,
154-
#endif
155146
z_phys, m_geom[lev].CellSize(2), flux_comp);
156147
} else {
157148
custom_flux flux_comp(klo);
158149
compute_most_bcs(lev, mfs,
159-
#ifdef ERF_EXPLICIT_MOST_STRESS
160150
xzmom_flux, xzmom_flux,
161151
yzmom_flux, yzmom_flux,
162152
heat_flux,
163-
#else
164153
eddyDiffs,
165-
#endif
166154
z_phys, m_geom[lev].CellSize(2), flux_comp);
167155
}
168156
}
@@ -180,13 +168,10 @@ template<typename FluxCalc>
180168
void
181169
ABLMost::compute_most_bcs (const int& lev,
182170
const Vector<MultiFab*>& mfs,
183-
#ifdef ERF_EXPLICIT_MOST_STRESS
184171
MultiFab* xzmom_flux, MultiFab* zxmom_flux,
185172
MultiFab* yzmom_flux, MultiFab* zymom_flux,
186173
MultiFab* heat_flux,
187-
#else
188174
MultiFab* eddyDiffs,
189-
#endif
190175
MultiFab* z_phys,
191176
const Real& dz_no_terrain,
192177
const FluxCalc& flux_comp)
@@ -199,36 +184,38 @@ ABLMost::compute_most_bcs (const int& lev,
199184
// Valid CC box
200185
Box vbx = mfi.validbox(); vbx.makeSlab(2,klo-1);
201186

202-
#ifdef ERF_EXPLICIT_MOST_STRESS
203187
Box vbxx = surroundingNodes(vbx,0);
204188
Box vbxy = surroundingNodes(vbx,1);
205-
#endif
189+
190+
// Expose for GPU
191+
bool exp_most = m_exp_most;
206192

207193
// Get field arrays
208194
const auto cons_arr = mfs[Vars::cons]->array(mfi);
209195
const auto velx_arr = mfs[Vars::xvel]->array(mfi);
210196
const auto vely_arr = mfs[Vars::yvel]->array(mfi);
211-
#ifdef ERF_EXPLICIT_MOST_STRESS
212-
auto t13_arr = xzmom_flux->array(mfi);
213-
auto t23_arr = yzmom_flux->array(mfi);
214-
auto t31_arr = (zxmom_flux) ? zxmom_flux->array(mfi) : Array4<Real>{};
215-
auto t32_arr = (zymom_flux) ? zymom_flux->array(mfi) : Array4<Real>{};
216-
auto hfx_arr = heat_flux->array(mfi);
217-
#else
197+
198+
auto t13_arr = (m_exp_most) ? xzmom_flux->array(mfi) : Array4<Real>{};
199+
auto t23_arr = (m_exp_most) ? yzmom_flux->array(mfi) : Array4<Real>{};
200+
auto t31_arr = (zxmom_flux && m_exp_most) ? zxmom_flux->array(mfi) : Array4<Real>{};
201+
auto t32_arr = (zymom_flux && m_exp_most) ? zymom_flux->array(mfi) : Array4<Real>{};
202+
auto hfx_arr = (m_exp_most) ? heat_flux->array(mfi) : Array4<Real>{};
203+
218204
const auto eta_arr = eddyDiffs->array(mfi);
219-
#endif
205+
220206
const auto zphys_arr = (z_phys) ? z_phys->const_array(mfi) : Array4<const Real>{};
221207

222208
// Get average arrays
223209
const auto *const u_mean = m_ma.get_average(lev,0);
224210
const auto *const v_mean = m_ma.get_average(lev,1);
225211
const auto *const t_mean = m_ma.get_average(lev,2);
226-
// const auto *const q_mean = m_ma.get_average(lev,3);
212+
const auto *const q_mean = m_ma.get_average(lev,3);
227213
const auto *const u_mag_mean = m_ma.get_average(lev,4);
228214

229215
const auto um_arr = u_mean->array(mfi);
230216
const auto vm_arr = v_mean->array(mfi);
231217
const auto tm_arr = t_mean->array(mfi);
218+
const auto qm_arr = q_mean->array(mfi);
232219
const auto umm_arr = u_mag_mean->array(mfi);
233220

234221
// Get derived arrays
@@ -254,54 +241,35 @@ ABLMost::compute_most_bcs (const int& lev,
254241
int n = RhoTheta_comp;
255242
ParallelFor(b2d, [=] AMREX_GPU_DEVICE (int i, int j, int k)
256243
{
257-
Real dz = (zphys_arr) ? ( zphys_arr(i,j,klo) - zphys_arr(i,j,klo-1) ) : dz_no_terrain;
258-
259-
#ifdef ERF_EXPLICIT_MOST_STRESS
260-
Real dz1 = (zphys_arr) ? ( zphys_arr(i,j,klo+1) - zphys_arr(i,j,klo) ) : dz_no_terrain;
261-
Real Tflux = flux_comp.compute_t_flux(i, j, k, n, icomp, dz, dz1,
262-
cons_arr, velx_arr, vely_arr,
263-
umm_arr, tm_arr, u_star_arr, t_star_arr, t_surf_arr,
264-
dest_arr);
265-
#else
266-
Real Tflux = flux_comp.compute_t_flux(i, j, k, n, icomp, dz,
244+
Real dz = (zphys_arr) ? ( zphys_arr(i,j,klo ) - zphys_arr(i,j,klo-1) ) : dz_no_terrain;
245+
Real dz1 = (zphys_arr) ? ( zphys_arr(i,j,klo+1) - zphys_arr(i,j,klo ) ) : dz_no_terrain;
246+
Real Tflux = flux_comp.compute_t_flux(i, j, k, n, icomp, dz, dz1, exp_most, eta_arr,
267247
cons_arr, velx_arr, vely_arr,
268-
eta_arr,
269248
umm_arr, tm_arr, u_star_arr, t_star_arr, t_surf_arr,
270249
dest_arr);
271-
#endif
272250

273251

274252
// TODO: make sure not to double-count surface heat flux if using a LSM
275253
int is_land = (lmask_arr) ? lmask_arr(i,j,klo) : 1;
276254
if (is_land && lsm_flux_arr && vbx.contains(i,j,k)) {
277255
lsm_flux_arr(i,j,klo) = Tflux;
278256
}
279-
#ifdef ERF_EXPLICIT_MOST_STRESS
280-
else if ((k == klo-1) && vbx.contains(i,j,k)) {
257+
else if ((k == klo-1) && vbx.contains(i,j,k) && exp_most) {
281258
hfx_arr(i,j,klo-1) = Tflux;
282259
}
283-
#endif
284260
});
285261

286262
// TODO: Generalize MOST q flux with MOENG & DONELAN flux types
287263
if ((flux_type == FluxCalcType::CUSTOM) && use_moisture) {
288264
n = RhoQ1_comp;
289265
ParallelFor(b2d, [=] AMREX_GPU_DEVICE (int i, int j, int k)
290266
{
291-
Real dz = (zphys_arr) ? ( zphys_arr(i,j,klo) - zphys_arr(i,j,klo-1) ) : dz_no_terrain;
292-
#ifdef ERF_EXPLICIT_MOST_STRESS
293-
Real dz1 = (zphys_arr) ? ( zphys_arr(i,j,klo+1) - zphys_arr(i,j,klo) ) : dz_no_terrain;
294-
Real Qflux = flux_comp.compute_q_flux(i, j, k, n, icomp, dz, dz1,
267+
Real dz = (zphys_arr) ? ( zphys_arr(i,j,klo ) - zphys_arr(i,j,klo-1) ) : dz_no_terrain;
268+
Real dz1 = (zphys_arr) ? ( zphys_arr(i,j,klo+1) - zphys_arr(i,j,klo ) ) : dz_no_terrain;
269+
Real Qflux = flux_comp.compute_q_flux(i, j, k, n, icomp, dz, dz1, exp_most, eta_arr,
295270
cons_arr, velx_arr, vely_arr,
296-
umm_arr, tm_arr, u_star_arr, q_star_arr, t_surf_arr,
271+
umm_arr, qm_arr, u_star_arr, q_star_arr, t_surf_arr,
297272
dest_arr);
298-
#else
299-
Real Qflux = flux_comp.compute_q_flux(i, j, k, n, icomp, dz,
300-
cons_arr, velx_arr, vely_arr,
301-
eta_arr,
302-
umm_arr, tm_arr, u_star_arr, q_star_arr, t_surf_arr,
303-
dest_arr);
304-
#endif
305273
amrex::ignore_unused(Qflux);
306274
});
307275
}
@@ -313,24 +281,16 @@ ABLMost::compute_most_bcs (const int& lev,
313281

314282
ParallelFor(xb2d, [=] AMREX_GPU_DEVICE (int i, int j, int k)
315283
{
316-
Real dz = (zphys_arr) ? ( zphys_arr(i,j,klo) - zphys_arr(i,j,klo-1) ) : dz_no_terrain;
317-
#ifdef ERF_EXPLICIT_MOST_STRESS
318-
Real dz1 = (zphys_arr) ? ( zphys_arr(i,j,klo+1) - zphys_arr(i,j,klo) ) : dz_no_terrain;
319-
Real stressx = flux_comp.compute_u_flux(i, j, k, icomp, dz, dz1,
284+
Real dz = (zphys_arr) ? ( zphys_arr(i,j,klo ) - zphys_arr(i,j,klo-1) ) : dz_no_terrain;
285+
Real dz1 = (zphys_arr) ? ( zphys_arr(i,j,klo+1) - zphys_arr(i,j,klo ) ) : dz_no_terrain;
286+
Real stressx = flux_comp.compute_u_flux(i, j, k, icomp, dz, dz1, exp_most, eta_arr,
320287
cons_arr, velx_arr, vely_arr,
321288
umm_arr, um_arr, u_star_arr,
322289
dest_arr);
323-
if ((k == klo-1) && vbxx.contains(i,j,k)) {
290+
if ((k == klo-1) && vbxx.contains(i,j,k) && exp_most) {
324291
t13_arr(i,j,klo) = -stressx;
325292
if (t31_arr) t31_arr(i,j,klo) = -stressx;
326293
}
327-
#else
328-
Real stressx = flux_comp.compute_u_flux(i, j, k, icomp, dz,
329-
cons_arr, velx_arr, vely_arr,
330-
eta_arr,
331-
umm_arr, um_arr, u_star_arr,
332-
dest_arr);
333-
#endif
334294
amrex::ignore_unused(stressx);
335295
});
336296

@@ -341,24 +301,16 @@ ABLMost::compute_most_bcs (const int& lev,
341301

342302
ParallelFor(yb2d, [=] AMREX_GPU_DEVICE (int i, int j, int k)
343303
{
344-
Real dz = (zphys_arr) ? ( zphys_arr(i,j,klo) - zphys_arr(i,j,klo-1) ) : dz_no_terrain;
345-
#ifdef ERF_EXPLICIT_MOST_STRESS
346-
Real dz1 = (zphys_arr) ? ( zphys_arr(i,j,klo+1) - zphys_arr(i,j,klo) ) : dz_no_terrain;
347-
Real stressy = flux_comp.compute_v_flux(i, j, k, icomp, dz, dz1,
304+
Real dz = (zphys_arr) ? ( zphys_arr(i,j,klo ) - zphys_arr(i,j,klo-1) ) : dz_no_terrain;
305+
Real dz1 = (zphys_arr) ? ( zphys_arr(i,j,klo+1) - zphys_arr(i,j,klo ) ) : dz_no_terrain;
306+
Real stressy = flux_comp.compute_v_flux(i, j, k, icomp, dz, dz1, exp_most, eta_arr,
348307
cons_arr, velx_arr, vely_arr,
349308
umm_arr, vm_arr, u_star_arr,
350309
dest_arr);
351-
if ((k == klo-1) && vbxy.contains(i,j,k)) {
310+
if ((k == klo-1) && vbxy.contains(i,j,k) && exp_most) {
352311
t23_arr(i,j,klo) = -stressy;
353312
if (t32_arr) t32_arr(i,j,klo) = -stressy;
354313
}
355-
#else
356-
Real stressy = flux_comp.compute_v_flux(i, j, k, icomp, dz,
357-
cons_arr, velx_arr, vely_arr,
358-
eta_arr,
359-
umm_arr, vm_arr, u_star_arr,
360-
dest_arr);
361-
#endif
362314
amrex::ignore_unused(stressy);
363315
});
364316
}

‎Source/BoundaryConditions/ERF_FillPatch.cpp

-3
Original file line numberDiff line numberDiff line change
@@ -383,13 +383,10 @@ ERF::FillIntermediatePatch (int lev, Real time,
383383
// MOST boundary conditions
384384
if (!(cons_only && ncomp_cons == 1) && m_most && allow_most_bcs) {
385385
m_most->impose_most_bcs(lev,mfs_vel,
386-
#ifdef ERF_EXPLICIT_MOST_STRESS
387386
Tau13_lev[lev].get(), Tau31_lev[lev].get(),
388387
Tau23_lev[lev].get(), Tau32_lev[lev].get(),
389388
SFS_hfx3_lev[lev].get(),
390-
#else
391389
eddyDiffs_lev[lev].get(),
392-
#endif
393390
z_phys_nd[lev].get());
394391
}
395392

0 commit comments

Comments
 (0)