Skip to content

Commit 6f94c65

Browse files
asalmgrenasalmgren
authored
This commit 1) always builds ERF with USE_EB=TRUE, 2) uses terrain_ty… (#2044)
* This commit 1) always builds ERF with USE_EB=TRUE, 2) uses terrain_type to distinguish between terrain-fitted coords vs EB representation of terrain (still a WIP) and the immersed forcing representation * get rid of EB cylinder * fix commas * fix for gpu * fix for gpu (take 2)
1 parent 9689125 commit 6f94c65

36 files changed

+430
-480
lines changed

‎Build/cmake_with_eb.sh

-19
This file was deleted.

‎CMake/BuildERFExe.cmake

+7-13
Original file line numberDiff line numberDiff line change
@@ -35,19 +35,6 @@ function(build_erf_lib erf_lib_name)
3535
target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_PARTICLES)
3636
endif()
3737

38-
if(ERF_ENABLE_EB)
39-
target_sources(${erf_lib_name} PRIVATE
40-
${SRC_DIR}/EB/ERF_InitEB.cpp
41-
${SRC_DIR}/EB/ERF_EBBox.cpp
42-
${SRC_DIR}/EB/ERF_EBCylinder.cpp
43-
${SRC_DIR}/EB/ERF_EBRegular.cpp
44-
${SRC_DIR}/EB/ERF_InitEB.cpp
45-
${SRC_DIR}/EB/ERF_WriteEBSurface.cpp
46-
${SRC_DIR}/LinearSolvers/ERF_SolveWithEBMLMG.cpp)
47-
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/EB>)
48-
target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_EB)
49-
endif()
50-
5138
if(ERF_ENABLE_FFT)
5239
target_sources(${erf_lib_name} PRIVATE
5340
${SRC_DIR}/LinearSolvers/ERF_SolveWithFFT.cpp)
@@ -138,6 +125,11 @@ function(build_erf_lib erf_lib_name)
138125
${SRC_DIR}/Diffusion/ERF_ComputeStrain_N.cpp
139126
${SRC_DIR}/Diffusion/ERF_ComputeStrain_T.cpp
140127
${SRC_DIR}/Diffusion/ERF_ComputeTurbulentViscosity.cpp
128+
${SRC_DIR}/EB/ERF_InitEB.cpp
129+
${SRC_DIR}/EB/ERF_EBBox.cpp
130+
${SRC_DIR}/EB/ERF_EBRegular.cpp
131+
${SRC_DIR}/EB/ERF_InitEB.cpp
132+
${SRC_DIR}/EB/ERF_WriteEBSurface.cpp
141133
${SRC_DIR}/Initialization/ERF_InitBCs.cpp
142134
${SRC_DIR}/Initialization/ERF_InitCustom.cpp
143135
${SRC_DIR}/Initialization/ERF_InitFromHSE.cpp
@@ -192,6 +184,7 @@ function(build_erf_lib erf_lib_name)
192184
${SRC_DIR}/LinearSolvers/ERF_PoissonSolve_tb.cpp
193185
${SRC_DIR}/LinearSolvers/ERF_PoissonWallDist.cpp
194186
${SRC_DIR}/LinearSolvers/ERF_ComputeDivergence.cpp
187+
${SRC_DIR}/LinearSolvers/ERF_SolveWithEBMLMG.cpp
195188
${SRC_DIR}/LinearSolvers/ERF_SolveWithGMRES.cpp
196189
${SRC_DIR}/LinearSolvers/ERF_SolveWithMLMG.cpp
197190
${SRC_DIR}/LinearSolvers/ERF_TerrainPoisson.cpp
@@ -247,6 +240,7 @@ endif()
247240
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/BoundaryConditions>)
248241
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/DataStructs>)
249242
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/Diffusion>)
243+
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/EB>)
250244
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/Initialization>)
251245
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/IO>)
252246
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/LinearSolvers>)

‎CMake/SetAmrexOptions.cmake

+1-5
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ set(AMReX_CUDA ${ERF_ENABLE_CUDA})
2626
set(AMReX_ACC OFF)
2727
set(AMReX_PLOTFILE_TOOLS ${ERF_ENABLE_FCOMPARE})
2828
set(AMReX_FORTRAN OFF)
29+
set(AMReX_EB ON)
2930

3031
set(AMReX_LINEAR_SOLVERS ON)
3132
set(AMReX_LINEAR_SOLVERS_EM OFF)
@@ -41,11 +42,6 @@ if(ERF_ENABLE_FFT)
4142
set(AMReX_FFT ON)
4243
endif()
4344

44-
set(AMReX_EB OFF)
45-
if(ERF_ENABLE_EB)
46-
set(AMReX_EB ON)
47-
endif()
48-
4945
if(ERF_ENABLE_CUDA)
5046
set(AMReX_GPU_BACKEND CUDA CACHE STRING "AMReX GPU type" FORCE)
5147
set(AMReX_CUDA_WARN_CAPTURE_THIS OFF)

‎Exec/DevTests/EB_Test/ERF_Prob.cpp

+11-8
Original file line numberDiff line numberDiff line change
@@ -169,14 +169,17 @@ Problem::init_custom_pert(
169169
dxInv[2] = 1. / dx[2];
170170

171171
// Set the z-velocity from impenetrable condition
172-
ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
173-
{
174-
#ifdef ERF_USE_EB
175-
z_vel_pert(i, j, k) = 0.0;
176-
#else
177-
z_vel_pert(i, j, k) = WFromOmega(i, j, k, 0.0, x_vel_pert, y_vel_pert, z_nd, dxInv);
178-
#endif
179-
});
172+
if (sc.terrain_type == TerrainType::EB) {
173+
ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
174+
{
175+
z_vel_pert(i, j, k) = 0.0;
176+
});
177+
} else {
178+
ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
179+
{
180+
z_vel_pert(i, j, k) = WFromOmega(i, j, k, 0.0, x_vel_pert, y_vel_pert, z_nd, dxInv);
181+
});
182+
}
180183

181184
amrex::Gpu::streamSynchronize();
182185
}

‎Exec/DevTests/EB_Test/GNUmakefile

-2
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@ DEBUG = FALSE
2424
TEST = TRUE
2525
USE_ASSERTION = TRUE
2626

27-
USE_EB = TRUE
28-
2927
# GNU Make
3028
Bpack := ./Make.package
3129
Blocs := .

‎Exec/Make.ERF

+44-61
Original file line numberDiff line numberDiff line change
@@ -2,66 +2,59 @@ AMREX_HOME ?= $(ERF_HOME)/Submodules/AMReX
22

33
BL_NO_FORT = TRUE
44

5+
USE_EB = TRUE
6+
57
include $(AMREX_HOME)/Tools/GNUMake/Make.defs
68

79
EBASE = ERF
810

911
ERF_SOURCE_DIR = $(ERF_HOME)/Source
1012
include $(ERF_SOURCE_DIR)/Make.package
11-
1213
VPATH_LOCATIONS += $(ERF_SOURCE_DIR)
1314
INCLUDE_LOCATIONS += $(ERF_SOURCE_DIR)
1415

1516
ERF_BC_DIR = $(ERF_SOURCE_DIR)/BoundaryConditions
1617
include $(ERF_BC_DIR)/Make.package
17-
1818
VPATH_LOCATIONS += $(ERF_BC_DIR)
1919
INCLUDE_LOCATIONS += $(ERF_BC_DIR)
2020

2121
ERF_ADVECTION_DIR = $(ERF_SOURCE_DIR)/Advection
2222
include $(ERF_ADVECTION_DIR)/Make.package
23+
VPATH_LOCATIONS += $(ERF_ADVECTION_DIR)
24+
INCLUDE_LOCATIONS += $(ERF_ADVECTION_DIR)
2325

2426
ERF_DIFFUSION_DIR = $(ERF_SOURCE_DIR)/Diffusion
2527
include $(ERF_DIFFUSION_DIR)/Make.package
28+
VPATH_LOCATIONS += $(ERF_DIFFUSION_DIR)
29+
INCLUDE_LOCATIONS += $(ERF_DIFFUSION_DIR)
2630

2731
ERF_PBL_DIR = $(ERF_SOURCE_DIR)/PBL
2832
include $(ERF_PBL_DIR)/Make.package
33+
VPATH_LOCATIONS += $(ERF_PBL_DIR)
34+
INCLUDE_LOCATIONS += $(ERF_PBL_DIR)
2935

3036
ERF_INIT_DIR = $(ERF_SOURCE_DIR)/Initialization
3137
include $(ERF_INIT_DIR)/Make.package
38+
VPATH_LOCATIONS += $(ERF_INIT_DIR)
39+
INCLUDE_LOCATIONS += $(ERF_INIT_DIR)
3240

3341
ERF_DATA_DIR = $(ERF_SOURCE_DIR)/DataStructs
3442
include $(ERF_DATA_DIR)/Make.package
43+
VPATH_LOCATIONS += $(ERF_DATA_DIR)
44+
INCLUDE_LOCATIONS += $(ERF_DATA_DIR)
3545

3646
ERF_SOLVERS_DIR = $(ERF_SOURCE_DIR)/LinearSolvers
3747
include $(ERF_SOLVERS_DIR)/Make.package
48+
VPATH_LOCATIONS += $(ERF_SOLVERS_DIR)
49+
INCLUDE_LOCATIONS += $(ERF_SOLVERS_DIR)
3850

3951
ERF_UTIL_DIR = $(ERF_SOURCE_DIR)/Utils
4052
include $(ERF_UTIL_DIR)/Make.package
41-
42-
ERF_MULTIBLOCK_DIR = $(ERF_SOURCE_DIR)/MultiBlock
43-
include $(ERF_MULTIBLOCK_DIR)/Make.package
44-
45-
VPATH_LOCATIONS += $(ERF_ADVECTION_DIR)
46-
INCLUDE_LOCATIONS += $(ERF_ADVECTION_DIR)
47-
48-
VPATH_LOCATIONS += $(ERF_DIFFUSION_DIR)
49-
INCLUDE_LOCATIONS += $(ERF_DIFFUSION_DIR)
50-
51-
VPATH_LOCATIONS += $(ERF_PBL_DIR)
52-
INCLUDE_LOCATIONS += $(ERF_PBL_DIR)
53-
54-
VPATH_LOCATIONS += $(ERF_INIT_DIR)
55-
INCLUDE_LOCATIONS += $(ERF_INIT_DIR)
56-
57-
VPATH_LOCATIONS += $(ERF_DATA_DIR)
58-
INCLUDE_LOCATIONS += $(ERF_DATA_DIR)
59-
6053
VPATH_LOCATIONS += $(ERF_UTIL_DIR)
6154
INCLUDE_LOCATIONS += $(ERF_UTIL_DIR)
6255

63-
VPATH_LOCATIONS += $(ERF_SOLVERS_DIR)
64-
INCLUDE_LOCATIONS += $(ERF_SOLVERS_DIR)
56+
ERF_MULTIBLOCK_DIR = $(ERF_SOURCE_DIR)/MultiBlock
57+
include $(ERF_MULTIBLOCK_DIR)/Make.package
6558

6659
ifeq ($(USE_MULTIBLOCK),TRUE)
6760
VPATH_LOCATIONS += $(ERF_MULTIBLOCK_DIR)
@@ -75,60 +68,30 @@ VPATH_LOCATIONS += $(ERF_PARTICLES_DIR)
7568
INCLUDE_LOCATIONS += $(ERF_PARTICLES_DIR)
7669
endif
7770

78-
ifeq ($(USE_EB),TRUE)
71+
include $(ERF_PROBLEM_DIR)/Make.package
72+
VPATH_LOCATIONS += $(ERF_PROBLEM_DIR)
73+
INCLUDE_LOCATIONS += $(ERF_PROBLEM_DIR)
74+
7975
ERF_EB_DIR = $(ERF_SOURCE_DIR)/EB
8076
include $(ERF_EB_DIR)/Make.package
8177
VPATH_LOCATIONS += $(ERF_EB_DIR)
8278
INCLUDE_LOCATIONS += $(ERF_EB_DIR)
83-
endif
8479

8580
ERF_SRCTERMS_DIR = $(ERF_SOURCE_DIR)/SourceTerms
8681
include $(ERF_SRCTERMS_DIR)/Make.package
87-
8882
VPATH_LOCATIONS += $(ERF_SRCTERMS_DIR)
8983
INCLUDE_LOCATIONS += $(ERF_SRCTERMS_DIR)
9084

9185
ERF_TIMEINT_DIR = $(ERF_SOURCE_DIR)/TimeIntegration
9286
include $(ERF_TIMEINT_DIR)/Make.package
93-
9487
VPATH_LOCATIONS += $(ERF_TIMEINT_DIR)
9588
INCLUDE_LOCATIONS += $(ERF_TIMEINT_DIR)
9689

9790
ERF_IO_DIR = $(ERF_SOURCE_DIR)/IO
9891
include $(ERF_IO_DIR)/Make.package
99-
10092
VPATH_LOCATIONS += $(ERF_IO_DIR)
10193
INCLUDE_LOCATIONS += $(ERF_IO_DIR)
10294

103-
include $(ERF_PROBLEM_DIR)/Make.package
104-
105-
VPATH_LOCATIONS += $(ERF_PROBLEM_DIR)
106-
INCLUDE_LOCATIONS += $(ERF_PROBLEM_DIR)
107-
108-
include $(AMREX_HOME)/Src/Base/Make.package
109-
110-
AMReXdirs := Base Boundary AmrCore
111-
112-
ifeq ($(USE_PARTICLES),TRUE)
113-
AMReXdirs += Particle
114-
endif
115-
116-
ifeq ($(USE_EB),TRUE)
117-
AMReXdirs += EB
118-
endif
119-
120-
USE_LINEAR_SOLVERS_INCFLO = TRUE
121-
USE_LINEAR_SOLVERS_EM = FALSE
122-
AMReXdirs += LinearSolvers
123-
124-
ifeq ($(USE_FFT),TRUE)
125-
AMReXdirs += FFT
126-
endif
127-
128-
AMReXpack += $(foreach dir, $(AMReXdirs), $(AMREX_HOME)/Src/$(dir)/Make.package)
129-
130-
include $(AMReXpack)
131-
13295
ERF_MOISTURE_DIR = $(ERF_SOURCE_DIR)/Microphysics
13396
include $(ERF_MOISTURE_DIR)/Make.package
13497
VPATH_LOCATIONS += $(ERF_MOISTURE_DIR)
@@ -238,6 +201,30 @@ ERF_LSM_MM5_DIR = $(ERF_SOURCE_DIR)/LandSurfaceModel/MM5
238201
include $(ERF_LSM_MM5_DIR)/Make.package
239202
VPATH_LOCATIONS += $(ERF_LSM_MM5_DIR)
240203
INCLUDE_LOCATIONS += $(ERF_LSM_MM5_DIR)
204+
205+
include $(AMREX_HOME)/Src/Base/Make.package
206+
207+
AMReXdirs := Base Boundary AmrCore
208+
209+
ifeq ($(USE_PARTICLES),TRUE)
210+
AMReXdirs += Particle
211+
endif
212+
213+
ifeq ($(USE_EB),TRUE)
214+
AMReXdirs += EB
215+
endif
216+
217+
USE_LINEAR_SOLVERS_INCFLO = TRUE
218+
USE_LINEAR_SOLVERS_EM = FALSE
219+
AMReXdirs += LinearSolvers
220+
221+
ifeq ($(USE_FFT),TRUE)
222+
AMReXdirs += FFT
223+
endif
224+
225+
AMReXpack += $(foreach dir, $(AMReXdirs), $(AMREX_HOME)/Src/$(dir)/Make.package)
226+
227+
include $(AMReXpack)
241228

242229
ifeq ($(COMPUTE_ERROR), TRUE)
243230
DEFINES += -DERF_COMPUTE_ERROR
@@ -247,10 +234,6 @@ ifeq ($(USE_PARTICLES), TRUE)
247234
DEFINES += -DERF_USE_PARTICLES
248235
endif
249236

250-
ifeq ($(USE_EB), TRUE)
251-
DEFINES += -DERF_USE_EB
252-
endif
253-
254237
ifeq ($(USE_MULTIBLOCK), TRUE)
255238
DEFINES += -DERF_USE_MULTIBLOCK
256239
endif

‎Source/Advection/ERF_Advection.H

+3-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#include <AMReX_MultiFab.H>
66
#include <AMReX_BCRec.H>
77
#include <AMReX_YAFluxRegister.H>
8+
89
#include <ERF_DataStruct.H>
910
#include <ERF_IndexDefines.H>
1011
#include <ERF_ABLMost.H>
@@ -74,7 +75,8 @@ void AdvectionSrcForMom (const amrex::Box& bx,
7475
const amrex::Array4<const amrex::Real>& mf_v,
7576
const AdvType horiz_adv_type, const AdvType vert_adv_type,
7677
const amrex::Real horiz_upw_frac, const amrex::Real vert_upw_frac,
77-
const bool use_terrain, const int lo_z_face, const int hi_z_face,
78+
TerrainType& terrain_type,
79+
const int lo_z_face, const int hi_z_face,
7880
const amrex::Box& domain,
7981
const amrex::BCRec* bc_ptr_h);
8082

0 commit comments

Comments
 (0)