Skip to content

Commit

Permalink
Merge HAFS coupling (ufs-community#715)
Browse files Browse the repository at this point in the history 
- HAFS FV3ATM-HYCOM coupling, which is currently used in HAFSv0.2A/D HFIP real-time parallel experiments for 2021 hurricane season (this will require adding HYCOM, e.g., the [NOAA-EMC fork of HYCOM-src](https://github.com/NOAA-EMC/HYCOM-src), as a submodule of ufs-weather-model)
- HAFS FV3ATM-WW3 coupling, for which technical testing has been conducted and the WW3 related changes have already been merged back to WW3's develop branch
- HAFS related CDEPS data component coupling development (with some CDEPS data component coupling regression tests added)
- HAFSv0.2 related physics enhancement specialized for the UFS-HAFS (hurricane) application (namelist controlled options added and does not affect other applications unless the related options are used)

The following five HAFS related regression tests have been added into the standard regression tests. 
- hafs_regional_atm: HAFS regional atmosphere only
- hafs_regional_atm_ocn: HAFS regional atmosphere-ocean coupled with HYCOM
- hafs_regional_docn: HAFS regional coupled with regional data ocean from MOM6
- hafs_regional_docn_oisst: HAFS regional coupled with global data ocean from OISST
- hafs_regional_datm_cdeps: HAFS regional coupled CDEPS data atmosphere from ERA5 with regional HYCOM

Co-authored-by: Dom Heinzeller <[email protected]>
Co-authored-by: Grant Firl <[email protected]>
Co-authored-by: Daniel Rosen <[email protected]>
Co-authored-by: Ufuk Turuncoglu <[email protected]>
Co-authored-by: Biju.Thomas <[email protected]>
  • Loading branch information
@BinLiu-NOAA @climbfuji
@grantfirl @danrosen25 @BijuThomas-NOAA
6 people authored Aug 5, 2021
1 parent 85fa626 commit 1c2d16e
Show file tree
Hide file tree
Showing 50 changed files with 4,049 additions and 2,853 deletions.
6 changes: 5 additions & 1 deletion .gitmodules
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,11 @@
branch = develop
[submodule "CMEPS"]
path = CMEPS-interface/CMEPS
url = https://github.com/NOAA-EMC/CMEPS.git
url = https://github.com/NOAA-EMC/CMEPS
branch = emc/develop
[submodule "HYCOM"]
path = HYCOM-interface/HYCOM
url = https://github.com/NOAA-EMC/HYCOM-src
branch = emc/develop
[submodule "MOM6"]
path = MOM6-interface/MOM6
Expand Down
2 changes: 1 addition & 1 deletion CDEPS-interface/CDEPS
Submodule CDEPS updated 15 files
+3 −1 .github/workflows/extbuild.yml
+31 −8 cime_config/buildlib
+12 −5 cime_config/stream_cdeps.py
+1 −1 cime_config/testdefs/testlist_cdeps.xml
+55 −4 datm/cime_config/buildnml
+5 −6 datm/cime_config/stream_definition_datm.xml
+105 −107 datm/datm_datamode_era5_mod.F90
+1 −1 dice/cime_config/buildnml
+1 −1 dlnd/cime_config/buildnml
+1 −1 docn/cime_config/buildnml
+1 −1 drof/cime_config/buildnml
+24 −11 dshr/dshr_mod.F90
+1 −1 dwav/cime_config/buildnml
+14 −11 streams/dshr_strdata_mod.F90
+4 −4 streams/dshr_stream_mod.F90
2 changes: 1 addition & 1 deletion CMEPS-interface/CMEPS
Submodule CMEPS updated 6 files
+239 −710 mediator/esmFldsExchange_hafs_mod.F90
+51 −9 mediator/med.F90
+21 −5 mediator/med_fraction_mod.F90
+54 −19 mediator/med_map_mod.F90
+37 −10 mediator/med_methods_mod.F90
+12 −1 mediator/med_phases_post_wav_mod.F90
17 changes: 15 additions & 2 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMakeModules/Modules)
###############################################################################

# Valid applications and choices
list(APPEND VALID_APPS ATM ATMAERO ATMW S2S S2SW NG-GODAS NG-GODAS-NEMSDATM)
list(APPEND VALID_APPS ATM ATMAERO ATMW S2S S2SW HAFS HAFSW HAFS-ALL NG-GODAS NG-GODAS-NEMSDATM)
set(APP NONE CACHE BOOL "Application Name")
if(NOT (APP IN_LIST VALID_APPS))
message(FATAL_ERROR "${APP} is not a valid application.\nValid Applications are: ${VALID_APPS}")
Expand All @@ -26,6 +26,7 @@ set(FMS OFF CACHE BOOL "Enable FMS")
set(FV3 OFF CACHE BOOL "Enable FV3")
set(UFS_GOCART OFF CACHE BOOL "Enable GOCART")
set(MOM6 OFF CACHE BOOL "Enable MOM6")
set(HYCOM OFF CACHE BOOL "Enable HYCOM")
set(CICE6 OFF CACHE BOOL "Enable CICE6")
set(WW3 OFF CACHE BOOL "Enable WW3")
set(STOCH_PHYS OFF CACHE BOOL "Enable Stochastic Physics")
Expand All @@ -42,6 +43,7 @@ message("FMS .............. ${FMS}")
message("FV3 .............. ${FV3}")
message("GOCART ........... ${UFS_GOCART}")
message("MOM6 ............. ${MOM6}")
message("HYCOM ............ ${HYCOM}")
message("CICE6 ............ ${CICE6}")
message("WW3 .............. ${WW3}")
message("STOCH_PHYS ....... ${STOCH_PHYS}")
Expand Down Expand Up @@ -181,12 +183,16 @@ if(WW3)
endif()

###############################################################################
### Marine Components [MOM6, CICE6]
### Marine Components [MOM6, HYCOM, CICE6]
###############################################################################
if(MOM6)
add_subdirectory(MOM6-interface)
endif()

if(HYCOM)
add_subdirectory(HYCOM-interface)
endif()

if(CICE6)
add_subdirectory(CICE-interface)
endif()
Expand Down Expand Up @@ -263,6 +269,12 @@ if(MOM6)
list(APPEND _ufs_libs_public mom6)
endif()

if(HYCOM)
list(APPEND _ufs_defs_private FRONT_HYCOM=HYCOM_Mod)
add_dependencies(ufs hycom)
target_link_libraries(ufs PUBLIC hycom)
endif()

if(CICE6)
add_dependencies(ufs cice)
list(APPEND _ufs_defs_private FRONT_CICE6=ice_comp_nuopc)
Expand All @@ -278,6 +290,7 @@ endif()
if(CDEPS)
add_dependencies(ufs cdeps::cdeps)
list(APPEND _ufs_defs_private FRONT_CDEPS_DATM=cdeps_datm_comp)
list(APPEND _ufs_defs_private FRONT_CDEPS_DOCN=cdeps_docn_comp)
target_link_libraries(ufs PUBLIC cdeps::cdeps)
endif()

Expand Down
2 changes: 1 addition & 1 deletion FV3
Submodule FV3 updated 12 files
+44 −5 atmos_model.F90
+28 −2 ccpp/data/GFS_typedefs.F90
+38 −0 ccpp/data/GFS_typedefs.meta
+1 −1 ccpp/physics
+95 −0 ccpp/suites/suite_FV3_HAFS_v0_gfdlmp_tedmf.xml
+93 −0 ccpp/suites/suite_FV3_HAFS_v0_gfdlmp_tedmf_nonsst.xml
+1 −1 ccpp/suites/suite_FV3_HAFS_v0_hwrf.xml
+1 −1 ccpp/suites/suite_FV3_HAFS_v0_hwrf_thompson.xml
+13 −3 cpl/module_cap_cpl.F90
+2 −2 fv3_cap.F90
+1 −1 io/module_write_netcdf.F90
+207 −12 module_fcst_grid_comp.F90
124 changes: 124 additions & 0 deletions HYCOM-interface/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
get_filename_component(hycom_dir "${CMAKE_CURRENT_SOURCE_DIR}/HYCOM" ABSOLUTE)

### HYCOM Fortran compiler flags
if(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
set(CMAKE_Fortran_FLAGS "-g -fbacktrace")
set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8 -fdefault-double-8")
set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Waliasing -fcray-pointer -fconvert=big-endian -ffree-line-length-none -fno-range-check -fbacktrace")
set(CMAKE_Fortran_FLAGS_RELEASE "-O3")
set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -fcheck=bounds -ffpe-trap=invalid,zero,overflow,underflow" )
set(CMAKE_Fortran_LINK_FLAGS "")
elseif(CMAKE_Fortran_COMPILER_ID MATCHES "Intel")
set(CMAKE_Fortran_FLAGS "-g -traceback -xSSE4.2 -mcmodel=small -r8")
set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -fp-model source -warn nogeneral")
set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check -check noarg_temp_created -check nopointer -fpe0 -ftrapuv -link_mpi=dbg")
set(CMAKE_Fortran_LINK_FLAGS "-V ${CMAKE_Fortran_FLAGS} -static-intel")
else()
message(WARNING "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options")
endif()

### HYCOM C compiler flags
if(CMAKE_C_COMPILER_ID MATCHES "GNU")
set(CMAKE_C_FLAGS "-g -fbacktrace")
set(CMAKE_C_FLAGS_RELEASE "-O3")
set(CMAKE_C_FLAGS_DEBUG "-O0")
set(CMAKE_C_LINK_FLAGS "")
elseif(CMAKE_C_COMPILER_ID MATCHES "Intel")
set(CMAKE_C_FLAGS "-g -traceback -xSSE4.2 -mcmodel=small")
set(CMAKE_C_FLAGS_RELEASE "-O")
set(CMAKE_C_FLAGS_DEBUG "-O0 -ftrapuv")
set(CMAKE_C_LINK_FLAGS "-V ${CMAKE_C_FLAGS} -static-intel")
else()
message(WARNING "C compiler with ID ${CMAKE_C_COMPILER_ID} will be used with CMake default options")
endif()

list(APPEND _hycom_defs IA32
REAL8
MPI
SERIAL_IO
ENDIAN_IO
NAN2003
TIME
RELO
EOS_SIG2
EOS_17T
ESPC_COUPLE
)

# User option to build HYCOM offline executable
set(HYCOMOFFLINE OFF CACHE BOOL "Build HYCOM offline")

message("Build HYCOM:")
message(" in: ${hycom_dir}")
message(" HYCOMOFFLINE: ${HYCOMOFFLINE}")
message("")

# Too many files to list, so include them via this file
include("hycom_files.cmake")

### Use common object library for building target library
add_library(hycom_obj OBJECT ${_hycom_src_files})
set_target_properties(hycom_obj PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/HYCOM)
set_target_properties(hycom_obj PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
target_include_directories(hycom_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>)
target_include_directories(hycom_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/HYCOM>)
target_compile_definitions(hycom_obj PRIVATE "${_hycom_defs}")


### Use NUOPC object library for building target library
add_library(hycom_nuopc_obj OBJECT ${_hycom_nuopc_src_files})
add_dependencies(hycom_nuopc_obj hycom_obj)
set_target_properties(hycom_nuopc_obj PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/HYCOM)
set_target_properties(hycom_nuopc_obj PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
target_include_directories(hycom_nuopc_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>)
target_include_directories(hycom_nuopc_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/HYCOM>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/HYCOM/NUOPC>)
target_compile_definitions(hycom_nuopc_obj PRIVATE "${_hycom_defs}")
target_link_libraries(hycom_nuopc_obj PRIVATE hycom_obj
esmf
NetCDF::NetCDF_Fortran)

### Create target library and set PUBLIC interfaces on the library
add_library(hycom STATIC $<TARGET_OBJECTS:hycom_obj>
$<TARGET_OBJECTS:hycom_nuopc_obj>)
target_include_directories(hycom PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>
$<INSTALL_INTERFACE:mod>)
target_include_directories(hycom PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/HYCOM>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/HYCOM/NUOPC>)
target_compile_definitions(hycom PRIVATE "${_hycom_defs}")
target_link_libraries(hycom PUBLIC esmf
NetCDF::NetCDF_Fortran)

### Create HYCOM offline executable
if(HYCOMOFFLINE)
message("Building HYCOM offline executable")
add_executable(hycomoffline ${_hycom_offline_src_files})
add_dependencies(hycomoffline hycom_obj)
set_target_properties(hycomoffline PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod_offline)
target_include_directories(hycomoffline PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>)
target_include_directories(hycomoffline PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod_offline>)
target_include_directories(hycomoffline PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/HYCOM>)

target_link_libraries(hycomoffline PRIVATE hycom_obj)
endif()

###############################################################################
### Install
###############################################################################

install(
TARGETS hycom
EXPORT hycom-config
RUNTIME DESTINATION bin
LIBRARY DESTINATION lib
ARCHIVE DESTINATION lib
COMPONENT Library)

install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod DESTINATION ${CMAKE_INSTALL_PREFIX}/HYCOM)

install(EXPORT hycom-config
DESTINATION lib/cmake)

if(HYCOMOFFLINE)
install(TARGETS hycomoffline RUNTIME DESTINATION bin)
endif()
1 change: 1 addition & 0 deletions HYCOM-interface/HYCOM
Submodule HYCOM added at b4f4ee
68 changes: 68 additions & 0 deletions HYCOM-interface/hycom_files.cmake
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
list(APPEND _hycom_src_files
HYCOM/mod_dimensions.F90
HYCOM/mod_xc.F90
HYCOM/mod_za.F90
HYCOM/mod_cb_arrays.F90
HYCOM/mod_pipe.F90
HYCOM/mod_incupd.F90
HYCOM/mod_floats.F90
HYCOM/mod_stokes.F90
HYCOM/mod_tides.F90
HYCOM/mod_mean.F90
HYCOM/mod_archiv.F90
HYCOM/mod_tsadvc.F90
HYCOM/mod_momtum.F90
HYCOM/mod_barotp.F90
HYCOM/mod_asselin.F90
HYCOM/mod_restart.F90
HYCOM/mod_import.F90
HYCOM/mod_hycom.F90

HYCOM/bigrid.F90
HYCOM/blkdat.F90
HYCOM/cnuity.F90
HYCOM/convec.F90
HYCOM/diapfl.F90
HYCOM/dpthuv.F90
HYCOM/dpudpv.F90
HYCOM/forfun.F90
HYCOM/geopar.F90
HYCOM/hybgen.F90
HYCOM/icloan.F90
HYCOM/inicon.F90
HYCOM/inigiss.F90
HYCOM/inikpp.F90
HYCOM/inimy.F90
HYCOM/latbdy.F90
HYCOM/matinv.F90
HYCOM/mxkprf.F90
HYCOM/mxkrt.F90
HYCOM/mxkrtm.F90
HYCOM/mxpwp.F90
HYCOM/overtn.F90
HYCOM/poflat.F90
HYCOM/prtmsk.F90
HYCOM/psmoo.F90
HYCOM/thermf.F90
HYCOM/trcupd.F90
HYCOM/machine.F90
HYCOM/wtime.F90
HYCOM/machi_c.c
HYCOM/isnan.F90
HYCOM/s8gefs.F90
)

list(APPEND _hycom_nuopc_src_files
HYCOM/NUOPC/HYCOM_OceanComp.F90
HYCOM/NUOPC/HYCOM_ESMF_Extensions.F90
HYCOM/NUOPC/hycom_couple.F90
HYCOM/NUOPC/read_impexp_config_mod.F90
HYCOM/NUOPC/impexpField_cdf_mod.F90
HYCOM/NUOPC/export_from_hycom_tiled.F90
HYCOM/NUOPC/hycom_read_latlon.F90
HYCOM/NUOPC/hycom_nuopc_flags.F90
)

list(APPEND _hycom_offline_src_files
HYCOM/hycom.F90
)
16 changes: 16 additions & 0 deletions cmake/configure_apps.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,22 @@ if(APP MATCHES "^(S2S|S2SW)$")
endif()
endif()

if(APP MATCHES "^(HAFS|HAFSW|HAFS-ALL)$")
set(CMEPS ON CACHE BOOL "Enable CMEPS" FORCE)
if(APP MATCHES "^(HAFS-ALL)$")
set(CDEPS ON CACHE BOOL "Enable CDEPS" FORCE)
message("Configuring UFS app in HAFS with CDEPS mode")
endif()
set(FMS ON CACHE BOOL "Enable FMS" FORCE)
set(FV3 ON CACHE BOOL "Enable FV3" FORCE)
set(STOCH_PHYS ON CACHE BOOL "Enable Stochastic Physics" FORCE)
set(HYCOM ON CACHE BOOL "Enable HYCOM" FORCE)
if(APP MATCHES "^(HAFSW|HAFS-ALL)$")
set(WW3 ON CACHE BOOL "Enable WAVEWATCH III" FORCE)
message("Configuring UFS app in HAFS with Waves mode")
endif()
endif()

if(APP MATCHES "^(ATMAERO)$")
set(FMS ON CACHE BOOL "Enable FMS" FORCE)
set(FV3 ON CACHE BOOL "Enable FV3" FORCE)
Expand Down
1 change: 1 addition & 0 deletions modulefiles/ufs_wcoss_cray
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ setenv PNG_ROOT /usrx/local/prod//png/1.2.49/intel/sandybridge
## NCEP libraries
##
module use /usrx/local/nceplibs/NCEPLIBS/cmake/install/NCEPLIBS-v1.3.0/modules
module load pio/2.5.2
module load bacio/2.4.1
module load crtm/2.3.0
module load g2/3.4.1
Expand Down
1 change: 1 addition & 0 deletions modulefiles/ufs_wcoss_cray_debug
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ setenv PNG_ROOT /usrx/local/prod//png/1.2.49/intel/sandybridge
## NCEP libraries
##
module use /usrx/local/nceplibs/NCEPLIBS/cmake/install/NCEPLIBS-v1.3.0/modules
module load pio/2.5.2
module load bacio/2.4.1
module load crtm/2.3.0
module load g2/3.4.1
Expand Down
Loading

0 comments on commit 1c2d16e

Please sign in to comment.