From d52977d94d684de0111303d4eb2e54d63c2b62a2 Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Wed, 22 Jan 2025 18:10:48 +0000 Subject: [PATCH 1/9] update fix for wave --- versions/fix.ver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions/fix.ver b/versions/fix.ver index db54701e77..7f69e657b2 100644 --- a/versions/fix.ver +++ b/versions/fix.ver @@ -23,6 +23,6 @@ export reg2grb2_ver=20220805 export sfc_climo_ver=20220805 export ugwd_ver=20240624 export verif_ver=20220805 -export wave_ver=20240105 +export wave_ver=20250110 export orog_nest_ver=global-nest.20240419 export ugwd_nest_ver=global-nest.20240419 From 26765f6b82b3a8a62442e14064f23d8a446c58e8 Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Fri, 24 Jan 2025 17:58:22 +0000 Subject: [PATCH 2/9] do not copy temp files to COM as they are not needed fix broken logic so that we do not try to copy somethign to COM --- ush/wave_grid_interp_sbs.sh | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/ush/wave_grid_interp_sbs.sh b/ush/wave_grid_interp_sbs.sh index c046afb2e9..6f9d110ac6 100755 --- a/ush/wave_grid_interp_sbs.sh +++ b/ush/wave_grid_interp_sbs.sh @@ -142,14 +142,6 @@ source "${USHgfs}/preamble.sh" "${EXECgfs}/${pgm}" 1> gint.${grdID}.out 2>&1 export err=$?;err_chk -# Write interpolation file to main TEMP dir area if not there yet - if [ "wht_OK" = 'no' ] # FIXME: This is never going to evaluate to true, wht_OK is a string and needs to be ${wht_OK}. With ${wht_OK}, the next line is trying to copy into ${FIXgfs} space. This leads to a Permission denied error. The logic here needs to be evaluated and recoded. #TODO - then - cp -f ./WHTGRIDINT.bin ${DATA}/ww3_gint.WHTGRIDINT.bin.${grdID} - cp -f ./WHTGRIDINT.bin ${FIXgfs}/wave/ww3_gint.WHTGRIDINT.bin.${grdID} - fi - - if [ "$err" != '0' ] then set +x @@ -168,25 +160,6 @@ source "${USHgfs}/preamble.sh" rm -f mod_def.* mv out_grd.$grdID ${DATA}/output_${ymdh}0000/out_grd.$grdID -# 1.c Save in /com - - set +x - echo " Saving GRID file as ${COMOUT_WAVE_PREP}/${WAV_MOD_TAG}.out_grd.${grdID}.${PDY}${cyc}" - set_trace - cp "${DATA}/output_${ymdh}0000/out_grd.${grdID}" "${COMOUT_WAVE_PREP}/${WAV_MOD_TAG}.out_grd.${grdID}.${PDY}${cyc}" - -# if [ "$SENDDBN" = 'YES' ] -# then -# set +x -# echo " Alerting GRID file as $COMOUT/rundata/$WAV_MOD_TAG.out_grd.$grdID.${PDY}${cyc} -# set_trace - -# -# PUT DBNET ALERT HERE .... -# - -# fi - # --------------------------------------------------------------------------- # # 2. Clean up the directory From ac141b6736cbf199ef8f5072adf0d1beffc398c0 Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Fri, 24 Jan 2025 18:09:11 +0000 Subject: [PATCH 3/9] remove un-used COMOUT_WAVE_PREP in JGLOBAL_WAVE_POST_SBS and subsequent scripts --- jobs/JGLOBAL_WAVE_POST_SBS | 3 +-- ush/wave_grid_interp_sbs.sh | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/jobs/JGLOBAL_WAVE_POST_SBS b/jobs/JGLOBAL_WAVE_POST_SBS index 171d160515..77886be2b0 100755 --- a/jobs/JGLOBAL_WAVE_POST_SBS +++ b/jobs/JGLOBAL_WAVE_POST_SBS @@ -13,10 +13,9 @@ export MP_PULSE=0 YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_WAVE_PREP:COM_WAVE_PREP_TMPL \ COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \ - COMOUT_WAVE_PREP:COM_WAVE_PREP_TMPL \ COMOUT_WAVE_GRID:COM_WAVE_GRID_TMPL -for out_dir in "${COMOUT_WAVE_PREP}" "${COMOUT_WAVE_GRID}"; do +for out_dir in "${COMOUT_WAVE_GRID}"; do if [[ ! -d "${out_dir}" ]]; then mkdir -p "${out_dir}"; fi done diff --git a/ush/wave_grid_interp_sbs.sh b/ush/wave_grid_interp_sbs.sh index 6f9d110ac6..fd7180add6 100755 --- a/ush/wave_grid_interp_sbs.sh +++ b/ush/wave_grid_interp_sbs.sh @@ -66,8 +66,7 @@ source "${USHgfs}/preamble.sh" set_trace if [[ -z "${PDY}" ]] || [[ -z "${cyc}" ]] || [[ -z "${cycle}" ]] || [[ -z "${EXECgfs}" ]] || \ - [[ -z "${COMOUT_WAVE_PREP}" ]] || [[ -z "${WAV_MOD_TAG}" ]] || [[ -z "${SENDDBN}" ]] || \ - [ -z "${waveGRD}" ] + [[ -z "${waveGRD}" ]] || [[ -z "${WAV_MOD_TAG}" ]] || [[ -z "${SENDDBN}" ]] then set +x echo ' ' @@ -75,7 +74,7 @@ source "${USHgfs}/preamble.sh" echo '*** EXPORTED VARIABLES IN postprocessor NOT SET ***' echo '***************************************************' echo ' ' - echo "${PDY}${cyc} ${cycle} ${EXECgfs} ${COMOUT_WAVE_PREP} ${WAV_MOD_TAG} ${SENDDBN} ${waveGRD}" + echo "${PDY}${cyc} ${cycle} ${EXECgfs} ${WAV_MOD_TAG} ${SENDDBN} ${waveGRD}" set_trace exit 1 fi From 9ac3313c1facdb0c52d80c5672fb0e6c7afc4b37 Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Fri, 24 Jan 2025 18:09:54 +0000 Subject: [PATCH 4/9] change name of restart variables --- ush/forecast_postdet.sh | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 7b9bd0ee48..7cb2badadc 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -367,16 +367,10 @@ WW3_postdet() { elif [[ -s "${ww3_binary_restart_file}" ]]; then # found binary ww3 restart file export WW3_restart_from_binary=true - if [[ -f "${DATA}/ufs.cpld.cpl.r.nc" ]]; then - #if this is a cmeps continue then the wave restart name is different - seconds=$(to_seconds "${restart_date:8:2}0000") # convert HHMMSS to seconds - local ww3_restart_dest_file="ufs.cpld.ww3.r.${restart_date:0:4}-${restart_date:4:2}-${restart_date:6:2}-${seconds}" - ${NCP} "${ww3_binary_restart_file}" "${DATA}/${ww3_restart_dest_file}" \ - || ( echo "FATAL ERROR: Unable to copy binary WW3 IC, ABORT!"; exit 1 ) - else - ${NCP} "${ww3_binary_restart_file}" "${DATA}/restart.ww3" \ - || ( echo "FATAL ERROR: Unable to copy binary WW3 IC, ABORT!"; exit 1 ) - fi + seconds=$(to_seconds "${restart_date:8:2}0000") # convert HHMMSS to seconds + local ww3_restart_dest_file="ufs.cpld.ww3.r.${restart_date:0:4}-${restart_date:4:2}-${restart_date:6:2}-${seconds}" + ${NCP} "${ww3_binary_restart_file}" "${DATA}/${ww3_restart_dest_file}" \ + || ( echo "FATAL ERROR: Unable to copy binary WW3 IC, ABORT!"; exit 1 ) else if [[ "${RERUN}" == "YES" ]]; then # In the case of a RERUN, the WW3 restart file is required From 02ca6aeb199c92f1b1ebbe5910c2a7f8fb44d20f Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Mon, 27 Jan 2025 19:52:20 +0000 Subject: [PATCH 5/9] update ic date --- versions/ic.ver | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versions/ic.ver b/versions/ic.ver index e7bd8358ef..1994910c92 100644 --- a/versions/ic.ver +++ b/versions/ic.ver @@ -16,7 +16,7 @@ ic_versions['C384C192']=20240610 ic_versions['C384mx025']=20241120 ic_versions['C768']=20241120 -ic_versions['C768mx025']=20241120 +ic_versions['C768mx025']=20250115 ic_versions['C1152']=20241120 -ic_versions['C1152mx025']=20241120 +ic_versions['C1152mx025']=20250115 From 3342450a42b299d0750b5bcc701766d55fe98363 Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Wed, 29 Jan 2025 13:55:11 +0000 Subject: [PATCH 6/9] changes requested by NCO for v16 adding to develop --- scripts/exgfs_wave_post_pnt.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index 96c60fc96c..bad848c588 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -618,13 +618,13 @@ source "${USHgfs}/preamble.sh" if [ ${CFP_MP:-"NO"} = "YES" ] && [ "$DOBLL_WAV" = "YES" ]; then if [ "$DOBNDPNT_WAV" = YES ]; then if [ "$DOSPC_WAV" = YES ]; then - echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_ibp_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) fi if [ "$DOBLL_WAV" = YES ]; then - echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_ibpbull_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) - echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_ibpcbull_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) fi else @@ -633,28 +633,28 @@ source "${USHgfs}/preamble.sh" nm=$(( nm + 1 )) fi if [ "$DOBLL_WAV" = YES ]; then - echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_bull_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) - echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_cbull_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) fi fi else if [ "$DOBNDPNT_WAV" = YES ]; then if [ "$DOSPC_WAV" = YES ]; then - echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_ibp_tar.out 2>&1 " >> cmdtarfile fi if [ "$DOBLL_WAV" = YES ]; then - echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile - echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_ibpbull_tar.out 2>&1 " >> cmdtarfile + echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_ibpcbull_tar.out 2>&1 " >> cmdtarfile fi else if [ "$DOSPC_WAV" = YES ]; then echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG spec $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile fi if [ "$DOBLL_WAV" = YES ]; then - echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile - echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_bull_tar.out 2>&1 " >> cmdtarfile + echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_cbull_tar.out 2>&1 " >> cmdtarfile fi fi fi From a06de0f3a3411b3ed2798694eee92cceb699269e Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Wed, 29 Jan 2025 19:50:18 +0000 Subject: [PATCH 7/9] add environment variable for fcst job for hercules/orion for addressing hang issues --- env/HERCULES.env | 1 + env/ORION.env | 2 ++ 2 files changed, 3 insertions(+) diff --git a/env/HERCULES.env b/env/HERCULES.env index acfbe438ef..b565881a73 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -214,6 +214,7 @@ case ${step} in "fcst" | "efcs") export OMP_STACKSIZE=512M + export FI_MLX_INJECT_LIMIT=0 (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node )) (( ufs_ntasks = nnodes*tasks_per_node )) diff --git a/env/ORION.env b/env/ORION.env index fbe00c153c..64f3f3d555 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -208,6 +208,8 @@ elif [[ "${step}" = "eupd" ]]; then elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then + export FI_MLX_INJECT_LIMIT=0 + (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node )) (( ufs_ntasks = nnodes*tasks_per_node )) # With ESMF threading, the model wants to use the full node From a9d8f7ed6788e53e428a0e385323d789d089b9bc Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Thu, 30 Jan 2025 14:11:17 +0000 Subject: [PATCH 8/9] update WW3 IC error handling --- ush/forecast_postdet.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 7cb2badadc..65ab5695f9 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -372,13 +372,13 @@ WW3_postdet() { ${NCP} "${ww3_binary_restart_file}" "${DATA}/${ww3_restart_dest_file}" \ || ( echo "FATAL ERROR: Unable to copy binary WW3 IC, ABORT!"; exit 1 ) else - if [[ "${RERUN}" == "YES" ]]; then + if [[ "${RERUN}" == "YES" ]] || [[ -f "${DATA}/ufs.cpld.cpl.r.nc" ]]; then # In the case of a RERUN, the WW3 restart file is required - echo "FATAL ERROR: WW3 binary | netcdf restart file '${ww3_binary_restart_file}' | '${ww3_netcdf_restart_file}' not found for RERUN='${RERUN}', ABORT!" + # In the case of runtype=continue, if no wave restart when using PIO, the model will fail + echo "FATAL ERROR: WW3 binary | netcdf restart file '${ww3_binary_restart_file}' | '${ww3_netcdf_restart_file}' not found for RERUN='${RERUN}' or runtype=continue, ABORT!" exit 1 else echo "WARNING: WW3 binary | netcdf restart file '${ww3_binary_restart_file}' | '${ww3_netcdf_restart_file}' not found for warm_start='${warm_start}', will start from rest!" - export WW3_restart_from_binary=true fi fi From ee352fe65a76bb81011ad6601016577e1fa8ebca Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Thu, 30 Jan 2025 21:30:21 +0000 Subject: [PATCH 9/9] remove loop in JGLOBAL_WAVE_POST_SBS --- jobs/JGLOBAL_WAVE_POST_SBS | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jobs/JGLOBAL_WAVE_POST_SBS b/jobs/JGLOBAL_WAVE_POST_SBS index 77886be2b0..acd654d6fa 100755 --- a/jobs/JGLOBAL_WAVE_POST_SBS +++ b/jobs/JGLOBAL_WAVE_POST_SBS @@ -15,9 +15,7 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \ COMOUT_WAVE_GRID:COM_WAVE_GRID_TMPL -for out_dir in "${COMOUT_WAVE_GRID}"; do - if [[ ! -d "${out_dir}" ]]; then mkdir -p "${out_dir}"; fi -done +if [[ ! -d "${COMOUT_WAVE_GRID}" ]]; then mkdir -p "${COMOUT_WAVE_GRID}"; fi if [[ -n "${wavepostGRD}" || -n "${waveinterpGRD}" ]]; then for grdID in ${wavepostGRD} ${waveinterpGRD}; do