From f5ec7ec9526403d051749f2cf00b270257bde0f0 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Tue, 21 Jan 2025 03:09:59 -0600 Subject: [PATCH] Add sfs as valid system Adds sfs as a valid option for NET. To start, the GEFS system is generally just copied wholesale for SFS. This includes the extract_vars job. Other than base and resources, config files link to the GEFS versions, just as GEFS config files point to the GFS versions except where they have needed to be changed. The temporarily SFS_POST option has been removed. The sfs test case is moved from a separate sfs directory to the pr directory, but ICs are still needed before it is functional. Resolves #2271 --- ci/cases/{sfs => pr}/C96mx100_S2S.yaml | 2 +- ci/cases/yamls/sfs_defaults.yaml | 1 - parm/archive/gefs_arcdir.yaml.j2 | 16 +- parm/config/gefs/config.atmos_products | 13 +- parm/config/gefs/config.base | 1 - parm/config/gefs/config.fcst | 6 +- parm/config/gefs/config.resources | 16 +- parm/config/gefs/config.ufs | 78 ++- parm/config/gefs/yaml/defaults.yaml | 1 - parm/config/gfs/config.ufs | 78 ++- parm/config/sfs/config.aero | 1 + parm/config/sfs/config.arch | 1 + parm/config/sfs/config.atmos_ensstat | 1 + parm/config/sfs/config.atmos_products | 1 + parm/config/sfs/config.base | 354 +++++++++++ parm/config/sfs/config.cleanup | 1 + parm/config/sfs/config.com | 1 + parm/config/sfs/config.efcs | 1 + parm/config/sfs/config.extractvars | 1 + parm/config/sfs/config.fcst | 1 + parm/config/sfs/config.ice | 1 + parm/config/sfs/config.nsst | 1 + parm/config/sfs/config.oceanice_products | 1 + parm/config/sfs/config.ocn | 1 + parm/config/sfs/config.prep_emissions | 1 + parm/config/sfs/config.resources | 363 +++++++++++ parm/config/sfs/config.resources.AWSPW | 1 + parm/config/sfs/config.resources.AZUREPW | 1 + parm/config/sfs/config.resources.GOOGLEPW | 1 + parm/config/sfs/config.stage_ic | 1 + parm/config/sfs/config.ufs | 1 + parm/config/sfs/config.wave | 1 + parm/config/sfs/config.waveinit | 1 + parm/config/sfs/config.wavepostbndpnt | 1 + parm/config/sfs/config.wavepostbndpntbll | 1 + parm/config/sfs/config.wavepostpnt | 1 + parm/config/sfs/config.wavepostsbs | 1 + parm/config/sfs/yaml/defaults.yaml | 34 + ush/forecast_postdet.sh | 262 ++++---- ush/forecast_predet.sh | 47 +- ush/python/pygfs/task/archive.py | 2 +- workflow/applications/application_factory.py | 2 + workflow/applications/sfs.py | 98 +++ workflow/rocoto/gefs_tasks.py | 132 ++-- workflow/rocoto/gefs_xml.py | 2 - workflow/rocoto/rocoto_xml_factory.py | 3 +- workflow/rocoto/sfs_tasks.py | 632 +++++++++++++++++++ workflow/rocoto/sfs_xml.py | 32 + workflow/rocoto/tasks_factory.py | 2 + workflow/setup_expt.py | 33 +- 50 files changed, 1926 insertions(+), 309 deletions(-) rename ci/cases/{sfs => pr}/C96mx100_S2S.yaml (96%) create mode 120000 parm/config/sfs/config.aero create mode 120000 parm/config/sfs/config.arch create mode 120000 parm/config/sfs/config.atmos_ensstat create mode 120000 parm/config/sfs/config.atmos_products create mode 100644 parm/config/sfs/config.base create mode 120000 parm/config/sfs/config.cleanup create mode 120000 parm/config/sfs/config.com create mode 120000 parm/config/sfs/config.efcs create mode 120000 parm/config/sfs/config.extractvars create mode 120000 parm/config/sfs/config.fcst create mode 120000 parm/config/sfs/config.ice create mode 120000 parm/config/sfs/config.nsst create mode 120000 parm/config/sfs/config.oceanice_products create mode 120000 parm/config/sfs/config.ocn create mode 120000 parm/config/sfs/config.prep_emissions create mode 100644 parm/config/sfs/config.resources create mode 120000 parm/config/sfs/config.resources.AWSPW create mode 120000 parm/config/sfs/config.resources.AZUREPW create mode 120000 parm/config/sfs/config.resources.GOOGLEPW create mode 120000 parm/config/sfs/config.stage_ic create mode 120000 parm/config/sfs/config.ufs create mode 120000 parm/config/sfs/config.wave create mode 120000 parm/config/sfs/config.waveinit create mode 120000 parm/config/sfs/config.wavepostbndpnt create mode 120000 parm/config/sfs/config.wavepostbndpntbll create mode 120000 parm/config/sfs/config.wavepostpnt create mode 120000 parm/config/sfs/config.wavepostsbs create mode 100644 parm/config/sfs/yaml/defaults.yaml create mode 100644 workflow/applications/sfs.py create mode 100644 workflow/rocoto/sfs_tasks.py create mode 100644 workflow/rocoto/sfs_xml.py diff --git a/ci/cases/sfs/C96mx100_S2S.yaml b/ci/cases/pr/C96mx100_S2S.yaml similarity index 96% rename from ci/cases/sfs/C96mx100_S2S.yaml rename to ci/cases/pr/C96mx100_S2S.yaml index 6bdb9a4887..ddc9b83829 100644 --- a/ci/cases/sfs/C96mx100_S2S.yaml +++ b/ci/cases/pr/C96mx100_S2S.yaml @@ -1,5 +1,5 @@ experiment: - system: gefs + system: sfs mode: forecast-only arguments: diff --git a/ci/cases/yamls/sfs_defaults.yaml b/ci/cases/yamls/sfs_defaults.yaml index 43f27d9239..1312d715fb 100644 --- a/ci/cases/yamls/sfs_defaults.yaml +++ b/ci/cases/yamls/sfs_defaults.yaml @@ -21,7 +21,6 @@ base: USE_ATM_ENS_PERTURB_FILES: "YES" HPSSARCH: "NO" LOCALARCH: "NO" - SFS_POST: "YES" ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} fcst: TYPE: "hydro" diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 index d1008bc5c4..12338ceda8 100644 --- a/parm/archive/gefs_arcdir.yaml.j2 +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -24,15 +24,13 @@ {% endif %} # Select ensstat files to copy to the arcdir -{% if RUN == "gefs" %} - {% set ensstat_files = [] %} - {% if path_exists(COMIN_ATMOS_ENSSTAT_1p00) %} - {% for fhr in range(ofst_hr, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ "/" ~ head ~ "mean.pres_." ~ - "1p00" ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2", - GEFS_ARCH]) %} - {% endfor %} - {% endif %} +{% set ensstat_files = [] %} +{% if path_exists(COMIN_ATMOS_ENSSTAT_1p00) %} + {% for fhr in range(ofst_hr, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ "/" ~ head ~ "mean.pres_." ~ + "1p00" ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2", + GEFS_ARCH]) %} + {% endfor %} {% endif %} {% set file_set = ensstat_files %} # Actually write the yaml diff --git a/parm/config/gefs/config.atmos_products b/parm/config/gefs/config.atmos_products index d1f36a7bc9..11c144636d 100644 --- a/parm/config/gefs/config.atmos_products +++ b/parm/config/gefs/config.atmos_products @@ -25,14 +25,9 @@ fi export FLXGF="NO" # Create interpolated sflux.1p00 file # paramlist files for the different forecast hours and downsets -if [[ ${SFS_POST} == "YES" ]]; then - export post_prefix='sfs' -else - export post_prefix='gefs' -fi -export paramlista="${PARMgfs}/product/${post_prefix}.0p25.fFFF.paramlist.a.txt" -export paramlista_anl="${PARMgfs}/product/${post_prefix}.0p25.anl.paramlist.a.txt" -export paramlista_f000="${PARMgfs}/product/${post_prefix}.0p25.f000.paramlist.a.txt" -export paramlistb="${PARMgfs}/product/${post_prefix}.0p25.fFFF.paramlist.b.txt" +export paramlista="${PARMgfs}/product/${NET}.0p25.fFFF.paramlist.a.txt" +export paramlista_anl="${PARMgfs}/product/${NET}.0p25.anl.paramlist.a.txt" +export paramlista_f000="${PARMgfs}/product/${NET}.0p25.f000.paramlist.a.txt" +export paramlistb="${PARMgfs}/product/${NET}.0p25.fFFF.paramlist.b.txt" echo "END: config.atmos_products" diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 400be7eb17..6f1fe04a3e 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -64,7 +64,6 @@ export REALTIME="YES" # Experiment mode (cycled or forecast-only) export MODE="@MODE@" # cycled/forecast-only -export SFS_POST="@SFS_POST@" # TODO, place holder until RUN=SFS is developed export DO_TEST_MODE="@DO_TEST_MODE@" # option to change configuration for automated testing #################################################### diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst index 2fef1ba0a0..c470aacb14 100644 --- a/parm/config/gefs/config.fcst +++ b/parm/config/gefs/config.fcst @@ -51,11 +51,7 @@ export esmf_logkind="ESMF_LOGKIND_MULTI_ON_ERROR" #Options: ESMF_LOGKIND_MULTI_O export FORECASTSH="${SCRgfs}/exglobal_forecast.sh" #export FORECASTSH="${SCRgfs}/exglobal_forecast.py" # Temp. while this is worked on -if [[ "${SFS_POST:-}" == "YES" ]]; then - export FCSTEXEC="sfs_model.x" -else - export FCSTEXEC="gefs_model.x" -fi +export FCSTEXEC="${NET}_model.x" ####################################################################### # Model configuration diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index bb33f3eb02..b6474364f4 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -312,14 +312,14 @@ case ${step} in ;; "extractvars") - export walltime_gefs="00:30:00" - export ntasks_gefs=1 - export threads_per_task_gefs=1 - export tasks_per_node_gefs="${ntasks_gefs}" - export walltime_gfs="${walltime_gefs}" - export ntasks_gfs="${ntasks_gefs}" - export threads_per_tasks_gfs="${threads_per_task_gefs}" - export tasks_per_node_gfs="${tasks_per_node_gefs}" + export walltime="00:30:00" + export ntasks=1 + export threads_per_task=1 + export tasks_per_node="${ntasks}" + export walltime_gfs="${walltime}" + export ntasks_gfs="${ntasks}" + export threads_per_tasks_gfs="${threads_per_task}" + export tasks_per_node_gfs="${tasks_per_node}" export is_exclusive=False ;; diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs index f2ee7b8619..365eb9704f 100644 --- a/parm/config/gefs/config.ufs +++ b/parm/config/gefs/config.ufs @@ -326,11 +326,16 @@ if [[ "${skip_mom6}" == "false" ]]; then CHLCLIM="seawifs_1998-2006_smoothed_2X.nc" MOM6_RESTART_SETTING='r' MOM6_RIVER_RUNOFF='False' - if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then - MOM6_DIAG_MISVAL="-1e34" - else - MOM6_DIAG_MISVAL="0.0" - fi + case ${RUN} in + gfs|gefs|sfs) + MOM6_DIAG_MISVAL="-1e34";; + gdas|enkf*) + MOM6_DIAG_MISVAL="0.0";; + *) + echo "FATAL ERROR: Unsupported RUN ${RUN} for ${mom6_res}" + exit 10 + ;; + esac eps_imesh="4.0e-1" MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_25L.nc" MOM6_ALLOW_LANDMASK_CHANGES='False' @@ -349,13 +354,20 @@ if [[ "${skip_mom6}" == "false" ]]; then MOM6_RIVER_RUNOFF='False' eps_imesh="2.5e-1" TOPOEDITS="ufs.topo_edits_011818.nc" - if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then - MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" - MOM6_DIAG_MISVAL="-1e34" - else - MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc" - MOM6_DIAG_MISVAL="0.0" - fi + case ${RUN} in + gfs|gefs|sfs) + MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" + MOM6_DIAG_MISVAL="-1e34" + ;; + gdas|enkf*) + MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc" + MOM6_DIAG_MISVAL="0.0" + ;; + *) + echo "FATAL ERROR: Unsupported RUN ${RUN} for ${mom6_res}" + exit 10 + ;; + esac MOM6_ALLOW_LANDMASK_CHANGES='True' ;; "050") @@ -370,13 +382,20 @@ if [[ "${skip_mom6}" == "false" ]]; then MOM6_RESTART_SETTING='n' MOM6_RIVER_RUNOFF='True' eps_imesh="1.0e-1" - if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then - MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" - MOM6_DIAG_MISVAL="-1e34" - else - MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc" - MOM6_DIAG_MISVAL="0.0" - fi + case ${RUN} in + gfs|gefs|sfs) + MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" + MOM6_DIAG_MISVAL="-1e34" + ;; + gdas|enkf*) + MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc" + MOM6_DIAG_MISVAL="0.0" + ;; + *) + echo "FATAL ERROR: Unsupported RUN ${RUN} for ${mom6_res}" + exit 10 + ;; + esac MOM6_ALLOW_LANDMASK_CHANGES='False' TOPOEDITS="" ;; @@ -392,13 +411,20 @@ if [[ "${skip_mom6}" == "false" ]]; then MOM6_RIVER_RUNOFF='True' MOM6_RESTART_SETTING="r" eps_imesh="1.0e-1" - if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then - MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" - MOM6_DIAG_MISVAL="-1e34" - else - MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc" - MOM6_DIAG_MISVAL="0.0" - fi + case ${RUN} in + gfs|gefs|sfs) + MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" + MOM6_DIAG_MISVAL="-1e34" + ;; + gdas|enkf*) + MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc" + MOM6_DIAG_MISVAL="0.0" + ;; + *) + echo "FATAL ERROR: Unsupported RUN ${RUN} for ${mom6_res}" + exit 10 + ;; + esac MOM6_ALLOW_LANDMASK_CHANGES='False' TOPOEDITS="" ;; diff --git a/parm/config/gefs/yaml/defaults.yaml b/parm/config/gefs/yaml/defaults.yaml index df12b16282..d918d12ebc 100644 --- a/parm/config/gefs/yaml/defaults.yaml +++ b/parm/config/gefs/yaml/defaults.yaml @@ -19,7 +19,6 @@ base: FHOUT_ICE_GFS: 6 HPSSARCH: "NO" LOCALARCH: "NO" - SFS_POST: "NO" USE_OCN_ENS_PERTURB_FILES: "NO" USE_ATM_ENS_PERTURB_FILES: "NO" DO_TEST_MODE: "NO" diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index 5f2b6675cf..77374a3762 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -436,11 +436,16 @@ if [[ "${skip_mom6}" == "false" ]]; then CHLCLIM="seawifs_1998-2006_smoothed_2X.nc" MOM6_RESTART_SETTING='r' MOM6_RIVER_RUNOFF='False' - if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then - MOM6_DIAG_MISVAL="-1e34" - else - MOM6_DIAG_MISVAL="0.0" - fi + case ${RUN} in + gfs|gefs|sfs) + MOM6_DIAG_MISVAL="-1e34";; + gdas|enkf*) + MOM6_DIAG_MISVAL="0.0";; + *) + echo "FATAL ERROR: Unsupported RUN ${RUN} for ${mom6_res}" + exit 10 + ;; + esac eps_imesh="4.0e-1" MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_25L.nc" MOM6_ALLOW_LANDMASK_CHANGES='False' @@ -459,13 +464,20 @@ if [[ "${skip_mom6}" == "false" ]]; then MOM6_RIVER_RUNOFF='False' eps_imesh="2.5e-1" TOPOEDITS="ufs.topo_edits_011818.nc" - if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then - MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" - MOM6_DIAG_MISVAL="-1e34" - else - MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc" - MOM6_DIAG_MISVAL="0.0" - fi + case ${RUN} in + gfs|gefs|sfs) + MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" + MOM6_DIAG_MISVAL="-1e34" + ;; + gdas|enkf*) + MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc" + MOM6_DIAG_MISVAL="0.0" + ;; + *) + echo "FATAL ERROR: Unsupported RUN ${RUN} for ${mom6_res}" + exit 10 + ;; + esac MOM6_ALLOW_LANDMASK_CHANGES='True' ;; "050") @@ -480,13 +492,20 @@ if [[ "${skip_mom6}" == "false" ]]; then MOM6_RESTART_SETTING='n' MOM6_RIVER_RUNOFF='True' eps_imesh="1.0e-1" - if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then - MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" - MOM6_DIAG_MISVAL="-1e34" - else - MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc" - MOM6_DIAG_MISVAL="0.0" - fi + case ${RUN} in + gfs|gefs|sfs) + MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" + MOM6_DIAG_MISVAL="-1e34" + ;; + gdas|enkf*) + MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc" + MOM6_DIAG_MISVAL="0.0" + ;; + *) + echo "FATAL ERROR: Unsupported RUN ${RUN} for ${mom6_res}" + exit 10 + ;; + esac MOM6_ALLOW_LANDMASK_CHANGES='False' TOPOEDITS="" ;; @@ -502,13 +521,20 @@ if [[ "${skip_mom6}" == "false" ]]; then MOM6_RIVER_RUNOFF='True' MOM6_RESTART_SETTING="r" eps_imesh="1.0e-1" - if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then - MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" - MOM6_DIAG_MISVAL="-1e34" - else - MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc" - MOM6_DIAG_MISVAL="0.0" - fi + case ${RUN} in + gfs|gefs|sfs) + MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" + MOM6_DIAG_MISVAL="-1e34" + ;; + gdas|enkf*) + MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc" + MOM6_DIAG_MISVAL="0.0" + ;; + *) + echo "FATAL ERROR: Unsupported RUN ${RUN} for ${mom6_res}" + exit 10 + ;; + esac MOM6_ALLOW_LANDMASK_CHANGES='False' TOPOEDITS="" ;; diff --git a/parm/config/sfs/config.aero b/parm/config/sfs/config.aero new file mode 120000 index 0000000000..f5b2edb643 --- /dev/null +++ b/parm/config/sfs/config.aero @@ -0,0 +1 @@ +../gefs/config.aero \ No newline at end of file diff --git a/parm/config/sfs/config.arch b/parm/config/sfs/config.arch new file mode 120000 index 0000000000..4b26138131 --- /dev/null +++ b/parm/config/sfs/config.arch @@ -0,0 +1 @@ +../gefs/config.arch \ No newline at end of file diff --git a/parm/config/sfs/config.atmos_ensstat b/parm/config/sfs/config.atmos_ensstat new file mode 120000 index 0000000000..63738267f2 --- /dev/null +++ b/parm/config/sfs/config.atmos_ensstat @@ -0,0 +1 @@ +../gefs/config.atmos_ensstat \ No newline at end of file diff --git a/parm/config/sfs/config.atmos_products b/parm/config/sfs/config.atmos_products new file mode 120000 index 0000000000..41192d1cf2 --- /dev/null +++ b/parm/config/sfs/config.atmos_products @@ -0,0 +1 @@ +../gefs/config.atmos_products \ No newline at end of file diff --git a/parm/config/sfs/config.base b/parm/config/sfs/config.base new file mode 100644 index 0000000000..da85f5761f --- /dev/null +++ b/parm/config/sfs/config.base @@ -0,0 +1,354 @@ +#! /usr/bin/env bash + +########## config.base ########## +# Common to all steps + +echo "BEGIN: config.base" + +# Machine environment +export machine="@MACHINE@" + +# EMC parallel or NCO production +export RUN_ENVIR="emc" + +# Account, queue, etc. +export ACCOUNT="@ACCOUNT@" +export QUEUE="@QUEUE@" +export QUEUE_SERVICE="@QUEUE_SERVICE@" +export PARTITION_BATCH="@PARTITION_BATCH@" +export PARTITION_SERVICE="@PARTITION_SERVICE@" +export RESERVATION="@RESERVATION@" + +# Project to use in mass store: +export HPSS_PROJECT="@HPSS_PROJECT@" + +# Directories relative to installation areas: +export HOMEgfs=@HOMEgfs@ +export EXECgfs=${HOMEgfs}/exec +export FIXgfs=${HOMEgfs}/fix +export PARMgfs=${HOMEgfs}/parm +export SCRgfs=${HOMEgfs}/scripts +export USHgfs=${HOMEgfs}/ush +export FIXorog=${FIXgfs}/orog +export FIXugwd=${FIXgfs}/ugwd + +######################################################################## + +# GLOBAL static environment parameters +export PACKAGEROOT="@PACKAGEROOT@" # TODO: set via prod_envir in Ops +export COMROOT="@COMROOT@" # TODO: set via prod_envir in Ops +export COMINsyn="@COMINsyn@" + +# USER specific paths +export HOMEDIR="@HOMEDIR@" +export STMP="@STMP@" +export PTMP="@PTMP@" +export NOSCRUB="@NOSCRUB@" + +# Base directories for various builds +export BASE_GIT="@BASE_GIT@" + +# Base directory for staged data +export BASE_DATA="@BASE_DATA@" + +# Toggle to turn on/off GFS downstream processing. +export DO_BUFRSND="@DO_BUFRSND@" # BUFR sounding products +export DO_GEMPAK="@DO_GEMPAK@" # GEMPAK products +export DO_AWIPS="@DO_AWIPS@" # AWIPS products + +# NO for retrospective parallel; YES for real-time parallel +# arch.sh uses REALTIME for MOS. Need to set REALTIME=YES +# if want MOS written to HPSS. Should update arch.sh to +# use RUNMOS flag +export REALTIME="YES" + +# Experiment mode (cycled or forecast-only) +export MODE="@MODE@" # cycled/forecast-only +export DO_TEST_MODE="@DO_TEST_MODE@" # option to change configuration for automated testing + +#################################################### +# DO NOT ADD MACHINE DEPENDENT STUFF BELOW THIS LINE +# IF YOU HAVE TO MAKE MACHINE SPECIFIC CHANGES BELOW +# FEEL FREE TO MOVE THEM ABOVE THIS LINE TO KEEP IT +# CLEAR +#################################################### +# Build paths relative to $HOMEgfs +export HOMEpost="${HOMEgfs}" + +# CONVENIENT utility scripts and other environment parameters +export NCP="/bin/cp -p" +export NMV="/bin/mv" +export NLN="/bin/ln -sf" +export VERBOSE="YES" +export KEEPDATA="@KEEPDATA@" +export DEBUG_POSTSCRIPT="NO" # PBS only; sets debug=true +export CHGRP_RSTPROD="@CHGRP_RSTPROD@" +export CHGRP_CMD="@CHGRP_CMD@" +export NCDUMP="${NETCDF:-${netcdf_c_ROOT:-}}/bin/ncdump" +export NCLEN="${HOMEgfs}/ush/getncdimlen" + +# Machine environment, jobs, and other utility scripts +export BASE_ENV="${HOMEgfs}/env" +export BASE_JOB="${HOMEgfs}/jobs/rocoto" + +# EXPERIMENT specific environment parameters +export SDATE=@SDATE@ +export EDATE=@EDATE@ +export EXP_WARM_START="@EXP_WARM_START@" +export assim_freq=6 +export PSLOT="@PSLOT@" +export EXPDIR="@EXPDIR@/${PSLOT}" +export ROTDIR="@COMROOT@/${PSLOT}" + +export ARCDIR="${NOSCRUB}/archive/${PSLOT}" +export ATARDIR="@ATARDIR@" + +# Commonly defined parameters in JJOBS +export envir=${envir:-"prod"} +export NET="sfs" # NET is defined in the job-card (ecf) +export RUN="sfs" # RUN is defined in the job-card (ecf) + +# Get all the COM path templates +source "${EXPDIR}/config.com" + +# shellcheck disable=SC2016 +export ERRSCRIPT=${ERRSCRIPT:-'eval [[ $err = 0 ]]'} +export LOGSCRIPT=${LOGSCRIPT:-""} +#export ERRSCRIPT=${ERRSCRIPT:-"err_chk"} +#export LOGSCRIPT=${LOGSCRIPT:-"startmsg"} +export REDOUT="1>" +export REDERR="2>" + +export SENDECF=${SENDECF:-"NO"} +export SENDSDM=${SENDSDM:-"NO"} +export SENDDBN_NTC=${SENDDBN_NTC:-"NO"} +export SENDDBN=${SENDDBN:-"NO"} +export DBNROOT=${DBNROOT:-${UTILROOT:-}/fakedbn} + +# APP settings +export APP=@APP@ + +# Defaults: +export DO_ATM="YES" +export DO_COUPLED="NO" +export DO_WAVE="NO" +export DO_OCN="NO" +export DO_ICE="NO" +export DO_EXTRACTVARS="@DO_EXTRACTVARS@" # Option to process and extract a subset of products to save on disk +export DO_AERO_FCST="NO" +export DOBNDPNT_WAVE="NO" # The GEFS buoys file does not currently have any boundary points +export DOIBP_WAV="NO" # Option to create point outputs from input boundary points +export FRAC_GRID=".true." +export DO_NEST="NO" # Whether to run a global-nested domain +if [[ "${DO_NEST:-NO}" == "YES" ]] ; then + export ntiles=7 + export NEST_OUTPUT_GRID="regional_latlon" +else + export ntiles=6 +fi + +# Set operational resolution +export OPS_RES="C768" # Do not change + +# Resolution specific parameters +export LEVS=128 +export CASE="@CASECTL@" # CASE is required in GEFS to determine ocean/ice/wave resolutions +export CASE_ENS="@CASEENS@" +export OCNRES="@OCNRES@" +export ICERES="${OCNRES}" +# These are the currently recommended grid-combinations +case "${CASE}" in + "C48") + export waveGRD='glo_500' + ;; + "C96" | "C192") + export waveGRD='glo_100' + ;; + "C384") + export waveGRD='glo_025' + ;; + "C768" | "C1152") + export waveGRD='mx025' + ;; + *) + echo "FATAL ERROR: Unrecognized CASE ${CASE}, ABORT!" + exit 1 + ;; +esac + +case "${APP}" in + ATM) + ;; + ATMA) + export DO_AERO_FCST="YES" + ;; + ATMW) + export DO_COUPLED="YES" + export DO_WAVE="YES" + ;; + NG-GODAS) + export DO_ATM="NO" + export DO_OCN="YES" + export DO_ICE="YES" + ;; + S2S*) + export DO_COUPLED="YES" + export DO_OCN="YES" + export DO_ICE="YES" + + if [[ "${APP}" =~ A$ ]]; then + export DO_AERO_FCST="YES" + fi + + if [[ "${APP}" =~ ^S2SW ]]; then + export DO_WAVE="YES" + fi + ;; + *) + echo "FATAL ERROR: Unrecognized APP: '${APP}'" + exit 2 + ;; +esac + +# Output frequency of the forecast model (for cycling) +export FHMIN=0 +export FHMAX=9 +export FHOUT=3 # Will be changed to 1 in config.base if (DOHYBVAR set to NO and l4densvar set to false) +export FHOUT_OCN=3 +export FHOUT_ICE=3 +export FHOUT_AERO=3 + +# GFS cycle info +export INTERVAL_GFS=@INTERVAL_GFS@ # Frequency of GFS forecast +export SDATE_GFS=@SDATE_GFS@ + +# set variables needed for use with REPLAY ICs +export REPLAY_ICS=@REPLAY_ICS@ +if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then + export OFFSET_START_HOUR=$(( assim_freq / 2 )) + echo "WARNING: Replay ICs require perturbation files, ignoring any previous settings" + export USE_OCN_ENS_PERTURB_FILES="YES" + export USE_ATM_ENS_PERTURB_FILES="YES" +else + export OFFSET_START_HOUR=0 + export USE_OCN_ENS_PERTURB_FILES=@USE_OCN_ENS_PERTURB_FILES@ + export USE_ATM_ENS_PERTURB_FILES=@USE_ATM_ENS_PERTURB_FILES@ +fi + +# GFS output and frequency +export FHMIN_GFS=0 +export FHMAX_GFS="@FHMAX_GFS@" +# Intermediate times to stop forecast when running in segments +breakpnts="@FCST_BREAKPOINTS@" +export FCST_SEGMENTS="${FHMIN_GFS},${breakpnts:+${breakpnts},}${FHMAX_GFS}" + +export FHOUT_GFS=@FHOUT_GFS@ +export FHMAX_HF_GFS=@FHMAX_HF_GFS@ +export FHOUT_HF_GFS=@FHOUT_HF_GFS@ +export FHOUT_OCN_GFS=@FHOUT_OCN_GFS@ +export FHOUT_ICE_GFS=@FHOUT_ICE_GFS@ +export FHMIN_WAV=${OFFSET_START_HOUR:-0} +export FHOUT_WAV=3 +export FHMAX_HF_WAV=120 +export FHOUT_HF_WAV=1 +export FHMAX_WAV=${FHMAX_GFS} +export FHMAX_WAV_GFS=${FHMAX_GFS} +export FHMAX_HF_GFS=$(( FHMAX_HF_GFS > FHMAX_GFS ? FHMAX_GFS : FHMAX_HF_GFS )) +export FHMAX_WAV_GFS=$(( FHMAX_WAV_GFS > FHMAX_GFS ? FHMAX_GFS : FHMAX_WAV_GFS )) +export FHMAX_HF_WAV=$(( FHMAX_HF_WAV > FHMAX_WAV_GFS ? FHMAX_WAV_GFS : FHMAX_HF_WAV )) +export ILPOST=1 # gempak output frequency up to F120 + +export FHMIN_ENKF=${FHMIN_GFS} +export FHMAX_ENKF=${FHMAX_GFS} +export FHOUT_ENKF=${FHOUT_GFS} +export FHOUT_OCN=${FHOUT_OCN_GFS} +export FHOUT_ICE=${FHOUT_ICE_GFS} + +# GFS restart interval in hours +export restart_interval_gfs=12 +export restart_interval_enkfgfs=12 +# NOTE: Do not set this to zero. Instead set it to $FHMAX_GFS +# TODO: Remove this variable from config.base and reference from config.fcst +# TODO: rework logic in config.wave and push it to parsing_nameslist_WW3.sh where it is actually used + +export QUILTING=".true." +export OUTPUT_GRID="gaussian_grid" +export WRITE_DOPOST=".true." # WRITE_DOPOST=true, use inline POST +export WRITE_NSFLIP=".true." + +# Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL +export imp_physics=8 + +# Shared parameters +# DA engine +export DO_JEDIATMVAR="NO" +export DO_JEDIATMENS="NO" +export DO_JEDIOCNVAR="NO" +export DO_JEDISNOWDA="NO" +export DO_MERGENSST="NO" +export DO_STARTMEM_FROM_JEDIICE="NO" + +# Hybrid related +export NMEM_ENS=@NMEM_ENS@ + +# set default member number memdir for control +# this will be overwridden for the perturbed members +export ENSMEM=${ENSMEM:-"000"} +export MEMDIR="mem${ENSMEM}" + +# cellular automata +if (( ENSMEM == 0 )); then + export DO_CA="YES" +else + export DO_CA="YES" +fi + +export DOIAU="NO" # While we are not doing IAU, we may want to warm start w/ IAU in the future + +# Check if cycle is warm starting with IAU +if [[ "${EXP_WARM_START}" = ".true." ]]; then + if [[ "${DOIAU}" = "YES" ]]; then + export IAU_FHROT=3 + else + export IAU_FHROT=0 + fi +fi + +# turned on nsst in anal and/or fcst steps, and turn off rtgsst +export DONST="YES" +if [[ ${DONST} = "YES" ]]; then export FNTSFA=" "; fi + +# The switch to apply SST elevation correction or not +export nst_anl=.true. + +# Make the nsstbufr file on the fly or use the GDA version +export MAKE_NSSTBUFR="@MAKE_NSSTBUFR@" + +# Make the aircraft prepbufr file on the fly or use the GDA version +export MAKE_ACFTBUFR="@MAKE_ACFTBUFR@" + +# Verification options +export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config.metp +export DO_FIT2OBS="NO" # Run fit to observations package + +# Archiving options +export HPSSARCH="@HPSSARCH@" # save data to HPSS archive +export LOCALARCH="@LOCALARCH@" # save data to local archive +if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then + echo "Both HPSS and local archiving selected. Please choose one or the other." + exit 3 +fi +export ARCH_CYC=00 # Archive data at this cycle for warm start and/or forecast-only capabilities +export ARCH_WARMICFREQ=4 # Archive frequency in days for warm start capability +export ARCH_FCSTICFREQ=1 # Archive frequency in days for gdas and gfs forecast-only capability +export ARCH_EXPDIR='YES' # Archive the EXPDIR configs, XML, and database +export ARCH_EXPDIR_FREQ=0 # How often to archive the EXPDIR in hours or 0 for first and last cycle only +export ARCH_HASHES='YES' # Archive the hashes of the GW and submodules and 'git status' for each; requires ARCH_EXPDIR +export ARCH_DIFFS='NO' # Archive the output of 'git diff' for the GW; requires ARCH_EXPDIR + +export DELETE_COM_IN_ARCHIVE_JOB="YES" # NO=retain ROTDIR. YES default in arch.sh and earc.sh. + +# Number of regional collectives to create soundings for +export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9} + +echo "END: config.base" diff --git a/parm/config/sfs/config.cleanup b/parm/config/sfs/config.cleanup new file mode 120000 index 0000000000..6665dc7bfd --- /dev/null +++ b/parm/config/sfs/config.cleanup @@ -0,0 +1 @@ +../gefs/config.cleanup \ No newline at end of file diff --git a/parm/config/sfs/config.com b/parm/config/sfs/config.com new file mode 120000 index 0000000000..6f0e485d3e --- /dev/null +++ b/parm/config/sfs/config.com @@ -0,0 +1 @@ +../gefs/config.com \ No newline at end of file diff --git a/parm/config/sfs/config.efcs b/parm/config/sfs/config.efcs new file mode 120000 index 0000000000..886d065854 --- /dev/null +++ b/parm/config/sfs/config.efcs @@ -0,0 +1 @@ +../gefs/config.efcs \ No newline at end of file diff --git a/parm/config/sfs/config.extractvars b/parm/config/sfs/config.extractvars new file mode 120000 index 0000000000..0ab4b598d5 --- /dev/null +++ b/parm/config/sfs/config.extractvars @@ -0,0 +1 @@ +../gefs/config.extractvars \ No newline at end of file diff --git a/parm/config/sfs/config.fcst b/parm/config/sfs/config.fcst new file mode 120000 index 0000000000..efa8406ad9 --- /dev/null +++ b/parm/config/sfs/config.fcst @@ -0,0 +1 @@ +../gefs/config.fcst \ No newline at end of file diff --git a/parm/config/sfs/config.ice b/parm/config/sfs/config.ice new file mode 120000 index 0000000000..f613b7bfa6 --- /dev/null +++ b/parm/config/sfs/config.ice @@ -0,0 +1 @@ +../gefs/config.ice \ No newline at end of file diff --git a/parm/config/sfs/config.nsst b/parm/config/sfs/config.nsst new file mode 120000 index 0000000000..3a015c1d65 --- /dev/null +++ b/parm/config/sfs/config.nsst @@ -0,0 +1 @@ +../gefs/config.nsst \ No newline at end of file diff --git a/parm/config/sfs/config.oceanice_products b/parm/config/sfs/config.oceanice_products new file mode 120000 index 0000000000..4d69db6f26 --- /dev/null +++ b/parm/config/sfs/config.oceanice_products @@ -0,0 +1 @@ +../gefs/config.oceanice_products \ No newline at end of file diff --git a/parm/config/sfs/config.ocn b/parm/config/sfs/config.ocn new file mode 120000 index 0000000000..c2f19253fb --- /dev/null +++ b/parm/config/sfs/config.ocn @@ -0,0 +1 @@ +../gefs/config.ocn \ No newline at end of file diff --git a/parm/config/sfs/config.prep_emissions b/parm/config/sfs/config.prep_emissions new file mode 120000 index 0000000000..1f1aff64a3 --- /dev/null +++ b/parm/config/sfs/config.prep_emissions @@ -0,0 +1 @@ +../gefs/config.prep_emissions \ No newline at end of file diff --git a/parm/config/sfs/config.resources b/parm/config/sfs/config.resources new file mode 100644 index 0000000000..b6474364f4 --- /dev/null +++ b/parm/config/sfs/config.resources @@ -0,0 +1,363 @@ +#! /usr/bin/env bash + +########## config.resources ########## +# Set resource information for job tasks +# e.g. walltime, node, cores per node, memory etc. + +if (( $# != 1 )); then + + echo "Must specify an input task argument to set resource variables!" + exit 1 + +fi + +step=$1 + +echo "BEGIN: config.resources" + +case ${machine} in + "WCOSS2") max_tasks_per_node=128;; + "HERA") max_tasks_per_node=40;; + "ORION") max_tasks_per_node=40;; + "HERCULES") max_tasks_per_node=80;; + "JET") + case ${PARTITION_BATCH} in + "xjet") max_tasks_per_node=24;; + "vjet" | "sjet") max_tasks_per_node=16;; + "kjet") max_tasks_per_node=40;; + *) + echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}" + exit 3 + esac + ;; + "S4") + case ${PARTITION_BATCH} in + "s4") max_tasks_per_node=32;; + "ivy") max_tasks_per_node=20;; + *) + echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}" + exit 3 + esac + ;; + "AWSPW") + export PARTITION_BATCH="compute" + max_tasks_per_node=48 + ;; + "AZUREPW") + export PARTITION_BATCH="compute" + max_tasks_per_node=36 + ;; + "GOOGLEPW") + export PARTITION_BATCH="compute" + max_tasks_per_node=30 + ;; + *) + echo "FATAL ERROR: Unknown machine encountered by ${BASH_SOURCE[0]}" + exit 2 + ;; +esac +export max_tasks_per_node + +case ${step} in + + "stage_ic") + export walltime="00:15:00" + export ntasks=1 + export tasks_per_node=1 + export threads_per_task=1 + export memory="4096M" + ;; + + "waveinit") + export walltime="00:10:00" + export ntasks=12 + export threads_per_task=1 + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export NTASKS=${ntasks} + export memory="2GB" + ;; + + "prep_emissions") + export walltime="00:10:00" + export ntasks=1 + export threads_per_task=1 + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export memory="1GB" + ;; + + "fcst" | "efcs") + export is_exclusive=True + + export layout_x=${layout_x_gfs} + export layout_y=${layout_y_gfs} + export WRITE_GROUP=${WRITE_GROUP_GFS} + export WRTTASK_PER_GROUP_PER_THREAD=${WRTTASK_PER_GROUP_PER_THREAD_GFS} + ntasks_fv3=${ntasks_fv3_gfs} + ntasks_quilt=${ntasks_quilt_gfs} + nthreads_fv3=${nthreads_fv3_gfs} + nthreads_ufs=${nthreads_ufs_gfs} + + # Determine if using ESMF-managed threading or traditional threading + # If using traditional threading, set them to 1 + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + export UFS_THREADS=1 + else # traditional threading + export UFS_THREADS=${nthreads_ufs:-1} + nthreads_fv3=1 + nthreads_mediator=1 + [[ "${DO_WAVE}" == "YES" ]] && nthreads_ww3=1 + [[ "${DO_OCN}" == "YES" ]] && nthreads_mom6=1 + [[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1 + fi + + # FV3 + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + (( FV3THREADS = nthreads_fv3 )) + (( FV3PETS = ntasks_fv3 * nthreads_fv3 )) + else + (( FV3THREADS = UFS_THREADS )) + (( FV3PETS = ntasks_fv3 )) + fi + echo "FV3 using (nthreads, PETS) = (${FV3THREADS}, ${FV3PETS})" + + # Write grid component + QUILTPETS=0; QUILTTHREADS=0 + if [[ "${QUILTING:-}" == ".true." ]]; then + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + (( QUILTTHREADS = nthreads_fv3 )) + (( QUILTPETS = ntasks_quilt * nthreads_fv3 )) + else + (( QUILTTHREADS = UFS_THREADS )) + (( QUILTPETS = ntasks_quilt )) + fi + (( WRTTASK_PER_GROUP = WRTTASK_PER_GROUP_PER_THREAD )) + export WRTTASK_PER_GROUP + fi + echo "QUILT using (nthreads, PETS) = (${QUILTTHREADS}, ${QUILTPETS})" + + # Total PETS for the atmosphere component + ATMTHREADS=${FV3THREADS} + (( ATMPETS = FV3PETS + QUILTPETS )) + export ATMPETS ATMTHREADS + echo "FV3ATM using (nthreads, PETS) = (${ATMTHREADS}, ${ATMPETS})" + + # Total PETS for the coupled model (starting w/ the atmosphere) + NTASKS_TOT=${ATMPETS} + + # Mediator + # The mediator PETS can overlap with other components, usually it lands on the atmosphere tasks. + # However, it is suggested limiting mediator PETS to 300, as it may cause the slow performance. + # See https://docs.google.com/document/d/1bKpi-52t5jIfv2tuNHmQkYUe3hkKsiG_DG_s6Mnukog/edit + # TODO: Update reference when moved to ufs-weather-model RTD + MEDTHREADS=${nthreads_mediator:-1} + MEDPETS=${MEDPETS:-${FV3PETS}} + (( "${MEDPETS}" > 300 )) && MEDPETS=300 + export MEDPETS MEDTHREADS + echo "MEDIATOR using (threads, PETS) = (${MEDTHREADS}, ${MEDPETS})" + + # GOCART + CHMPETS=0; CHMTHREADS=0 + if [[ "${DO_AERO_FCST}" == "YES" ]]; then + # GOCART shares the same grid and forecast tasks as FV3 (do not add write grid component tasks). + (( CHMTHREADS = ATMTHREADS )) + (( CHMPETS = FV3PETS )) + # Do not add to NTASKS_TOT + echo "GOCART using (threads, PETS) = (${CHMTHREADS}, ${CHMPETS})" + fi + export CHMPETS CHMTHREADS + + # Waves + WAVPETS=0; WAVTHREADS=0 + if [[ "${DO_WAVE}" == "YES" ]]; then + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + (( WAVTHREADS = nthreads_ww3 )) + (( WAVPETS = ntasks_ww3 * nthreads_ww3 )) + else + (( WAVTHREADS = UFS_THREADS )) + (( WAVPETS = ntasks_ww3 )) + fi + echo "WW3 using (threads, PETS) = (${WAVTHREADS}, ${WAVPETS})" + (( NTASKS_TOT = NTASKS_TOT + WAVPETS )) + fi + export WAVPETS WAVTHREADS + + # Ocean + OCNPETS=0; OCNTHREADS=0 + if [[ "${DO_OCN}" == "YES" ]]; then + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + (( OCNTHREADS = nthreads_mom6 )) + (( OCNPETS = ntasks_mom6 * nthreads_mom6 )) + else + (( OCNTHREADS = UFS_THREADS )) + (( OCNPETS = ntasks_mom6 )) + fi + echo "MOM6 using (threads, PETS) = (${OCNTHREADS}, ${OCNPETS})" + (( NTASKS_TOT = NTASKS_TOT + OCNPETS )) + fi + export OCNPETS OCNTHREADS + + # Ice + ICEPETS=0; ICETHREADS=0 + if [[ "${DO_ICE}" == "YES" ]]; then + if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then + (( ICETHREADS = nthreads_cice6 )) + (( ICEPETS = ntasks_cice6 * nthreads_cice6 )) + else + (( ICETHREADS = UFS_THREADS )) + (( ICEPETS = ntasks_cice6 )) + fi + echo "CICE6 using (threads, PETS) = (${ICETHREADS}, ${ICEPETS})" + (( NTASKS_TOT = NTASKS_TOT + ICEPETS )) + fi + export ICEPETS ICETHREADS + + echo "Total PETS = ${NTASKS_TOT}" + + declare -x "ntasks"="${NTASKS_TOT}" + declare -x "threads_per_task"="${UFS_THREADS}" + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + + case "${CASE}" in + "C48" | "C96" | "C192") + declare -x "walltime"="04:00:00" + ;; + "C384" | "C768" | "C1152") + declare -x "walltime"="06:00:00" + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + exit 4 + ;; + esac + + unset NTASKS_TOT + ;; + + "atmos_products") + # Walltime is per forecast hour; will be multipled by group size + export walltime="00:15:00" + export ntasks=24 + export threads_per_task=1 + export tasks_per_node="${ntasks}" + export is_exclusive=True + ;; + + "atmos_ensstat") + # Walltime is per forecast hour; will be multipled by group size + export walltime="00:15:00" + export ntasks=6 + export threads_per_task=1 + export tasks_per_node="${ntasks}" + export is_exclusive=True + ;; + + "oceanice_products") + # Walltime is per forecast hour; will be multipled by group size + export walltime="00:15:00" + export ntasks=1 + export tasks_per_node=1 + export threads_per_task=1 + export memory="96GB" + ;; + + "wavepostsbs") + # Walltime is per forecast hour; will be multipled by group size + export walltime="00:15:00" + export ntasks=1 + export threads_per_task=1 + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export NTASKS=${ntasks} + export memory="10GB" + ;; + + # The wavepost*pnt* jobs are I/O heavy and do not scale well to large nodes. + # Limit the number of tasks/node to 40. + "wavepostbndpnt") + export walltime="03:00:00" + export ntasks=240 + export threads_per_task=1 + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + if [[ ${tasks_per_node} -gt 40 ]]; then + export tasks_per_node=40 + export is_exclusive=False + fi + export NTASKS=${ntasks} + ;; + + "wavepostbndpntbll") + export walltime="01:00:00" + export ntasks=448 + export threads_per_task=1 + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + if [[ ${tasks_per_node} -gt 40 ]]; then + export tasks_per_node=40 + export is_exclusive=False + fi + export NTASKS=${ntasks} + ;; + + "wavepostpnt") + export walltime="04:00:00" + export ntasks=200 + export threads_per_task=1 + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + if [[ ${tasks_per_node} -gt 40 ]]; then + export tasks_per_node=40 + export is_exclusive=False + fi + export NTASKS=${ntasks} + ;; + + "extractvars") + export walltime="00:30:00" + export ntasks=1 + export threads_per_task=1 + export tasks_per_node="${ntasks}" + export walltime_gfs="${walltime}" + export ntasks_gfs="${ntasks}" + export threads_per_tasks_gfs="${threads_per_task}" + export tasks_per_node_gfs="${tasks_per_node}" + export is_exclusive=False + ;; + + "arch") + export walltime="06:00:00" + export ntasks=1 + export tasks_per_node=1 + export threads_per_task=1 + export memory="4096M" + ;; + + "cleanup") + export walltime="00:30:00" + export ntasks=1 + export tasks_per_node=1 + export threads_per_task=1 + export memory="4096M" + ;; + *) + echo "FATAL ERROR: Invalid job ${step} passed to ${BASH_SOURCE[0]}" + exit 1 + ;; + +esac + +# Get machine-specific resources, overriding/extending the above assignments +if [[ -f "${EXPDIR}/config.resources.${machine}" ]]; then + source "${EXPDIR}/config.resources.${machine}" +fi + +# Check for RUN-specific variables and export them +for resource_var in threads_per_task ntasks tasks_per_node NTASKS memory walltime; do + run_resource_var="${resource_var}_${RUN}" + if [[ -n "${!run_resource_var+0}" ]]; then + declare -x "${resource_var}"="${!run_resource_var}" + elif [[ -n "${!resource_var+0}" ]]; then + export "${resource_var?}" + fi +done + +echo "END: config.resources" diff --git a/parm/config/sfs/config.resources.AWSPW b/parm/config/sfs/config.resources.AWSPW new file mode 120000 index 0000000000..c2b08a2b1c --- /dev/null +++ b/parm/config/sfs/config.resources.AWSPW @@ -0,0 +1 @@ +../gefs/config.resources.AWSPW \ No newline at end of file diff --git a/parm/config/sfs/config.resources.AZUREPW b/parm/config/sfs/config.resources.AZUREPW new file mode 120000 index 0000000000..8f8688696d --- /dev/null +++ b/parm/config/sfs/config.resources.AZUREPW @@ -0,0 +1 @@ +../gefs/config.resources.AZUREPW \ No newline at end of file diff --git a/parm/config/sfs/config.resources.GOOGLEPW b/parm/config/sfs/config.resources.GOOGLEPW new file mode 120000 index 0000000000..10cb8e6311 --- /dev/null +++ b/parm/config/sfs/config.resources.GOOGLEPW @@ -0,0 +1 @@ +../gefs/config.resources.GOOGLEPW \ No newline at end of file diff --git a/parm/config/sfs/config.stage_ic b/parm/config/sfs/config.stage_ic new file mode 120000 index 0000000000..1bba74ba62 --- /dev/null +++ b/parm/config/sfs/config.stage_ic @@ -0,0 +1 @@ +../gefs/config.stage_ic \ No newline at end of file diff --git a/parm/config/sfs/config.ufs b/parm/config/sfs/config.ufs new file mode 120000 index 0000000000..c26c7e5416 --- /dev/null +++ b/parm/config/sfs/config.ufs @@ -0,0 +1 @@ +../gefs/config.ufs \ No newline at end of file diff --git a/parm/config/sfs/config.wave b/parm/config/sfs/config.wave new file mode 120000 index 0000000000..fa2a8b2f00 --- /dev/null +++ b/parm/config/sfs/config.wave @@ -0,0 +1 @@ +../gefs/config.wave \ No newline at end of file diff --git a/parm/config/sfs/config.waveinit b/parm/config/sfs/config.waveinit new file mode 120000 index 0000000000..3c57a08c17 --- /dev/null +++ b/parm/config/sfs/config.waveinit @@ -0,0 +1 @@ +../gefs/config.waveinit \ No newline at end of file diff --git a/parm/config/sfs/config.wavepostbndpnt b/parm/config/sfs/config.wavepostbndpnt new file mode 120000 index 0000000000..a9518b0961 --- /dev/null +++ b/parm/config/sfs/config.wavepostbndpnt @@ -0,0 +1 @@ +../gefs/config.wavepostbndpnt \ No newline at end of file diff --git a/parm/config/sfs/config.wavepostbndpntbll b/parm/config/sfs/config.wavepostbndpntbll new file mode 120000 index 0000000000..66a6853435 --- /dev/null +++ b/parm/config/sfs/config.wavepostbndpntbll @@ -0,0 +1 @@ +../gefs/config.wavepostbndpntbll \ No newline at end of file diff --git a/parm/config/sfs/config.wavepostpnt b/parm/config/sfs/config.wavepostpnt new file mode 120000 index 0000000000..c6deaf44aa --- /dev/null +++ b/parm/config/sfs/config.wavepostpnt @@ -0,0 +1 @@ +../gefs/config.wavepostpnt \ No newline at end of file diff --git a/parm/config/sfs/config.wavepostsbs b/parm/config/sfs/config.wavepostsbs new file mode 120000 index 0000000000..4858683d7f --- /dev/null +++ b/parm/config/sfs/config.wavepostsbs @@ -0,0 +1 @@ +../gefs/config.wavepostsbs \ No newline at end of file diff --git a/parm/config/sfs/yaml/defaults.yaml b/parm/config/sfs/yaml/defaults.yaml new file mode 100644 index 0000000000..d918d12ebc --- /dev/null +++ b/parm/config/sfs/yaml/defaults.yaml @@ -0,0 +1,34 @@ +base: + DO_JEDIATMVAR: "NO" + DO_JEDIATMENS: "NO" + DO_JEDIOCNVAR: "NO" + DO_JEDISNOWDA: "NO" + DO_MERGENSST: "NO" + DO_BUFRSND: "NO" + DO_GEMPAK: "NO" + DO_AWIPS: "NO" + KEEPDATA: "NO" + DO_EXTRACTVARS: "NO" + FHMAX_GFS: 120 + FHMAX_HF_GFS: 0 + FHOUT_HF_GFS: 1 + FCST_BREAKPOINTS: "48" + REPLAY_ICS: "NO" + FHOUT_GFS: 6 + FHOUT_OCN_GFS: 6 + FHOUT_ICE_GFS: 6 + HPSSARCH: "NO" + LOCALARCH: "NO" + USE_OCN_ENS_PERTURB_FILES: "NO" + USE_ATM_ENS_PERTURB_FILES: "NO" + DO_TEST_MODE: "NO" +fcst: + reforecast: "NO" + FHZER: 6 + TYPE: "nh" + MONO: "non-mono" +ocn: + MOM6_INTERP_ICS: "NO" +# config.aero has just a system-specific path to add. +# This is handled by the setup_expt.py, but it has to be told to write to it. +aero: {} diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 7b9bd0ee48..0be7bbd2a1 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -305,18 +305,24 @@ FV3_out() { local restart_date restart_dates restart_dates=() - # Copy restarts in the assimilation window for RUN=gdas|enkfgdas|enkfgfs - if [[ "${RUN}" =~ "gdas" || "${RUN}" == "enkfgfs" ]]; then - restart_date="${model_start_date_next_cycle}" - while (( restart_date <= forecast_end_cycle )); do - restart_dates+=("${restart_date:0:8}.${restart_date:8:2}0000") - restart_date=$(date --utc -d "${restart_date:0:8} ${restart_date:8:2} + ${restart_interval} hours" +%Y%m%d%H) - done - elif [[ "${RUN}" == "gfs" || "${RUN}" == "gefs" ]]; then # Copy restarts at the end of the forecast segment for RUN=gfs|gefs - if [[ "${COPY_FINAL_RESTARTS}" == "YES" ]]; then - restart_dates+=("${forecast_end_cycle:0:8}.${forecast_end_cycle:8:2}0000") - fi - fi + case ${RUN} in + gdas|enkf*) # Copy restarts in the assimilation window for RUN=gdas|enkfgdas|enkfgfs + restart_date="${model_start_date_next_cycle}" + while (( restart_date <= forecast_end_cycle )); do + restart_dates+=("${restart_date:0:8}.${restart_date:8:2}0000") + restart_date=$(date --utc -d "${restart_date:0:8} ${restart_date:8:2} + ${restart_interval} hours" +%Y%m%d%H) + done + ;; + gfs|gefs|sfs) # Copy restarts at the end of the forecast segment for RUN=gfs|gefs|sfs + if [[ "${COPY_FINAL_RESTARTS}" == "YES" ]]; then + restart_dates+=("${forecast_end_cycle:0:8}.${forecast_end_cycle:8:2}0000") + fi + ;; + *) + echo "FATAL ERROR: Not sure how to copy restart files for RUN ${RUN}" + exit 25 + ;; + esac ### Check that there are restart files to copy if [[ ${#restart_dates[@]} -gt 0 ]]; then @@ -536,52 +542,57 @@ MOM6_postdet() { fi # if [[ "${RERUN}" == "NO" ]]; then # Link output files - if [[ "${RUN}" =~ "gfs" || "${RUN}" == "gefs" ]]; then # Link output files for RUN=gfs|enkfgfs|gefs - - # Looping over MOM6 output hours - local fhr fhr3 last_fhr interval midpoint vdate vdate_mid source_file dest_file - for fhr in ${MOM6_OUTPUT_FH}; do - fhr3=$(printf %03i "${fhr}") - - if [[ -z ${last_fhr:-} ]]; then - last_fhr=${fhr} - continue - fi + case ${RUN} in + *gfs|gefs|sfs) # Link output files for RUN=gfs|enkfgfs|gefs|sfs + # Looping over MOM6 output hours + local fhr fhr3 last_fhr interval midpoint vdate vdate_mid source_file dest_file + for fhr in ${MOM6_OUTPUT_FH}; do + fhr3=$(printf %03i "${fhr}") + + if [[ -z ${last_fhr:-} ]]; then + last_fhr=${fhr} + continue + fi - (( interval = fhr - last_fhr )) - (( midpoint = last_fhr + interval/2 )) + (( interval = fhr - last_fhr )) + (( midpoint = last_fhr + interval/2 )) - vdate=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${fhr} hours" +%Y%m%d%H) - #If OFFSET_START_HOUR is greater than 0, OFFSET_START_HOUR should be added to the midpoint for first lead time - if (( OFFSET_START_HOUR > 0 )) && (( fhr == FHOUT_OCN ));then - vdate_mid=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + $(( midpoint + OFFSET_START_HOUR )) hours" +%Y%m%d%H) - else - vdate_mid=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${midpoint} hours" +%Y%m%d%H) - fi - - # Native model output uses window midpoint in the filename, but we are mapping that to the end of the period for COM - if (( OFFSET_START_HOUR > 0 )) && (( fhr == FHOUT_OCN ));then - source_file="ocn_lead1_${vdate_mid:0:4}_${vdate_mid:4:2}_${vdate_mid:6:2}_${vdate_mid:8:2}.nc" - else - source_file="ocn_${vdate_mid:0:4}_${vdate_mid:4:2}_${vdate_mid:6:2}_${vdate_mid:8:2}.nc" - fi - dest_file="${RUN}.ocean.t${cyc}z.${interval}hr_avg.f${fhr3}.nc" - ${NLN} "${COMOUT_OCEAN_HISTORY}/${dest_file}" "${DATA}/MOM6_OUTPUT/${source_file}" + vdate=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${fhr} hours" +%Y%m%d%H) + #If OFFSET_START_HOUR is greater than 0, OFFSET_START_HOUR should be added to the midpoint for first lead time + if (( OFFSET_START_HOUR > 0 )) && (( fhr == FHOUT_OCN ));then + vdate_mid=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + $(( midpoint + OFFSET_START_HOUR )) hours" +%Y%m%d%H) + else + vdate_mid=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${midpoint} hours" +%Y%m%d%H) + fi - last_fhr=${fhr} + # Native model output uses window midpoint in the filename, but we are mapping that to the end of the period for COM + if (( OFFSET_START_HOUR > 0 )) && (( fhr == FHOUT_OCN ));then + source_file="ocn_lead1_${vdate_mid:0:4}_${vdate_mid:4:2}_${vdate_mid:6:2}_${vdate_mid:8:2}.nc" + else + source_file="ocn_${vdate_mid:0:4}_${vdate_mid:4:2}_${vdate_mid:6:2}_${vdate_mid:8:2}.nc" + fi + dest_file="${RUN}.ocean.t${cyc}z.${interval}hr_avg.f${fhr3}.nc" + ${NLN} "${COMOUT_OCEAN_HISTORY}/${dest_file}" "${DATA}/MOM6_OUTPUT/${source_file}" - done + last_fhr=${fhr} - elif [[ "${RUN}" =~ "gdas" ]]; then # Link output files for RUN=gdas|enkfgdas + done + ;; - # Save (instantaneous) MOM6 backgrounds - local fhr3 vdatestr - for fhr in ${MOM6_OUTPUT_FH}; do - fhr3=$(printf %03i "${fhr}") - vdatestr=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${fhr} hours" +%Y_%m_%d_%H) - ${NLN} "${COMOUT_OCEAN_HISTORY}/${RUN}.ocean.t${cyc}z.inst.f${fhr3}.nc" "${DATA}/MOM6_OUTPUT/ocn_da_${vdatestr}.nc" - done - fi + *gdas) # Link output files for RUN=gdas|enkfgdas + # Save (instantaneous) MOM6 backgrounds + local fhr3 vdatestr + for fhr in ${MOM6_OUTPUT_FH}; do + fhr3=$(printf %03i "${fhr}") + vdatestr=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${fhr} hours" +%Y_%m_%d_%H) + ${NLN} "${COMOUT_OCEAN_HISTORY}/${RUN}.ocean.t${cyc}z.inst.f${fhr3}.nc" "${DATA}/MOM6_OUTPUT/ocn_da_${vdatestr}.nc" + done + ;; + *) + echo "FATAL ERROR: Don't know how to copy MOM output files for RUN ${RUN}" + exit 25 + ;; + esac echo "SUB ${FUNCNAME[0]}: MOM6 input data linked/copied" @@ -614,30 +625,33 @@ MOM6_out() { *) ;; esac - # Copy MOM6 restarts at the end of the forecast segment to COM for RUN=gfs|gefs - if [[ "${COPY_FINAL_RESTARTS}" == "YES" ]]; then - local restart_file - if [[ "${RUN}" == "gfs" || "${RUN}" == "gefs" ]]; then - echo "Copying MOM6 restarts for 'RUN=${RUN}' at ${forecast_end_cycle}" + case ${RUN} in + gdas|enkf*) # Copy restarts for the next cycle for RUN=gdas|enkfgdas|enkfgfs + local restart_date + restart_date="${model_start_date_next_cycle}" + echo "Copying MOM6 restarts for 'RUN=${RUN}' at ${restart_date}" for mom6_restart_file in "${mom6_restart_files[@]}"; do - restart_file="${forecast_end_cycle:0:8}.${forecast_end_cycle:8:2}0000.${mom6_restart_file}" + restart_file="${restart_date:0:8}.${restart_date:8:2}0000.${mom6_restart_file}" ${NCP} "${DATArestart}/MOM6_RESTART/${restart_file}" \ "${COMOUT_OCEAN_RESTART}/${restart_file}" done - fi - fi - - # Copy restarts for the next cycle for RUN=gdas|enkfgdas|enkfgfs - if [[ "${RUN}" =~ "gdas" || "${RUN}" == "enkfgfs" ]]; then - local restart_date - restart_date="${model_start_date_next_cycle}" - echo "Copying MOM6 restarts for 'RUN=${RUN}' at ${restart_date}" - for mom6_restart_file in "${mom6_restart_files[@]}"; do - restart_file="${restart_date:0:8}.${restart_date:8:2}0000.${mom6_restart_file}" - ${NCP} "${DATArestart}/MOM6_RESTART/${restart_file}" \ - "${COMOUT_OCEAN_RESTART}/${restart_file}" - done - fi + ;; + gfs|gefs|sfs) # Copy MOM6 restarts at the end of the forecast segment to COM for RUN=gfs|gefs|sfs + if [[ "${COPY_FINAL_RESTARTS}" == "YES" ]]; then + local restart_file + echo "Copying MOM6 restarts for 'RUN=${RUN}' at ${forecast_end_cycle}" + for mom6_restart_file in "${mom6_restart_files[@]}"; do + restart_file="${forecast_end_cycle:0:8}.${forecast_end_cycle:8:2}0000.${mom6_restart_file}" + ${NCP} "${DATArestart}/MOM6_RESTART/${restart_file}" \ + "${COMOUT_OCEAN_RESTART}/${restart_file}" + done + fi + ;; + *) + echo "FATAL ERROR: Not sure how to copy restart files for RUN ${RUN}" + exit 25 + ;; + esac } CICE_postdet() { @@ -716,31 +730,31 @@ CICE_out() { # Copy ice_in namelist from DATA to COMOUT_CONF after the forecast is run (and successfull) ${NCP} "${DATA}/ice_in" "${COMOUT_CONF}/ufs.ice_in" - # Copy CICE restarts at the end of the forecast segment to COM for RUN=gfs|gefs - if [[ "${COPY_FINAL_RESTARTS}" == "YES" ]]; then - local seconds source_file target_file - if [[ "${RUN}" == "gfs" || "${RUN}" == "gefs" ]]; then + case ${RUN} in + gdas|enkf*) # Copy restarts for next cycle for RUN=gdas|enkfgdas|enkfgfs + local restart_date + restart_date="${model_start_date_next_cycle}" + echo "Copying CICE restarts for 'RUN=${RUN}' at ${restart_date}" + seconds=$(to_seconds "${restart_date:8:2}0000") # convert HHMMSS to seconds + source_file="cice_model.res.${restart_date:0:4}-${restart_date:4:2}-${restart_date:6:2}-${seconds}.nc" + target_file="${restart_date:0:8}.${restart_date:8:2}0000.cice_model.res.nc" + ${NCP} "${DATArestart}/CICE_RESTART/${source_file}" \ + "${COMOUT_ICE_RESTART}/${target_file}" + ;; + gfs|gefs|sfs) # Copy CICE restarts at the end of the forecast segment to COM for RUN=gfs|gefs|sfs + local seconds source_file target_file echo "Copying CICE restarts for 'RUN=${RUN}' at ${forecast_end_cycle}" seconds=$(to_seconds "${forecast_end_cycle:8:2}0000") # convert HHMMSS to seconds source_file="cice_model.res.${forecast_end_cycle:0:4}-${forecast_end_cycle:4:2}-${forecast_end_cycle:6:2}-${seconds}.nc" target_file="${forecast_end_cycle:0:8}.${forecast_end_cycle:8:2}0000.cice_model.res.nc" ${NCP} "${DATArestart}/CICE_RESTART/${source_file}" \ "${COMOUT_ICE_RESTART}/${target_file}" - fi - fi - - # Copy restarts for next cycle for RUN=gdas|enkfgdas|enkfgfs - if [[ "${RUN}" =~ "gdas" || "${RUN}" == "enkfgfs" ]]; then - local restart_date - restart_date="${model_start_date_next_cycle}" - echo "Copying CICE restarts for 'RUN=${RUN}' at ${restart_date}" - seconds=$(to_seconds "${restart_date:8:2}0000") # convert HHMMSS to seconds - source_file="cice_model.res.${restart_date:0:4}-${restart_date:4:2}-${restart_date:6:2}-${seconds}.nc" - target_file="${restart_date:0:8}.${restart_date:8:2}0000.cice_model.res.nc" - ${NCP} "${DATArestart}/CICE_RESTART/${source_file}" \ - "${COMOUT_ICE_RESTART}/${target_file}" - fi - + ;; + *) + echo "FATAL ERROR: Not sure how to copy restart files for RUN ${RUN}" + exit 25 + ;; + esac } GOCART_rc() { @@ -846,35 +860,39 @@ CMEPS_postdet() { CMEPS_out() { echo "SUB ${FUNCNAME[0]}: Copying output data for CMEPS mediator" - # Copy mediator restarts at the end of the forecast segment to COM for RUN=gfs|gefs - if [[ "${COPY_FINAL_RESTARTS}" == "YES" ]]; then - echo "Copying mediator restarts for 'RUN=${RUN}' at ${forecast_end_cycle}" - local seconds source_file target_file - seconds=$(to_seconds "${forecast_end_cycle:8:2}"0000) - source_file="ufs.cpld.cpl.r.${forecast_end_cycle:0:4}-${forecast_end_cycle:4:2}-${forecast_end_cycle:6:2}-${seconds}.nc" - target_file="${forecast_end_cycle:0:8}.${forecast_end_cycle:8:2}0000.ufs.cpld.cpl.r.nc" - if [[ -f "${DATArestart}/CMEPS_RESTART/${source_file}" ]]; then - ${NCP} "${DATArestart}/CMEPS_RESTART/${source_file}" \ - "${COMOUT_MED_RESTART}/${target_file}" - else - echo "Mediator restart '${DATArestart}/CMEPS_RESTART/${source_file}' not found." - fi - fi - - # Copy restarts for the next cycle to COM for RUN=gdas|enkfgdas|enkfgfs - if [[ "${RUN}" =~ "gdas" || "${RUN}" == "enkfgfs" ]]; then - local restart_date - restart_date="${model_start_date_next_cycle}" - echo "Copying mediator restarts for 'RUN=${RUN}' at ${restart_date}" - seconds=$(to_seconds "${restart_date:8:2}"0000) - source_file="ufs.cpld.cpl.r.${restart_date:0:4}-${restart_date:4:2}-${restart_date:6:2}-${seconds}.nc" - target_file="${restart_date:0:8}.${restart_date:8:2}0000.ufs.cpld.cpl.r.nc" - if [[ -f "${DATArestart}/CMEPS_RESTART/${source_file}" ]]; then - ${NCP} "${DATArestart}/CMEPS_RESTART/${source_file}" \ - "${COMOUT_MED_RESTART}/${target_file}" - else - echo "Mediator restart '${DATArestart}/CMEPS_RESTART/${source_file}' not found." - fi - fi - + case ${RUN} in + gdas|enkf*) # Copy restarts for the next cycle to COM for RUN=gdas|enkfgdas|enkfgfs + local restart_date + restart_date="${model_start_date_next_cycle}" + echo "Copying mediator restarts for 'RUN=${RUN}' at ${restart_date}" + seconds=$(to_seconds "${restart_date:8:2}"0000) + source_file="ufs.cpld.cpl.r.${restart_date:0:4}-${restart_date:4:2}-${restart_date:6:2}-${seconds}.nc" + target_file="${restart_date:0:8}.${restart_date:8:2}0000.ufs.cpld.cpl.r.nc" + if [[ -f "${DATArestart}/CMEPS_RESTART/${source_file}" ]]; then + ${NCP} "${DATArestart}/CMEPS_RESTART/${source_file}" \ + "${COMOUT_MED_RESTART}/${target_file}" + else + echo "Mediator restart '${DATArestart}/CMEPS_RESTART/${source_file}' not found." + fi + ;; + gfs|gefs|sfs) # Copy mediator restarts at the end of the forecast segment to COM for RUN=gfs|gefs|sfs + if [[ "${COPY_FINAL_RESTARTS}" == "YES" ]]; then + echo "Copying mediator restarts for 'RUN=${RUN}' at ${forecast_end_cycle}" + local seconds source_file target_file + seconds=$(to_seconds "${forecast_end_cycle:8:2}"0000) + source_file="ufs.cpld.cpl.r.${forecast_end_cycle:0:4}-${forecast_end_cycle:4:2}-${forecast_end_cycle:6:2}-${seconds}.nc" + target_file="${forecast_end_cycle:0:8}.${forecast_end_cycle:8:2}0000.ufs.cpld.cpl.r.nc" + if [[ -f "${DATArestart}/CMEPS_RESTART/${source_file}" ]]; then + ${NCP} "${DATArestart}/CMEPS_RESTART/${source_file}" \ + "${COMOUT_MED_RESTART}/${target_file}" + else + echo "Mediator restart '${DATArestart}/CMEPS_RESTART/${source_file}' not found." + fi + fi + ;; + *) + echo "FATAL ERROR: Not sure how to copy restart files for RUN ${RUN}" + exit 25 + ;; + esac } diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index e08b84d932..f86c6ddb71 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -564,28 +564,31 @@ FV3_predet(){ ${NCP} "${POSTGRB2TBL:-${PARMgfs}/post/params_grib2_tbl_new}" "${DATA}/params_grib2_tbl_new" ${NCP} "${PARMgfs}/ufs/post_itag_gfs" "${DATA}/itag" # TODO: Need a GEFS version when available in the UFS-weather-model # TODO: These should be replaced with ones from the ufs-weather-model when available there - if [[ "${RUN}" =~ "gdas" || "${RUN}" =~ "gfs" ]]; then # RUN = gdas | enkfgdas | gfs | enkfgfs - ${NCP} "${PARMgfs}/post/gfs/postxconfig-NT-gfs-two.txt" "${DATA}/postxconfig-NT.txt" - ${NCP} "${PARMgfs}/post/gfs/postxconfig-NT-gfs-f00-two.txt" "${DATA}/postxconfig-NT_FH00.txt" - elif [[ "${RUN}" == "gefs" && "${SFS_POST:-NO}" == "NO" ]]; then # RUN = gefs - ${NCP} "${PARMgfs}/post/gefs/postxconfig-NT-gefs.txt" "${DATA}/postxconfig-NT.txt" - ${NCP} "${PARMgfs}/post/gefs/postxconfig-NT-gefs-f00.txt" "${DATA}/postxconfig-NT_FH00.txt" - elif [[ "${RUN}" == "gefs" && "${SFS_POST:-NO}" == "YES" ]]; then # RUN = sfs output - ${NCP} "${PARMgfs}/post/sfs/postxconfig-NT-sfs.txt" "${DATA}/postxconfig-NT.txt" - ${NCP} "${PARMgfs}/post/sfs/postxconfig-NT-sfs.txt" "${DATA}/postxconfig-NT_FH00.txt" - fi - - # For gefs run, provide ensemble header information - if [[ "${RUN}" == "gefs" ]]; then - if [[ "${ENSMEM}" == "000" ]]; then - export e1=1 - else - export e1=3 - fi - export e2="${ENSMEM:1:2}" - export e3="${NMEM_ENS}" - fi - + case ${NET} in + gfs) + ${NCP} "${PARMgfs}/post/gfs/postxconfig-NT-gfs-two.txt" "${DATA}/postxconfig-NT.txt" + ${NCP} "${PARMgfs}/post/gfs/postxconfig-NT-gfs-f00-two.txt" "${DATA}/postxconfig-NT_FH00.txt" + ;; + gefs) + ${NCP} "${PARMgfs}/post/gefs/postxconfig-NT-gefs.txt" "${DATA}/postxconfig-NT.txt" + ${NCP} "${PARMgfs}/post/gefs/postxconfig-NT-gefs-f00.txt" "${DATA}/postxconfig-NT_FH00.txt" + # Provide ensemble header information for GEFS + if [[ "${ENSMEM}" == "000" ]]; then + export e1=1 + else + export e1=3 + fi + export e2="${ENSMEM:1:2}" + export e3="${NMEM_ENS}" + ;; + sfs) + ${NCP} "${PARMgfs}/post/sfs/postxconfig-NT-sfs.txt" "${DATA}/postxconfig-NT.txt" + ${NCP} "${PARMgfs}/post/sfs/postxconfig-NT-sfs.txt" "${DATA}/postxconfig-NT_FH00.txt" + ;; + *) + echo "FATAL ERROR: Unknown RUN ${RUN}, unable to determine appropriate post files" + exit 20 + esac fi } diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index ed63a22230..34c9b39334 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -458,7 +458,7 @@ def _archive_expdir(self, arch_dict: Dict[str, Any]) -> bool: EDATE: Datetime Ending cycle date. NET: str - The workflow type (gfs or gefs) + The workflow type (gfs, gefs, or sfs) ARCH_EXPDIR_FREQ: int Frequency to perform EXPDIR archiving ROTDIR: str diff --git a/workflow/applications/application_factory.py b/workflow/applications/application_factory.py index ff6b6992f4..805878a6c7 100644 --- a/workflow/applications/application_factory.py +++ b/workflow/applications/application_factory.py @@ -2,9 +2,11 @@ from applications.gfs_cycled import GFSCycledAppConfig from applications.gfs_forecast_only import GFSForecastOnlyAppConfig from applications.gefs import GEFSAppConfig +from applications.sfs import SFSAppConfig app_config_factory = Factory('AppConfig') app_config_factory.register('gfs_cycled', GFSCycledAppConfig) app_config_factory.register('gfs_forecast-only', GFSForecastOnlyAppConfig) app_config_factory.register('gefs_forecast-only', GEFSAppConfig) +app_config_factory.register('sfs_forecast-only', SFSAppConfig) diff --git a/workflow/applications/sfs.py b/workflow/applications/sfs.py new file mode 100644 index 0000000000..13f7a51cdf --- /dev/null +++ b/workflow/applications/sfs.py @@ -0,0 +1,98 @@ +from applications.applications import AppConfig +from typing import Dict, Any +from wxflow import Configuration + + +class SFSAppConfig(AppConfig): + ''' + Class to define SFS configurations + ''' + + def __init__(self, conf: Configuration): + super().__init__(conf) + + base = conf.parse_config('config.base') + self.run = base.get('RUN', 'sfs') + self.runs = [self.run] + + def _get_run_options(self, conf: Configuration) -> Dict[str, Any]: + + run_options = super()._get_run_options(conf) + + run_options[self.run]['nens'] = conf.parse_config('config.base').get('NMEM_ENS', 0) + + return run_options + + def _get_app_configs(self, run): + """ + Returns the config_files that are involved in sfs + """ + options = self.run_options[run] + configs = ['stage_ic', 'fcst', 'atmos_products', 'arch', 'cleanup'] + + if options['nens'] > 0: + configs += ['efcs', 'atmos_ensstat'] + + if options['do_wave']: + configs += ['waveinit', 'wavepostsbs', 'wavepostpnt'] + if options['do_wave_bnd']: + configs += ['wavepostbndpnt', 'wavepostbndpntbll'] + + if options['do_ocean'] or options['do_ice']: + configs += ['oceanice_products'] + + if options['do_aero_fcst']: + configs += ['prep_emissions'] + + if options['do_extractvars']: + configs += ['extractvars'] + + return configs + + @staticmethod + def _update_base(base_in): + + base_out = base_in.copy() + base_out['RUN'] = 'sfs' + + return base_out + + def get_task_names(self): + + options = self.run_options[self.run] + tasks = ['stage_ic'] + + if options['do_wave']: + tasks += ['waveinit'] + + if options['do_aero_fcst']: + tasks += ['prep_emissions'] + + tasks += ['fcst'] + + if options['nens'] > 0: + tasks += ['efcs'] + + tasks += ['atmos_prod'] + + if options['nens'] > 0: + tasks += ['atmos_ensstat'] + + if options['do_ocean']: + tasks += ['ocean_prod'] + + if options['do_ice']: + tasks += ['ice_prod'] + + if options['do_wave']: + tasks += ['wavepostsbs'] + if options['do_wave_bnd']: + tasks += ['wavepostbndpnt', 'wavepostbndpntbll'] + tasks += ['wavepostpnt'] + + if options['do_extractvars']: + tasks += ['extractvars', 'arch'] + + tasks += ['cleanup'] + + return {f"{self.run}": tasks} diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index f1b1cd1ea2..8d1c144299 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -11,11 +11,11 @@ def __init__(self, app_config: AppConfig, run: str) -> None: def stage_ic(self): resources = self.get_resource('stage_ic') - task_name = f'gefs_stage_ic' + task_name = f'{self.run}_stage_ic' task_dict = {'task_name': task_name, 'resources': resources, 'envars': self.envars, - 'cycledef': 'gefs', + 'cycledef': f'{self.run}', 'command': f'{self.HOMEgfs}/jobs/rocoto/stage_ic.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -28,11 +28,11 @@ def stage_ic(self): def waveinit(self): resources = self.get_resource('waveinit') - task_name = f'gefs_wave_init' + task_name = f'{self.run}_wave_init' task_dict = {'task_name': task_name, 'resources': resources, 'envars': self.envars, - 'cycledef': 'gefs', + 'cycledef': f'{self.run}', 'command': f'{self.HOMEgfs}/jobs/rocoto/waveinit.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -45,11 +45,11 @@ def waveinit(self): def prep_emissions(self): resources = self.get_resource('prep_emissions') - task_name = 'gefs_prep_emissions' + task_name = f'{self.run}_prep_emissions' task_dict = {'task_name': task_name, 'resources': resources, 'envars': self.envars, - 'cycledef': 'gefs', + 'cycledef': f'{self.run}', 'command': f'{self.HOMEgfs}/jobs/rocoto/prep_emissions.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -61,15 +61,15 @@ def prep_emissions(self): def fcst(self): dependencies = [] - dep_dict = {'type': 'task', 'name': f'gefs_stage_ic'} + dep_dict = {'type': 'task', 'name': f'{self.run}_stage_ic'} dependencies.append(rocoto.add_dependency(dep_dict)) if self.options['do_wave']: - dep_dict = {'type': 'task', 'name': f'gefs_wave_init'} + dep_dict = {'type': 'task', 'name': f'{self.run}_wave_init'} dependencies.append(rocoto.add_dependency(dep_dict)) if self.options['do_aero_fcst']: - dep_dict = {'type': 'task', 'name': f'gefs_prep_emissions'} + dep_dict = {'type': 'task', 'name': f'{self.run}_prep_emissions'} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) @@ -82,12 +82,12 @@ def fcst(self): fcst_vars.append(rocoto.create_envar(name=key, value=str(value))) resources = self.get_resource('fcst') - task_name = f'gefs_fcst_mem000_seg#seg#' + task_name = f'{self.run}_fcst_mem000_seg#seg#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': fcst_vars, - 'cycledef': 'gefs', + 'cycledef': f'{self.run}', 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -95,7 +95,7 @@ def fcst(self): } seg_var_dict = {'seg': ' '.join([f"{seg}" for seg in range(0, num_fcst_segments)])} - metatask_dict = {'task_name': f'gefs_fcst_mem000', + metatask_dict = {'task_name': f'{self.run}_fcst_mem000', 'is_serial': True, 'var_dict': seg_var_dict, 'task_dict': task_dict @@ -107,15 +107,15 @@ def fcst(self): def efcs(self): dependencies = [] - dep_dict = {'type': 'task', 'name': f'gefs_stage_ic'} + dep_dict = {'type': 'task', 'name': f'{self.run}_stage_ic'} dependencies.append(rocoto.add_dependency(dep_dict)) if self.options['do_wave']: - dep_dict = {'type': 'task', 'name': f'gefs_wave_init'} + dep_dict = {'type': 'task', 'name': f'{self.run}_wave_init'} dependencies.append(rocoto.add_dependency(dep_dict)) if self.options['do_aero_fcst']: - dep_dict = {'type': 'task', 'name': f'gefs_prep_emissions'} + dep_dict = {'type': 'task', 'name': f'{self.run}_prep_emissions'} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) @@ -139,12 +139,12 @@ def efcs(self): for key, value in efcsenvars_dict.items(): efcsenvars.append(rocoto.create_envar(name=key, value=str(value))) - task_name = f'gefs_fcst_mem{member}_seg#seg#' + task_name = f'{self.run}_fcst_mem{member}_seg#seg#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': efcsenvars, - 'cycledef': 'gefs', + 'cycledef': f'{self.run}', 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -152,7 +152,7 @@ def efcs(self): } seg_var_dict = {'seg': ' '.join([f"{seg}" for seg in range(0, num_fcst_segments)])} - seg_metatask_dict = {'task_name': f'gefs_fcst_mem{member}', + seg_metatask_dict = {'task_name': f'{self.run}_fcst_mem{member}', 'is_serial': True, 'var_dict': seg_var_dict, 'task_dict': task_dict @@ -165,7 +165,7 @@ def efcs(self): # Keeping this in hopes the kludge is no longer necessary at some point # # member_var_dict = {'member': ' '.join([f"{mem:03d}" for mem in range(1, self.nmem + 1)])} - # mem_metatask_dict = {'task_name': 'gefs_fcst_ens', + # mem_metatask_dict = {'task_name': f'{self.run}_fcst_ens', # 'is_serial': False, # 'var_dict': member_var_dict, # 'task_dict': seg_metatask_dict @@ -228,7 +228,7 @@ def _atmosoceaniceprod(self, component: str): data = f'{history_path}/{history_file_tmpl}' dep_dict = {'type': 'data', 'data': data, 'age': 120} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': 'gefs_fcst_mem#member#_#seg_dep#'} + dep_dict = {'type': 'task', 'name': f'{self.run}_fcst_mem#member#_#seg_dep#'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps, dep_condition='or') @@ -240,23 +240,23 @@ def _atmosoceaniceprod(self, component: str): for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) - task_name = f'gefs_{component}_prod_mem#member#_#fhr_label#' + task_name = f'{self.run}_{component}_prod_mem#member#_#fhr_label#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': postenvars, - 'cycledef': 'gefs', + 'cycledef': f'{self.run}', 'command': f'{self.HOMEgfs}/jobs/rocoto/{config}.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;'} - fhr_metatask_dict = {'task_name': f'gefs_{component}_prod_#member#', + fhr_metatask_dict = {'task_name': f'{self.run}_{component}_prod_#member#', 'task_dict': task_dict, 'var_dict': fhr_var_dict} member_var_dict = {'member': ' '.join([f"{mem:03d}" for mem in range(0, self.nmem + 1)])} - member_metatask_dict = {'task_name': f'gefs_{component}_prod', + member_metatask_dict = {'task_name': f'{self.run}_{component}_prod', 'task_dict': fhr_metatask_dict, 'var_dict': member_var_dict} @@ -270,7 +270,7 @@ def atmos_ensstat(self): deps = [] for member in range(0, self.nmem + 1): - task = f'gefs_atmos_prod_mem{member:03d}_#fhr_label#' + task = f'{self.run}_atmos_prod_mem{member:03d}_#fhr_label#' dep_dict = {'type': 'task', 'name': task} deps.append(rocoto.add_dependency(dep_dict)) @@ -295,18 +295,18 @@ def atmos_ensstat(self): for key, value in postenvar_dict.items(): postenvars.append(rocoto.create_envar(name=key, value=str(value))) - task_name = f'gefs_atmos_ensstat_#fhr_label#' + task_name = f'{self.run}_atmos_ensstat_#fhr_label#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': postenvars, - 'cycledef': 'gefs', + 'cycledef': f'{self.run}', 'command': f'{self.HOMEgfs}/jobs/rocoto/atmos_ensstat.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;'} - fhr_metatask_dict = {'task_name': f'gefs_atmos_ensstat', + fhr_metatask_dict = {'task_name': f'{self.run}_atmos_ensstat', 'task_dict': task_dict, 'var_dict': fhr_var_dict} @@ -351,24 +351,24 @@ def wavepostsbs(self): largest_group = max([len(grp.split(',')) for grp in fhr_var_dict['fhr_list'].split(' ')]) resources['walltime'] = Tasks.multiply_HMS(resources['walltime'], largest_group) - task_name = f'gefs_wave_post_grid_mem#member#_#fhr_label#' + task_name = f'{self.run}_wave_post_grid_mem#member#_#fhr_label#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': wave_post_envars, - 'cycledef': 'gefs', + 'cycledef': f'{self.run}', 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostsbs.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' } - fhr_metatask_dict = {'task_name': f'gefs_wave_post_grid_#member#', + fhr_metatask_dict = {'task_name': f'{self.run}_wave_post_grid_#member#', 'task_dict': task_dict, 'var_dict': fhr_var_dict} member_var_dict = {'member': ' '.join([f"{mem:03d}" for mem in range(0, self.nmem + 1)])} - member_metatask_dict = {'task_name': f'gefs_wave_post_grid', + member_metatask_dict = {'task_name': f'{self.run}_wave_post_grid', 'task_dict': fhr_metatask_dict, 'var_dict': member_var_dict} @@ -378,7 +378,7 @@ def wavepostsbs(self): def wavepostbndpnt(self): deps = [] - dep_dict = {'type': 'metatask', 'name': f'gefs_fcst_mem#member#'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_fcst_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -390,12 +390,12 @@ def wavepostbndpnt(self): wave_post_bndpnt_envars.append(rocoto.create_envar(name=key, value=str(value))) resources = self.get_resource('wavepostbndpnt') - task_name = f'gefs_wave_post_bndpnt_mem#member#' + task_name = f'{self.run}_wave_post_bndpnt_mem#member#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': wave_post_bndpnt_envars, - 'cycledef': 'gefs', + 'cycledef': f'{self.run}', 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostbndpnt.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -403,7 +403,7 @@ def wavepostbndpnt(self): } member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} - member_metatask_dict = {'task_name': 'gefs_wave_post_bndpnt', + member_metatask_dict = {'task_name': f'{self.run}_wave_post_bndpnt', 'task_dict': task_dict, 'var_dict': member_var_dict } @@ -423,7 +423,7 @@ def wavepostbndpntbll(self): dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': f'gefs_fcst_mem#member#'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_fcst_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) @@ -435,12 +435,12 @@ def wavepostbndpntbll(self): wave_post_bndpnt_bull_envars.append(rocoto.create_envar(name=key, value=str(value))) resources = self.get_resource('wavepostbndpntbll') - task_name = f'gefs_wave_post_bndpnt_bull_mem#member#' + task_name = f'{self.run}_wave_post_bndpnt_bull_mem#member#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': wave_post_bndpnt_bull_envars, - 'cycledef': 'gefs', + 'cycledef': f'{self.run}', 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostbndpntbll.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -448,7 +448,7 @@ def wavepostbndpntbll(self): } member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} - member_metatask_dict = {'task_name': 'gefs_wave_post_bndpnt_bull', + member_metatask_dict = {'task_name': f'{self.run}_wave_post_bndpnt_bull', 'task_dict': task_dict, 'var_dict': member_var_dict } @@ -459,10 +459,10 @@ def wavepostbndpntbll(self): def wavepostpnt(self): deps = [] - dep_dict = {'type': 'metatask', 'name': f'gefs_fcst_mem#member#'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_fcst_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_wave_bnd']: - dep_dict = {'type': 'task', 'name': f'gefs_wave_post_bndpnt_bull_mem#member#'} + dep_dict = {'type': 'task', 'name': f'{self.run}_wave_post_bndpnt_bull_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -474,12 +474,12 @@ def wavepostpnt(self): wave_post_pnt_envars.append(rocoto.create_envar(name=key, value=str(value))) resources = self.get_resource('wavepostpnt') - task_name = f'gefs_wave_post_pnt_mem#member#' + task_name = f'{self.run}_wave_post_pnt_mem#member#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': wave_post_pnt_envars, - 'cycledef': 'gefs', + 'cycledef': f'{self.run}', 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostpnt.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -487,7 +487,7 @@ def wavepostpnt(self): } member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} - member_metatask_dict = {'task_name': 'gefs_wave_post_pnt', + member_metatask_dict = {'task_name': f'{self.run}_wave_post_pnt', 'task_dict': task_dict, 'var_dict': member_var_dict } @@ -499,16 +499,16 @@ def wavepostpnt(self): def extractvars(self): deps = [] if self.options['do_wave']: - dep_dict = {'type': 'metatask', 'name': 'gefs_wave_post_grid_#member#'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_wave_post_grid_#member#'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_ocean']: - dep_dict = {'type': 'metatask', 'name': 'gefs_ocean_prod_#member#'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_ocean_prod_#member#'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_ice']: - dep_dict = {'type': 'metatask', 'name': 'gefs_ice_prod_#member#'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_ice_prod_#member#'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_atm']: - dep_dict = {'type': 'metatask', 'name': 'gefs_atmos_prod_#member#'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_prod_#member#'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) extractvars_envars = self.envars.copy() @@ -519,12 +519,12 @@ def extractvars(self): extractvars_envars.append(rocoto.create_envar(name=key, value=str(value))) resources = self.get_resource('extractvars') - task_name = f'gefs_extractvars_mem#member#' + task_name = f'{self.run}_extractvars_mem#member#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': extractvars_envars, - 'cycledef': 'gefs', + 'cycledef': f'{self.run}', 'command': f'{self.HOMEgfs}/jobs/rocoto/extractvars.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -532,7 +532,7 @@ def extractvars(self): } member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} - member_metatask_dict = {'task_name': 'gefs_extractvars', + member_metatask_dict = {'task_name': f'{self.run}_extractvars', 'task_dict': task_dict, 'var_dict': member_var_dict } @@ -543,37 +543,37 @@ def extractvars(self): def arch(self): deps = [] - dep_dict = {'type': 'metatask', 'name': 'gefs_atmos_prod'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_prod'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'gefs_atmos_ensstat'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_ensstat'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_ice']: - dep_dict = {'type': 'metatask', 'name': 'gefs_ice_prod'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_ice_prod'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_ocean']: - dep_dict = {'type': 'metatask', 'name': 'gefs_ocean_prod'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_ocean_prod'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_wave']: - dep_dict = {'type': 'metatask', 'name': 'gefs_wave_post_grid'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_wave_post_grid'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'gefs_wave_post_pnt'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_wave_post_pnt'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_wave_bnd']: - dep_dict = {'type': 'metatask', 'name': 'gefs_wave_post_bndpnt'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_wave_post_bndpnt'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'gefs_wave_post_bndpnt_bull'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_wave_post_bndpnt_bull'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_extractvars']: - dep_dict = {'type': 'metatask', 'name': 'gefs_extractvars'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_extractvars'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps, dep_condition='and') resources = self.get_resource('arch') - task_name = 'gefs_arch' + task_name = f'{self.run}_arch' task_dict = {'task_name': task_name, 'resources': resources, 'envars': self.envars, - 'cycledef': 'gefs', + 'cycledef': f'{self.run}', 'dependency': dependencies, 'command': f'{self.HOMEgfs}/jobs/rocoto/arch.sh', 'job_name': f'{self.pslot}_{task_name}_@H', @@ -587,15 +587,15 @@ def arch(self): def cleanup(self): deps = [] - dep_dict = {'type': 'task', 'name': 'gefs_arch'} + dep_dict = {'type': 'task', 'name': f'{self.run}_arch'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('cleanup') - task_name = 'gefs_cleanup' + task_name = f'{self.run}_cleanup' task_dict = {'task_name': task_name, 'resources': resources, 'envars': self.envars, - 'cycledef': 'gefs', + 'cycledef': f'{self.run}', 'dependency': dependencies, 'command': f'{self.HOMEgfs}/jobs/rocoto/cleanup.sh', 'job_name': f'{self.pslot}_{task_name}_@H', diff --git a/workflow/rocoto/gefs_xml.py b/workflow/rocoto/gefs_xml.py index f0ea407e34..92a25336e9 100644 --- a/workflow/rocoto/gefs_xml.py +++ b/workflow/rocoto/gefs_xml.py @@ -6,8 +6,6 @@ from typing import Dict -# Copy of GFSForecastOnlyRocotoXML for now, other than changing cycledef names from 'gfs' to 'gefs' -# If it remains this way, we can consolidate into a single forecast-only class class GEFSRocotoXML(RocotoXML): def __init__(self, app_config: AppConfig, rocoto_config: Dict) -> None: diff --git a/workflow/rocoto/rocoto_xml_factory.py b/workflow/rocoto/rocoto_xml_factory.py index cb2d4c276c..4c21c0d268 100644 --- a/workflow/rocoto/rocoto_xml_factory.py +++ b/workflow/rocoto/rocoto_xml_factory.py @@ -2,9 +2,10 @@ from rocoto.gfs_cycled_xml import GFSCycledRocotoXML from rocoto.gfs_forecast_only_xml import GFSForecastOnlyRocotoXML from rocoto.gefs_xml import GEFSRocotoXML - +from rocoto.sfs_xml import SFSRocotoXML rocoto_xml_factory = Factory('RocotoXML') rocoto_xml_factory.register('gfs_cycled', GFSCycledRocotoXML) rocoto_xml_factory.register('gfs_forecast-only', GFSForecastOnlyRocotoXML) rocoto_xml_factory.register('gefs_forecast-only', GEFSRocotoXML) +rocoto_xml_factory.register('sfs_forecast-only', SFSRocotoXML) diff --git a/workflow/rocoto/sfs_tasks.py b/workflow/rocoto/sfs_tasks.py new file mode 100644 index 0000000000..4650fe6c89 --- /dev/null +++ b/workflow/rocoto/sfs_tasks.py @@ -0,0 +1,632 @@ +from applications.applications import AppConfig +from rocoto.tasks import Tasks +import rocoto.rocoto as rocoto + + +class SFSTasks(Tasks): + + def __init__(self, app_config: AppConfig, run: str) -> None: + super().__init__(app_config, run) + + def stage_ic(self): + + resources = self.get_resource('stage_ic') + task_name = f'{self.run}_stage_ic' + task_dict = {'task_name': task_name, + 'resources': resources, + 'envars': self.envars, + 'cycledef': f'{self.run}', + 'command': f'{self.HOMEgfs}/jobs/rocoto/stage_ic.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + task = rocoto.create_task(task_dict) + + return task + + def waveinit(self): + + resources = self.get_resource('waveinit') + task_name = f'{self.run}_wave_init' + task_dict = {'task_name': task_name, + 'resources': resources, + 'envars': self.envars, + 'cycledef': f'{self.run}', + 'command': f'{self.HOMEgfs}/jobs/rocoto/waveinit.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + task = rocoto.create_task(task_dict) + + return task + + def prep_emissions(self): + + resources = self.get_resource('prep_emissions') + task_name = f'{self.run}_prep_emissions' + task_dict = {'task_name': task_name, + 'resources': resources, + 'envars': self.envars, + 'cycledef': f'{self.run}', + 'command': f'{self.HOMEgfs}/jobs/rocoto/prep_emissions.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + task = rocoto.create_task(task_dict) + + return task + + def fcst(self): + dependencies = [] + dep_dict = {'type': 'task', 'name': f'{self.run}_stage_ic'} + dependencies.append(rocoto.add_dependency(dep_dict)) + + if self.options['do_wave']: + dep_dict = {'type': 'task', 'name': f'{self.run}_wave_init'} + dependencies.append(rocoto.add_dependency(dep_dict)) + + if self.options['do_aero_fcst']: + dep_dict = {'type': 'task', 'name': f'{self.run}_prep_emissions'} + dependencies.append(rocoto.add_dependency(dep_dict)) + + dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) + + num_fcst_segments = len(self.options['fcst_segments']) - 1 + + fcst_vars = self.envars.copy() + fcst_envars_dict = {'FCST_SEGMENT': '#seg#'} + for key, value in fcst_envars_dict.items(): + fcst_vars.append(rocoto.create_envar(name=key, value=str(value))) + + resources = self.get_resource('fcst') + task_name = f'{self.run}_fcst_mem000_seg#seg#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': fcst_vars, + 'cycledef': f'{self.run}', + 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + seg_var_dict = {'seg': ' '.join([f"{seg}" for seg in range(0, num_fcst_segments)])} + metatask_dict = {'task_name': f'{self.run}_fcst_mem000', + 'is_serial': True, + 'var_dict': seg_var_dict, + 'task_dict': task_dict + } + + task = rocoto.create_task(metatask_dict) + + return task + + def efcs(self): + dependencies = [] + dep_dict = {'type': 'task', 'name': f'{self.run}_stage_ic'} + dependencies.append(rocoto.add_dependency(dep_dict)) + + if self.options['do_wave']: + dep_dict = {'type': 'task', 'name': f'{self.run}_wave_init'} + dependencies.append(rocoto.add_dependency(dep_dict)) + + if self.options['do_aero_fcst']: + dep_dict = {'type': 'task', 'name': f'{self.run}_prep_emissions'} + dependencies.append(rocoto.add_dependency(dep_dict)) + + dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) + + num_fcst_segments = len(self.options['fcst_segments']) - 1 + resources = self.get_resource('efcs') + + # Kludge to work around bug in rocoto with serial metatasks nested + # in a parallel one (see christopherwharrop/rocoto#109). For now, + # loop over member to create a separate metatask for each instead + # of a metatask of a metatask. + # + tasks = [] + for member in [f"{mem:03d}" for mem in range(1, self.nmem + 1)]: + + efcsenvars = self.envars.copy() + efcsenvars_dict = {'ENSMEM': f'{member}', + 'MEMDIR': f'mem{member}', + 'FCST_SEGMENT': '#seg#' + } + for key, value in efcsenvars_dict.items(): + efcsenvars.append(rocoto.create_envar(name=key, value=str(value))) + + task_name = f'{self.run}_fcst_mem{member}_seg#seg#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': efcsenvars, + 'cycledef': f'{self.run}', + 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + seg_var_dict = {'seg': ' '.join([f"{seg}" for seg in range(0, num_fcst_segments)])} + seg_metatask_dict = {'task_name': f'{self.run}_fcst_mem{member}', + 'is_serial': True, + 'var_dict': seg_var_dict, + 'task_dict': task_dict + } + + tasks.append(rocoto.create_task(seg_metatask_dict)) + + return '\n'.join(tasks) + + # Keeping this in hopes the kludge is no longer necessary at some point + # + # member_var_dict = {'member': ' '.join([f"{mem:03d}" for mem in range(1, self.nmem + 1)])} + # mem_metatask_dict = {'task_name': f'{self.run}_fcst_ens', + # 'is_serial': False, + # 'var_dict': member_var_dict, + # 'task_dict': seg_metatask_dict + # } + + # task = rocoto.create_task(mem_metatask_dict) + + # return task + + def atmos_prod(self): + return self._atmosoceaniceprod('atmos') + + def ocean_prod(self): + return self._atmosoceaniceprod('ocean') + + def ice_prod(self): + return self._atmosoceaniceprod('ice') + + def _atmosoceaniceprod(self, component: str): + + fhout_ocn_gfs = self._configs['base']['FHOUT_OCN_GFS'] + fhout_ice_gfs = self._configs['base']['FHOUT_ICE_GFS'] + products_dict = {'atmos': {'config': 'atmos_products', + 'history_path_tmpl': 'COM_ATMOS_MASTER_TMPL', + 'history_file_tmpl': f'{self.run}.t@Hz.master.grb2f#fhr3_last#'}, + 'ocean': {'config': 'oceanice_products', + 'history_path_tmpl': 'COM_OCEAN_HISTORY_TMPL', + 'history_file_tmpl': f'{self.run}.ocean.t@Hz.{fhout_ocn_gfs}hr_avg.f#fhr3_next#.nc'}, + 'ice': {'config': 'oceanice_products', + 'history_path_tmpl': 'COM_ICE_HISTORY_TMPL', + 'history_file_tmpl': f'{self.run}.ice.t@Hz.{fhout_ice_gfs}hr_avg.f#fhr3_last#.nc'}} + + component_dict = products_dict[component] + config = component_dict['config'] + history_path_tmpl = component_dict['history_path_tmpl'] + history_file_tmpl = component_dict['history_file_tmpl'] + + max_tasks = self._configs[config]['MAX_TASKS'] + resources = self.get_resource(config) + + fhrs = self._get_forecast_hours('gefs', self._configs[config], component) + + # when replaying, atmos component does not have fhr 0, therefore remove 0 from fhrs + is_replay = self._configs[config]['REPLAY_ICS'] + if is_replay and component in ['atmos'] and 0 in fhrs: + fhrs.remove(0) + + # ocean/ice components do not have fhr 0 as they are averaged output + if component in ['ocean', 'ice'] and 0 in fhrs: + fhrs.remove(0) + + fhr_var_dict = self.get_grouped_fhr_dict(fhrs=fhrs, ngroups=max_tasks) + + # Adjust walltime based on the largest group + largest_group = max([len(grp.split(',')) for grp in fhr_var_dict['fhr_list'].split(' ')]) + resources['walltime'] = Tasks.multiply_HMS(resources['walltime'], largest_group) + + history_path = self._template_to_rocoto_cycstring(self._base[history_path_tmpl], {'MEMDIR': 'mem#member#'}) + deps = [] + data = f'{history_path}/{history_file_tmpl}' + dep_dict = {'type': 'data', 'data': data, 'age': 120} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{self.run}_fcst_mem#member#_#seg_dep#'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps, dep_condition='or') + + postenvars = self.envars.copy() + postenvar_dict = {'ENSMEM': '#member#', + 'MEMDIR': 'mem#member#', + 'FHR_LIST': '#fhr_list#', + 'COMPONENT': component} + for key, value in postenvar_dict.items(): + postenvars.append(rocoto.create_envar(name=key, value=str(value))) + + task_name = f'{self.run}_{component}_prod_mem#member#_#fhr_label#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': postenvars, + 'cycledef': f'{self.run}', + 'command': f'{self.HOMEgfs}/jobs/rocoto/{config}.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;'} + + fhr_metatask_dict = {'task_name': f'{self.run}_{component}_prod_#member#', + 'task_dict': task_dict, + 'var_dict': fhr_var_dict} + + member_var_dict = {'member': ' '.join([f"{mem:03d}" for mem in range(0, self.nmem + 1)])} + member_metatask_dict = {'task_name': f'{self.run}_{component}_prod', + 'task_dict': fhr_metatask_dict, + 'var_dict': member_var_dict} + + task = rocoto.create_task(member_metatask_dict) + + return task + + def atmos_ensstat(self): + + resources = self.get_resource('atmos_ensstat') + + deps = [] + for member in range(0, self.nmem + 1): + task = f'{self.run}_atmos_prod_mem{member:03d}_#fhr_label#' + dep_dict = {'type': 'task', 'name': task} + deps.append(rocoto.add_dependency(dep_dict)) + + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + fhrs = self._get_forecast_hours('gefs', self._configs['atmos_ensstat']) + + # when replaying, atmos component does not have fhr 0, therefore remove 0 from fhrs + is_replay = self._configs['atmos_ensstat']['REPLAY_ICS'] + if is_replay and 0 in fhrs: + fhrs.remove(0) + + max_tasks = self._configs['atmos_ensstat']['MAX_TASKS'] + fhr_var_dict = self.get_grouped_fhr_dict(fhrs=fhrs, ngroups=max_tasks) + + # Adjust walltime based on the largest group + largest_group = max([len(grp.split(',')) for grp in fhr_var_dict['fhr_list'].split(' ')]) + resources['walltime'] = Tasks.multiply_HMS(resources['walltime'], largest_group) + + postenvars = self.envars.copy() + postenvar_dict = {'FHR_LIST': '#fhr_list#'} + for key, value in postenvar_dict.items(): + postenvars.append(rocoto.create_envar(name=key, value=str(value))) + + task_name = f'{self.run}_atmos_ensstat_#fhr_label#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': postenvars, + 'cycledef': f'{self.run}', + 'command': f'{self.HOMEgfs}/jobs/rocoto/atmos_ensstat.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;'} + + fhr_metatask_dict = {'task_name': f'{self.run}_atmos_ensstat', + 'task_dict': task_dict, + 'var_dict': fhr_var_dict} + + task = rocoto.create_task(fhr_metatask_dict) + + return task + + def wavepostsbs(self): + + wave_grid = self._configs['base']['waveGRD'] + history_path = self._template_to_rocoto_cycstring(self._base['COM_WAVE_HISTORY_TMPL'], {'MEMDIR': 'mem#member#'}) + history_file = f'/{self.run}wave.out_grd.{wave_grid}.@Y@m@d.@H@M@S' + + deps = [] + dep_dict = {'type': 'data', 'data': [history_path, history_file], 'offset': [None, '#fhr3_next#:00:00']} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{self.run}_fcst_mem#member#_#seg_dep#'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps, dep_condition='or') + + fhrs = self._get_forecast_hours('gefs', self._configs['wavepostsbs'], 'wave') + + # When using replay, output does not start until hour 3 + is_replay = self._configs['wavepostsbs']['REPLAY_ICS'] + if is_replay: + fhrs = [fhr for fhr in fhrs if fhr not in [0, 1, 2]] + + max_tasks = self._configs['wavepostsbs']['MAX_TASKS'] + fhr_var_dict = self.get_grouped_fhr_dict(fhrs=fhrs, ngroups=max_tasks) + + wave_post_envars = self.envars.copy() + postenvar_dict = {'ENSMEM': '#member#', + 'MEMDIR': 'mem#member#', + 'FHR_LIST': '#fhr_list#', + } + for key, value in postenvar_dict.items(): + wave_post_envars.append(rocoto.create_envar(name=key, value=str(value))) + + resources = self.get_resource('wavepostsbs') + + # Adjust walltime based on the largest group + largest_group = max([len(grp.split(',')) for grp in fhr_var_dict['fhr_list'].split(' ')]) + resources['walltime'] = Tasks.multiply_HMS(resources['walltime'], largest_group) + + task_name = f'{self.run}_wave_post_grid_mem#member#_#fhr_label#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': wave_post_envars, + 'cycledef': f'{self.run}', + 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostsbs.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + fhr_metatask_dict = {'task_name': f'{self.run}_wave_post_grid_#member#', + 'task_dict': task_dict, + 'var_dict': fhr_var_dict} + + member_var_dict = {'member': ' '.join([f"{mem:03d}" for mem in range(0, self.nmem + 1)])} + member_metatask_dict = {'task_name': f'{self.run}_wave_post_grid', + 'task_dict': fhr_metatask_dict, + 'var_dict': member_var_dict} + + task = rocoto.create_task(member_metatask_dict) + + return task + + def wavepostbndpnt(self): + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{self.run}_fcst_mem#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + wave_post_bndpnt_envars = self.envars.copy() + postenvar_dict = {'ENSMEM': '#member#', + 'MEMDIR': 'mem#member#', + } + for key, value in postenvar_dict.items(): + wave_post_bndpnt_envars.append(rocoto.create_envar(name=key, value=str(value))) + + resources = self.get_resource('wavepostbndpnt') + task_name = f'{self.run}_wave_post_bndpnt_mem#member#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': wave_post_bndpnt_envars, + 'cycledef': f'{self.run}', + 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostbndpnt.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} + member_metatask_dict = {'task_name': f'{self.run}_wave_post_bndpnt', + 'task_dict': task_dict, + 'var_dict': member_var_dict + } + + task = rocoto.create_task(member_metatask_dict) + + return task + + def wavepostbndpntbll(self): + deps = [] + atmos_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"], {'MEMDIR': 'mem#member#'}) + + # The wavepostbndpntbll job runs on forecast hours up to FHMAX_WAV_IBP + last_fhr = self._configs['wave']['FHMAX_WAV_IBP'] + + data = f'{atmos_hist_path}/{self.run}.t@Hz.atm.logf{last_fhr:03d}.txt' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + + dep_dict = {'type': 'metatask', 'name': f'{self.run}_fcst_mem#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) + + wave_post_bndpnt_bull_envars = self.envars.copy() + postenvar_dict = {'ENSMEM': '#member#', + 'MEMDIR': 'mem#member#', + } + for key, value in postenvar_dict.items(): + wave_post_bndpnt_bull_envars.append(rocoto.create_envar(name=key, value=str(value))) + + resources = self.get_resource('wavepostbndpntbll') + task_name = f'{self.run}_wave_post_bndpnt_bull_mem#member#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': wave_post_bndpnt_bull_envars, + 'cycledef': f'{self.run}', + 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostbndpntbll.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} + member_metatask_dict = {'task_name': f'{self.run}_wave_post_bndpnt_bull', + 'task_dict': task_dict, + 'var_dict': member_var_dict + } + + task = rocoto.create_task(member_metatask_dict) + + return task + + def wavepostpnt(self): + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{self.run}_fcst_mem#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_wave_bnd']: + dep_dict = {'type': 'task', 'name': f'{self.run}_wave_post_bndpnt_bull_mem#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + wave_post_pnt_envars = self.envars.copy() + postenvar_dict = {'ENSMEM': '#member#', + 'MEMDIR': 'mem#member#', + } + for key, value in postenvar_dict.items(): + wave_post_pnt_envars.append(rocoto.create_envar(name=key, value=str(value))) + + resources = self.get_resource('wavepostpnt') + task_name = f'{self.run}_wave_post_pnt_mem#member#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': wave_post_pnt_envars, + 'cycledef': f'{self.run}', + 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostpnt.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} + member_metatask_dict = {'task_name': f'{self.run}_wave_post_pnt', + 'task_dict': task_dict, + 'var_dict': member_var_dict + } + + task = rocoto.create_task(member_metatask_dict) + + return task + + def extractvars(self): + deps = [] + if self.options['do_wave']: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_wave_post_grid_#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_ocean']: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_ocean_prod_#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_ice']: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_ice_prod_#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_atm']: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_prod_#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + extractvars_envars = self.envars.copy() + extractvars_dict = {'ENSMEM': '#member#', + 'MEMDIR': 'mem#member#', + } + for key, value in extractvars_dict.items(): + extractvars_envars.append(rocoto.create_envar(name=key, value=str(value))) + + resources = self.get_resource('extractvars') + task_name = f'{self.run}_extractvars_mem#member#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': extractvars_envars, + 'cycledef': f'{self.run}', + 'command': f'{self.HOMEgfs}/jobs/rocoto/extractvars.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} + member_metatask_dict = {'task_name': f'{self.run}_extractvars', + 'task_dict': task_dict, + 'var_dict': member_var_dict + } + + task = rocoto.create_task(member_metatask_dict) + + return task + + def arch(self): + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_prod'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_ensstat'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_ice']: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_ice_prod'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_ocean']: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_ocean_prod'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_wave']: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_wave_post_grid'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': f'{self.run}_wave_post_pnt'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_wave_bnd']: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_wave_post_bndpnt'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': f'{self.run}_wave_post_bndpnt_bull'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_extractvars']: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_extractvars'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps, dep_condition='and') + + resources = self.get_resource('arch') + task_name = 'arch' + task_dict = {'task_name': task_name, + 'resources': resources, + 'envars': self.envars, + 'cycledef': f'{self.run}', + 'dependency': dependencies, + 'command': f'{self.HOMEgfs}/jobs/rocoto/arch.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + + def cleanup(self): + deps = [] + if self.options['do_extractvars']: + dep_dict = {'type': 'task', 'name': 'arch'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + else: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_prod'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_ensstat'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_ice']: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_ice_prod'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_ocean']: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_ocean_prod'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_wave']: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_wave_post_grid'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': f'{self.run}_wave_post_pnt'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_wave_bnd']: + dep_dict = {'type': 'metatask', 'name': f'{self.run}_wave_post_bndpnt'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': f'{self.run}_wave_post_bndpnt_bull'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps, dep_condition='and') + + resources = self.get_resource('cleanup') + task_name = f'{self.run}_cleanup' + task_dict = {'task_name': task_name, + 'resources': resources, + 'envars': self.envars, + 'cycledef': f'{self.run}', + 'dependency': dependencies, + 'command': f'{self.HOMEgfs}/jobs/rocoto/cleanup.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task diff --git a/workflow/rocoto/sfs_xml.py b/workflow/rocoto/sfs_xml.py new file mode 100644 index 0000000000..c63beef204 --- /dev/null +++ b/workflow/rocoto/sfs_xml.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +from rocoto.workflow_xml import RocotoXML +from applications.applications import AppConfig +from wxflow import to_timedelta, timedelta_to_HMS +from typing import Dict + + +class SFSRocotoXML(RocotoXML): + + def __init__(self, app_config: AppConfig, rocoto_config: Dict) -> None: + super().__init__(app_config, rocoto_config) + + def get_cycledefs(self): + sdate = self._base['SDATE_GFS'] + edate = self._base['EDATE'] + interval = self._base['interval_gfs'] + sdate_str = sdate.strftime("%Y%m%d%H%M") + edate_str = edate.strftime("%Y%m%d%H%M") + interval_str = timedelta_to_HMS(interval) + strings = [] + strings.append(f'\t{sdate_str} {edate_str} {interval_str}') + + date2 = sdate + interval + if date2 <= edate: + date2_str = date2.strftime("%Y%m%d%H%M") + strings.append(f'\t{date2_str} {edate_str} {interval_str}') + + strings.append('') + strings.append('') + + return '\n'.join(strings) diff --git a/workflow/rocoto/tasks_factory.py b/workflow/rocoto/tasks_factory.py index 38cf0d0bd1..573cdb277a 100644 --- a/workflow/rocoto/tasks_factory.py +++ b/workflow/rocoto/tasks_factory.py @@ -1,8 +1,10 @@ from wxflow import Factory from rocoto.gfs_tasks import GFSTasks from rocoto.gefs_tasks import GEFSTasks +from rocoto.sfs_tasks import SFSTasks tasks_factory = Factory('Tasks') tasks_factory.register('gfs', GFSTasks) tasks_factory.register('gefs', GEFSTasks) +tasks_factory.register('sfs', SFSTasks) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 09bc1c90ac..88958b13b6 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -100,7 +100,7 @@ def _update_defaults(dict_in: dict) -> dict: def edit_baseconfig(host, inputs, yaml_dict): """ - Parses and populates the templated `HOMEgfs/parm/config//config.base` + Parses and populates the templated `HOMEgfs/parm/config//config.base` to `EXPDIR/pslot/config.base` """ @@ -259,14 +259,14 @@ def _gfs_cycled_args(parser): parser.add_argument('--sdate_gfs', help='date to start GFS', type=lambda dd: to_datetime(dd), required=False, default=None) return parser - def _gfs_or_gefs_ensemble_args(parser): + def _any_ensemble_args(parser): parser.add_argument('--resensatmos', help='atmosphere resolution of the ensemble model forecast', type=int, required=False, default=192) parser.add_argument('--nens', help='number of ensemble members', type=int, required=False, default=20) return parser - def _gfs_or_gefs_forecast_args(parser): + def _any_forecast_args(parser): parser.add_argument('--app', help='UFS application', type=str, choices=ufs_apps, required=False, default='ATM') return parser @@ -280,6 +280,15 @@ def _gefs_args(parser): default=os.path.join(_top, 'parm/config/gefs/yaml/defaults.yaml')) return parser + def _sfs_args(parser): + parser.add_argument('--start', help='restart mode: warm or cold', type=str, + choices=['warm', 'cold'], required=False, default='cold') + parser.add_argument('--configdir', help=SUPPRESS, type=str, required=False, + default=os.path.join(_top, 'parm/config/sfs')) + parser.add_argument('--yaml', help='Defaults to substitute from', type=str, required=False, + default=os.path.join(_top, 'parm/config/sfs/yaml/defaults.yaml')) + return parser + description = """ Setup files and directories to start a GFS parallel.\n Create EXPDIR, copy config files.\n @@ -293,6 +302,7 @@ def _gefs_args(parser): sysparser = parser.add_subparsers(dest='system') gfs = sysparser.add_parser('gfs', help='arguments for GFS') gefs = sysparser.add_parser('gefs', help='arguments for GEFS') + sfs = sysparser.add_parser('sfs', help='arguments for SFS') gfsmodeparser = gfs.add_subparsers(dest='mode') gfscycled = gfsmodeparser.add_parser('cycled', help='arguments for cycled mode') @@ -301,8 +311,11 @@ def _gefs_args(parser): gefsmodeparser = gefs.add_subparsers(dest='mode') gefsforecasts = gefsmodeparser.add_parser('forecast-only', help='arguments for forecast-only mode') + sfsmodeparser = sfs.add_subparsers(dest='mode') + sfsforecasts = sfsmodeparser.add_parser('forecast-only', help='arguments for forecast-only mode') + # Common arguments across all modes - for subp in [gfscycled, gfsforecasts, gefsforecasts]: + for subp in [gfscycled, gfsforecasts, gefsforecasts, sfsforecasts]: subp = _common_args(subp) # GFS-only arguments @@ -310,12 +323,12 @@ def _gefs_args(parser): subp = _gfs_args(subp) # ensemble-only arguments - for subp in [gfscycled, gefsforecasts]: - subp = _gfs_or_gefs_ensemble_args(subp) + for subp in [gfscycled, gefsforecasts, sfsforecasts]: + subp = _any_ensemble_args(subp) # GFS/GEFS forecast-only additional arguments - for subp in [gfsforecasts, gefsforecasts]: - subp = _gfs_or_gefs_forecast_args(subp) + for subp in [gfsforecasts, gefsforecasts, sfsforecasts]: + subp = _any_forecast_args(subp) # cycled mode additional arguments for subp in [gfscycled]: @@ -325,6 +338,10 @@ def _gefs_args(parser): for subp in [gefsforecasts]: subp = _gefs_args(subp) + # SFS arguments + for subp in [sfsforecasts]: + subp = _sfs_args(subp) + inputs = parser.parse_args(list(*argv) if len(argv) else None) # Validate dates