diff --git a/dev/ci/cases/pr/C48_S2SWA_gefs_RT.yaml b/dev/ci/cases/pr/C48_S2SWA_gefs_RT.yaml index 4bd99d07b00..c5bed57239b 100644 --- a/dev/ci/cases/pr/C48_S2SWA_gefs_RT.yaml +++ b/dev/ci/cases/pr/C48_S2SWA_gefs_RT.yaml @@ -9,12 +9,13 @@ experiment: nens: 30 interval: 24 start: cold - gefstype: near-real-time + gefstype: gefs-real-time comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR idate: 2024112500 edate: 2024112500 yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/gefs_defaults_ci.yaml + icsdir: /lfs/h2/emc/ens/noscrub/eric.sinsky/RETRO_ICS # TODO run on supported platforms once the gefs forecast and subsequent tasks can succeed with RETRO ICs # Add this to run the stage ic job 'icsdir: /lfs/h2/emc/ens/noscrub/eric.sinsky/RETRO_ICS' @@ -22,7 +23,6 @@ skip_ci_on_hosts: - gaeac6 - gaeac5 - hercules - - wcoss2 - hera - ursa - orion diff --git a/dev/parm/config/gefs/config.base.j2 b/dev/parm/config/gefs/config.base.j2 index 4e9cf02f71c..56a924fe104 100644 --- a/dev/parm/config/gefs/config.base.j2 +++ b/dev/parm/config/gefs/config.base.j2 @@ -64,7 +64,7 @@ export DO_AWIPS="{{ DO_AWIPS }}" # AWIPS products # Experiment mode (cycled or forecast-only) export MODE="{{ MODE }}" # cycled/forecast-only export DO_TEST_MODE="{{ DO_TEST_MODE }}" # option to change configuration for automated testing -export GEFSTYPE="{{ GEFSTYPE }}" # near-real-time/gefs-offline +export GEFSTYPE="{{ GEFSTYPE }}" # gefs-real-time/gefs-offline #################################################### # DO NOT ADD MACHINE DEPENDENT STUFF BELOW THIS LINE # IF YOU HAVE TO MAKE MACHINE SPECIFIC CHANGES BELOW diff --git a/dev/parm/config/gefs/config.stage_ic.j2 b/dev/parm/config/gefs/config.stage_ic.j2 index 4d6514cf9f4..8693fc170c8 100644 --- a/dev/parm/config/gefs/config.stage_ic.j2 +++ b/dev/parm/config/gefs/config.stage_ic.j2 @@ -11,7 +11,7 @@ export ICSDIR="{{ ICSDIR }}" # User provided ICSDIR; blank if not provided export BASE_IC="{{ BASE_IC }}" # Platform home for staged ICs if [[ ${RUN} == "gefs" ]] ; then - if [[ ${GEFSTYPE} = "near-real-time" ]] ; then + if [[ ${GEFSTYPE} = "gefs-real-time" ]] ; then export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/master_gefs_RT.yaml.j2" elif [[ ${GEFSTYPE} = "gefs-offline" ]] ; then export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/master_gefs.yaml.j2" diff --git a/dev/workflow/applications/applications.py b/dev/workflow/applications/applications.py index e49d053c1e9..3d0ebc6e36c 100644 --- a/dev/workflow/applications/applications.py +++ b/dev/workflow/applications/applications.py @@ -198,7 +198,7 @@ def _get_run_options(self, conf: Configuration) -> Dict[str, Any]: run_options[run]['use_aero_anl'] = run_base.get('USE_AERO_ANL', False) run_options[run]['do_aero_fcst'] = run_base.get('DO_AERO_FCST', False) - if run_base.get('GEFSTYPE', "") == "near-real-time": + if run_base.get('GEFSTYPE', "") == "gefs-real-time": run_options[run]['do_gefs_real_time'] = True elif run_base.get('GEFSTYPE', "") == "gefs-offline": run_options[run]['do_gefs_real_time'] = False diff --git a/dev/workflow/rocoto/gefs_tasks.py b/dev/workflow/rocoto/gefs_tasks.py index 61438252e58..3cd9be020ea 100644 --- a/dev/workflow/rocoto/gefs_tasks.py +++ b/dev/workflow/rocoto/gefs_tasks.py @@ -85,7 +85,7 @@ def fcst(self): if self.app_config.gefstype in ['gefs-offline']: dep_dict = {'type': 'task', 'name': f'{self.run}_stage_ic'} dependencies.append(rocoto.add_dependency(dep_dict)) - elif self.app_config.gefstype in ['near-real-time']: + elif self.app_config.gefstype in ['gefs-real-time']: dep_dict = {'type': 'task', 'name': f'{self.run}_gen_control_ic'} dependencies.append(rocoto.add_dependency(dep_dict)) diff --git a/dev/workflow/setup_expt.py b/dev/workflow/setup_expt.py index 1cd9ad1785c..d668ea415a5 100755 --- a/dev/workflow/setup_expt.py +++ b/dev/workflow/setup_expt.py @@ -244,7 +244,7 @@ def _common_args(parser): parser.add_argument('--interval', help='frequency of forecast (in hours); must be a multiple of 6 or 0 for no forecasts', type=_validate_interval, required=False, default=6) parser.add_argument('--icsdir', help='full path to user initial condition directory', type=str, required=False, default='') - parser.add_argument('--gefstype', help='type of the gefs experiment: near-real-time or gefs-offline', type=str, required=False, default='') + parser.add_argument('--gefstype', help='type of the gefs experiment: gefs-real-time or gefs-offline', type=str, required=False, default='') parser.add_argument('--overwrite', help='overwrite previously created experiment (if it exists)', action='store_true', required=False) return parser diff --git a/parm/stage/aero.yaml.j2 b/parm/stage/aero.yaml.j2 index 422c67a0852..01656f82334 100644 --- a/parm/stage/aero.yaml.j2 +++ b/parm/stage/aero.yaml.j2 @@ -1,17 +1,9 @@ aero: mkdir: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_CHEM_ANALYSIS_MEM = COMOUT_CHEM_ANALYSIS_MEM_list[imem] %} - "{{ COMOUT_CHEM_ANALYSIS_MEM }}" - {% endfor %} + link_req: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_CHEM_ANALYSIS_MEM = COMOUT_CHEM_ANALYSIS_MEM_list[imem] %} - {% for ftype in ["aero_varbc_params.tar"] %} {% if path_exists(ICSDIR ~ "/" ~ COMOUT_CHEM_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z." ~ ftype) %} - - ["{{ ICSDIR }}/{{ COMOUT_CHEM_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_CHEM_ANALYSIS_MEM }}"] + - ["{{ ICSDIR }}/{{ COMOUT_CHEM_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.aero_varbc_params.tar", + "{{ COMOUT_CHEM_ANALYSIS_MEM }}"] {% endif %} - {% endfor %} - {% endfor %} # mem loop diff --git a/parm/stage/analysis.yaml.j2 b/parm/stage/analysis.yaml.j2 index 852dab62ca4..24a23324648 100644 --- a/parm/stage/analysis.yaml.j2 +++ b/parm/stage/analysis.yaml.j2 @@ -1,16 +1,8 @@ -{% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM_list[0] | relpath(ROTDIR)) %} +{% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR)) %} analysis: mkdir: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} - "{{ COMOUT_ATMOS_ANALYSIS_MEM }}" - {% endfor %} link_req: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} - {% if DO_JEDIATMVAR %} {% for itile in range(6) %} @@ -81,7 +73,6 @@ analysis: "{{ COMOUT_ATMOS_ANALYSIS_MEM }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ dest_ftype }}"] {% endif %} {% endfor %} - {% if DO_LAND_IAU %} {% for itile in range(1,7) %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/sfc_inc.tile{{ itile }}.nc", @@ -90,5 +81,4 @@ analysis: {% endif %} {% endif %} - {% endfor %} # mem loop {% endif %} diff --git a/parm/stage/atmosphere_cold.yaml.j2 b/parm/stage/atmosphere_cold.yaml.j2 index c742b28c8d3..e1ac103fbc5 100644 --- a/parm/stage/atmosphere_cold.yaml.j2 +++ b/parm/stage/atmosphere_cold.yaml.j2 @@ -1,18 +1,10 @@ atmosphere_cold: mkdir: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %} - "{{ COMOUT_ATMOS_INPUT_MEM }}" - {% endfor %} # mem loop link_req: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_INPUT_MEM | relpath(ROTDIR) }}/gfs_ctrl.nc", "{{ COMOUT_ATMOS_INPUT_MEM }}"] + - ["{{ ICSDIR }}/{{ COMIN_ATMOS_INPUT_MEM | relpath(ROTDIR) }}/gfs_ctrl.nc", "{{ COMOUT_ATMOS_INPUT_MEM }}"] {% for ftype in ["gfs_data", "sfc_data"] %} {% for ntile in range(1, ntiles + 1) %} - - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_INPUT_MEM | relpath(ROTDIR) }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_INPUT_MEM }}"] + - ["{{ ICSDIR }}/{{ COMIN_ATMOS_INPUT_MEM | relpath(ROTDIR) }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_INPUT_MEM }}"] {% endfor %} # ntile {% endfor %} # ftype - {% endfor %} # mem loop diff --git a/parm/stage/atmosphere_ens_perturbations.yaml.j2 b/parm/stage/atmosphere_ens_perturbations.yaml.j2 index 8f9fc857042..437cbe3cb6c 100644 --- a/parm/stage/atmosphere_ens_perturbations.yaml.j2 +++ b/parm/stage/atmosphere_ens_perturbations.yaml.j2 @@ -1,13 +1,5 @@ atmosphere_ens_perturbation: mkdir: - {% for mem in range(first_mem + 1, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} - "{{ COMOUT_ATMOS_ANALYSIS_MEM }}" - {% endfor %} # mem loop link_req: - {% for mem in range(first_mem + 1, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.fv3_perturbation.nc", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}/{{ RUN }}.t{{ current_cycle_HH }}z.increment.atm.i006.nc"] - {% endfor %} # mem loop diff --git a/parm/stage/atmosphere_nest.yaml.j2 b/parm/stage/atmosphere_nest.yaml.j2 index 3ef8a7c3919..51afcf19e11 100644 --- a/parm/stage/atmosphere_nest.yaml.j2 +++ b/parm/stage/atmosphere_nest.yaml.j2 @@ -2,35 +2,19 @@ atmosphere_nest: {% set ntile = 7 %} {% if EXP_WARM_START == True %} mkdir: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %} - "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}" - {% endfor %} # mem loop link_req: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %} {% for ftype in ["fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}/{{ m_prefix }}.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] {% if DO_CA %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.ca_data.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}/{{ m_prefix }}.ca_data.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] {% endif %} {% endfor %} - {% endfor %} # mem loop {% else %} # cold start mkdir: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %} - "{{ COMOUT_ATMOS_INPUT_MEM }}" - {% endfor %} # mem loop link_req: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %} {% for ftype in ["gfs_data", "sfc_data"] %} - ["{{ COMOUT_ATMOS_INPUT_MEM }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_INPUT_MEM }}/{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] {% endfor %} - {% endfor %} # mem loop {% endif %} diff --git a/parm/stage/atmosphere_warm.yaml.j2 b/parm/stage/atmosphere_warm.yaml.j2 index 501826391ce..02c7954b97b 100644 --- a/parm/stage/atmosphere_warm.yaml.j2 +++ b/parm/stage/atmosphere_warm.yaml.j2 @@ -1,17 +1,8 @@ atmosphere_warm: mkdir: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %} - "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}" - {% set COMOUT_ATMOS_RESTART_MEM = COMOUT_ATMOS_RESTART_MEM_list[imem] %} - "{{ COMOUT_ATMOS_RESTART_MEM }}" - {% endfor %} # mem loop link_req: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %} - {% set COMOUT_ATMOS_RESTART_MEM = COMOUT_ATMOS_RESTART_MEM_list[imem] %} {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) ~ "/" ~ m_prefix ~ ".atm_stoch.res.nc") %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.atm_stoch.res.nc", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"] {% endif %} # path_exists @@ -31,4 +22,3 @@ atmosphere_warm: - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.sfcanl_data.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_MEM }}"] {% endif %} # path_exists {% endfor %} # ntile - {% endfor %} # mem loop diff --git a/parm/stage/atmosphere_warm_RT.yaml.j2 b/parm/stage/atmosphere_warm_RT.yaml.j2 index d432f12a128..bf798147b15 100644 --- a/parm/stage/atmosphere_warm_RT.yaml.j2 +++ b/parm/stage/atmosphere_warm_RT.yaml.j2 @@ -4,11 +4,8 @@ atmosphere_warm_RT: - "{{ COMOUT_ATMOS_ANALYSIS_MEM }}" link_req: # select restart members files (80) specific to the cycle - {% set mid_cyc = ("%02d" | format(previous_cycle_HH | int + half_window)) ~ "0000" %} - {% set restart_mem_path = ICSDIR ~ '/' ~ 'enkfgdas.' ~ previous_cycle_YMD ~ '/' ~ previous_cycle_HH ~ '/' ~ 'mem%03d' | format(gfs_member) ~ '/model/atmos/restart/' ~ previous_cycle_YMD ~ '.' ~ mid_cyc ~ '.' %} + {% set restart_mem_path = ICSDIR ~ '/' ~ 'enkfgdas.' ~ previous_cycle_YMD ~ '/' ~ previous_cycle_HH ~ '/' ~ 'mem%03d' | format(gfs_member) ~ '/model/atmos/restart/' ~ previous_cycle_YMD ~ '.' ~ mid_cyc ~ '0000.' %} {% set increment_mem_path = ICSDIR ~ '/' ~ 'enkfgfs.' ~ current_cycle_YMD ~ '/' ~ current_cycle_HH ~ '/' ~ 'mem%03d' | format(gfs_member) ~ '/analysis/atmos/enkfgfs.t' ~ current_cycle_HH ~ 'z.' %} - {% set restart_destination_path = COMOUT_ATMOS_RESTART_PREV_MEM %} - {% set increment_destination_path = COMOUT_ATMOS_ANALYSIS_MEM %} # Include increment files # TODO: make these filenames the same after RE-staging inputs with EE2-compliant filenames @@ -24,15 +21,15 @@ atmosphere_warm_RT: {% if path_exists(ICSDIR) %} {% set file = restart_mem_path ~ 'coupler.res' %} - - ["{{ file }}", "{{ restart_destination_path }}"] + - ["{{ file }}", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"] {% set file = restart_mem_path ~ 'fv_core.res.nc' %} - - ["{{ file }}", "{{ restart_destination_path }}"] + - ["{{ file }}", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"] {% set ntiles = 6 %} {% for ftype in ["fv_core.res", "fv_srf_wnd.res", "phy_data", "sfc_data"] %} # Include restart tile files (e.g., .tile1 to .tile6) {% for tile in range(1, ntiles + 1) %} {% set file = restart_mem_path ~ ftype ~ '.tile' ~ tile ~ '.nc' %} - - ["{{ file }}", "{{ restart_destination_path }}"] + - ["{{ file }}", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"] {% endfor %} {% endfor %} {% endif %} diff --git a/parm/stage/ice.yaml.j2 b/parm/stage/ice.yaml.j2 index 36fef4d1d8b..5943cd20528 100644 --- a/parm/stage/ice.yaml.j2 +++ b/parm/stage/ice.yaml.j2 @@ -1,35 +1,14 @@ -{% set START_ICE_FROM_ANA = False %} -{% if DO_JEDIOCNVAR == True and RUN == 'gdas' %} - {% set START_ICE_FROM_ANA = True %} -{% endif %} -{% if DO_STARTMEM_FROM_JEDIICE == True and RUN == 'enkfgdas' %} - {% set START_ICE_FROM_ANA = True %} -{% endif %} ice: {% if START_ICE_FROM_ANA == True %} mkdir: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_ICE_ANALYSIS_MEM = COMOUT_ICE_ANALYSIS_MEM_list[imem] %} - "{{ COMOUT_ICE_ANALYSIS_MEM }}" - {% endfor %} # mem loop link_req: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_ICE_ANALYSIS_MEM = COMOUT_ICE_ANALYSIS_MEM_list[imem] %} - - ["{{ ICSDIR }}/{{ COMOUT_ICE_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.cice_model_anl.res.nc", "{{ COMOUT_ICE_ANALYSIS_MEM }}/{{ m_prefix }}.analysis.cice_model.res.nc"] - {% endfor %} # mem loop + {% set COMOUT_ICE_ANALYSIS_MEM = COMOUT_ICE_ANALYSIS_MEM %} + - ["{{ ICSDIR }}/{{ COMOUT_ICE_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.cice_model_anl.res.nc", + "{{ COMOUT_ICE_ANALYSIS_MEM }}/{{ m_prefix }}.analysis.cice_model.res.nc"] {% else %} mkdir: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_ICE_RESTART_PREV_MEM = COMOUT_ICE_RESTART_PREV_MEM_list[imem] %} - "{{ COMOUT_ICE_RESTART_PREV_MEM }}" - {% endfor %} # mem loop link_req: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_ICE_RESTART_PREV_MEM = COMOUT_ICE_RESTART_PREV_MEM_list[imem] %} - ["{{ ICSDIR }}/{{ COMOUT_ICE_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.cice_model.res.nc", "{{ COMOUT_ICE_RESTART_PREV_MEM }}"] - {% endfor %} # mem loop {% endif %} diff --git a/parm/stage/ice_RT.yaml.j2 b/parm/stage/ice_RT.yaml.j2 index e5ceaaea480..e1429c41324 100644 --- a/parm/stage/ice_RT.yaml.j2 +++ b/parm/stage/ice_RT.yaml.j2 @@ -3,10 +3,8 @@ ice_RT: - "{{ COMOUT_ICE_ANALYSIS_MEM }}" link_req: # select restart members files (80) specific to the cycle - {% set mid_cyc = ("%02d" | format(previous_cycle_HH | int + half_window)) ~ "0000" %} - {% set increment_mem_path = ICSDIR ~ '/' ~ 'enkfgfs.' ~ current_cycle_YMD ~ '/' ~ current_cycle_HH ~ '/' ~ 'mem%03d' | format(gfs_member) ~ '/analysis/ice/' ~ previous_cycle_YMD ~ '.' ~ mid_cyc ~ '.analysis.cice_model.res.nc' %} + {% set increment_mem_path = ICSDIR ~ '/' ~ 'enkfgfs.' ~ current_cycle_YMD ~ '/' ~ current_cycle_HH ~ '/' ~ 'mem%03d' | format(gfs_member) ~ '/analysis/ice/' ~ previous_cycle_YMD ~ '.' ~ mid_cyc ~ '0000.analysis.cice_model.res.nc' %} {% set increment_destination_path = COMOUT_ICE_ANALYSIS_MEM %} # Include increment files - {% set file = increment_mem_path %} - - ["{{ file }}", "{{ increment_destination_path }}"] + - ["{{ increment_mem_file }}", "{{ COMOUT_ICE_ANALYSIS_MEM }}"] diff --git a/parm/stage/ice_control_RT.yaml.j2 b/parm/stage/ice_control_RT.yaml.j2 index ab55623f341..050418b13e0 100644 --- a/parm/stage/ice_control_RT.yaml.j2 +++ b/parm/stage/ice_control_RT.yaml.j2 @@ -2,7 +2,6 @@ ice_control_RT: mkdir: - "{{ COMOUT_ICE_ANALYSIS_MEM }}" link_req: - {% set mid_cyc = ("%02d" | format(previous_cycle_HH | int + half_window)) ~ "0000" %} - {% set increment_file = ICSDIR ~ '/' ~ 'gfs.' ~ current_cycle_YMD ~ '/' ~ current_cycle_HH ~ '/analysis/ice/' ~ previous_cycle_YMD ~ '.' ~ mid_cyc ~ '.analysis.cice_model.res.nc' %} + {% set increment_file = ICSDIR ~ '/' ~ 'gfs.' ~ current_cycle_YMD ~ '/' ~ current_cycle_HH ~ '/analysis/ice/' ~ previous_cycle_YMD ~ '.' ~ mid_cyc ~ '0000.analysis.cice_model.res.nc' %} {% set increment_destination_path = COMOUT_ICE_ANALYSIS_MEM %} - ["{{ increment_file }}", "{{ increment_destination_path }}"] diff --git a/parm/stage/master_gcafs.yaml.j2 b/parm/stage/master_gcafs.yaml.j2 index 5faf462a95c..36cc43f4714 100644 --- a/parm/stage/master_gcafs.yaml.j2 +++ b/parm/stage/master_gcafs.yaml.j2 @@ -26,111 +26,14 @@ # - DO_ATM, DO_OCN, DO_ICE, etc. # For a full list see scripts/exglobal_stage_ic.py ################################################################### +# Initial condition to stage - include components based on switches +################################################################### -# Set cycle date variables -# ------------------------ -{% set half_window = assim_freq // 2 %} -{% set half_window_begin = (-half_window | string + "H") | to_timedelta %} -{% set half_window_end = (half_window | string + "H") | to_timedelta %} -{% if DOIAU and MODE == "cycled" %} - {% set model_start_date_current_cycle = current_cycle | add_to_datetime(half_window_begin) %} -{% else %} - {% set model_start_date_current_cycle = current_cycle %} -{% endif %} - -{% set current_cycle_YMD = current_cycle | to_YMD %} -{% set current_cycle_HH = current_cycle | strftime("%H") %} -{% set previous_cycle_YMD = previous_cycle | to_YMD %} -{% set previous_cycle_HH = previous_cycle | strftime("%H") %} -{% set p_prefix = previous_cycle | strftime("%Y%m%d.%H0000") %} -{% set m_prefix = model_start_date_current_cycle | strftime("%Y%m%d.%H0000") %} - -# Determine restart RUN -# --------------------- # always use GDAS for now, fix once we have staged GCDAS/GCAFS ICs +# TODO: move this to stage_ic.py +# once GCDAS/GCAFS ICs are available {% set rRUN = "gdas" %} -# Set first/last mem for loop -# --------------------------- -{% if RUN == "enkfgdas" %} # Ensemble RUN - {% set first_mem = 1 %} - {% set last_mem = NMEM_ENS %} -{% else %} # Deterministic RUN - {% set first_mem = -1 %} - {% set last_mem = -1 %} -{% endif %} - -# Declare to-be-filled lists of member COM directories -# ---------------------------------------------------- -{% set COMIN_ATMOS_INPUT_MEM_list = [] %} -{% set COMOUT_ATMOS_INPUT_MEM_list = [] %} -{% set COMOUT_ATMOS_RESTART_PREV_MEM_list = [] %} -{% set COMOUT_ATMOS_RESTART_MEM_list = [] %} -{% set COMOUT_ATMOS_ANALYSIS_MEM_list = [] %} -{% set COMOUT_ICE_ANALYSIS_MEM_list = [] %} -{% set COMOUT_ICE_RESTART_PREV_MEM_list = [] %} -{% set COMOUT_OCEAN_RESTART_PREV_MEM_list = [] %} -{% set COMOUT_OCEAN_ANALYSIS_MEM_list = [] %} -{% set COMOUT_MED_RESTART_PREV_MEM_list = [] %} -{% set COMOUT_WAVE_RESTART_PREV_MEM_list = [] %} - -# Construct member COM directory lists -# ------------------------------------ -{% for mem in range(first_mem, last_mem + 1) %} - - {% if mem >= 0 %} - {% set mem_char = 'mem%03d' | format(mem) %} - {% else %} - {% set mem_char = '' %} - {% endif %} - - {% set current_cycle_dict_in = ({ '${ROTDIR}':ROTDIR, - '${RUN}':rRUN, - '${YMD}':current_cycle_YMD, - '${HH}':current_cycle_HH, - '${MEMDIR}': mem_char }) %} - {% set current_cycle_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':RUN, - '${YMD}':current_cycle_YMD, - '${HH}':current_cycle_HH, - '${MEMDIR}': mem_char }) %} - {% set previous_cycle_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':rRUN, - '${YMD}':previous_cycle_YMD, - '${HH}':previous_cycle_HH, - '${MEMDIR}': mem_char }) %} - - {% set COMIN_ATMOS_INPUT_MEM = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_cycle_dict_in) %} - {% set COMOUT_ATMOS_INPUT_MEM = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_cycle_dict) %} - {% set COMOUT_ATMOS_RESTART_PREV_MEM = COM_ATMOS_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - {% set COMOUT_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_TMPL | replace_tmpl(current_cycle_dict) %} - {% set COMOUT_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} - {% set COMOUT_ICE_ANALYSIS_MEM = COM_ICE_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} - {% set COMOUT_ICE_RESTART_PREV_MEM = COM_ICE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - {% set COMOUT_OCEAN_RESTART_PREV_MEM = COM_OCEAN_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - {% set COMOUT_OCEAN_ANALYSIS_MEM = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} - {% set COMOUT_MED_RESTART_PREV_MEM = COM_MED_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - {% set COMOUT_WAVE_RESTART_PREV_MEM = COM_WAVE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - - # Append the member COM directories - {% do COMIN_ATMOS_INPUT_MEM_list.append(COMIN_ATMOS_INPUT_MEM)%} - {% do COMOUT_ATMOS_INPUT_MEM_list.append(COMOUT_ATMOS_INPUT_MEM)%} - {% do COMOUT_ATMOS_RESTART_PREV_MEM_list.append(COMOUT_ATMOS_RESTART_PREV_MEM)%} - {% do COMOUT_ATMOS_RESTART_MEM_list.append(COMOUT_ATMOS_RESTART_MEM)%} - {% do COMOUT_ATMOS_ANALYSIS_MEM_list.append(COMOUT_ATMOS_ANALYSIS_MEM)%} - {% do COMOUT_ICE_ANALYSIS_MEM_list.append(COMOUT_ICE_ANALYSIS_MEM)%} - {% do COMOUT_ICE_RESTART_PREV_MEM_list.append(COMOUT_ICE_RESTART_PREV_MEM)%} - {% do COMOUT_OCEAN_RESTART_PREV_MEM_list.append(COMOUT_OCEAN_RESTART_PREV_MEM)%} - {% do COMOUT_OCEAN_ANALYSIS_MEM_list.append(COMOUT_OCEAN_ANALYSIS_MEM)%} - {% do COMOUT_MED_RESTART_PREV_MEM_list.append(COMOUT_MED_RESTART_PREV_MEM)%} - {% do COMOUT_WAVE_RESTART_PREV_MEM_list.append(COMOUT_WAVE_RESTART_PREV_MEM)%} - -{% endfor %} - -################################################################### -# Initial condition to stage - include components based on switches -################################################################### - {% if MODE == "cycled" %} {% filter indent(width=4) %} {% include "analysis.yaml.j2" %} @@ -142,25 +45,9 @@ {% include "atmosphere_warm.yaml.j2" %} {% endfilter %} {% else %} # cold start - atmosphere_cold: - mkdir: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %} - - "{{ COMOUT_ATMOS_INPUT_MEM }}" - {% endfor %} # mem loop - link: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMIN_ATMOS_INPUT_MEM = COMIN_ATMOS_INPUT_MEM_list[imem] %} - {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %} - - ["{{ ICSDIR }}/{{ COMIN_ATMOS_INPUT_MEM | relpath(ROTDIR) }}/gfs_ctrl.nc", "{{ COMOUT_ATMOS_INPUT_MEM }}"] - {% for ftype in ["gfs_data", "sfc_data"] %} - {% for ntile in range(1, ntiles + 1) %} - - ["{{ ICSDIR }}/{{ COMIN_ATMOS_INPUT_MEM | relpath(ROTDIR) }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_INPUT_MEM }}"] - {% endfor %} # ntile - {% endfor %} # ftype - {% endfor %} # mem loop +{% filter indent(width=4) %} +{% include "atmosphere_cold.yaml.j2" %} +{% endfilter %} {% endif %} {% if DO_NEST %} diff --git a/parm/stage/master_gefs.yaml.j2 b/parm/stage/master_gefs.yaml.j2 index 4b848ba30aa..90c33245db0 100644 --- a/parm/stage/master_gefs.yaml.j2 +++ b/parm/stage/master_gefs.yaml.j2 @@ -25,81 +25,6 @@ # - COMOUT_ # - DO_ATM, DO_OCN, DO_ICE, etc. # For a full list see scripts/exglobal_stage_ic.py -################################################################### - -# Set cycle variables -# ------------------------ -{% set half_window = assim_freq // 2 %} -{% set half_window_begin = (-half_window | string + "H") | to_timedelta %} -{% set half_window_end = (half_window | string + "H") | to_timedelta %} -{% if DOIAU and MODE == "cycled" %} - {% set model_start_date_current_cycle = current_cycle | add_to_datetime(half_window_begin) %} -{% else %} - {% set model_start_date_current_cycle = current_cycle %} -{% endif %} - -{% set current_cycle_YMD = current_cycle | to_YMD %} -{% set current_cycle_HH = current_cycle | strftime("%H") %} -{% set previous_cycle_YMD = previous_cycle | to_YMD %} -{% set previous_cycle_HH = previous_cycle | strftime("%H") %} -{% set p_prefix = previous_cycle | strftime("%Y%m%d.%H0000") %} -{% set m_prefix = model_start_date_current_cycle | strftime("%Y%m%d.%H0000") %} - -# Set first/last mem for loop -# --------------------------- -{% set first_mem = 0 %} -{% set last_mem = NMEM_ENS %} - -# Declare to-be-filled lists of member COM directories -# ---------------------------------------------------- -{% set COMOUT_ATMOS_INPUT_MEM_list = [] %} -{% set COMOUT_ATMOS_RESTART_PREV_MEM_list = [] %} -{% set COMOUT_ATMOS_ANALYSIS_MEM_list = [] %} -{% set COMOUT_ICE_ANALYSIS_MEM_list = [] %} -{% set COMOUT_ICE_RESTART_PREV_MEM_list = [] %} -{% set COMOUT_OCEAN_RESTART_PREV_MEM_list = [] %} -{% set COMOUT_OCEAN_ANALYSIS_MEM_list = [] %} -{% set COMOUT_MED_RESTART_PREV_MEM_list = [] %} -{% set COMOUT_WAVE_RESTART_PREV_MEM_list = [] %} - -# Construct member COM directory lists -# ------------------------------------ -{% for mem in range(first_mem, last_mem + 1) %} - - {% set current_cycle_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':RUN, - '${YMD}':current_cycle_YMD, - '${HH}':current_cycle_HH, - '${MEMDIR}': 'mem%03d' | format(mem) }) %} - {% set previous_cycle_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':RUN, - '${YMD}':previous_cycle_YMD, - '${HH}':previous_cycle_HH, - '${MEMDIR}': 'mem%03d' | format(mem) }) %} - - {% set COMOUT_ATMOS_INPUT_MEM = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_cycle_dict) %} - {% set COMOUT_ATMOS_RESTART_PREV_MEM = COM_ATMOS_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - {% set COMOUT_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} - {% set COMOUT_ICE_ANALYSIS_MEM = COM_ICE_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} - {% set COMOUT_ICE_RESTART_PREV_MEM = COM_ICE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - {% set COMOUT_OCEAN_RESTART_PREV_MEM = COM_OCEAN_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - {% set COMOUT_OCEAN_ANALYSIS_MEM = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} - {% set COMOUT_MED_RESTART_PREV_MEM = COM_MED_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - {% set COMOUT_WAVE_RESTART_PREV_MEM = COM_WAVE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - - # Append the member COM directories - {% do COMOUT_ATMOS_INPUT_MEM_list.append(COMOUT_ATMOS_INPUT_MEM)%} - {% do COMOUT_ATMOS_RESTART_PREV_MEM_list.append(COMOUT_ATMOS_RESTART_PREV_MEM)%} - {% do COMOUT_ATMOS_ANALYSIS_MEM_list.append(COMOUT_ATMOS_ANALYSIS_MEM)%} - {% do COMOUT_ICE_ANALYSIS_MEM_list.append(COMOUT_ICE_ANALYSIS_MEM)%} - {% do COMOUT_ICE_RESTART_PREV_MEM_list.append(COMOUT_ICE_RESTART_PREV_MEM)%} - {% do COMOUT_OCEAN_RESTART_PREV_MEM_list.append(COMOUT_OCEAN_RESTART_PREV_MEM)%} - {% do COMOUT_OCEAN_ANALYSIS_MEM_list.append(COMOUT_OCEAN_ANALYSIS_MEM)%} - {% do COMOUT_MED_RESTART_PREV_MEM_list.append(COMOUT_MED_RESTART_PREV_MEM)%} - {% do COMOUT_WAVE_RESTART_PREV_MEM_list.append(COMOUT_WAVE_RESTART_PREV_MEM)%} - -{% endfor %} - ################################################################### # Initial condition to stage - include components based on switches ################################################################### diff --git a/parm/stage/master_gefs_RT.yaml.j2 b/parm/stage/master_gefs_RT.yaml.j2 index 1531422825e..ef42231542a 100644 --- a/parm/stage/master_gefs_RT.yaml.j2 +++ b/parm/stage/master_gefs_RT.yaml.j2 @@ -25,63 +25,7 @@ # - DO_ATM, DO_OCN, DO_ICE, etc. # For a full list see scripts/exglobal_stage_ic.py ################################################################### -# Set cycle variables -# ------------------------ -{% set half_window = assim_freq // 2 %} -{% set half_window_begin = (-half_window | string + "H") | to_timedelta %} -{% set half_window_end = (half_window | string + "H") | to_timedelta %} -{% set model_start_date_current_cycle = current_cycle %} -{% set current_cycle_YMD = current_cycle | to_YMD %} -{% set current_cycle_HH = current_cycle | strftime("%H") %} -{% set previous_cycle_YMD = previous_cycle | to_YMD %} -{% set previous_cycle_HH = previous_cycle | strftime("%H") %} -{% set p_prefix = previous_cycle | strftime("%Y%m%d.%H0000") %} -{% set m_prefix = model_start_date_current_cycle | strftime("%Y%m%d.%H0000") %} - -################################################################### -# TODO -# this script should run for 30 members (+ control member) -# logic and variables in this script will move to stage_ic.py -# until then this will run only for ENSMEM = 0 (control) -{% set ENSMEM = 0 %} -################################################################### -# Construct member COM directory lists -# ------------------------------------ -{% set current_cycle_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':RUN, - '${YMD}':current_cycle_YMD, - '${HH}':current_cycle_HH, - '${MEMDIR}': 'mem%03d' | format(ENSMEM) }) %} -{% set previous_cycle_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':RUN, - '${YMD}':previous_cycle_YMD, - '${HH}':previous_cycle_HH, - '${MEMDIR}': 'mem%03d' | format(ENSMEM) }) %} - - -{% set COMOUT_ATMOS_INPUT_MEM = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_cycle_dict) %} -{% set COMOUT_ATMOS_RESTART_PREV_MEM = COM_ATMOS_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} -{% set COMOUT_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} -{% set COMOUT_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_TMPL | replace_tmpl(previous_cycle_dict) %} -{% set COMOUT_ICE_ANALYSIS_MEM = COM_ICE_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} -{% set COMOUT_ICE_RESTART_PREV_MEM = COM_ICE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} -{% set COMOUT_OCEAN_RESTART_PREV_MEM = COM_OCEAN_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} -{% set COMOUT_OCEAN_ANALYSIS_MEM = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} -{% set COMOUT_MED_RESTART_PREV_MEM = COM_MED_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} -{% set COMOUT_WAVE_RESTART_PREV_MEM = COM_WAVE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} -################################################################### -# Initial condition to stage - include components based on switches -################################################################### -{% set m_index = (current_cycle_HH | int) // 6 %} - -{% if ENSMEM != 0 %} -# GFS members (80) -{% set cyc_ranges = [(range(1, 31) | list), (range(21, 51) | list), (range(41, 71) | list), (range(61, 81) | list) + (range(1, 11) | list)] %} -# select the relevant member for each GEFS member from GFS outputs -{% set gfs_member = cyc_ranges[m_index][(ENSMEM - 1)] %} -{% endif %} - -{% if ENSMEM == 0 %} +{% if memdir == 0 %} {% include "atmosphere_cold_RT.yaml.j2" %} {% include "ocean_control_RT.yaml.j2" %} {% include "ice_control_RT.yaml.j2" %} diff --git a/parm/stage/master_gfs.yaml.j2 b/parm/stage/master_gfs.yaml.j2 index 35fa21cd8af..c14e6fad4ae 100644 --- a/parm/stage/master_gfs.yaml.j2 +++ b/parm/stage/master_gfs.yaml.j2 @@ -25,105 +25,6 @@ # - COMOUT_ # - DO_ATM, DO_OCN, DO_ICE, etc. # For a full list see scripts/exglobal_stage_ic.py -################################################################### - -# Set cycle date variables -# ------------------------ -{% set half_window = assim_freq // 2 %} -{% set half_window_begin = (-half_window | string + "H") | to_timedelta %} -{% set half_window_end = (half_window | string + "H") | to_timedelta %} -{% if DOIAU and MODE == "cycled" %} - {% set model_start_date_current_cycle = current_cycle | add_to_datetime(half_window_begin) %} -{% else %} - {% set model_start_date_current_cycle = current_cycle %} -{% endif %} - -{% set current_cycle_YMD = current_cycle | to_YMD %} -{% set current_cycle_HH = current_cycle | strftime("%H") %} -{% set previous_cycle_YMD = previous_cycle | to_YMD %} -{% set previous_cycle_HH = previous_cycle | strftime("%H") %} -{% set p_prefix = previous_cycle | strftime("%Y%m%d.%H0000") %} -{% set m_prefix = model_start_date_current_cycle | strftime("%Y%m%d.%H0000") %} - -# Determine restart RUN -# --------------------- -{% set rRUN = RUN %} -{% if RUN == "gfs" %} - {% set rRUN = "gdas" %} -{% endif %} - -# Set first/last mem for loop -# --------------------------- -{% if RUN == "enkfgdas" %} # Ensemble RUN - {% set first_mem = 1 %} - {% set last_mem = NMEM_ENS %} -{% else %} # Deterministic RUN - {% set first_mem = -1 %} - {% set last_mem = -1 %} -{% endif %} - -# Declare to-be-filled lists of member COM directories -# ---------------------------------------------------- -{% set COMOUT_ATMOS_INPUT_MEM_list = [] %} -{% set COMOUT_ATMOS_RESTART_PREV_MEM_list = [] %} -{% set COMOUT_ATMOS_RESTART_MEM_list = [] %} -{% set COMOUT_ATMOS_ANALYSIS_MEM_list = [] %} -{% set COMOUT_ICE_ANALYSIS_MEM_list = [] %} -{% set COMOUT_ICE_RESTART_PREV_MEM_list = [] %} -{% set COMOUT_OCEAN_RESTART_PREV_MEM_list = [] %} -{% set COMOUT_OCEAN_ANALYSIS_MEM_list = [] %} -{% set COMOUT_MED_RESTART_PREV_MEM_list = [] %} -{% set COMOUT_WAVE_RESTART_PREV_MEM_list = [] %} -{% set COMOUT_CHEM_ANALYSIS_MEM_list = [] %} - -# Construct member COM directory lists -# ------------------------------------ -{% for mem in range(first_mem, last_mem + 1) %} - - {% if mem >= 0 %} - {% set mem_char = 'mem%03d' | format(mem) %} - {% else %} - {% set mem_char = '' %} - {% endif %} - - {% set current_cycle_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':RUN, - '${YMD}':current_cycle_YMD, - '${HH}':current_cycle_HH, - '${MEMDIR}': mem_char }) %} - {% set previous_cycle_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':rRUN, - '${YMD}':previous_cycle_YMD, - '${HH}':previous_cycle_HH, - '${MEMDIR}': mem_char }) %} - - {% set COMOUT_ATMOS_INPUT_MEM = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_cycle_dict) %} - {% set COMOUT_ATMOS_RESTART_PREV_MEM = COM_ATMOS_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - {% set COMOUT_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_TMPL | replace_tmpl(current_cycle_dict) %} - {% set COMOUT_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} - {% set COMOUT_ICE_ANALYSIS_MEM = COM_ICE_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} - {% set COMOUT_ICE_RESTART_PREV_MEM = COM_ICE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - {% set COMOUT_OCEAN_RESTART_PREV_MEM = COM_OCEAN_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - {% set COMOUT_OCEAN_ANALYSIS_MEM = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} - {% set COMOUT_MED_RESTART_PREV_MEM = COM_MED_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - {% set COMOUT_WAVE_RESTART_PREV_MEM = COM_WAVE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - {% set COMOUT_CHEM_ANALYSIS_MEM = COM_CHEM_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} - - # Append the member COM directories - {% do COMOUT_ATMOS_INPUT_MEM_list.append(COMOUT_ATMOS_INPUT_MEM)%} - {% do COMOUT_ATMOS_RESTART_PREV_MEM_list.append(COMOUT_ATMOS_RESTART_PREV_MEM)%} - {% do COMOUT_ATMOS_RESTART_MEM_list.append(COMOUT_ATMOS_RESTART_MEM)%} - {% do COMOUT_ATMOS_ANALYSIS_MEM_list.append(COMOUT_ATMOS_ANALYSIS_MEM)%} - {% do COMOUT_ICE_ANALYSIS_MEM_list.append(COMOUT_ICE_ANALYSIS_MEM)%} - {% do COMOUT_ICE_RESTART_PREV_MEM_list.append(COMOUT_ICE_RESTART_PREV_MEM)%} - {% do COMOUT_OCEAN_RESTART_PREV_MEM_list.append(COMOUT_OCEAN_RESTART_PREV_MEM)%} - {% do COMOUT_OCEAN_ANALYSIS_MEM_list.append(COMOUT_OCEAN_ANALYSIS_MEM)%} - {% do COMOUT_MED_RESTART_PREV_MEM_list.append(COMOUT_MED_RESTART_PREV_MEM)%} - {% do COMOUT_WAVE_RESTART_PREV_MEM_list.append(COMOUT_WAVE_RESTART_PREV_MEM)%} - {% do COMOUT_CHEM_ANALYSIS_MEM_list.append(COMOUT_CHEM_ANALYSIS_MEM)%} - -{% endfor %} - ################################################################### # Initial condition to stage - include components based on switches ################################################################### diff --git a/parm/stage/ocean.yaml.j2 b/parm/stage/ocean.yaml.j2 index 135d5056bc4..ba28eaa1573 100644 --- a/parm/stage/ocean.yaml.j2 +++ b/parm/stage/ocean.yaml.j2 @@ -1,18 +1,10 @@ ocean: mkdir: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_OCEAN_RESTART_PREV_MEM = COMOUT_OCEAN_RESTART_PREV_MEM_list[imem] %} - "{{ COMOUT_OCEAN_RESTART_PREV_MEM }}" - {% endfor %} # mem loop link_req: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_OCEAN_RESTART_PREV_MEM = COMOUT_OCEAN_RESTART_PREV_MEM_list[imem] %} - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.MOM.res.nc", "{{ COMOUT_OCEAN_RESTART_PREV_MEM }}"] {% if OCNRES == "025" %} {% for nn in range(1, 4) %} - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.MOM.res_{{ nn }}.nc", "{{ COMOUT_OCEAN_RESTART_PREV_MEM }}"] {% endfor %} {% endif %} - {% endfor %} # mem loop diff --git a/parm/stage/ocean_RT.yaml.j2 b/parm/stage/ocean_RT.yaml.j2 index 3957388eba4..c5f82ddc7e1 100644 --- a/parm/stage/ocean_RT.yaml.j2 +++ b/parm/stage/ocean_RT.yaml.j2 @@ -4,20 +4,17 @@ ocean_RT: - "{{ COMOUT_OCEAN_ANALYSIS_MEM }}" link_req: # select restart members files (80) specific to the cycle - {% set mid_cyc = ("%02d" | format(previous_cycle_HH | int + half_window)) ~ "0000" %} - {% set restart_mem_path = ICSDIR ~ '/' ~ 'enkfgdas.' ~ previous_cycle_YMD ~ '/' ~ previous_cycle_HH ~ '/' ~ 'mem%03d' | format(gfs_member) ~ '/model/ocean/restart/' ~ previous_cycle_YMD ~ '.' ~ mid_cyc ~ '.' %} + {% set restart_mem_path = ICSDIR ~ '/' ~ 'enkfgdas.' ~ previous_cycle_YMD ~ '/' ~ previous_cycle_HH ~ '/' ~ 'mem%03d' | format(gfs_member) ~ '/model/ocean/restart/' ~ previous_cycle_YMD ~ '.' ~ mid_cyc ~ '0000.' %} {% set increment_mem_path = ICSDIR ~ '/' ~ 'enkfgfs.' ~ current_cycle_YMD ~ '/' ~ current_cycle_HH ~ '/' ~ 'mem%03d' | format(gfs_member) ~ '/analysis/ocean/enkfgfs.t' ~ current_cycle_HH ~ 'z.ocninc.nc' %} - {% set restart_destination_path = COMOUT_OCEAN_RESTART_PREV_MEM %} - {% set increment_destination_path = COMOUT_OCEAN_ANALYSIS_MEM %} # Include increment files {% set file = increment_mem_path %} - - ["{{ file }}", "{{ increment_destination_path }}"] + - ["{{ file }}", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}"] {% if path_exists(ICSDIR) %} {% for ftype in ["MOM.res.nc", "MOM.res_1.nc", "MOM.res_2.nc", "MOM.res_3.nc"] %} # Include base file for restart files {% set file = restart_mem_path ~ ftype %} - - ["{{ file }}", "{{ restart_destination_path }}"] + - ["{{ file }}", "{{ COMOUT_OCEAN_RESTART_PREV_MEM }}"] {% endfor %} {% endif %} diff --git a/parm/stage/ocean_control_RT.yaml.j2 b/parm/stage/ocean_control_RT.yaml.j2 index 249c8940650..6a1cae34456 100644 --- a/parm/stage/ocean_control_RT.yaml.j2 +++ b/parm/stage/ocean_control_RT.yaml.j2 @@ -3,16 +3,13 @@ ocean_control_RT: - "{{ COMOUT_OCEAN_RESTART_PREV_MEM }}" - "{{ COMOUT_OCEAN_ANALYSIS_MEM }}" link_req: - {% set mid_cyc = ("%02d" | format(previous_cycle_HH | int + half_window)) ~ "0000" %} - {% set restart_path = ICSDIR ~ '/' ~ 'gdas.' ~ previous_cycle_YMD ~ '/' ~ previous_cycle_HH ~ '/model/ocean/restart/' ~ previous_cycle_YMD ~ '.' ~ mid_cyc ~ '.' %} + {% set restart_path = ICSDIR ~ '/' ~ 'gdas.' ~ previous_cycle_YMD ~ '/' ~ previous_cycle_HH ~ '/model/ocean/restart/' ~ previous_cycle_YMD ~ '.' ~ mid_cyc ~ '0000.' %} {% set increment_file = ICSDIR ~ '/' ~ 'gfs.' ~ current_cycle_YMD ~ '/' ~ current_cycle_HH ~ '/analysis/ocean/gfs.t' ~ current_cycle_HH ~ 'z.ocninc.nc' %} - {% set restart_destination_path = COMOUT_OCEAN_RESTART_PREV_MEM %} - {% set increment_destination_path = COMOUT_OCEAN_ANALYSIS_MEM %} # Restart increment files {% for ftype in ["MOM.res.nc", "MOM.res_1.nc", "MOM.res_2.nc", "MOM.res_3.nc"] %} {% set file = restart_path ~ ftype %} - - ["{{ file }}", "{{ restart_destination_path }}"] + - ["{{ file }}", "{{ COMOUT_OCEAN_RESTART_PREV_MEM }}"] {% endfor %} # Include increment files - - ["{{ increment_file }}", "{{ increment_destination_path }}"] + - ["{{ increment_file }}", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}"] diff --git a/parm/stage/ocean_ens_perturbations.yaml.j2 b/parm/stage/ocean_ens_perturbations.yaml.j2 index d551d89694f..a7e6076716e 100644 --- a/parm/stage/ocean_ens_perturbations.yaml.j2 +++ b/parm/stage/ocean_ens_perturbations.yaml.j2 @@ -1,14 +1,6 @@ ocean_ens_perturbation: mkdir: - {% for mem in range(first_mem + 1, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} - "{{ COMOUT_OCEAN_ANALYSIS_MEM }}" - {% endfor %} # mem loop link_req: - {% for mem in range(first_mem + 1, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}/{{ RUN }}.t{{ current_cycle_HH }}z.mom6_increment.i006.nc"] - {% endfor %} # mem loop diff --git a/parm/stage/ocean_mediator.yaml.j2 b/parm/stage/ocean_mediator.yaml.j2 index e72f973ac54..57889197e3b 100644 --- a/parm/stage/ocean_mediator.yaml.j2 +++ b/parm/stage/ocean_mediator.yaml.j2 @@ -1,15 +1,7 @@ -{% if path_exists(ICSDIR ~ "/" ~ COMOUT_MED_RESTART_PREV_MEM_list[0] | relpath(ROTDIR) ~ "/" ~ m_prefix ~ ".ufs.cpld.cpl.r.nc") %} +{% if path_exists(ICSDIR ~ "/" ~ COMOUT_MED_RESTART_PREV_MEM | relpath(ROTDIR) ~ "/" ~ m_prefix ~ ".ufs.cpld.cpl.r.nc") %} ocean_mediator: mkdir: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_MED_RESTART_PREV_MEM = COMOUT_MED_RESTART_PREV_MEM_list[imem] %} - "{{ COMOUT_MED_RESTART_PREV_MEM }}" - {% endfor %} # mem loop link_req: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_MED_RESTART_PREV_MEM = COMOUT_MED_RESTART_PREV_MEM_list[imem] %} - ["{{ ICSDIR }}/{{ COMOUT_MED_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.ufs.cpld.cpl.r.nc", "{{ COMOUT_MED_RESTART_PREV_MEM }}"] - {% endfor %} # mem loop {% endif %} # path exists diff --git a/parm/stage/ocean_rerun.yaml.j2 b/parm/stage/ocean_rerun.yaml.j2 index 3766f9fa476..d37896a425a 100644 --- a/parm/stage/ocean_rerun.yaml.j2 +++ b/parm/stage/ocean_rerun.yaml.j2 @@ -1,14 +1,6 @@ ocean_rerun: mkdir: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} - "{{ COMOUT_OCEAN_ANALYSIS_MEM }}" - {% endfor %} # mem loop link_req: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.ocninc.nc", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}/{{ RUN }}.t{{ current_cycle_HH }}z.mom6_increment.i006.nc"] - {% endfor %} # mem loop diff --git a/parm/stage/wave.yaml.j2 b/parm/stage/wave.yaml.j2 index f7d42a9a9cf..613b31a72d7 100644 --- a/parm/stage/wave.yaml.j2 +++ b/parm/stage/wave.yaml.j2 @@ -1,20 +1,12 @@ wave: mkdir: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_WAVE_RESTART_PREV_MEM = COMOUT_WAVE_RESTART_PREV_MEM_list[imem] %} - "{{ COMOUT_WAVE_RESTART_PREV_MEM }}" - {% endfor %} # mem loop link_req: - {% for mem in range(first_mem, last_mem + 1) %} - {% set imem = mem - first_mem %} - {% set COMOUT_WAVE_RESTART_PREV_MEM = COMOUT_WAVE_RESTART_PREV_MEM_list[imem] %} {% set ww3_file = ICSDIR ~ "/" ~ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) ~ "/" ~ m_prefix ~ ".restart.ww3" %} - {% if path_exists(ww3_file ~ ".nc") %} + {% if path_exists(ww3_file ~ ".nc") %} - ["{{ ww3_file }}.nc" , "{{ COMOUT_WAVE_RESTART_PREV_MEM }}/{{ m_prefix }}.restart.ww3.nc"] {% else %} {% if path_exists(ww3_file) %} - ["{{ ww3_file }}" , "{{ COMOUT_WAVE_RESTART_PREV_MEM }}/{{ m_prefix }}.restart.ww3"] {% endif %} {% endif %} - {% endfor %} # mem loop diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index e0af18b42fa..5fc20801e85 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -3,7 +3,7 @@ import os from pygfs.task.stage_ic import Stage -from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, logit +from wxflow import Logger, cast_strdict_as_dtypedict, logit # Initialize root logger logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True) @@ -12,36 +12,16 @@ @logit(logger) def main(): - config = cast_strdict_as_dtypedict(os.environ) - # Instantiate the Stage object - stage = Stage(config) - - # Pull out all the configuration keys needed to run stage job - keys = ['RUN', 'MODE', 'EXP_WARM_START', 'NMEM_ENS', - 'assim_freq', 'current_cycle', 'previous_cycle', - 'ROTDIR', 'ICSDIR', 'STAGE_IC_YAML_TMPL', 'DO_JEDIATMVAR', - 'OCNRES', 'waveGRD', 'ntiles', 'DOIAU', - 'DO_JEDIOCNVAR', 'DO_STARTMEM_FROM_JEDIICE', - 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST', 'DO_CA', 'DO_AERO_ANL', - 'USE_ATM_ENS_PERTURB_FILES', 'USE_OCN_ENS_PERTURB_FILES', 'DO_GSISOILDA', 'DO_LAND_IAU'] - - stage_dict = AttrDict() - for key in keys: - # Make sure OCNRES is three digits - if key == "OCNRES": - stage.task_config.OCNRES = f"{stage.task_config.OCNRES :03d}" - stage_dict[key] = stage.task_config[key] - - # Also import all COM* directory and template variables - for key in stage.task_config.keys(): - if key.startswith("COM"): - stage_dict[key] = stage.task_config[key] - if "ENSMEM" in stage.task_config: - stage_dict["ENSMEM"] = stage.task_config["ENSMEM"] - - # Stage ICs - stage.execute_stage(stage_dict) + stage = Stage(cast_strdict_as_dtypedict(os.environ)) + + # Calculate member configuration + stage.calculate_member() + + # Loop through members and stage ICs for each + for member in range(stage.task_config.first_mem, stage.task_config.last_mem + 1): + logger.info(f"Staging initial conditions for member: {member}") + stage.execute_stage(stage.task_config, member=member) if __name__ == '__main__': diff --git a/ush/python/pygfs/task/stage_ic.py b/ush/python/pygfs/task/stage_ic.py index d1e95b8d120..715538f459d 100644 --- a/ush/python/pygfs/task/stage_ic.py +++ b/ush/python/pygfs/task/stage_ic.py @@ -1,10 +1,79 @@ #!/usr/bin/env python3 +""" +Stage Initial Conditions (IC) Task +Overview +-------- +This module constructs cycle and member-specific COM directory path variables +required for initial conditions for the Stage IC task. + +High-Level Responsibilities +--------------------------- +1. Create cycle variables for: + - GCAFS (cycled or forecast-only) + - GFS (deterministic and enkfgdas variant) + - GEFS (ensemble: real-time and offline modes) + + 1.1 Provide a method (calculate_general_cycle_variables) that computes + variables common to all supported applications (time windows, cycle + date strings, template substitution dictionaries, etc.). + + 1.2 Provide a method (calculate_member) that derives + application / RUN specific attributes (member indexing rules, GEFSTYPE + handling etc.). + +2. Provide separate methods for populating member-level COM path variables: + - calculate_member_com_paths_gfs + - calculate_member_com_paths_gefs_offline + - calculate_member_com_paths_gefs_rt + - calculate_member_com_paths_gcafs + +Key Methods +----------- +calculate_member(): + Establishes RUN-dependent settings (member ranges, GEFSTYPE logic, replay / + IAU offsets, and rRUN mapping for coupled cases). + +calculate_general_cycle_variables(): + Builds on case-specific variables and computes: + - Half-window assimilation times + - Current / previous cycle formatted strings (YMD, HH) + - Model start date logic (IAU vs replay vs standard start) + - Template substitution dictionaries for current and previous cycles + +calculate_member_com_paths_*(): + For a given member index, formats and injects member-specific COM directory + paths derived from template variables defined in the master configuration. + +execute_stage(stage_dict): + Renders a YAML (Jinja2) template describing required files, then synchronizes + (copies/links) those files into the ROTDIR. + +Extensibility Notes +------------------- +To add a new application: + 1. Extend calculate_member() for RUN-specific member logic. + 2. Implement a new calculate_member_com_paths_() variant if COM path + semantics differ from existing cases. + 3. Add dispatch logic inside calculate_stage_vars(). + +Performance Considerations +-------------------------- +The staging loop processes each member sequentially. If significant scaling +issues arise for very large ensembles, a future enhancement could introduce +parallelization or batched FileHandler operations. + +Logging +------- +All public operational methods are decorated with @logit(logger, ...), +providing entry/exit logging. + +""" import os from logging import getLogger -from typing import Any, Dict - -from wxflow import FileHandler, Task, logit, parse_j2yaml +from typing import Any, Dict, Optional, Tuple +from datetime import timedelta +from wxflow import FileHandler, Task, logit, parse_j2yaml, AttrDict logger = getLogger(__name__.split('.')[-1]) @@ -30,14 +99,79 @@ def __init__(self, config: Dict[str, Any]) -> None: """ super().__init__(config) + # assign rRUN to RUN for gfs cases + self.task_config.rRUN = self.task_config.RUN + # case specific rRUN + if self.task_config.RUN in ['gfs', 'gcafs']: + self.task_config.rRUN = "gdas" + else: + # RUN not gfs leave rRUN unchanged and continue + logger.debug("No rRUN remap applied: RUN='%s' (rRUN stays '%s')", self.task_config.RUN, self.task_config.rRUN) + + if "OCNRES" in self.task_config: + self.task_config.OCNRES = f"{int(self.task_config.OCNRES):03d}" + + # START_ICE_FROM_ANA logic (only if DO_ICE is True) + if self.task_config.get("DO_ICE", False): + self.task_config.START_ICE_FROM_ANA = False + if self.task_config.get("DO_JEDIOCNVAR", False) and self.task_config.RUN == "gdas": + self.task_config.START_ICE_FROM_ANA = True + if self.task_config.get("DO_STARTMEM_FROM_JEDIICE", False) and self.task_config.RUN == "enkfgdas": + self.task_config.START_ICE_FROM_ANA = True + + # Calculate half window variables + self.task_config.half_window = self.task_config.assim_freq // 2 + + # Calculate current cycle variables + if self.task_config.current_cycle: + self.task_config.current_cycle_HH = self.task_config.current_cycle.strftime("%H") + self.task_config.current_cycle_YMD = self.task_config.current_cycle.strftime("%Y%m%d") + if self.task_config.DOIAU and self.task_config.MODE == "cycled": + self.task_config.model_start_date_current_cycle = self.task_config.current_cycle + timedelta(hours=-self.task_config.half_window) + else: + if 'REPLAY_ICS' in self.task_config and self.task_config['REPLAY_ICS']: + self.task_config.model_start_date_current_cycle = self.task_config.current_cycle + timedelta(hours=self.task_config.half_window) + else: + self.task_config.model_start_date_current_cycle = self.task_config.current_cycle + + # Calculate YMD and HH formats + self.task_config.m_prefix = self.task_config.model_start_date_current_cycle.strftime("%Y%m%d.%H0000") + + # Calculate previous cycle variables + if self.task_config.previous_cycle: + self.task_config.m_index = self.task_config.current_cycle.hour // self.task_config.assim_freq + self.task_config.p_prefix = self.task_config.previous_cycle.strftime("%Y%m%d.%H0000") + self.task_config.previous_cycle_HH = self.task_config.previous_cycle.strftime("%H") + self.task_config.previous_cycle_YMD = self.task_config.previous_cycle.strftime("%Y%m%d") + self.task_config.mid_cyc = int(self.task_config.previous_cycle_HH) + int(self.task_config.half_window) + # Define cycle directories to update com paths + self.task_config.current_cycle_dict = { + "${ROTDIR}": self.task_config.ROTDIR, + "${RUN}": self.task_config.RUN, + "${YMD}": self.task_config.current_cycle.strftime("%Y%m%d"), + "${HH}": self.task_config.current_cycle.strftime("%H"), + } + self.task_config.previous_cycle_dict = { + "${ROTDIR}": self.task_config.ROTDIR, + "${RUN}": self.task_config.RUN, + "${YMD}": self.task_config.previous_cycle.strftime("%Y%m%d"), + "${HH}": self.task_config.previous_cycle.strftime("%H"), + } + @logit(logger) - def execute_stage(self, stage_dict: Dict[str, Any]) -> None: + def execute_stage(self, stage_dict: AttrDict, member: Optional[int] = None) -> None: """Perform local staging of initial condition files. + This method calculates member-specific COM paths if member is provided, + then performs file staging based on the YAML template configuration. + Parameters ---------- - stage_dict : Dict[str, Any] - Configuration dictionary + stage_dict : AttrDict + Configuration dictionary with attribute-style access + member : int, optional + Member directory number. If provided, calculates member-specific COM paths. + If None, skips member COM path calculation (for deterministic runs). Returns ------- @@ -47,6 +181,26 @@ def execute_stage(self, stage_dict: Dict[str, Any]) -> None: if not os.path.isdir(stage_dict.ROTDIR): raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({stage_dict.ROTDIR}) does not exist!") + # Calculate member-specific COM paths if member is provided + if member is not None: + self.task_config.member = member + run = self.task_config.get('RUN', None) + + if run == 'gefs': + gefstype = self.task_config.get('GEFSTYPE', None) + if gefstype == 'gefs-real-time': + self.calculate_member_com_paths_gefs_rt(member) + elif gefstype == 'gefs-offline': + self.calculate_member_com_paths_gefs_offline(member) + else: + raise ValueError(f"Invalid GEFSTYPE '{gefstype}' for RUN 'gefs'.") + elif run in ('gcafs', 'enkfgdas', 'gcdas', 'gdas'): + self.calculate_member_com_paths_gcafs(member) + elif run == 'gfs': + self.calculate_member_com_paths_gfs(member) + else: + raise ValueError(f"Unknown RUN type: {run}") + # Add the os.path.exists function to the dict for yaml parsing stage_dict['path_exists'] = os.path.exists @@ -56,3 +210,253 @@ def execute_stage(self, stage_dict: Dict[str, Any]) -> None: # stage files to ROTDIR for key in stage_set.keys(): FileHandler(stage_set[key]).sync() + + @logit(logger) + def calculate_member(self) -> None: + """ + Calculate member for master YAML templates + - gfs + - gefs + - gcafs + - enkfgdas + - gdas + + Returns + ------- + None + Updates the task_config with member-specific variables for staging. + """ + # Assign last_mem and first_mem to run members + self.task_config.last_mem = self.task_config.NMEM_ENS + if self.task_config.RUN in ['enkfgdas']: + self.task_config.first_mem = 1 + elif self.task_config.RUN in ['gefs']: # GEFS Ensemble RUN (both regular and RT) + self.task_config.first_mem = 0 + if self.task_config.GEFSTYPE == "gefs-offline": + pass + elif self.task_config.GEFSTYPE == "gefs-real-time": + # select the relevant member for each GEFS member from GFS outputs + self.task_config.cyc_ranges = [list(range(1, 31)), list(range(21, 51)), + list(range(41, 71)), list(range(61, 81)) + list(range(1, 11))] + else: + # Error handling for unknown GEFSTYPE + valid_types = ['gefs-offline', 'gefs-real-time'] + raise ValueError(f"Invalid GEFSTYPE '{self.task_config.GEFSTYPE}' for RUN '{self.task_config.RUN}'. " + f"Valid options are: {valid_types}") + else: # Deterministic RUN (GFS and GCAFS) + self.task_config.first_mem = -1 + self.task_config.last_mem = -1 + + def _paths_from_templates(self, com_path_tuples: Tuple[Tuple[str, str, Dict[str, Any]], ...]) -> Dict[str, str]: + """Generate COM paths from template configurations. + + Parameters + ---------- + com_path_tuples : Tuple[Tuple[str, str, Dict[str, Any]], ...] + Tuple of tuples, each containing: + - COM path key (e.g., 'COMOUT_MED_RESTART_PREV_MEM') + - Template key from task_config (e.g., 'COM_MED_RESTART_TMPL') + - Variable substitution dictionary (e.g., previous_cycle_mem_dict) + + Returns + ------- + Dict[str, str] + Dictionary mapping COM path keys to resolved file paths + + Examples + -------- + >>> com_paths = self._paths_from_templates(( + ... ('COMOUT_ATMOS_RESTART', 'COM_ATMOS_RESTART_TMPL', current_cycle_dict), + ... ('COMIN_ATMOS_RESTART_PREV', 'COM_ATMOS_RESTART_TMPL', previous_cycle_dict), + ... )) + >>> print(com_paths['COMOUT_ATMOS_RESTART']) + """ + path_dict = {} + for com_key, template_key, substitution_dict in com_path_tuples: + template_str = self.task_config.get(template_key, '') + if not template_str: + logger.warning("Template key '%s' not found in task_config for COM key '%s'", template_key, com_key) + path_dict[com_key] = '' + else: + path_dict[com_key] = self._replace_template_vars(template_str, substitution_dict) + return path_dict + + @logit(logger) + def calculate_member_com_paths_gfs(self, member) -> None: + """Calculate member COM paths for GFS + + Parameters + ---------- + member : int + The member directory number + + Returns + ------- + None + Updates the task_config with member-specific COM paths for GFS. + """ + self.calculate_member() + member = f"mem{member:03d}" if member >= 0 else '' + current_cycle_mem_dict = {**self.task_config.current_cycle_dict, "${MEMDIR}": member} + previous_cycle_mem_dict = {**self.task_config.previous_cycle_dict, "${MEMDIR}": member, "${RUN}": self.task_config.rRUN} + + # Define all COM path mappings as tuple of tuples + com_paths = ( + ('COMIN_ATMOS_INPUT_MEM', 'COM_ATMOS_INPUT_TMPL', current_cycle_mem_dict), + ('COMOUT_ATMOS_INPUT_MEM', 'COM_ATMOS_INPUT_TMPL', current_cycle_mem_dict), + ('COMOUT_ATMOS_RESTART_PREV_MEM', 'COM_ATMOS_RESTART_TMPL', previous_cycle_mem_dict), + ('COMOUT_ATMOS_RESTART_MEM', 'COM_ATMOS_RESTART_TMPL', current_cycle_mem_dict), + ('COMOUT_ATMOS_ANALYSIS_MEM', 'COM_ATMOS_ANALYSIS_TMPL', current_cycle_mem_dict), + ('COMOUT_ICE_ANALYSIS_MEM', 'COM_ICE_ANALYSIS_TMPL', current_cycle_mem_dict), + ('COMOUT_ICE_RESTART_PREV_MEM', 'COM_ICE_RESTART_TMPL', previous_cycle_mem_dict), + ('COMOUT_OCEAN_RESTART_PREV_MEM', 'COM_OCEAN_RESTART_TMPL', previous_cycle_mem_dict), + ('COMOUT_OCEAN_ANALYSIS_MEM', 'COM_OCEAN_ANALYSIS_TMPL', current_cycle_mem_dict), + ('COMOUT_MED_RESTART_PREV_MEM', 'COM_MED_RESTART_TMPL', previous_cycle_mem_dict), + ('COMOUT_CHEM_ANALYSIS_MEM', 'COM_CHEM_ANALYSIS_TMPL', current_cycle_mem_dict), + ('COMOUT_WAVE_RESTART_PREV_MEM', 'COM_WAVE_RESTART_TMPL', previous_cycle_mem_dict), + ) + + # Generate paths and update task_config with returned dictionary + com_path_dict = self._paths_from_templates(com_paths) + self.task_config.update(com_path_dict) + + @logit(logger) + def calculate_member_com_paths_gefs_offline(self, member) -> None: + """Calculate member COM paths for GEFS offline + + Parameters + ---------- + member : int + The member directory number + + Returns + ------- + None + Updates the task_config with member-specific COM paths for GEFS offline. + """ + self.calculate_member() + member = f"mem{member:03d}" if member >= 0 else '' + current_cycle = {**self.task_config.current_cycle_dict, "${MEMDIR}": member} + previous_cycle = {**self.task_config.previous_cycle_dict, "${MEMDIR}": member} + + # Define all COM path mappings as tuple of tuples + com_paths = ( + ('COMIN_ATMOS_INPUT_MEM', 'COM_ATMOS_INPUT_TMPL', current_cycle), + ('COMOUT_ATMOS_INPUT_MEM', 'COM_ATMOS_INPUT_TMPL', current_cycle), + ('COMOUT_ATMOS_RESTART_PREV_MEM', 'COM_ATMOS_RESTART_TMPL', previous_cycle), + ('COMOUT_ATMOS_RESTART_MEM', 'COM_ATMOS_RESTART_TMPL', current_cycle), + ('COMOUT_ATMOS_ANALYSIS_MEM', 'COM_ATMOS_ANALYSIS_TMPL', current_cycle), + ('COMOUT_ATMOS_HISTORY_MEM', 'COM_ATMOS_HISTORY_TMPL', previous_cycle), + ('COMOUT_ICE_ANALYSIS_MEM', 'COM_ICE_ANALYSIS_TMPL', current_cycle), + ('COMOUT_ICE_RESTART_PREV_MEM', 'COM_ICE_RESTART_TMPL', previous_cycle), + ('COMOUT_OCEAN_RESTART_PREV_MEM', 'COM_OCEAN_RESTART_TMPL', previous_cycle), + ('COMOUT_OCEAN_ANALYSIS_MEM', 'COM_OCEAN_ANALYSIS_TMPL', current_cycle), + ('COMOUT_MED_RESTART_PREV_MEM', 'COM_MED_RESTART_TMPL', previous_cycle), + ('COMOUT_WAVE_RESTART_PREV_MEM', 'COM_WAVE_RESTART_TMPL', previous_cycle), + ) + + # Generate paths and update task_config with returned dictionary + com_path_dict = self._paths_from_templates(com_paths) + self.task_config.update(com_path_dict) + + @logit(logger) + def calculate_member_com_paths_gefs_rt(self, member) -> None: + """Calculate member COM paths for GEFS real-time + + Parameters + ---------- + member : int + The member directory number + + Returns + ------- + None + Updates the task_config with member-specific COM paths for GEFS real-time. + """ + self.calculate_member() + if member != 0: + self.task_config.gfs_member = self.task_config.cyc_ranges[self.task_config.m_index][(member - 1)] + member = f"mem{member:03d}" if member >= 0 else '' + current_cycle = {**self.task_config.current_cycle_dict, "${MEMDIR}": member} + previous_cycle = {**self.task_config.previous_cycle_dict, "${MEMDIR}": member} + + # Define all COM path mappings as tuple of tuples + com_paths = ( + ('COMIN_ATMOS_INPUT_MEM', 'COM_ATMOS_INPUT_TMPL', current_cycle), + ('COMOUT_ATMOS_INPUT_MEM', 'COM_ATMOS_INPUT_TMPL', current_cycle), + ('COMOUT_ATMOS_RESTART_PREV_MEM', 'COM_ATMOS_RESTART_TMPL', previous_cycle), + ('COMOUT_ATMOS_ANALYSIS_MEM', 'COM_ATMOS_ANALYSIS_TMPL', current_cycle), + ('COMOUT_ATMOS_HISTORY_MEM', 'COM_ATMOS_HISTORY_TMPL', previous_cycle), + ('COMOUT_ICE_ANALYSIS_MEM', 'COM_ICE_ANALYSIS_TMPL', current_cycle), + ('COMOUT_ICE_RESTART_PREV_MEM', 'COM_ICE_RESTART_TMPL', previous_cycle), + ('COMOUT_OCEAN_RESTART_PREV_MEM', 'COM_OCEAN_RESTART_TMPL', previous_cycle), + ('COMOUT_OCEAN_ANALYSIS_MEM', 'COM_OCEAN_ANALYSIS_TMPL', current_cycle), + ('COMOUT_MED_RESTART_PREV_MEM', 'COM_MED_RESTART_TMPL', previous_cycle), + ('COMOUT_WAVE_RESTART_PREV_MEM', 'COM_WAVE_RESTART_TMPL', previous_cycle), + ) + + # Generate paths and update task_config with returned dictionary + com_path_dict = self._paths_from_templates(com_paths) + self.task_config.update(com_path_dict) + + @logit(logger) + def calculate_member_com_paths_gcafs(self, member) -> None: + """Calculate member COM paths for GCAFS + + Parameters + ---------- + member : int + The member directory number + + Returns + ------- + None + Updates the task_config with member-specific COM paths for GCAFS. + """ + self.calculate_member() + member = f"mem{member:03d}" if member >= 0 else '' + + # Three contexts for GCAFS path generation + current_cycle_in = {**self.task_config.current_cycle_dict, "${MEMDIR}": member, "${RUN}": self.task_config.rRUN} + current_cycle = {**current_cycle_in, "${RUN}": self.task_config.rRUN} + previous_cycle = {**self.task_config.previous_cycle_dict, "${MEMDIR}": member, "${RUN}": self.task_config.rRUN} + + # Define all COM path mappings as tuple of tuples + com_paths = ( + ('COMIN_ATMOS_INPUT_MEM', 'COM_ATMOS_INPUT_TMPL', current_cycle_in), + ('COMOUT_ATMOS_INPUT_MEM', 'COM_ATMOS_INPUT_TMPL', current_cycle), + ('COMOUT_ATMOS_RESTART_PREV_MEM', 'COM_ATMOS_RESTART_TMPL', previous_cycle), + ('COMOUT_ATMOS_RESTART_MEM', 'COM_ATMOS_RESTART_TMPL', current_cycle), + ('COMOUT_ATMOS_ANALYSIS_MEM', 'COM_ATMOS_ANALYSIS_TMPL', current_cycle), + ('COMOUT_ICE_ANALYSIS_MEM', 'COM_ICE_ANALYSIS_TMPL', current_cycle), + ('COMOUT_ICE_RESTART_PREV_MEM', 'COM_ICE_RESTART_TMPL', previous_cycle), + ('COMOUT_OCEAN_RESTART_PREV_MEM', 'COM_OCEAN_RESTART_TMPL', previous_cycle), + ('COMOUT_OCEAN_ANALYSIS_MEM', 'COM_OCEAN_ANALYSIS_TMPL', current_cycle), + ('COMOUT_MED_RESTART_PREV_MEM', 'COM_MED_RESTART_TMPL', previous_cycle), + ('COMOUT_WAVE_RESTART_PREV_MEM', 'COM_WAVE_RESTART_TMPL', previous_cycle), + ) + + # Generate paths and update task_config with returned dictionary + com_path_dict = self._paths_from_templates(com_paths) + self.task_config.update(com_path_dict) + + @staticmethod + def _replace_template_vars(template: str, var_dict: Dict[str, Any]) -> str: + """Replace template variables in string with actual values + + Parameters + ---------- + template : str + Template string with variables to replace + var_dict : Dict[str, Any] + Dictionary of variable names and values + + Returns + ------- + str + String with variables replaced + """ + replaced_com = template + for var, value in var_dict.items(): + replaced_com = replaced_com.replace(var, value) + return replaced_com