Add capability to run diff resolutions for marine anl and background (N…

…OAA-EMC#3238) This PR mostly includes configuration changes to allow the use a B-matrix at a defined resolution that may or may not be the same as the background. - Addition of an interpolation step in the B-matrix job - version update for the fix files - **the `gdas.cd` update should be the correct one now.** Resolves NOAA-EMC#3230 Resolves NOAA-EMC/GDASApp/issues/1375 # Dependencies - NOAA-EMC/GDASApp/pull/1441 - NOAA-EMC/jcb-gdas/pull/66 --------- Co-authored-by: Anna Shlyaeva <[email protected]> Co-authored-by: RussTreadon-NOAA <[email protected]> Co-authored-by: Rahul Mahajan <[email protected]>
sbanihash · Feb 20, 2025 · 6a287b7 · 6a287b7
1 parent a1c0e7d
commit 6a287b7
Show file tree

Hide file tree

Showing 14 changed files with 89 additions and 42 deletions.
diff --git a/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml b/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml
@@ -8,12 +8,13 @@ arguments:
   resdetatmos: 384
   resdetocean: 0.25
   nens: 0
-  interval: 6
+  interval: 0
   start: warm
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2021063018
   edate: 2021070306
   #icsdir: /scratch1/NCEPDEV/climate/Jessica.Meixner/cycling/IC_2021063000_V2
-  icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/
+  #icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/
+  icsdir: /scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/prepics/hybrid-test/
   yaml: {{ HOMEgfs }}/ci/cases/gfsv17/marine3dvar.yaml
diff --git a/ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml b/ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml
@@ -15,5 +15,6 @@ arguments:
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2021063018
   edate: 2021070306
-  icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/
+  #icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/
+  icsdir: /scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/prepics/hybrid-test/
   yaml: {{ HOMEgfs }}/ci/cases/gfsv17/marinehyb.yaml
diff --git a/ci/cases/gfsv17/marine3dvar.yaml b/ci/cases/gfsv17/marine3dvar.yaml
@@ -16,7 +16,12 @@ base:
   ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
   DO_TEST_MODE: "YES"
 
+prepoceanobs:
+  use_exp_obs: "YES"
+  dmpdir_exp: /scratch1/NCEPDEV/da/common/
+
 marineanl:
   SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca
+  SOCA_ANL_GEOM: {{ HOMEgfs }}/fix/gdas/soca/720x540x75/soca
   SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
   SOCA_NINNER: 100
diff --git a/ci/cases/gfsv17/marinehyb.yaml b/ci/cases/gfsv17/marinehyb.yaml
@@ -15,6 +15,11 @@ base:
   FHMAX_GFS: 240
   ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
 
+prepoceanobs:
+  use_exp_obs: "YES"
+  dmpdir_exp: /scratch1/NCEPDEV/da/common/
+
 marineanl:
   SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca
-  SOCA_NINNER: 20  # revert to ~100 after the memory leak is fixed
+  SOCA_ANL_GEOM: {{ HOMEgfs }}/fix/gdas/soca/720x540x75/soca
+  SOCA_NINNER: 100
diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN
@@ -20,6 +20,11 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
    COMIN_ICE_RESTART:COM_ICE_RESTART_TMPL \
    COMOUT_ICE_ANALYSIS:COM_ICE_ANALYSIS_TMPL
 
+export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}"
+export DATAens="${DATAjob}/ensdata"
+export GDUMP="gdas"
+export GDUMP_ENS="enkf${GDUMP}"
+
 ##############################################
 # Begin JOB SPECIFIC work
 ##############################################

diff --git a/parm/config/gfs/config.marineanl b/parm/config/gfs/config.marineanl
@@ -11,6 +11,7 @@ export JCB_ALGO_YAML_VAR=@JCB_ALGO_YAML_VAR@
 export MARINE_OBS_YAML_DIR="${PARMgfs}/gdas/soca/obs/config"
 export MARINE_OBS_LIST_YAML=@SOCA_OBS_LIST@
 export SOCA_INPUT_FIX_DIR=@SOCA_INPUT_FIX_DIR@
+export SOCA_ANL_GEOM=@SOCA_ANL_GEOM@
 export SOCA_NINNER=@SOCA_NINNER@
 export DOMAIN_STACK_SIZE=116640000  #TODO: Make the stack size resolution dependent
 export SOCA_ENS_BKG_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_ens_bkg_stage.yaml.j2"

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
@@ -509,23 +509,26 @@ case ${step} in
     walltime="00:10:00"
     ntasks=1
     threads_per_task=1
-    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    tasks_per_node=${max_tasks_per_node}
     memory="24GB"
     ;;
 
   "prepoceanobs")
     walltime="00:10:00"
     ntasks=1
     threads_per_task=1
-    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    tasks_per_node=${max_tasks_per_node}
     memory="48GB"
     ;;
 
   "marinebmat")
     npes=16
     ntasks=16
     case ${OCNRES} in
-      "025") ntasks=480;;
+      "025")
+        ntasks=480
+        memory="256GB"
+	;;
       "050")  ntasks=16;;
       "100")  ntasks=16;;
       "500")  ntasks=16;;
@@ -537,15 +540,15 @@ case ${step} in
     walltime="00:30:00"
     threads_per_task=1
     export is_exclusive=True
-    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    tasks_per_node=$(( max_tasks_per_node / 2 ))
     ;;
 
   "marineanlvar")
     ntasks=16
     case ${OCNRES} in
       "025")
         ntasks=480
-        memory="96GB"
+        memory="256GB"
         ;;
       "050")
         ntasks=16
@@ -564,18 +567,18 @@ case ${step} in
         exit 4
     esac
 
-    walltime="00:15:00"
+    walltime="00:30:00"
     threads_per_task=1
     export is_exclusive=True
-    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    tasks_per_node=$(( max_tasks_per_node / 2 ))
     ;;
 
   "ocnanalecen")
     ntasks=16
     case ${OCNRES} in
       "025")
         ntasks=40
-        memory="96GB"
+        memory="256GB"
         ;;
       "050")
         ntasks=16
@@ -595,17 +598,17 @@ case ${step} in
     esac
 
     walltime="00:10:00"
-    threads_per_task=1
     export is_exclusive=True
-    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    threads_per_task=1
+    tasks_per_node=$(( max_tasks_per_node / 2 ))
     ;;
 
   "marineanlletkf")
     ntasks=16
     case ${OCNRES} in
       "025")
         ntasks=480
-        memory="96GB"
+        memory="256GB"
         ;;
       "050")
         ntasks=16
@@ -624,18 +627,18 @@ case ${step} in
         exit 4
     esac
 
-    walltime="00:10:00"
-    threads_per_task=1
+    walltime="00:30:00"
     export is_exclusive=True
-    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    threads_per_task=1
+    tasks_per_node=$(( max_tasks_per_node / 2 ))
     ;;
 
 
   "marineanlchkpt")
     walltime="00:10:00"
     ntasks=1
     threads_per_task=1
-    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    tasks_per_node=${max_tasks_per_node}
     case ${OCNRES} in
       "025")
         memory="128GB"
@@ -659,7 +662,7 @@ case ${step} in
     walltime="00:30:00"
     ntasks=${max_tasks_per_node}
     threads_per_task=1
-    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    tasks_per_node=${max_tasks_per_node}
     ;;
 
   "ocnanalvrfy")

diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml
@@ -53,6 +53,7 @@ snowanl:
 
 marineanl:
   SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca"
+  SOCA_ANL_GEOM: "${FIXgfs}/gdas/soca/72x35x25/soca"
   SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml"  # TODO: This is also repeated in oceanprepobs
   SOCA_NINNER: 100
   JCB_ALGO_YAML_VAR: "${PARMgfs}/gdas/soca/marine-jcb-3dfgat.yaml.j2"

diff --git a/parm/gdas/soca_bmat_jedi_config.yaml.j2 b/parm/gdas/soca_bmat_jedi_config.yaml.j2
@@ -8,35 +8,42 @@ soca_diagb:
   rundir: '{{ DATA }}'
   exe_src: '{{ EXECgfs }}/gdas_soca_diagb.x'
   mpi_cmd: '{{ APRUN_MARINEBMAT }}'
-  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'  
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
   jcb_algo: soca_diagb
+soca_chgres:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ EXECgfs }}/gdas.x'
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'
+  jedi_args: ['soca', 'convertstate']
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
+  jcb_algo: soca_chgres
 soca_parameters_diffusion_vt:
   rundir: '{{ DATA }}'
   exe_src: '{{ EXECgfs }}/gdas_soca_error_covariance_toolbox.x'
-  mpi_cmd: '{{ APRUN_MARINEBMAT }}'  
-  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'  
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
   jcb_algo: soca_parameters_diffusion_vt
 soca_setcorscales:
   rundir: '{{ DATA }}'
   exe_src: '{{ EXECgfs }}/gdas_soca_setcorscales.x'
-  mpi_cmd: '{{ APRUN_MARINEBMAT }}'  
-  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'  
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
   jcb_algo: soca_setcorscales
 soca_parameters_diffusion_hz:
   rundir: '{{ DATA }}'
   exe_src: '{{ EXECgfs }}/gdas_soca_error_covariance_toolbox.x'
-  mpi_cmd: '{{ APRUN_MARINEBMAT }}'  
-  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'  
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
   jcb_algo: soca_parameters_diffusion_hz
 soca_ensb:
   rundir: '{{ DATA }}'
   exe_src: '{{ EXECgfs }}/gdas_ens_handler.x'
-  mpi_cmd: '{{ APRUN_MARINEBMAT }}'  
-  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'  
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
   jcb_algo: soca_ensb
 soca_ensweights:
   rundir: '{{ DATA }}'
   exe_src: '{{ EXECgfs }}/gdas_socahybridweights.x'
-  mpi_cmd: '{{ APRUN_MARINEBMAT }}'  
-  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'  
-  jcb_algo: soca_ensweights  
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
+  jcb_algo: soca_ensweights
diff --git a/sorc/gdas.cd b/sorc/gdas.cd
diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py
@@ -179,9 +179,13 @@ def _prep_scratch_dir(self: Task) -> None:
         soca_fix_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config)
         FileHandler(soca_fix_list).sync()
 
-        # prepare the MOM6 input.nml
+        # prepare the deterministic MOM6 input.nml
         mdau.prep_input_nml(self.task_config)
 
+        # prepare the input.nml for the analysis geometry
+        mdau.prep_input_nml(self.task_config, output_nml="./anl_geom/mom_input.nml",
+                            simple_geom=True, mom_input="./anl_geom/MOM_input")
+
         # stage the soca utility yamls (gridgen, fields and ufo mapping yamls)
         logger.info(f"Staging SOCA utility yaml files from {self.task_config.PARMsoca}")
         soca_utility_list = parse_j2yaml(self.task_config.MARINE_UTILITY_YAML_TMPL, self.task_config)
@@ -206,7 +210,7 @@ def _prep_variational_yaml(self: Task) -> None:
         envconfig_jcb['PARMgfs'] = self.task_config.PARMgfs
         envconfig_jcb['NMEM_ENS'] = self.task_config.NMEM_ENS
         envconfig_jcb['berror_model'] = 'marine_background_error_static_diffusion'
-        if self.task_config.NMEM_ENS >= 3:
+        if self.task_config.NMEM_ENS >= 2:
             envconfig_jcb['berror_model'] = 'marine_background_error_hybrid_diffusion_diffusion'
         envconfig_jcb['DATA'] = self.task_config.DATA
         envconfig_jcb['OPREFIX'] = self.task_config.OPREFIX
@@ -365,7 +369,7 @@ def list_all_files(dir_in, dir_out, wc='*', fh_list=[]):
         post_file_list = []
 
         # Make a copy the IAU increment
-        post_file_list.append([os.path.join(anl_dir, 'inc.nc'),
+        post_file_list.append([os.path.join(anl_dir, 'ocn.inc.nc'),
                                os.path.join(com_ocean_analysis, f'{RUN}.t{cyc}z.ocninc.nc')])
 
         domains = ['ocn', 'ice']

diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py
@@ -74,7 +74,7 @@ def __init__(self, config):
 
         # Create dictionary of Jedi objects
         expected_keys = ['gridgen', 'soca_diagb', 'soca_parameters_diffusion_vt', 'soca_setcorscales',
-                         'soca_parameters_diffusion_hz', 'soca_ensb', 'soca_ensweights']
+                         'soca_parameters_diffusion_hz', 'soca_ensb', 'soca_ensweights', 'soca_chgres']
         self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys)
 
     @logit(logger)
@@ -105,9 +105,13 @@ def initialize(self: Task) -> None:
         soca_fix_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config)
         FileHandler(soca_fix_list).sync()
 
-        # prepare the MOM6 input.nml
+        # prepare the deterministic MOM6 input.nml
         mdau.prep_input_nml(self.task_config)
 
+        # prepare the input.nml for the analysis geometry
+        mdau.prep_input_nml(self.task_config, output_nml="./anl_geom/mom_input.nml",
+                            simple_geom=True, mom_input="./anl_geom/MOM_input")
+
         # stage backgrounds
         # TODO(G): Check ocean backgrounds dates for consistency
         bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config)
@@ -127,6 +131,7 @@ def initialize(self: Task) -> None:
         # initialize JEDI applications
         self.jedi_dict['gridgen'].initialize(self.task_config)
         self.jedi_dict['soca_diagb'].initialize(self.task_config)
+        self.jedi_dict['soca_chgres'].initialize(self.task_config)
         self.jedi_dict['soca_parameters_diffusion_vt'].initialize(self.task_config)
         self.jedi_dict['soca_setcorscales'].initialize(self.task_config)
         self.jedi_dict['soca_parameters_diffusion_hz'].initialize(self.task_config)
@@ -163,11 +168,15 @@ def execute(self) -> None:
         None
         """
 
+        # soca grid generation
         self.jedi_dict['gridgen'].execute()
 
         # variance partitioning
         self.jedi_dict['soca_diagb'].execute()
 
+        # Interpolate f009 bkg to analysis geometry
+        self.jedi_dict['soca_chgres'].execute()
+
         # horizontal diffusion
         self.jedi_dict['soca_setcorscales'].execute()
         self.jedi_dict['soca_parameters_diffusion_hz'].execute()

diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py
@@ -45,7 +45,10 @@ def link_executable(task_config: AttrDict, exe_name: str) -> None:
 
 
 @logit(logger)
-def prep_input_nml(task_config: AttrDict) -> None:
+def prep_input_nml(task_config: AttrDict,
+                   output_nml: str = "mom_input.nml",
+                   simple_geom: bool = False,
+                   mom_input: str = "MOM_input") -> None:
     """Prepare the mom_input.nml file
     """
     # stage input.nml.j2
@@ -56,9 +59,11 @@ def prep_input_nml(task_config: AttrDict) -> None:
     # swap date and stacksize
     date_init = [int(s) for s in task_config.MARINE_WINDOW_END.strftime('%Y,%m,%d,%H,%M,%S').split(',')]
     input_nml_config = {'domain_stack_size': task_config.DOMAIN_STACK_SIZE,
-                        'date_init': date_init}
+                        'date_init': date_init,
+                        'simple_geom': simple_geom,
+                        'mom_input': mom_input}
     jinja_input_nml = jinja.Jinja(mom_input_nml_tmpl, input_nml_config)
-    jinja_input_nml.save('mom_input.nml')
+    jinja_input_nml.save(output_nml)
 
 
 @logit(logger)

diff --git a/versions/fix.ver b/versions/fix.ver
@@ -9,7 +9,7 @@ export cpl_ver=20230526
 export datm_ver=20220805
 export gdas_crtm_ver=20220805
 export gdas_fv3jedi_ver=20241115
-export gdas_soca_ver=20240802
+export gdas_soca_ver=20240919
 export gdas_gsibec_ver=20240416
 export gdas_obs_ver=20240213
 export gdas_aero_ver=20240806
+5 −0		build.sh
+1 −1		modulefiles/EVA/hera.lua
+1 −1		modulefiles/GDAS/hera.intel.lua
+2 −2		parm/aero/jcb-base.yaml.j2
+1 −1		parm/jcb-gdas
+60 −5		parm/soca/berror/soca_ensrecenter.yaml
+5 −11		parm/soca/fms/input.nml.j2
+5 −0		parm/soca/soca_fix_stage_025.yaml.j2
+5 −0		parm/soca/soca_fix_stage_100.yaml.j2
+5 −0		parm/soca/soca_fix_stage_500.yaml.j2
+1 −1		sorc/oops
+1 −1		sorc/saber
+1 −1		sorc/soca
+1 −1		sorc/ufo
+5 −0		test/gw-ci/CMakeLists.txt
+6 −1		ush/ioda/bufr2ioda/marine/b2i/bufr2ioda_insitu_profile_glider.py
+58 −5		ush/soca/marine_recenter.py
+1 −1		ush/soca/soca_vrfy.py
+38 −0		utils/soca/fig_gallery/README.md
+53 −16		utils/soca/fig_gallery/gdassoca_obsstats.py
+23 −24		utils/soca/fig_gallery/gdassoca_obsstats_template.html
+19 −53		utils/soca/fig_gallery/marine_vrfy_display/default.css
+586 −205		utils/soca/fig_gallery/marine_vrfy_display/index_vrfy_marine.html
+0 −45		utils/soca/fig_gallery/run_marine_analysis_vrfy_manual.job
+71 −0		utils/soca/fig_gallery/run_vrfy.py
+16 −0		utils/soca/fig_gallery/vrfy_config.yaml
+58 −0		utils/soca/fig_gallery/vrfy_jobcard.sh.j2
+93 −67		utils/soca/fig_gallery/vrfy_script.py
+56 −18		utils/soca/gdas_ens_handler.h
+8 −1		utils/soca/gdas_incr_handler.h
+75 −43		utils/soca/gdas_postprocincr.h
+10 −1		utils/soca/gdas_soca_diagb.h
+20 −8		utils/soca/gdas_socahybridweights.h