Add capability to run diff resolutions for marine anl and background

ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml

@@ -8,12 +8,13 @@ arguments:
   resdetatmos: 384
   resdetocean: 0.25
   nens: 0
-  interval: 6
+  interval: 0
   start: warm
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2021063018
   edate: 2021070306
   #icsdir: /scratch1/NCEPDEV/climate/Jessica.Meixner/cycling/IC_2021063000_V2
-  icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/
+  #icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/
+  icsdir: /scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/prepics/hybrid-test/
   yaml: {{ HOMEgfs }}/ci/cases/gfsv17/marine3dvar.yaml

ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml

@@ -15,5 +15,6 @@ arguments:
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2021063018
   edate: 2021070306
-  icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/
+  #icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/
+  icsdir: /scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/prepics/hybrid-test/
   yaml: {{ HOMEgfs }}/ci/cases/gfsv17/marinehyb.yaml

ci/cases/gfsv17/marine3dvar.yaml

@@ -17,7 +17,12 @@ base:
   ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
   DO_TEST_MODE: "YES"
 
+prepoceanobs:
+  use_exp_obs: "YES"
+  dmpdir_exp: /scratch1/NCEPDEV/da/common/
+
 marineanl:
   SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca
+  SOCA_ANL_GEOM: {{ HOMEgfs }}/fix/gdas/soca/720x540x75/soca
   SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
   SOCA_NINNER: 100

ci/cases/gfsv17/marinehyb.yaml

@@ -16,6 +16,11 @@ base:
   FHMAX_GFS: 240
   ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
 
+prepoceanobs:
+  use_exp_obs: "YES"
+  dmpdir_exp: /scratch1/NCEPDEV/da/common/
+
 marineanl:
   SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca
-  SOCA_NINNER: 20  # revert to ~100 after the memory leak is fixed
+  SOCA_ANL_GEOM: {{ HOMEgfs }}/fix/gdas/soca/720x540x75/soca
+  SOCA_NINNER: 100

parm/config/gfs/config.marineanl

@@ -11,6 +11,7 @@ export JCB_ALGO_YAML_VAR=@JCB_ALGO_YAML_VAR@
 export MARINE_OBS_YAML_DIR="${PARMgfs}/gdas/soca/obs/config"
 export MARINE_OBS_LIST_YAML=@SOCA_OBS_LIST@
 export SOCA_INPUT_FIX_DIR=@SOCA_INPUT_FIX_DIR@
+export SOCA_ANL_GEOM=@SOCA_ANL_GEOM@
 export SOCA_NINNER=@SOCA_NINNER@
 export DOMAIN_STACK_SIZE=116640000  #TODO: Make the stack size resolution dependent
 export SOCA_ENS_BKG_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_ens_bkg_stage.yaml.j2"

parm/config/gfs/config.resources

@@ -504,23 +504,26 @@ case ${step} in
     walltime="00:10:00"
     ntasks=1
     threads_per_task=1
-    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    tasks_per_node=${max_tasks_per_node}
     memory="24GB"
     ;;
 
   "prepoceanobs")
     walltime="00:10:00"
     ntasks=1
     threads_per_task=1
-    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    tasks_per_node=${max_tasks_per_node}
     memory="48GB"
     ;;
 
   "marinebmat")
     npes=16
     ntasks=16
     case ${OCNRES} in
-      "025") ntasks=480;;
+      "025")
+        ntasks=480
+        memory="256GB"
+	;;
       "050")  ntasks=16;;
       "100")  ntasks=16;;
       "500")  ntasks=16;;
@@ -532,15 +535,15 @@ case ${step} in
     walltime="00:30:00"
     threads_per_task=1
     export is_exclusive=True
-    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    tasks_per_node=$(( max_tasks_per_node / 2 ))
     ;;
 
   "marineanlvar")
     ntasks=16
     case ${OCNRES} in
       "025")
         ntasks=480
-        memory="96GB"
+        memory="256GB"
         ;;
       "050")
         ntasks=16
@@ -559,18 +562,18 @@ case ${step} in
         exit 4
     esac
 
-    walltime="00:15:00"
+    walltime="00:30:00"
     threads_per_task=1
     export is_exclusive=True
-    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    tasks_per_node=$(( max_tasks_per_node / 2 ))
     ;;
 
   "ocnanalecen")
     ntasks=16
     case ${OCNRES} in
       "025")
         ntasks=40
-        memory="96GB"
+        memory="256GB"
         ;;
       "050")
         ntasks=16
@@ -590,17 +593,17 @@ case ${step} in
     esac
 
     walltime="00:10:00"
-    threads_per_task=1
     export is_exclusive=True
-    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    threads_per_task=1
+    tasks_per_node=$(( max_tasks_per_node / 2 ))
     ;;
 
   "marineanlletkf")
     ntasks=16
     case ${OCNRES} in
       "025")
         ntasks=480
-        memory="96GB"
+        memory="256GB"
         ;;
       "050")
         ntasks=16
@@ -619,18 +622,18 @@ case ${step} in
         exit 4
     esac
 
-    walltime="00:10:00"
-    threads_per_task=1
+    walltime="00:30:00"
     export is_exclusive=True
-    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    threads_per_task=1
+    tasks_per_node=$(( max_tasks_per_node / 2 ))
     ;;
 
 
   "marineanlchkpt")
     walltime="00:10:00"
     ntasks=1
     threads_per_task=1
-    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    tasks_per_node=${max_tasks_per_node}
     case ${OCNRES} in
       "025")
         memory="128GB"
@@ -654,7 +657,7 @@ case ${step} in
     walltime="00:30:00"
     ntasks=${max_tasks_per_node}
     threads_per_task=1
-    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    tasks_per_node=${max_tasks_per_node}
     ;;
 
   "ocnanalvrfy")

parm/config/gfs/yaml/defaults.yaml

@@ -54,6 +54,7 @@ snowanl:
 
 marineanl:
   SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca"
+  SOCA_ANL_GEOM: "${FIXgfs}/gdas/soca/72x35x25/soca"
   SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml"  # TODO: This is also repeated in oceanprepobs
   SOCA_NINNER: 100
   JCB_ALGO_YAML_VAR: "${PARMgfs}/gdas/soca/marine-jcb-3dfgat.yaml.j2"

parm/gdas/soca_bmat_jedi_config.yaml.j2

@@ -8,35 +8,42 @@ soca_diagb:
   rundir: '{{ DATA }}'
   exe_src: '{{ EXECgfs }}/gdas_soca_diagb.x'
   mpi_cmd: '{{ APRUN_MARINEBMAT }}'
-  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'  
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
   jcb_algo: soca_diagb
+soca_chgres:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ EXECgfs }}/gdas.x'
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'
+  jedi_args: ['soca', 'convertstate']
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
+  jcb_algo: soca_chgres
 soca_parameters_diffusion_vt:
   rundir: '{{ DATA }}'
   exe_src: '{{ EXECgfs }}/gdas_soca_error_covariance_toolbox.x'
-  mpi_cmd: '{{ APRUN_MARINEBMAT }}'  
-  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'  
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
   jcb_algo: soca_parameters_diffusion_vt
 soca_setcorscales:
   rundir: '{{ DATA }}'
   exe_src: '{{ EXECgfs }}/gdas_soca_setcorscales.x'
-  mpi_cmd: '{{ APRUN_MARINEBMAT }}'  
-  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'  
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
   jcb_algo: soca_setcorscales
 soca_parameters_diffusion_hz:
   rundir: '{{ DATA }}'
   exe_src: '{{ EXECgfs }}/gdas_soca_error_covariance_toolbox.x'
-  mpi_cmd: '{{ APRUN_MARINEBMAT }}'  
-  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'  
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
   jcb_algo: soca_parameters_diffusion_hz
 soca_ensb:
   rundir: '{{ DATA }}'
   exe_src: '{{ EXECgfs }}/gdas_ens_handler.x'
-  mpi_cmd: '{{ APRUN_MARINEBMAT }}'  
-  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'  
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
   jcb_algo: soca_ensb
 soca_ensweights:
   rundir: '{{ DATA }}'
   exe_src: '{{ EXECgfs }}/gdas_socahybridweights.x'
-  mpi_cmd: '{{ APRUN_MARINEBMAT }}'  
-  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'  
-  jcb_algo: soca_ensweights  
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
+  jcb_algo: soca_ensweights

ush/python/pygfs/task/marine_analysis.py

@@ -179,9 +179,13 @@ def _prep_scratch_dir(self: Task) -> None:
         soca_fix_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config)
         FileHandler(soca_fix_list).sync()
 
-        # prepare the MOM6 input.nml
+        # prepare the deterministic MOM6 input.nml
         mdau.prep_input_nml(self.task_config)
 
+        # prepare the input.nml for the analysis geometry
+        mdau.prep_input_nml(self.task_config, output_nml="./anl_geom/mom_input.nml",
+                            simple_geom=True, mom_input="./anl_geom/MOM_input")
+
         # stage the soca utility yamls (gridgen, fields and ufo mapping yamls)
         logger.info(f"Staging SOCA utility yaml files from {self.task_config.PARMsoca}")
         soca_utility_list = parse_j2yaml(self.task_config.MARINE_UTILITY_YAML_TMPL, self.task_config)
@@ -206,7 +210,7 @@ def _prep_variational_yaml(self: Task) -> None:
         envconfig_jcb['PARMgfs'] = self.task_config.PARMgfs
         envconfig_jcb['NMEM_ENS'] = self.task_config.NMEM_ENS
         envconfig_jcb['berror_model'] = 'marine_background_error_static_diffusion'
-        if self.task_config.NMEM_ENS >= 3:
+        if self.task_config.NMEM_ENS >= 2:
             envconfig_jcb['berror_model'] = 'marine_background_error_hybrid_diffusion_diffusion'
         envconfig_jcb['DATA'] = self.task_config.DATA
         envconfig_jcb['OPREFIX'] = self.task_config.OPREFIX
@@ -364,7 +368,7 @@ def list_all_files(dir_in, dir_out, wc='*', fh_list=[]):
         post_file_list = []
 
         # Make a copy the IAU increment
-        post_file_list.append([os.path.join(anl_dir, 'inc.nc'),
+        post_file_list.append([os.path.join(anl_dir, 'ocn.inc.nc'),
                                os.path.join(com_ocean_analysis, f'{RUN}.t{cyc}z.ocninc.nc')])
 
         domains = ['ocn', 'ice']

ush/python/pygfs/task/marine_bmat.py

@@ -74,7 +74,7 @@ def __init__(self, config):
 
         # Create dictionary of Jedi objects
         expected_keys = ['gridgen', 'soca_diagb', 'soca_parameters_diffusion_vt', 'soca_setcorscales',
-                         'soca_parameters_diffusion_hz', 'soca_ensb', 'soca_ensweights']
+                         'soca_parameters_diffusion_hz', 'soca_ensb', 'soca_ensweights', 'soca_chgres']
         self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys)
 
     @logit(logger)
@@ -105,9 +105,13 @@ def initialize(self: Task) -> None:
         soca_fix_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config)
         FileHandler(soca_fix_list).sync()
 
-        # prepare the MOM6 input.nml
+        # prepare the deterministic MOM6 input.nml
         mdau.prep_input_nml(self.task_config)
 
+        # prepare the input.nml for the analysis geometry
+        mdau.prep_input_nml(self.task_config, output_nml="./anl_geom/mom_input.nml",
+                            simple_geom=True, mom_input="./anl_geom/MOM_input")
+
         # stage backgrounds
         # TODO(G): Check ocean backgrounds dates for consistency
         bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config)
@@ -127,6 +131,7 @@ def initialize(self: Task) -> None:
         # initialize JEDI applications
         self.jedi_dict['gridgen'].initialize(self.task_config)
         self.jedi_dict['soca_diagb'].initialize(self.task_config)
+        self.jedi_dict['soca_chgres'].initialize(self.task_config)
         self.jedi_dict['soca_parameters_diffusion_vt'].initialize(self.task_config)
         self.jedi_dict['soca_setcorscales'].initialize(self.task_config)
         self.jedi_dict['soca_parameters_diffusion_hz'].initialize(self.task_config)
@@ -163,11 +168,15 @@ def execute(self) -> None:
         None
         """
 
+        # soca grid generation
         self.jedi_dict['gridgen'].execute()
 
         # variance partitioning
         self.jedi_dict['soca_diagb'].execute()
 
+        # Interpolate f009 bkg to analysis geometry
+        self.jedi_dict['soca_chgres'].execute()
+
         # horizontal diffusion
         self.jedi_dict['soca_setcorscales'].execute()
         self.jedi_dict['soca_parameters_diffusion_hz'].execute()

ush/python/pygfs/utils/marine_da_utils.py

@@ -45,7 +45,10 @@ def link_executable(task_config: AttrDict, exe_name: str) -> None:
 
 
 @logit(logger)
-def prep_input_nml(task_config: AttrDict) -> None:
+def prep_input_nml(task_config: AttrDict,
+                   output_nml: str = "mom_input.nml",
+                   simple_geom: bool = False,
+                   mom_input: str = "MOM_input") -> None:
     """Prepare the mom_input.nml file
     """
     # stage input.nml.j2
@@ -56,9 +59,11 @@ def prep_input_nml(task_config: AttrDict) -> None:
     # swap date and stacksize
     date_init = [int(s) for s in task_config.MARINE_WINDOW_END.strftime('%Y,%m,%d,%H,%M,%S').split(',')]
     input_nml_config = {'domain_stack_size': task_config.DOMAIN_STACK_SIZE,
-                        'date_init': date_init}
+                        'date_init': date_init,
+                        'simple_geom': simple_geom,
+                        'mom_input': mom_input}
     jinja_input_nml = jinja.Jinja(mom_input_nml_tmpl, input_nml_config)
-    jinja_input_nml.save('mom_input.nml')
+    jinja_input_nml.save(output_nml)
 
 
 @logit(logger)