Merge remote-tracking branch 'origin/release/obsproc.v1.2.0' into dev…

…elop

docs/Release_Notes.txt

@@ -1,3 +1,243 @@
+obsproc v1.2.0  RELEASED        AUG ??,2023
+        v1.2.?  IMPLEMENTED     ??? ??,2023
+
+FILE CHANGES
+
+
+D docs/ReleaseNotes.txt
+M docs/Release_Notes.txt
+M jobs/JOBSPROC_CDAS_PREP1
+M jobs/JOBSPROC_GLOBAL_DUMP_POST
+M jobs/JOBSPROC_GLOBAL_PREP
+M jobs/JOBSPROC_GLOBAL_PREP_POST
+M jobs/JOBSPROC_RAP_PREP
+A modulefiles/obsproc_jet.lua
+M scripts/excdas_dump.sh 
+M scripts/exdump_monitor.sh
+M scripts/exdump_post.sh
+M scripts/exglobal_dump.sh
+M scripts/exnam_dump.sh
+M scripts/exrap_dump.sh 
+M scripts/exrtma_dump.sh
+M scripts/exurma_dump.sh
+M sorc/bufr_remorest.fd/remorest.f 
+M ush/build.sh
+D versions/README
+M versions/run.ver
+M versions/VERSION
+(M-modified; A-added; D-deleted)
+
+DOCS CHANGES
+
+ docs/ReleaseNotes.txt
+ Delete repetitive file with similar name
+
+ docs/Release_Notes.txt
+ Reflect this release changes.
+
+JOBS CHANGES
+
+ jobs/JOBSPROC_CDAS_PREP1
+ Increase errPREPDATA_limit in order to allow code to generate prepbufr file if/when one or more key files 
+ (adpsfc, adpupa, uprair) are missing
+
+ jobs/JOBSPROC_GLOBAL_DUMP_POST
+ Add functionality to unrestrict aircraft data (PROCESS_REMOREST_dm2)
+ Separate processing of uprair to its own dump group
+
+ jobs/JOBSPROC_GLOBAL_PREP
+ Instruct to add uprair dump files data to GDAS and GFS prepbufr files
+ Increase NSPLIT for faster prep step
+
+ jobs/JOBSPROC_GLOBAL_PREP_POST
+ Add functionality to unrestrict aircraft data (PROCESS_REMOREST_dm2) 
+ Correct COMIN1 and COMINgdas patha
+
+ jobs/JOBSPROC_RAP_PREP
+ Instruct to add uprair dump files data to RAP's prepbufr file
+ Increase NSPLIT for faster prep step
+
+MODULEFILES CHANGES
+
+ modulefiles/obsproc_jet.lua
+ Add support for functionality on Jet
+
+SCRIPTS CHANGES
+
+ scripts/excdas_dump.sh
+ Instruct NOT to dump uprair file for CDAS network (too slow)
+ Introduce msone1 group and redistribute group;add MAX_TANK* tank size check; Turn off msonet and msone1 for the time being.
+
+ scripts/exdump_monitor.sh
+ Add MAX_TANK* tank size check
+
+ scripts/exdump_post.sh
+ Add functionality to unrestrict aircraft data (PROCESS_REMOREST_dm2)
+
+ scripts/exglobal_dump.sh
+ Split groups 1 and 10 and add an additional DUMP_group12 to allow code to run on one more cpu and faster
+ Dump BUFR formated high-res rawinsonde profiles into *uprair* dump files
+ Add DBNet dbn_alert for uprair and satwhr dump files going to NOMADS
+ Redistribute groups, introduce NAP and NAP_adpupa to allow earlier job/cron kickoff
+
+ scripts/exrap_dump.sh
+ Dump BUFR formated high-res rawinsonde profiles into *uprair* dump files
+ Add DBNet dbn_alert for uprair dump files going to NOMADS
+ Introduce msone1 group and redistribute groups; add MAX_TANK* tank size check 
+ Redistribute groups, introduce NAP to allow earlier job/cron kickoff
+
+ scripts/exnam_dump.sh
+ Introduce msone1 group and redistribute groups/add cpus; add MAX_TANK* tank size check
+
+ scripts/exrtma_dump.sh
+ Introduce msone1 group and redistribute groups
+ Introduce max size treshold for mesonet tanks;add MAX_TANK* tank size check
+
+ scripts/exurma_dump.sh
+ Introduce msone1 group and redistribute groups
+ Introduce max size treshold for mesonet tanks;add MAX_TANK* tank size check
+
+SORC CHANGES
+
+ sorc/bufr_remorest.fd/remorest.f
+ Add functionality to unrestrict aircraft data (PROCESS_REMOREST_dm2)
+
+USH CHANGES
+
+ ush/build.sh
+ Add support for functionality on Jet
+
+VERSIONS CHANGES
+
+ versions/README
+ Delete old notes
+
+ versions/run.ver
+ Set bufr_dump_ver=1.2.0 and prepbufr_ver=1.1.0
+
+ versions/VERSION
+ Updated version to 1.2.0
+
+ECF 
+
+ Contents of ECF directory are not EMC's responsibility
+ However,note that obsproc.v1.2.0 , bufr-dump,v1.2.0 and prepobs.v1.2.0
+ will require more memory in the PBS card for:
+ gdas, gfs, cdas, rap runs (all 4 steps for each network)
+ See the example logs for : /lfs/h2/emc/stmp/iliana.genkova/CRON/R12/output 
+
+CHANGES TO OUTPUT AND FILE SIZES
+
+ Generate new dumps:
+ *satwhr* (gdas,gfs), *uprair* (gdas,gfs,cdas,rap) 
+ Update dumps (gdas, gfs):
+ *prepbufr* (when this release is synced with bufr-dump.v1.2.0 and prepobs.v1.1.0)
+ Remove dumps (gdas, gfs):
+ *saphir* , *sevcsr*
+
+ The newly generated dumps (*satwhr*, *uprair*) vary in size from 20Mb to 500Mb depending on network, cycle time, and season.
+ *uprair* will be largest for 00UTC and 12UTC.
+ *satwhr* will be largest during hurricane season.
+
+ The updated *prepbur* files  are up to ~20% larger in size
+ For examples, see: /lfs/h2/emc/stmp/iliana.genkova/CRON/R12/com
+
+ Both *saphir* and *sevcsr* dumps have been removed as they are no longer available.
+
+CHANGES TO RESOURCES
+
+ 1.
+ The gfs and gdas processing must start 10 min earlier than current processing. 
+ The NAP(600s) and NAP_adpupa(800s) variables in the obsproc script exglobal_dump.sh compensate for the earlier start,
+ such that only "uprair" dump stars early. This approach will be revisited in future obsproc updates. 
+
+ The rap processing must start 2 min earlier than current processing.
+ The NAP(120s) variables in the obsproc scripts exrap_dump.sh compensate for the earlier start,
+ such that only "uprair" dump stars early. This approach will be revisited in future obsproc updates.
+
+ Tests with this approach shows no loss of current opertaional obs data for gdas, gfs and rap.
+ The new uprair data will be captured and instrodyced with no loss for gdas and rap, and possibly minimal loss for gfs.
+
+ 2.
+ Memory and CPU usage was expanded for many steps in the processing.
+ EMC tests ran with the following recomended configurations (keep the ncpus values!)
+ gdas/gfs dump:      ncpus=14 mem=500gb 
+ gdas/gfs dump_post: ncpus=14 mem=4gb
+ gdas/gfs prep:      ncpus=14 mem=520gb
+ gdas/gfs prep_post: ncpus=14 mem=4gb
+
+ rap dump:      ncpus=11 mem=50gb
+ rap dump_post: ncpus=14 mem=6b
+ rap prep:      ncpus=4 mem=100gb
+ rap prep_post: ncpus=1 mem=3gb
+
+ cdas dump:      ncpus=10 mem=200gb
+ cdas dump_post: ncpus=12 mem=300gb
+ cdas prep1:     ncpus=12 mem=100gb
+ cdas prep2:     ncpus=16 mem=120gb
+ cdas prep1_post: ncpus=1 mem=6gb
+
+ nam dump:      ncpus=7 mem=20gb
+ nam dump2:     ncpus=3 mem=15gb
+ nam dump_post: ncpus=1 mem=3gb
+ nam prep:      ncpus=4 mem=220gb
+ nam prep_post: ncpus=1 mem=500Mb
+
+ rtma dump:     ncpus=5 mem=40gb
+ rtma dump_post:ncpus=1 mem=5000MB
+ rtma prep:     ncpus=1 mem=2gb
+ rtma prep_post:ncpus=1 mem=2gb
+
+ urma dump:     ncpus=6:mem=40gb
+ urma dump_post:ncpus=1:mem=6gb
+ urma prep:     ncpus=1:mem=6gb
+ urma prep_post:ncpus=1:mem=2gb
+
+ dump monitor:  ncpus=15:mem=400GB
+
+PRE-IMPLEMENTATION TESTING REQUIREMENTS
+
+ obsproc needs modules bufr-dump and prepobs in order to run
+ See docs/README.md for build instructions
+ See versions/build.ver and versions/run.ver for libaries
+
+ Run jobs/JOBSPROC_networkname_DUMP and jobs/JOBSPROC_networkname_DUMP_POST
+ for the following networkname values: GLOBAL, NAM, RAP, URMA, RTMA, CDAS
+
+ Parallel Production Testing (PPT)
+ After testing obsproc v1.2 in the /para space, we recommend doing for at least a month,
+ it is best to do a single cycle Parallel Production Testing (PPT) (i.e. obsproc + GFS + other models).
+ Copy the following directoris:
+ $model.YYYYMMDD, where model: gfs,gdas,nam,rap,rap_e,rap_p,urma,cdas,rtma,rtma_ru,dump
+ $model.YYYYMM,   where model: cdas
+ $model.YYYY,     where model:mods
+ $model,          where:gdas, sdm_rtdm
+ from the para run:
+ /para/com/obsproc/v1.2/
+ to
+ /prod/com/obsproc/v1.2/  for the PPT.
+
+ Good to know:
+ - the RAP uprair dumps might be 0 size for the hours not around 00,06,12,18 UTC
+ - the CDAS prepbufr files are 6 different flavours, it takes about 26 hours for all to be generated
+ - only GDAS and GFS networks will generate  *satwhr*
+ - only GDAS and GFS networks would support the unrestrictiom of aircraft data, and only on a DEV machine
+ - some *bufr_d files might be 0 size until the relevant data stream is recovered (airs, ombusv8) 
+ - please contact [email protected] should you have any Qs regarding testing
+
+DISSEMINATION INFORMATION
+
+  For NOMAD proposed changes, a SCN will be issued/submitted to Data Flow
+
+SPECIAL INSTRUCTIONS
+
+ 1.Implement release/obsproc.v1.2.0, release/bufr-dump.v1.2.0, and release/prepobs.v1.1.0 at the same time
+
+ 2.Implement release/obsproc.v1.2.0, release/bufr-dump.v1.2.0, and release/prepobs.v1.1.0 after confirming GFS global-workflow is ready for the new versions (first 2 digits)
+
+
+
+XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 obsproc v1.1.0	RELEASED	AUG 18,2022
 	v1.1.2	IMPLEMENTED	NOV 30,2022
 

jobs/JOBSPROC_CDAS_PREP1

@@ -251,6 +251,13 @@ export NSPLIT=${NSPLIT:-12}
 export BUFRLIST=${BUFRLIST:-"adpupa proflr aircar aircft satwnd adpsfc \
  sfcshp atovs"}
 
+# I.G. 07/27/2023
+export errPREPDATA_limit=4 # allow program PREPOBS_PREPDATA to continue at all
+                           # center hour (cycle) times if foreground exit
+                           # status is 4 (i.e., when either either "ADPUPA" or
+                           # "ADPSFC" dump is empty)
+
+
 # VARIABLES THAT CONTROL PROCESSING OF INDIVIDUAL PROGRAMS
 # --------------------------------------------------------
 

jobs/JOBSPROC_GLOBAL_DUMP_POST

@@ -157,16 +157,25 @@ if [[ "$RUN_ENVIR" == nco ]]; then
   export COMIN=${COMIN:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT})}
   export COMOUT=${COMOUT:-$(compath.py -o ${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT})}
   mkdir -m 775 -p $COMOUT
+# The following are used by $PROCESS_REMOREST_dm2 when it is YES
+# --------------------------------------------------------------
+  export COMINm2=${COMINm2:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT})}
+  export COMOUTm2=${COMOUTm2:-$(compath.py -o ${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT})}
 else
   export COMIN=${COMIN:-${COMIN_ROOT:?}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT}}
 
-  # COMOUT_ROOT for developers defaults to unique $DATA/com directory as root to
+# COMOUT_ROOT for developers defaults to unique $DATA/com directory as root to
 #  prevent job from writing into a non-unique directory that might already have
 #  output in it from a previous checkout run for the same cycle
 # -----------------------------------------------------------------------------
   export COMOUT_ROOT=${COMOUT_ROOT:-${DATA}/com}
   export COMOUT=${COMOUT:-${COMOUT_ROOT}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT}}
   mkdir -m 755 -p $COMOUT
+# The following are used by $PROCESS_REMOREST_dm2 when it is YES
+# --------------------------------------------------------------
+  export COMINm2=${COMINm2:-${COMIN_ROOT:?}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT}}
+  export COMOUTm2=${COMOUTm2:-${COMOUT_ROOT}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT}}
+  mkdir -m 755 -p $COMOUTm2
 fi
 
 # comin and comin_m1 are paths to dump status files for bufr_avgdata.sh

jobs/JOBSPROC_GLOBAL_PREP

@@ -233,11 +233,12 @@ export POE=${POE:-YES}
 #   (Applies only when either POE=YES or BACK=YES, and PREPDATA=YES)
 #   Consider the size of the backgrounds fields if using threads. 
 
-export NSPLIT=${NSPLIT:-4}
+#IG export NSPLIT=${NSPLIT:-4}
+export NSPLIT=${NSPLIT:-12} # ${NSPLIT:-8} #16 adds time to MERGE step :(
 
 #  BUFRLIST indicates the list of BUFR data dump file names to process
 
-export BUFRLIST=${BUFRLIST:-"adpupa proflr aircar aircft satwnd adpsfc \
+export BUFRLIST=${BUFRLIST:-"uprair proflr aircar aircft satwnd adpupa adpsfc \
  sfcshp vadwnd wdsatr ascatw rassda gpsipw"}
 
 export errPREPDATA_limit=4 # allow program PREPOBS_PREPDATA to continue at all

jobs/JOBSPROC_GLOBAL_PREP_POST

@@ -151,6 +151,10 @@ if [[ "$RUN_ENVIR" == nco ]]; then
   export COMIN=${COMIN:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT})}
   export COMOUT=${COMOUT:-$(compath.py -o ${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT})}
   mkdir -m 775 -p $COMOUT
+# The following are used by $PROCESS_REMOREST_dm2 when it is YES
+# --------------------------------------------------------------
+  export COMINm2=${COMINm2:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT})}
+  export COMOUTm2=${COMOUTm2:-$(compath.py -o ${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT})}
 else
   export COMIN=${COMIN:-${COMIN_ROOT:?}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT}}
 
@@ -161,12 +165,17 @@ else
   export COMOUT_ROOT=${COMOUT_ROOT:-${DATA}/com}
   export COMOUT=${COMOUT:-${COMOUT_ROOT}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT}}
   mkdir -m 755 -p $COMOUT
+# The following are used by $PROCESS_REMOREST_dm2 when it is YES
+# --------------------------------------------------------------
+  export COMINm2=${COMINm2:-${COMIN_ROOT:?}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT}}
+  export COMOUTm2=${COMOUTm2:-${COMOUT_ROOT}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT}}
+  mkdir -m 755 -p $COMOUTm2
 fi
 
 # The following are used by $PROCESS_TIMETWINS in the GDAS when it is YES
 # -----------------------------------------------------------------------
-export COMINgdas=${COMINgfs:-$(compath.py ${envir}/${NET}/${gfs_ver})/${RUN}.}
-export COMINGFS=${COMINgfs:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${NET}.${PDY}/${cyc}/${COMPONENT})}
+export COMINgdas=${COMINgdas:-$(compath.py ${envir}/${NET}/${gfs_ver})/${RUN}.}
+export COMINGFS=${COMINGFS:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${NET}.${PDY}/${cyc}/${COMPONENT})}
 #export COMIN1=${COMIN1:-$(compath.py ${envir}/${NET}/${gfs_ver})/${RUN}.}
 export COMIN1=${COMIN1:-$(compath.py ${envir}/${obsNET}/${obsproc_ver})/${RUN}.} #twinwind file now write in obsproc package
 

jobs/JOBSPROC_RAP_PREP

@@ -221,11 +221,11 @@ export POE=${POE:-NO}
 #   (Applies only when either POE=YES or BACK=YES, and PREPDATA=YES)
 #   Consider the size of the backgrounds fields if using threads.
 
-export NSPLIT=${NSPLIT:-1}
+export NSPLIT=${NSPLIT:-4}
 
 #  BUFRLIST indicates the list of BUFR data dump file names to process
 
-export BUFRLIST=${BUFRLIST:-"adpupa proflr aircar aircft satwnd adpsfc \
+export BUFRLIST=${BUFRLIST:-"uprair adpupa proflr aircar aircft satwnd adpsfc \
  sfcshp vadwnd goesnd msonet gpsipw rassda ascatw wdsatr"}
 
 export errPREPDATA_limit=4 # allow program PREPOBS_PREPDATA to continue at all

modulefiles/obsproc_jet.lua

@@ -0,0 +1,17 @@
+help([[
+Load environment to build obsproc on Jet
+]])
+
+load("cmake/3.20.1")
+
+prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/role.epic/hpc-stack/libs/intel-18.0.5.274/modulefiles/stack")
+load("hpc/1.2.0")
+load("hpc-intel/18.0.5.274")
+load("hpc-impi/2018.4.274")
+
+-- Load common modules for this package
+load("obsproc_common")
+
+setenv("FC", "mpiifort")
+
+whatis("Description: obsproc build environment")