diff --git a/docs/Release_Notes.txt b/docs/Release_Notes.txt index 342c973..fe694a4 100644 --- a/docs/Release_Notes.txt +++ b/docs/Release_Notes.txt @@ -1,3 +1,243 @@ +obsproc v1.2.0 RELEASED AUG ??,2023 + v1.2.? IMPLEMENTED ??? ??,2023 + +FILE CHANGES + + +D docs/ReleaseNotes.txt +M docs/Release_Notes.txt +M jobs/JOBSPROC_CDAS_PREP1 +M jobs/JOBSPROC_GLOBAL_DUMP_POST +M jobs/JOBSPROC_GLOBAL_PREP +M jobs/JOBSPROC_GLOBAL_PREP_POST +M jobs/JOBSPROC_RAP_PREP +A modulefiles/obsproc_jet.lua +M scripts/excdas_dump.sh +M scripts/exdump_monitor.sh +M scripts/exdump_post.sh +M scripts/exglobal_dump.sh +M scripts/exnam_dump.sh +M scripts/exrap_dump.sh +M scripts/exrtma_dump.sh +M scripts/exurma_dump.sh +M sorc/bufr_remorest.fd/remorest.f +M ush/build.sh +D versions/README +M versions/run.ver +M versions/VERSION +(M-modified; A-added; D-deleted) + +DOCS CHANGES + + docs/ReleaseNotes.txt + Delete repetitive file with similar name + + docs/Release_Notes.txt + Reflect this release changes. + +JOBS CHANGES + + jobs/JOBSPROC_CDAS_PREP1 + Increase errPREPDATA_limit in order to allow code to generate prepbufr file if/when one or more key files + (adpsfc, adpupa, uprair) are missing + + jobs/JOBSPROC_GLOBAL_DUMP_POST + Add functionality to unrestrict aircraft data (PROCESS_REMOREST_dm2) + Separate processing of uprair to its own dump group + + jobs/JOBSPROC_GLOBAL_PREP + Instruct to add uprair dump files data to GDAS and GFS prepbufr files + Increase NSPLIT for faster prep step + + jobs/JOBSPROC_GLOBAL_PREP_POST + Add functionality to unrestrict aircraft data (PROCESS_REMOREST_dm2) + Correct COMIN1 and COMINgdas patha + + jobs/JOBSPROC_RAP_PREP + Instruct to add uprair dump files data to RAP's prepbufr file + Increase NSPLIT for faster prep step + +MODULEFILES CHANGES + + modulefiles/obsproc_jet.lua + Add support for functionality on Jet + +SCRIPTS CHANGES + + scripts/excdas_dump.sh + Instruct NOT to dump uprair file for CDAS network (too slow) + Introduce msone1 group and redistribute group;add MAX_TANK* tank size check; Turn off msonet and msone1 for the time being. + + scripts/exdump_monitor.sh + Add MAX_TANK* tank size check + + scripts/exdump_post.sh + Add functionality to unrestrict aircraft data (PROCESS_REMOREST_dm2) + + scripts/exglobal_dump.sh + Split groups 1 and 10 and add an additional DUMP_group12 to allow code to run on one more cpu and faster + Dump BUFR formated high-res rawinsonde profiles into *uprair* dump files + Add DBNet dbn_alert for uprair and satwhr dump files going to NOMADS + Redistribute groups, introduce NAP and NAP_adpupa to allow earlier job/cron kickoff + + scripts/exrap_dump.sh + Dump BUFR formated high-res rawinsonde profiles into *uprair* dump files + Add DBNet dbn_alert for uprair dump files going to NOMADS + Introduce msone1 group and redistribute groups; add MAX_TANK* tank size check + Redistribute groups, introduce NAP to allow earlier job/cron kickoff + + scripts/exnam_dump.sh + Introduce msone1 group and redistribute groups/add cpus; add MAX_TANK* tank size check + + scripts/exrtma_dump.sh + Introduce msone1 group and redistribute groups + Introduce max size treshold for mesonet tanks;add MAX_TANK* tank size check + + scripts/exurma_dump.sh + Introduce msone1 group and redistribute groups + Introduce max size treshold for mesonet tanks;add MAX_TANK* tank size check + +SORC CHANGES + + sorc/bufr_remorest.fd/remorest.f + Add functionality to unrestrict aircraft data (PROCESS_REMOREST_dm2) + +USH CHANGES + + ush/build.sh + Add support for functionality on Jet + +VERSIONS CHANGES + + versions/README + Delete old notes + + versions/run.ver + Set bufr_dump_ver=1.2.0 and prepbufr_ver=1.1.0 + + versions/VERSION + Updated version to 1.2.0 + +ECF + + Contents of ECF directory are not EMC's responsibility + However,note that obsproc.v1.2.0 , bufr-dump,v1.2.0 and prepobs.v1.2.0 + will require more memory in the PBS card for: + gdas, gfs, cdas, rap runs (all 4 steps for each network) + See the example logs for : /lfs/h2/emc/stmp/iliana.genkova/CRON/R12/output + +CHANGES TO OUTPUT AND FILE SIZES + + Generate new dumps: + *satwhr* (gdas,gfs), *uprair* (gdas,gfs,cdas,rap) + Update dumps (gdas, gfs): + *prepbufr* (when this release is synced with bufr-dump.v1.2.0 and prepobs.v1.1.0) + Remove dumps (gdas, gfs): + *saphir* , *sevcsr* + + The newly generated dumps (*satwhr*, *uprair*) vary in size from 20Mb to 500Mb depending on network, cycle time, and season. + *uprair* will be largest for 00UTC and 12UTC. + *satwhr* will be largest during hurricane season. + + The updated *prepbur* files are up to ~20% larger in size + For examples, see: /lfs/h2/emc/stmp/iliana.genkova/CRON/R12/com + + Both *saphir* and *sevcsr* dumps have been removed as they are no longer available. + +CHANGES TO RESOURCES + + 1. + The gfs and gdas processing must start 10 min earlier than current processing. + The NAP(600s) and NAP_adpupa(800s) variables in the obsproc script exglobal_dump.sh compensate for the earlier start, + such that only "uprair" dump stars early. This approach will be revisited in future obsproc updates. + + The rap processing must start 2 min earlier than current processing. + The NAP(120s) variables in the obsproc scripts exrap_dump.sh compensate for the earlier start, + such that only "uprair" dump stars early. This approach will be revisited in future obsproc updates. + + Tests with this approach shows no loss of current opertaional obs data for gdas, gfs and rap. + The new uprair data will be captured and instrodyced with no loss for gdas and rap, and possibly minimal loss for gfs. + + 2. + Memory and CPU usage was expanded for many steps in the processing. + EMC tests ran with the following recomended configurations (keep the ncpus values!) + gdas/gfs dump: ncpus=14 mem=500gb + gdas/gfs dump_post: ncpus=14 mem=4gb + gdas/gfs prep: ncpus=14 mem=520gb + gdas/gfs prep_post: ncpus=14 mem=4gb + + rap dump: ncpus=11 mem=50gb + rap dump_post: ncpus=14 mem=6b + rap prep: ncpus=4 mem=100gb + rap prep_post: ncpus=1 mem=3gb + + cdas dump: ncpus=10 mem=200gb + cdas dump_post: ncpus=12 mem=300gb + cdas prep1: ncpus=12 mem=100gb + cdas prep2: ncpus=16 mem=120gb + cdas prep1_post: ncpus=1 mem=6gb + + nam dump: ncpus=7 mem=20gb + nam dump2: ncpus=3 mem=15gb + nam dump_post: ncpus=1 mem=3gb + nam prep: ncpus=4 mem=220gb + nam prep_post: ncpus=1 mem=500Mb + + rtma dump: ncpus=5 mem=40gb + rtma dump_post:ncpus=1 mem=5000MB + rtma prep: ncpus=1 mem=2gb + rtma prep_post:ncpus=1 mem=2gb + + urma dump: ncpus=6:mem=40gb + urma dump_post:ncpus=1:mem=6gb + urma prep: ncpus=1:mem=6gb + urma prep_post:ncpus=1:mem=2gb + + dump monitor: ncpus=15:mem=400GB + +PRE-IMPLEMENTATION TESTING REQUIREMENTS + + obsproc needs modules bufr-dump and prepobs in order to run + See docs/README.md for build instructions + See versions/build.ver and versions/run.ver for libaries + + Run jobs/JOBSPROC_networkname_DUMP and jobs/JOBSPROC_networkname_DUMP_POST + for the following networkname values: GLOBAL, NAM, RAP, URMA, RTMA, CDAS + + Parallel Production Testing (PPT) + After testing obsproc v1.2 in the /para space, we recommend doing for at least a month, + it is best to do a single cycle Parallel Production Testing (PPT) (i.e. obsproc + GFS + other models). + Copy the following directoris: + $model.YYYYMMDD, where model: gfs,gdas,nam,rap,rap_e,rap_p,urma,cdas,rtma,rtma_ru,dump + $model.YYYYMM, where model: cdas + $model.YYYY, where model:mods + $model, where:gdas, sdm_rtdm + from the para run: + /para/com/obsproc/v1.2/ + to + /prod/com/obsproc/v1.2/ for the PPT. + + Good to know: + - the RAP uprair dumps might be 0 size for the hours not around 00,06,12,18 UTC + - the CDAS prepbufr files are 6 different flavours, it takes about 26 hours for all to be generated + - only GDAS and GFS networks will generate *satwhr* + - only GDAS and GFS networks would support the unrestrictiom of aircraft data, and only on a DEV machine + - some *bufr_d files might be 0 size until the relevant data stream is recovered (airs, ombusv8) + - please contact obsproc_support@noaa.gov should you have any Qs regarding testing + +DISSEMINATION INFORMATION + + For NOMAD proposed changes, a SCN will be issued/submitted to Data Flow + +SPECIAL INSTRUCTIONS + + 1.Implement release/obsproc.v1.2.0, release/bufr-dump.v1.2.0, and release/prepobs.v1.1.0 at the same time + + 2.Implement release/obsproc.v1.2.0, release/bufr-dump.v1.2.0, and release/prepobs.v1.1.0 after confirming GFS global-workflow is ready for the new versions (first 2 digits) + + + +XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX obsproc v1.1.0 RELEASED AUG 18,2022 v1.1.2 IMPLEMENTED NOV 30,2022 diff --git a/jobs/JOBSPROC_CDAS_PREP1 b/jobs/JOBSPROC_CDAS_PREP1 index a4233e6..5072451 100755 --- a/jobs/JOBSPROC_CDAS_PREP1 +++ b/jobs/JOBSPROC_CDAS_PREP1 @@ -251,6 +251,13 @@ export NSPLIT=${NSPLIT:-12} export BUFRLIST=${BUFRLIST:-"adpupa proflr aircar aircft satwnd adpsfc \ sfcshp atovs"} +# I.G. 07/27/2023 +export errPREPDATA_limit=4 # allow program PREPOBS_PREPDATA to continue at all + # center hour (cycle) times if foreground exit + # status is 4 (i.e., when either either "ADPUPA" or + # "ADPSFC" dump is empty) + + # VARIABLES THAT CONTROL PROCESSING OF INDIVIDUAL PROGRAMS # -------------------------------------------------------- diff --git a/jobs/JOBSPROC_GLOBAL_DUMP_POST b/jobs/JOBSPROC_GLOBAL_DUMP_POST index 3fe7b27..9399bae 100755 --- a/jobs/JOBSPROC_GLOBAL_DUMP_POST +++ b/jobs/JOBSPROC_GLOBAL_DUMP_POST @@ -157,16 +157,25 @@ if [[ "$RUN_ENVIR" == nco ]]; then export COMIN=${COMIN:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT})} export COMOUT=${COMOUT:-$(compath.py -o ${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT})} mkdir -m 775 -p $COMOUT +# The following are used by $PROCESS_REMOREST_dm2 when it is YES +# -------------------------------------------------------------- + export COMINm2=${COMINm2:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT})} + export COMOUTm2=${COMOUTm2:-$(compath.py -o ${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT})} else export COMIN=${COMIN:-${COMIN_ROOT:?}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT}} - # COMOUT_ROOT for developers defaults to unique $DATA/com directory as root to +# COMOUT_ROOT for developers defaults to unique $DATA/com directory as root to # prevent job from writing into a non-unique directory that might already have # output in it from a previous checkout run for the same cycle # ----------------------------------------------------------------------------- export COMOUT_ROOT=${COMOUT_ROOT:-${DATA}/com} export COMOUT=${COMOUT:-${COMOUT_ROOT}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT}} mkdir -m 755 -p $COMOUT +# The following are used by $PROCESS_REMOREST_dm2 when it is YES +# -------------------------------------------------------------- + export COMINm2=${COMINm2:-${COMIN_ROOT:?}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT}} + export COMOUTm2=${COMOUTm2:-${COMOUT_ROOT}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT}} + mkdir -m 755 -p $COMOUTm2 fi # comin and comin_m1 are paths to dump status files for bufr_avgdata.sh diff --git a/jobs/JOBSPROC_GLOBAL_PREP b/jobs/JOBSPROC_GLOBAL_PREP index d0275b7..f8b6819 100755 --- a/jobs/JOBSPROC_GLOBAL_PREP +++ b/jobs/JOBSPROC_GLOBAL_PREP @@ -233,11 +233,12 @@ export POE=${POE:-YES} # (Applies only when either POE=YES or BACK=YES, and PREPDATA=YES) # Consider the size of the backgrounds fields if using threads. -export NSPLIT=${NSPLIT:-4} +#IG export NSPLIT=${NSPLIT:-4} +export NSPLIT=${NSPLIT:-12} # ${NSPLIT:-8} #16 adds time to MERGE step :( # BUFRLIST indicates the list of BUFR data dump file names to process -export BUFRLIST=${BUFRLIST:-"adpupa proflr aircar aircft satwnd adpsfc \ +export BUFRLIST=${BUFRLIST:-"uprair proflr aircar aircft satwnd adpupa adpsfc \ sfcshp vadwnd wdsatr ascatw rassda gpsipw"} export errPREPDATA_limit=4 # allow program PREPOBS_PREPDATA to continue at all diff --git a/jobs/JOBSPROC_GLOBAL_PREP_POST b/jobs/JOBSPROC_GLOBAL_PREP_POST index fe0c885..626cb6f 100755 --- a/jobs/JOBSPROC_GLOBAL_PREP_POST +++ b/jobs/JOBSPROC_GLOBAL_PREP_POST @@ -151,6 +151,10 @@ if [[ "$RUN_ENVIR" == nco ]]; then export COMIN=${COMIN:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT})} export COMOUT=${COMOUT:-$(compath.py -o ${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT})} mkdir -m 775 -p $COMOUT +# The following are used by $PROCESS_REMOREST_dm2 when it is YES +# -------------------------------------------------------------- + export COMINm2=${COMINm2:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT})} + export COMOUTm2=${COMOUTm2:-$(compath.py -o ${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT})} else export COMIN=${COMIN:-${COMIN_ROOT:?}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT}} @@ -161,12 +165,17 @@ else export COMOUT_ROOT=${COMOUT_ROOT:-${DATA}/com} export COMOUT=${COMOUT:-${COMOUT_ROOT}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT}} mkdir -m 755 -p $COMOUT +# The following are used by $PROCESS_REMOREST_dm2 when it is YES +# -------------------------------------------------------------- + export COMINm2=${COMINm2:-${COMIN_ROOT:?}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT}} + export COMOUTm2=${COMOUTm2:-${COMOUT_ROOT}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT}} + mkdir -m 755 -p $COMOUTm2 fi # The following are used by $PROCESS_TIMETWINS in the GDAS when it is YES # ----------------------------------------------------------------------- -export COMINgdas=${COMINgfs:-$(compath.py ${envir}/${NET}/${gfs_ver})/${RUN}.} -export COMINGFS=${COMINgfs:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${NET}.${PDY}/${cyc}/${COMPONENT})} +export COMINgdas=${COMINgdas:-$(compath.py ${envir}/${NET}/${gfs_ver})/${RUN}.} +export COMINGFS=${COMINGFS:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${NET}.${PDY}/${cyc}/${COMPONENT})} #export COMIN1=${COMIN1:-$(compath.py ${envir}/${NET}/${gfs_ver})/${RUN}.} export COMIN1=${COMIN1:-$(compath.py ${envir}/${obsNET}/${obsproc_ver})/${RUN}.} #twinwind file now write in obsproc package diff --git a/jobs/JOBSPROC_RAP_PREP b/jobs/JOBSPROC_RAP_PREP index fc0241c..75d010a 100755 --- a/jobs/JOBSPROC_RAP_PREP +++ b/jobs/JOBSPROC_RAP_PREP @@ -221,11 +221,11 @@ export POE=${POE:-NO} # (Applies only when either POE=YES or BACK=YES, and PREPDATA=YES) # Consider the size of the backgrounds fields if using threads. -export NSPLIT=${NSPLIT:-1} +export NSPLIT=${NSPLIT:-4} # BUFRLIST indicates the list of BUFR data dump file names to process -export BUFRLIST=${BUFRLIST:-"adpupa proflr aircar aircft satwnd adpsfc \ +export BUFRLIST=${BUFRLIST:-"uprair adpupa proflr aircar aircft satwnd adpsfc \ sfcshp vadwnd goesnd msonet gpsipw rassda ascatw wdsatr"} export errPREPDATA_limit=4 # allow program PREPOBS_PREPDATA to continue at all diff --git a/modulefiles/obsproc_jet.lua b/modulefiles/obsproc_jet.lua new file mode 100644 index 0000000..bab1f08 --- /dev/null +++ b/modulefiles/obsproc_jet.lua @@ -0,0 +1,17 @@ +help([[ +Load environment to build obsproc on Jet +]]) + +load("cmake/3.20.1") + +prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/role.epic/hpc-stack/libs/intel-18.0.5.274/modulefiles/stack") +load("hpc/1.2.0") +load("hpc-intel/18.0.5.274") +load("hpc-impi/2018.4.274") + +-- Load common modules for this package +load("obsproc_common") + +setenv("FC", "mpiifort") + +whatis("Description: obsproc build environment") diff --git a/scripts/excdas_dump.sh b/scripts/excdas_dump.sh index e58d1ea..72173e6 100755 --- a/scripts/excdas_dump.sh +++ b/scripts/excdas_dump.sh @@ -1,4 +1,5 @@ #!/bin/ksh +# ########################################################################### echo "--------------------------------------------------------------------" echo "excdas_dump.sh - CDAS network data dump processing " @@ -40,6 +41,11 @@ echo " Mar 08 2022 - Enable the dumping of 002017 in vadwnd dump " echo " group. " echo " Jul 30 2022 - Subpfl, saldrn, snocvr, and gmi1cr added " echo " to dump group #9. " +echo " Sep 30 2022 - Don't / Enable dumping of UPRAIR data in " +echo " group #3./ b/c it is too slow " +echo " Oct 17 2023 - Split msonet to msonet (#5) and msone1 (#10) " +echo " concatenate msonet and msone1 right after dump" +echo " Turn off msonet and msone1 - not needed " ########################################################################### set -xau @@ -62,7 +68,7 @@ set +u # # Dump group #3 (pb, TIME_TRIM = OFF) = # adpupa gpsipw ascatt -# +# # pull out uprair, too slow # Dump group #4 (pb, TIME_TRIM = default = ON) = # aircar aircft proflr vadwnd rassda # @@ -81,7 +87,10 @@ set +u # Dump group #9 (non-pb, TIME_TRIM = default = ON) = # geoimr subpfl saldrn snocvr gmi1cr # -# Dump group #10 STATUS FILE +# Dump group #10 (pb, TIME_TRIM = OFF) = +# msone1 # ONLY tank b255/xx030, the largest +# +# Dump group #11 STATUS FILE # ----------------------------------------------------------------------------- #VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV @@ -108,17 +117,21 @@ set -u DUMP_group7=${DUMP_group7:-"YES"} DUMP_group8=${DUMP_group8:-"NO"} DUMP_group9=${DUMP_group9:-"YES"} + DUMP_group10=${DUMP_group10:="NO"} else dump_ind=DUMP DUMP_group1=${DUMP_group1:-"NO"} DUMP_group2=${DUMP_group2:-"YES"} DUMP_group3=${DUMP_group3:-"YES"} DUMP_group4=${DUMP_group4:-"YES"} - DUMP_group5=${DUMP_group5:-"YES"} + #DUMP_group5=${DUMP_group5:-"YES"} + DUMP_group5=${DUMP_group5:-"NO"} DUMP_group6=${DUMP_group6:-"NO"} DUMP_group7=${DUMP_group7:-"NO"} DUMP_group8=${DUMP_group8:-"YES"} DUMP_group9=${DUMP_group9:-"NO"} + #DUMP_group10=${DUMP_group10:="YES"} + DUMP_group10=${DUMP_group10:="NO"} fi else dump_ind=DUMP @@ -126,11 +139,14 @@ else DUMP_group2=${DUMP_group2:-"YES"} DUMP_group3=${DUMP_group3:-"YES"} DUMP_group4=${DUMP_group4:-"YES"} - DUMP_group5=${DUMP_group5:-"YES"} + #DUMP_group5=${DUMP_group5:-"YES"} + DUMP_group5=${DUMP_group5:-"NO"} DUMP_group6=${DUMP_group6:-"YES"} DUMP_group7=${DUMP_group7:-"YES"} DUMP_group8=${DUMP_group8:-"YES"} DUMP_group9=${DUMP_group9:-"YES"} + #DUMP_group10=${DUMP_group10:="YES"} + DUMP_group10=${DUMP_group10:="NO"} fi @@ -293,6 +309,7 @@ echo "=======> Dump group 6 (thread_6) not executed." > $DATA/6.out echo "=======> Dump group 7 (thread_7) not executed." > $DATA/7.out echo "=======> Dump group 8 (thread_8) not executed." > $DATA/8.out echo "=======> Dump group 9 (thread_9) not executed." > $DATA/9.out +echo "=======> Dump group 10 (thread_10) not executed." > $DATA/10.out err1=0 err2=0 @@ -303,6 +320,33 @@ err6=0 err7=0 err8=0 err9=0 +err10=0 + +#restrict processing of unexpected big tanks +#this block appear in all /scripts/ex*_dump.sh proessing msonet and msone1 +TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb +TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb +TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb +if [ -s ${TANK}/${PDY}/b255/xx003 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then + export SKIP_255003=YES + msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov +fi +if [ -s ${TANK}/${PDY}/b255/xx004 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then + export SKIP_255004=YES + msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov + fi +if [ -s ${TANK}/${PDY}/b255/xx030 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then + export SKIP_255030=YES + msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov +fi +#end of block + if [ "$PROCESS_DUMP" = 'YES' ]; then #################################### @@ -497,21 +541,24 @@ export DUMP_NUMBER=3 # #-------------------------------------------------------------------------- # Dump #3: ADPUPA: 6 subtype(s) +# #UPRAIR: 5 subtype(s) - pull out, too slow # GPSIPW: 1 subtype(s) # ASCATT: 1 subtype(s) # --------------------- -# TOTAL NUMBER OF SUBTYPES = 8 +# TOTAL NUMBER OF SUBTYPES = 13 # #==================================================================== # other dump types # ---------------- DTIM_latest_adpupa=${DTIM_latest_adpupa:-"+2.99"} +DTIM_latest_uprair=${DTIM_latest_uprair:-"+2.99"} DTIM_latest_gpsipw=${DTIM_latest_gpsipw:-"+2.99"} DTIM_latest_ascatt=${DTIM_latest_ascatt:-"+2.99"} TIME_TRIM=off +# $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 adpupa uprair gpsipw ascatt $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 adpupa gpsipw ascatt error3=$? echo "$error3" > $DATA/error3 @@ -639,12 +686,17 @@ export DUMP_NUMBER=5 # #=================================================================== -DTIM_latest_msonet=${DTIM_latest_msonet:-"+2.99"} +#IG +DTIM_earliest_msonet=${DTIM_latest_msonet:-"-1.99"} +DTIM_latest_msonet=${DTIM_latest_msonet:-"+2.00"} + +#DTIM_latest_msonet=${DTIM_latest_msonet:-"+2.99"} export SKIP_255031=YES # Skip for port to Dell since no new data allowed. export SKIP_255101=YES # Also, b/c CDAS has not tested these providers. -TIME_TRIM=off +TIME_TRIM=on +#TIME_TRIM=off $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 msonet error5=$? @@ -1054,6 +1106,57 @@ set -x EOF set -x +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_10; chmod +x thread_10 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_10 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=10 + +#=================================================================== +# NOTES ABOUT THIS DUMP GROUP: +# (1) time window radius is -3.00 to +2.99 hours on all types +# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP +# +#-------------------------------------------------------------------------- +# Dump # 10 : MSONET: 1 subtype(s) 255/030 +# ------------------------ +# TOTAL NUMBER OF SUBTYPES = 1 +# +#=================================================================== + +DTIM_earliest_msone1=${DTIM_latest_msone1:-"-1.99"} +DTIM_latest_msone1=${DTIM_latest_msone1:-"+2.00"} + +TIME_TRIM=on #off + +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 msone1 +error10=$? +echo "$error10" > $DATA/error10 + +set +x +echo "********************************************************************" +echo Script thread_10 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/10.out 2>&1 +EOF +set -x #---------------------------------------------------------------- # Now launch the threads @@ -1079,12 +1182,13 @@ if [ "$launcher" = cfp ]; then [ $DUMP_group4 = YES ] && echo ./thread_4 >> $DATA/poe.cmdfile [ $DUMP_group7 = YES ] && echo ./thread_7 >> $DATA/poe.cmdfile [ $DUMP_group9 = YES ] && echo ./thread_9 >> $DATA/poe.cmdfile + [ $DUMP_group10 = YES ] && echo ./thread_10 >> $DATA/poe.cmdfile #msone1 if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes # ?? launcher_DUMP=${launcher_DUMP:-mpiexec} - #$launcher_DUMP -np 8 --cpu-bind verbose,core cfp $DATA/poe.cmdfile - NPROCS=${NPROCS:-8} + #$launcher_DUMP -np 10 --cpu-bind verbose,core cfp $DATA/poe.cmdfile + NPROCS=${NPROCS:-10} $launcher_DUMP -np $NPROCS --cpu-bind verbose,core cfp $DATA/poe.cmdfile errpoe=$? if [ $errpoe -ne 0 ]; then @@ -1106,9 +1210,10 @@ else [ $DUMP_group7 = YES ] && ./thread_7 [ $DUMP_group8 = YES ] && ./thread_8 [ $DUMP_group9 = YES ] && ./thread_9 + [ $DUMP_group10 = YES ] && ./thread_10 fi -cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out $DATA/7.out $DATA/8.out $DATA/9.out +cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.out set +x echo " " @@ -1124,11 +1229,11 @@ set -x [ -s $DATA/error7 ] && err7=`cat $DATA/error7` [ -s $DATA/error8 ] && err8=`cat $DATA/error8` [ -s $DATA/error9 ] && err9=`cat $DATA/error9` - +[ -s $DATA/error10 ] && err10=`cat $DATA/error10` #=============================================================================== export STATUS=YES -export DUMP_NUMBER=10 +export DUMP_NUMBER=11 $ushscript_dump/bufr_dump_obs.sh $dumptime 3.00 1 null if [ "$SENDCOM" = 'YES' -a "$COPY_TO_ARKV" = 'YES' ]; then @@ -1166,12 +1271,14 @@ echo " " >> $pgmout #================================================================ + if [ "$PROCESS_DUMP" = 'YES' ]; then if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o \ "$err4" -gt '5' -o "$err5" -gt '5' -o "$err6" -gt '5' -o \ - "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' ]; then - for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 + "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' -o \ + "$err10" -gt '5' ]; then + for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 do if [ "$n" -gt '5' ]; then if [ "$n" -ne '11' -a "$n" -ne '22' ]; then @@ -1182,7 +1289,7 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then echo echo " ###################################################### " echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7, $err8, $err9 " +$err5, $err6, $err7, $err8, $err9 $err10" echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " echo " ###################################################### " echo @@ -1200,7 +1307,7 @@ echo echo echo " ###################################################### " echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7, $err8, $err9 " +$err5, $err6, $err7, $err8, $err9 $err10 " echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " echo " ###################################################### " echo @@ -1210,6 +1317,9 @@ $err5, $err6, $err7, $err8, $err9 " # endif loop $PROCESS_DUMP fi +## concatenate msonet and msone1, b/c prepobs only wants one file +#cat ${COMSP}msone1.tm00.bufr_d >> ${COMSP}msonet.tm00.bufr_d + # # copy bufr_dumplist to $COMOUT per NCO SPA request # ------------------------------------------------- diff --git a/scripts/exdump_monitor.sh b/scripts/exdump_monitor.sh index 46123f6..d18df26 100755 --- a/scripts/exdump_monitor.sh +++ b/scripts/exdump_monitor.sh @@ -117,8 +117,35 @@ err5=0 err6=0 err7=0 err8=0 -if [ "$PROCESS_DUMP" = 'YES' ]; then +err9=0 +err10=0 + +#restrict processing of unexpected big tanks +#this block appear in all /scripts/ex*_dump.sh proessing msonet and msone1 +TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb +TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb +TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb +if [ -s ${TANK}/${PDY}/b255/xx003 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then + export SKIP_255003=YES + msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov +fi +if [ -s ${TANK}/${PDY}/b255/xx004 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then + export SKIP_255004=YES + msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov + fi +if [ -s ${TANK}/${PDY}/b255/xx030 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then + export SKIP_255030=YES + msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov +fi +#end of block +if [ "$PROCESS_DUMP" = 'YES' ]; then ########################### ########################### # The data "dump" script @@ -913,6 +940,94 @@ set -x EOF set -x +#-------NEW MESON1 + +set +x +cat<<\EOF>thread_9; chmod +x thread_9 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_9 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=9 + +#=================================================================== +# Dump # 9 : MSONE1 +# (1) +# -- TOTAL NUMBER OF SUBTYPES = 1 +# time window radius is -0.50 to +0.49 hours on MSONET +#=================================================================== +DTIM_latest_msone1=+0.49 +DTIM_earliest_msone1=-1.00 + +$ushscript_dump/bufr_dump_obs.sh $dumptime 0.50 1 msone1 +error9=$? +echo "$error9" > $DATA/error9 + +set +x +echo "********************************************************************" +echo Script thread_9 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/9.out 2>&1 +EOF +set -x + +#-------NEW UPRAIR + +set +x +cat<<\EOF>thread_10; chmod +x thread_10 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_10 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=10 + +#=================================================================== +# Dump # 10: UPRAIR +# (1) +# -- TOTAL NUMBER OF SUBTYPES = 1 +#=================================================================== +DTIM_latest_uprair=+0.49 +DTIM_earliest_uprair=-0.49 + +$ushscript_dump/bufr_dump_obs.sh $dumptime 0.50 1 uprair +error10=$? +echo "$error10" > $DATA/error10 + +set +x +echo "********************************************************************" +echo Script thread_10 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/10.out 2>&1 +EOF +set -x #---------------------------------------------------------------- @@ -930,11 +1045,14 @@ if [ "$launcher" = cfp ]; then echo ./thread_8 >> $DATA/poe.cmdfile # moved up echo ./thread_1 >> $DATA/poe.cmdfile echo ./thread_4 >> $DATA/poe.cmdfile - + echo ./thread_9 >> $DATA/poe.cmdfile + echo ./thread_10 >> $DATA/poe.cmdfile + if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes # ?? launcher_DUMP=${launcher_DUMP:-mpiexec} - $launcher_DUMP -np 7 --cpu-bind verbose,core cfp $DATA/poe.cmdfile + NPROCS=${NPROCS:-10} + $launcher_DUMP -np $NPROCS --cpu-bind verbose,core cfp $DATA/poe.cmdfile errpoe=$? if [ $errpoe -ne 0 ]; then $DATA/err_exit "***FATAL: EXIT STATUS $errpoe RUNNING POE COMMAND FILE" @@ -954,9 +1072,11 @@ else ./thread_6 ./thread_7 ./thread_8 + ./thread_9 + ./thread_10 fi cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out \ - $DATA/7.out $DATA/8.out + $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.out set +x echo " " @@ -971,12 +1091,13 @@ err5=`cat $DATA/error5` err6=`cat $DATA/error6` err7=`cat $DATA/error7` err8=`cat $DATA/error8` - +err9=`cat $DATA/error9` +err10=`cat $DATA/error10` #=============================================================================== export STATUS=YES -export DUMP_NUMBER=9 +export DUMP_NUMBER=11 $ushscript_dump/bufr_dump_obs.sh $dumptime 0.50 1 null @@ -1009,8 +1130,9 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o \ "$err4" -gt '5' -o "$err5" -gt '5' -o "$err6" -gt '5' -o \ - "$err7" -gt '5' -o "$err8" -gt '5' ]; then - for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 + "$err7" -gt '5' -o "$err8" -gt '5' -o \ + "$err9" -gt '5' -o "$err10" -gt '5' ]; then + for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 do if [ "$n" -gt '5' ]; then if [ "$n" -ne '11' -a "$n" -ne '22' ]; then @@ -1021,7 +1143,7 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then echo echo " ###################################################### " echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7 $err8 " +$err5, $err6, $err7 $err8 $err9 $err10" echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " echo " ###################################################### " echo @@ -1039,7 +1161,7 @@ echo echo echo " ###################################################### " echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7 $err8 " +$err5, $err6, $err7 $err8 $err9 $err10 " echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " echo " ###################################################### " echo diff --git a/scripts/exdump_post.sh b/scripts/exdump_post.sh index c014a24..c1e9a45 100755 --- a/scripts/exdump_post.sh +++ b/scripts/exdump_post.sh @@ -4,11 +4,14 @@ echo "----------------------------------------------------------" echo "exdump_post.sh version $obsproc_ver - If requested: " echo " 1) Generates combined dump STATUS file " echo " 2) Prepares data counts for the SDM " -echo " 3) Removes or masks restricted data from dump files" -echo " 4) Converts dump files to unblocked format " +echo " 3) Removes or masks restricted data from today's " +echo " dump files " +echo " 4) Removes restricted data from 2-day old aircar " +echo " and aircft dump files " +echo " 5) Converts dump files to unblocked format " echo " (meaningless on WCOSS) " -echo " 5) Lists the contents of dump files " -echo " 6) Updates dump data count average tables " +echo " 6) Lists the contents of dump files " +echo " 7) Updates dump data count average tables " echo "----------------------------------------------------------" ##################################################################### # @@ -59,6 +62,8 @@ echo "----------------------------------------------------------" # whose reports are retained and instead station IDs # are masked. # 09 Dec 2021 Esposito - Updated for use on WCOSS2. +# ?? ??? 202? Keyser - Added new "remorest" processing to remove restricted +# data from 2-day old "aircar" and "aircft" dump files. ##################################################################### # NOTE: NET is changed to gdas in the parent Job script for the gdas RUN @@ -81,6 +86,7 @@ SITE=${SITE:-""} cat break > $pgmout +PROCESS_REMOREST_dm2=${PROCESS_REMOREST_dm2:-NO} MPMD=${MPMD:-YES} CHGRP_RSTPROD=${CHGRP_RSTPROD:-YES} UPDATE_AVERAGE_FILE=${UPDATE_AVERAGE_FILE:-YES} @@ -204,25 +210,60 @@ fi if [ "$PROCESS_REMOREST" = 'YES' ]; then +######################################################################## +# Remove or Mask Restricted Data from today's BUFR Data Dump Files # +######################################################################## + +# For dump files "adpsfc", "aircar", "aircft", "msonet", "lghtng" and "sfcshp" +# for today (this network, run and cycle): +# If either the restriction indicator (mnemonic RSRD) is set for a report, or +# the report is in a Table A entry pre-determined to contain only restricted +# reports, then the report is always restricted {regardless of the time in +# hours for the expiration on restriction (mnemonic EXPRSRD)}. +# ---> Since we are running in near-realtime, there is no need to test on +# the value of EXPRSRD. This will always be > current time difference. +# ---> For all Table A entries listed in parm card namelist switch +# "MSG_RESTR" (see below): +# All reports are considered to be restricted by definition of this +# switch. They will be skipped by program BUFR_REMOREST. +# ---> For all Table A entries listed in parm card namelist switch +# "MSG_MIXED" (see below): +# RSRD may not be set for all reports. Those with RSRD set are +# considered to be restricted regardless of EXPRSRD because +# "DIFF_HR" (the difference in hours between the current UTC wall- +# clock date and the dump file center date),is exported as 0 into +# program BUFR_REMOREST (EXPRSRD will always be > actual DIFF_HR +# since we are running in near-realtime). They will be skipped by +# program BUFR_REMOREST. Those reports that do not have RSRD set +# are considered to be non-restricted and will be retained by +# program BUFR_REMOREST. +# ---> For all Table A entries listed in parm card namelist switch +# "MSG_MASKA" (see below): +# All reports are considered to be restricted by definition of this +# switch. Reports are not removed but all occurrences of their +# report ids are unilaterally masked out by program BUFR_REMOREST. +# In addition, program BUFR_REMOREST re-sets their values for RSRD +# and EXPRSRD to missing so that the reports are no longer +# considered to be restricted. +# ----------------------------------------------------------------------------- + msg="REMOVE OR MASK RESTRICTED DATA FROM $tmmark_uc $net_uc DATA DUMPS \ CENTERED ON $dumptime" $DATA/postmsg "$jlogfile" "$msg" -######################################################################## -# Remove or Mask Restricted Data from BUFR Data Dump Files # -######################################################################## + export DIFF_HR=0 cat <<\EOFparm > bufr_remorest.datadump.parm ========================================================================= - Cards for DUMP Version of BUFR_REMOREST -- Version 18 November 2008 + Cards for DUMP Version of BUFR_REMOREST -- Version 18 November 202? &SWITCHES - MSG_RESTR = 'NC000000', ! These are the Table A Entries for - 'NC000100', ! BUFR messages for which ALL reports - 'NC004003', ! are RESTRICTED and will be REMOVED. - 'NC004004', ! (up to 20) - 'NC004006', + MSG_RESTR = 'NC000000', ! These are the Table A Entries for BUFR messages + 'NC000100', ! for which ALL reports are considered to be + 'NC004003', ! restricted. All reports will be REMOVED by + 'NC004004', ! program BUFR_REMOREST. + 'NC004006', ! (up to 20 Tabel A entries) 'NC004007', 'NC004008', 'NC004009', @@ -234,56 +275,64 @@ cat <<\EOFparm > bufr_remorest.datadump.parm 'NC007001', 'NC007002', 'NC021242', - 'NC255xxx', - MSG_MIXED = 'NC012004', ! These are the Table A Entries for - 'NC003010', ! BUFR messages which contain a MIXTURE - ' ', ! of restricted and unrestricted - ' ', ! reports (based on mnemonic "RSRD"). All - ' ', ! restricted reports will be REMOVED. - ' ', ! (up to 20) + 'NC255xxx', ! move this to MSG_MIXED when we know for sure that RSRD values are correct for each provider/sub-provider + MSG_MIXED = 'NC012004', ! These are the Table A Entries for BUFR messages + 'NC003010', ! which may contain a MIXTURE of reports with and + ' ', ! without mnemonic "RSRD" being set. If "RSRD" is + ' ', ! not set -or- it is set and mnemonic "EXPRSRD" + ' ', ! is also set and has a value less than "DIFF_HR" + ' ', ! (the difference in hrs between the current UTC + ' ', ! wall-clock date and the dump file center date) + ' ', ! minus 4, the report will be RETAINED by program + ' ', ! BUFR_REMOREST. Otherwise, it will be REMOVED by + ' ', ! program BUFR_REMOREST. + ' ', ! (up to 20 Tabel A entries) ' ', - MSG_MASKA = 'NC001001', ! These are the Table A Entries for - 'NC001101' ! BUFR messages for which ALL reports - ! are RESTRICTED. They will not be removed, - ! but all occurrences of their report ids will be - ! unilaterally changed to either "MASKSTID" - ! (where the id is stored by itself) or to all - ! "X"'s where the number of "X"'s corresponds to - ! the number of characters in the original report - ! id (where the id is embedded in the raw report - ! bulletin header string) - ! (up to 20) + MSG_MASKA = 'NC001001', ! These are the Table A Entries for BUFR messages + 'NC001101' ! for which ALL reports are considered to be + ! restricted. They will not be removed, but all + ! occurrences of their report ids will be + ! unilaterally CHANGED, by program BUFR_REMOREST, + ! to either "MASKSTID" (where the id is stored by + ! itself) or to all "X"'s where the number of + ! "X"'s corresponds to the number of characters + ! in the original report id (where the id is + ! embedded in the raw report bulletin header + ! string). In addition, program BUFR_REMOREST re- + ! sets their values for "RSRD" and "EXPRSRD" to + ! missing so that the reports are no longer + ! considered to be restricted. + ! (up to 20 Table A entries) / - Note 1: A particular Table A entry should NEVER appear in more than one - of MSG_RESTR, MSG_MIXED or MSG_MASKA. + Note 1: A particular Table A entry should NEVER appear in more than one of + MSG_RESTR, MSG_MIXED or MSG_MASKA. Note 2: Any Table A entry not in either MSG_RESTR, MSG_MIXED or MSG_MASKA - is assumed to be a Table A entry for BUFR messages for which - ALL reports are UNRESTRICTED (these messages are copied - intact, no reports are unpacked). - Note 3: Always fill in these arrays beginning with word 1. If - there are less than 20 words filled in an array, either set - the extra words to " " (8 blank characters) or do not - specify them here (they default to " "). - Note 4: For data dump Table A entries in the form "NCtttsss", - where "ttt" is the BUFR message type and "sss" is the - BUFR message subtype, if the last three characters (the - subtype) is specified as 'xxx', then ALL BUFR messages - of that type are either treated as having all restricted - data all which is to be removed (if in MSG_RESTR), mixed data - some of which is to be removed (if in MSG_MIXED) or all restricted - data all of which is to have its report id masked (if in - MSG_MASKA), regardless of the message subtype. (For example, - if MSG_RESTR(1)='NC255xxx', then ALL mesonet BUFR messages are + is assumed to be a Table A entry for BUFR messages for which ALL + reports are NON-RESTRICTED (these messages are copied intact, no + reports are unpacked). + Note 3: Always fill in these arrays beginning with word 1. If there are + less than 20 words filled in an array, either set the extra words + to " " (8 blank characters) or do not specify them here + (they default to " "). + Note 4: For data dump Table A entries in the form "NCtttsss", where "ttt" + is the BUFR message type and "sss" is the BUFR message subtype, if + the last three characters (the subtype) is specified as 'xxx', then + ALL BUFR messages of that type are either treated as having all + restricted data all which is to be removed (if in MSG_RESTR), mixed + data some of which is to be removed (if in MSG_MIXED) or all + restricted data all of which is to have its report id masked (if in + MSG_MASKA), regardless of the message subtype. (For example, if + MSG_RESTR(1)='NC255xxx', then ALL mesonet BUFR messages are considered to have all restricted data and are all removed regardless of their subtype.) ========================================================================= EOFparm -REMX=${REMX:-$EXECobsproc/bufr_remorest} -REMC=${REMC:-bufr_remorest.datadump.parm} + REMX=${REMX:-$EXECobsproc/bufr_remorest} + REMC=${REMC:-bufr_remorest.datadump.parm} for file in adpsfc aircar aircft msonet sfcshp lghtng gpsipw saphir gpsro do @@ -296,22 +345,22 @@ REMC=${REMC:-bufr_remorest.datadump.parm} rc=$? if [ $rc -gt 0 ] ; then [ $rc -gt $retcode ] && retcode=$rc - msg="**WARNING: ERROR generating unrestricted $file BUFR file \ + msg="**WARNING: ERROR generating non-restricted $file BUFR file \ (rc = $rc)" $DATA/postmsg "$jlogfile" "$msg" else - msg="Successful generation of unrestricted $file BUFR file" + msg="Successful generation of non-restricted $file BUFR file" $DATA/postmsg "$jlogfile" "$msg" cp $filestem $COMOUT/$filestem.nr chmod 664 $COMOUT/$filestem.nr - if [ "$SENDDBN" = "YES" ] ; then + if [ "$SENDDBN" = "YES" ] ; then NETUP=`echo $RUN | tr {a-z} {A-Z}` if [[ $NETUP != 'GDAS' ]] || [[ $file != "saphir" ]]; then ### no alert gdas.tCCz.saphir.tm00.bufr_d.nr if [[ $NETUP != 'CDAS' ]] || [[ $file != "gpsro" ]]; then ### no alert cdas.tCCz.gpsro.tm00.bufr_d.nr if [[ $NETUP != 'RAP' ]] || [[ $file != "gpsro" ]]; then ### no alert rap.tCCz.gpsro.tm00.bufr_d.nr $DBNROOT/bin/dbn_alert MODEL ${NETUP}_BUFR_${file}_nr $job \ $COMOUT/$filestem.nr - fi + fi fi fi fi @@ -319,9 +368,116 @@ REMC=${REMC:-bufr_remorest.datadump.parm} done -# endif loop $PROCESS_REMOREST -fi +fi # endif loop $PROCESS_REMOREST + + +# PROCESS_REMOREST_dm2 = YES runs only on demand, on dev machines +# Unrestrict 48h old aircraft obs(exc. TAMDAR), add *.ur file to $COMOUTm2 +dev_m=$(grep backup /lfs/h1/ops/prod/config/prodmachinefile | cut -d: -f2) +this_m=$(cat /etc/cluster_name) + +if [ $PROCESS_REMOREST_dm2 = YES -a $this_m = $dev_m ]; then +######################################################################## +# Remove Restriction on Data in 2-day old "aircar" and "aircft" # +# BUFR Data Dump files # +######################################################################## +# For dump files "aircar" and "aircft" for 2-days ago (this network, run and +# cycle): +# ---> For all Table A entries listed in parm card namelist switch +# "MSG_MIXED" (see below): +# If the restriction indicator (mnemonic RSRD) is set for a report, +# then its value for time in hours for the expiration on restriction +# (mnemonic EXPRSRD) is examined. If "DIFF_HR" (the difference in +# hours between the current UTC wall-clock date and the dump file +# center date, exported into program BUFR_REMOREST) minus 4 is less +# than or equal to EXPRSRD, then the report is considered to be +# restricted and will be skipped by program BUFR_REMOREST. +# Otherwise, the report is considered to be non-restricted and will +# be retained by program BUFR_REMOREST. +# ---> Since we are running 2-days late, we want to test on the value of +# EXPRSRD (which currently should be set to 48 hours for any reports in +# these dumps which have RSRD set). This will allow non-rstprod users +# to have access to full "aircar" and "aircft" dumps after EXPRSRD + 4 +# (52) hours. +# ----------------------------------------------------------------------------- + + dumptimeM2=`$NDATE -$tmhr $PDYm2$cyc` + + msg="REMOVE OR MASK RESTRICTED DATA FROM $tmmark_uc $net_uc DATA DUMPS \ +CENTERED ON $dumptimeM2 (2-days ago)" + $DATA/postmsg "$jlogfile" "$msg" + + ymdh=$(date -u +'%Y%m%d%H') + export DIFF_HR=`$NHOUR $ymdh $dumptimeM2` +# current_date dump_date + + msg="Any reports with EXPRSRD less than `expr $DIFF_HR - 4 ` hrs will now \ +be retained" + $DATA/postmsg "$jlogfile" "$msg" + +cat <<\EOF_EXPRSRDparm > bufr_remorest.datadump_EXPRSRD.parm +========================================================================= + + Cards for DUMP Version of BUFR_REMOREST -- Version 1.1.0 (09 September 2015) + (documentation in bufr_remorest.datadump.parm above for $PROCESS_REMOREST + also applies here) + + &SWITCHES + MSG_RESTR = ' ', + MSG_MIXED = 'NC004003', + 'NC004004', + 'NC004006', + 'NC004007', + 'NC004008', + 'NC004009', + 'NC004010', + 'NC004011', + 'NC004012', + 'NC004013', + 'NC004014', + 'NC004015', + 'NC004103', + MSG_MASKA = ' ' + + / + +========================================================================= +EOF_EXPRSRDparm + + REMX=${REMX:-$EXECobsproc_shared_bufr_remorest/bufr_remorest} + REMC=${REMC_EXPRSRD:-bufr_remorest.datadump_EXPRSRD.parm} + + for file in aircar aircft + do + filestem=$RUN.$cycle.$file.$tmmark.bufr_d + [ -f $COMINm2/$filestem ] || continue + + cp $COMINm2/$filestem $filestem + + $USHobsproc/bufr_remorest.sh $filestem + rc=$? + if [ $rc -gt 0 ] ; then + [ $rc -gt $retcode ] && retcode=$rc + msg="**WARNING: ERROR generating non-restricted $file BUFR file \ +from 2-days ago (rc = $rc) -- existing file made 2-days ago is not overwritten" + $DATA/postmsg "$jlogfile" "$msg" + else + msg="Successful generation of non-restricted $file BUFR file from \ +2-days ago -- overwrite existing file made 2-days ago" + $DATA/postmsg "$jlogfile" "$msg" + cp $filestem $COMOUTm2/$filestem.ur + chmod 664 $COMOUTm2/$filestem.ur + if [ $SENDDBN = "YES" ] ; then + NETUP=`echo $RUN | tr {a-z} {A-Z}` # can this be net_uc? + $DBNROOT/bin/dbn_alert MODEL ${NETUP}_BUFR_${file}_nr $job \ + $COMOUTm2/$filestem.ur + fi + fi + + done + +fi # endif loop $PROCESS_REMOREST_dm2 if [ "$PROCESS_UNBLKBUFR" = 'YES' ]; then @@ -332,23 +488,43 @@ $dumptime" ######################################################################## # Unblock BUFR Data Dump Files # +# {Note: will also unblock non-restricted aircar and aircft dump files # +# from 2-days ago if they were overwritten in REMOREST above # +# (when $PROCESS_REMOREST_dm2 = YES)} # # ---> ON WCOSS dump files are already unblocked, so this whole set of # # processing can hopefully be removed someday!! # ######################################################################## + aircraft_nr_dm2="" + [ $PROCESS_REMOREST_dm2 = YES -a $this_m = $dev_m ] && \ + aircraft_nr_dm2="aircar_nr_dm2 aircft_nr_dm2" + for file in adpsfc adpupa aircar aircft satwnd sfcshp spssmi proflr \ vadwnd goesnd erscat sfcbog erswnd ssmip ssmipn ssmit \ atovs qkscat qkswnd trmm sptrmm geoimr 1bamua 1bamub \ 1bhrs3 1bmhs 1bhrs4 airs airswm amsre gpsipw msonet \ rassda nexrad gpsro airsev goesfv wndsat wdsatr osbuv8 \ ascatt ascatw mtiasi avcsam avcspm gome lghtng omi \ - esamua esamub eshrs3 esmhs ssmisu sevcsr lgycld efclam + esamua esamub eshrs3 esmhs ssmisu sevcsr lgycld efclam \ + $aircraft_nr_dm2 # --> don't add any new dumps here since files are already unblocked # on WCOSS!! do + file_orig=$file + if [ $PROCESS_REMOREST_dm2 = YES -a $this_m = $dev_m ]; then + if [ $file = aircar_nr_dm2 -o $file = aircft_nr_dm2 ]; then + file=`echo $file | cut -d"_" -f1` + COMIN_save=$COMIN + COMIN=$COMINm2 + COMOUT_save=$COMOUT + COMOUT=$COMOUTm2 + fi + fi for qual in "" ".nr" do + [ \( $file_orig = aircar_nr_dm2 -o $file_orig = aircft_nr_dm2 \) -a \ + "$qual" != .nr ] && continue qualdbn=$qual COMIN_here=$COMIN if [ "$qual" = .nr ]; then @@ -372,7 +548,12 @@ $dumptime" $file.unblock$qual (rc = $rc)" $DATA/postmsg "$jlogfile" "$msg" else - msg="$file$qual BUFR file SUCCESSFULLY copied to $file.unblock$qual" + if [ $file_orig = aircar_nr_dm2 -o $file_orig = aircft_nr_dm2 ];then + msg="$file$qual BUFR file from 2-days ago SUCCESSFULLY copied \ +to $file.unblock$qual -- overwrite existing file made 2-days ago" + else + msg="$file$qual BUFR file SUCCESSFULLY copied to $file.unblock$qual" + fi $DATA/postmsg "$jlogfile" "$msg" cp $file.unblock$qual $COMOUT/$filestem.unblok$qual chmod 664 $COMOUT/$filestem.unblok$qual @@ -420,11 +601,16 @@ a null file is copied in its place" fi done + if [ $PROCESS_REMOREST_dm2 = YES -a $this_m = $dev_m ]; then + if [ $file_orig = aircar_nr_dm2 -o $file_orig = aircft_nr_dm2 ]; then + COMIN=$COMIN_save + COMOUT=$COMOUT_save + fi + fi done -# endif loop $PROCESS_UNBLKBUFR -fi +fi # endif loop $PROCESS_UNBLKBUFR @@ -447,9 +633,9 @@ $dumptime" EOFlistdumps cat <<\EOFthread > $DATA/thread -#VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV -# Make herefile "thread" which will run in background shells to list dumps -#VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV +#VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV +# Herefile "thread" which can run in parallel (via MPMD methods) to list dumps +#VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV { echo set -x mkdir -p $DATA/thread${1} @@ -578,7 +764,7 @@ EOFthread else which cfp which_cfp_err=$? - mpiexec -np 1 --cpu-bind verbose,core cfp $DATA/mpmd.cmdfile + mpiexec -np 14 --cpu-bind verbose,core cfp $DATA/mpmd.cmdfile fi else echo @@ -601,8 +787,7 @@ fi [ $err -gt $retcode ] && retcode=$err done -# endif loop $PROCESS_LISTERS -fi +fi # endif loop $PROCESS_LISTERS @@ -690,8 +875,7 @@ by choice" fi done -# endif loop $PROCESS_AVGTABLES -fi +fi # endif loop $PROCESS_AVGTABLES ####################### diff --git a/scripts/exglobal_dump.sh b/scripts/exglobal_dump.sh index e906d8e..d4b9472 100755 --- a/scripts/exglobal_dump.sh +++ b/scripts/exglobal_dump.sh @@ -87,6 +87,7 @@ echo " DBN alerts are also enabled for subpfl,saldrn, " echo " gmi1cr,and snocvr " echo " Oct 17 2022 - Split up groups 1 and 10 into a new group 12 " echo " for better optimization. " +echo " Sep 30 2022 - Enable dumping of UPRAIR data in group #3. " ############################################################################# # NOTE: NET is changed to gdas in the parent Job script for the gdas RUN @@ -115,11 +116,11 @@ set +u # JOB_NUMBER not present indicates dump BOTH prepbufr and non-prepbufr data. # ----------------------------------------------------------------------------- # Dump group #1 (non-pb, TIME_TRIM defaults to OFF) = -# avcsam eshrs3 ssmisu saphir 1bhrs4 sevcsr tesac mls +# avcsam eshrs3 ssmisu 1bhrs4 tesac mls # esatms gsrcsr ahicsr sstvcw subpfl saldrn -# +# Stop: sevcsr, saphir in v1.2.0 # Dump group #2 (pb, TIME_TRIM defaults to OFF) = -# sfcshp tideg atovs* adpsfc ascatt +# sfcshp tideg atovs* adpsfc ascatt snocvr # * - for GDAS only # # Dump group #3 (pb, TIME_TRIM defaults to OFF) = @@ -142,19 +143,20 @@ set +u # satwnd # # Dump group #9 (non-pb, TIME_TRIM defaults to ON) = -# geoimr -# +# geoimr gmi1cr satwhr # Dump group #10 (non-pb, TIME_TRIM defaults to OFF) = -# esiasi mtiasi esamua sevasr 1bamua bathy osbuv8 -# ompst8 ompsn8 gsrasr ompslp sstvpw +# esiasi mtiasi esamua sevasr 1bamua bathy +# osbuv8 ompst8 ompsn8 gsrasr ompslp sstvpw # # Dump group #11 (non-pb, TIME_TRIM defaults to OFF) = # amsr2 # # Dump group #12 crisfs atms (previously group1) # crsfdb iasidb (previously group10) +# Dump group #13 (pb, TIME_TRIM defaults to OFF) = +# uprair # -# Dump group #13 STATUS FILE +# Dump group #14 STATUS FILE # ----------------------------------------------------------------------------- #VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV @@ -184,6 +186,7 @@ set -u DUMP_group10=${DUMP_group10:-"YES"} DUMP_group11=${DUMP_group11:-"YES"} DUMP_group12=${DUMP_group12:-"YES"} + DUMP_group13=${DUMP_group13:-"NO"} else dump_ind=DUMP DUMP_group1=${DUMP_group1:-"NO"} @@ -198,6 +201,7 @@ set -u DUMP_group10=${DUMP_group10:-"NO"} DUMP_group11=${DUMP_group11:-"NO"} DUMP_group12=${DUMP_group12:-"NO"} + DUMP_group13=${DUMP_group13:-"YES"} fi else dump_ind=DUMP @@ -213,16 +217,20 @@ else DUMP_group10=${DUMP_group10:-"YES"} DUMP_group11=${DUMP_group11:-"YES"} DUMP_group12=${DUMP_group12:-"YES"} + DUMP_group13=${DUMP_group13:-"YES"} fi +# NAP and NAP_adpupa instroduced so that uprair can run early on his own +NAP=${NAP:-600} #b/c cron is moved to run 10min (600s) early if [ "$NET" = 'gfs' ]; then ADPUPA_wait=${ADPUPA_wait:-"YES"} + NAP_adpupa=${NAP_adpupa:-800} #600s(compensate early cron) + 300s(for adpupa data to come) ########ADPUPA_wait=${ADPUPA_wait:-"NO"} # saves time if ADPUPA_wait=NO else ADPUPA_wait=${ADPUPA_wait:-"NO"} + NAP_adpupa=${NAP_adpupa:-600} #like other dump groups fi - # send extra output of DUMP2 for monitoring purposes. set +u if [ -n "$JOB_NUMBER" ]; then @@ -519,7 +527,7 @@ echo "=======> Dump group 9 (thread_9) not executed." > $DATA/9.out echo "=======> Dump group 10 (thread_10) not executed." > $DATA/10.out echo "=======> Dump group 11 (thread_11) not executed." > $DATA/11.out echo "=======> Dump group 12 (thread_12) not executed." > $DATA/12.out - +echo "=======> Dump group 13 (thread_13) not executed." > $DATA/13.out err1=0 err2=0 @@ -533,6 +541,7 @@ err9=0 err10=0 err11=0 err12=0 +err13=0 if [ "$PROCESS_DUMP" = 'YES' ]; then #################################### @@ -561,6 +570,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=1 @@ -582,6 +592,8 @@ export DUMP_NUMBER=1 # GSRCSR: 1 subtype(s) # AHICSR: 1 subtype(s) # SSTVCW: 1 subtype(s) +# SUBPFL: 1 subtype(s) +# SALDRN: 1 subtype(s) # -------------------- # TOTAL NUMBER OF SUBTYPES = 14 # @@ -590,10 +602,10 @@ export DUMP_NUMBER=1 DTIM_latest_avcsam=${DTIM_latest_avcsam:-"+2.99"} DTIM_latest_eshrs3=${DTIM_latest_eshrs3:-"+2.99"} DTIM_latest_ssmisu=${DTIM_latest_ssmisu:-"+2.99"} -DTIM_latest_saphir=${DTIM_latest_saphir:-"+2.99"} +#DTIM_latest_saphir=${DTIM_latest_saphir:-"+2.99"} DTIM_latest_saldrn=${DTIM_latest_saldrn:-"+2.99"} DTIM_latest_1bhrs4=${DTIM_latest_1bhrs4:-"+2.99"} -DTIM_latest_sevcsr=${DTIM_latest_sevcsr:-"+2.99"} +#DTIM_latest_sevcsr=${DTIM_latest_sevcsr:-"+2.99"} DTIM_latest_tesac=${DTIM_latest_tesac:-"+2.99"} #----------------------------------------------- # check for mls tank presence in past 10 days @@ -624,7 +636,7 @@ DTIM_latest_sstvcw=${DTIM_latest_sstvcw:-"+2.99"} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM1:-off}} $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 avcsam eshrs3 ssmisu \ - saphir 1bhrs4 sevcsr tesac $mls $esatms gsrcsr ahicsr sstvcw subpfl saldrn + 1bhrs4 tesac $mls $esatms gsrcsr ahicsr sstvcw subpfl saldrn error1=$? echo "$error1" > $DATA/error1 @@ -635,14 +647,14 @@ if [ "$SENDDBN" = "YES" ]; then ${COMSP}eshrs3.tm00.bufr_d $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ssmisu $job \ ${COMSP}ssmisu.tm00.bufr_d - if [ "${NET}" = "gdas" ]; then - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_saphir $job \ - ${COMSP}saphir.tm00.bufr_d ### restricted, only GDAS, turn on 01/13/2020 - fi +# if [ "${NET}" = "gdas" ]; then +# $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_saphir $job \ +# ${COMSP}saphir.tm00.bufr_d ### restricted, only GDAS, turn on 01/13/2020 +# fi $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_1bhrs4 $job \ ${COMSP}1bhrs4.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sevcsr $job \ - ${COMSP}sevcsr.tm00.bufr_d +# $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sevcsr $job \ +# ${COMSP}sevcsr.tm00.bufr_d $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_tesac $job \ ${COMSP}tesac.tm00.bufr_d $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_saldrn $job \ @@ -693,6 +705,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=2 @@ -711,11 +724,12 @@ export DUMP_NUMBER=2 # ATOVS: 1 subtype(s) # ADPSFC: 7 subtype(s) # ASCATT: 1 subtype(s) +# SNOCVR: 1 subtype(s) # xxxxxxxxx WNDSAT: 1 subtype(s) (if present in past 10 days of tanks) # ===> Dumping of WNDSAT removed from here until new ingest feed is established # (had been dumped with a time window radius of -3.00 to +2.99 hours) # -------------------- -# TOTAL NUMBER OF SUBTYPES = 20 - 21 +# TOTAL NUMBER OF SUBTYPES = 21 - 22 # #-------------------------------------------------------------------------- # GFS: @@ -723,11 +737,12 @@ export DUMP_NUMBER=2 # TIDEG: 1 subtype(s) # ADPSFC: 7 subtype(s) # ASCATT: 1 subtype(s) +# SNOCVR: 1 subtype(s) # xxxxxxxxx WNDSAT: 1 subtype(s) (if present in past 10 days of tanks) # ===> Dumping of WNDSAT removed from here until new ingest feed is established # (had been dumped with a time window radius of -3.00 to +2.99 hours) # -------------------- -# TOTAL NUMBER OF SUBTYPES = 19 - 20 +# TOTAL NUMBER OF SUBTYPES = 21 - 22 # #========================================================================== DTIM_latest_snocvr=${DTIM_latest_snocvr:-"+2.99"} @@ -817,6 +832,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP_adpupa} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=3 @@ -873,6 +889,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=4 @@ -982,6 +999,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=5 @@ -1034,6 +1052,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=6 @@ -1177,6 +1196,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=7 @@ -1284,6 +1304,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=8 @@ -1395,6 +1416,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=9 @@ -1406,19 +1428,24 @@ export DUMP_NUMBER=9 # #-------------------------------------------------------------------------- # Dump # 9 : GEOIMR: 1 subtype(s) +# GMI1CR: 1 subtype(s) +# SATWHR: 1 subtype(s) # -------------------- -# TOTAL NUMBER OF SUBTYPES = 1 +# TOTAL NUMBER OF SUBTYPES = 3 # #======================================================================= DTIM_earliest_gmi1cr=${DTIM_earliest_gmi1cr:-"-3.00"} DTIM_latest_gmi1cr=${DTIM_latest_gmi1cr:-"+2.99"} +DTIM_earliest_satwhr=${DTIM_earliest_satwhr:-"-3.00"} +DTIM_latest_satwhr=${DTIM_latest_satwhr:-"+2.99"} + DTIM_earliest_geoimr=${DTIM_earliest_geoimr:-"-0.50"} DTIM_latest_geoimr=${DTIM_latest_geoimr:-"+0.50"} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM9:-on}} -$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 geoimr gmi1cr +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 geoimr gmi1cr satwhr error9=$? echo "$error9" > $DATA/error9 @@ -1428,6 +1455,8 @@ if [ "$SENDDBN" = "YES" ]; then ${COMSP}geoimr.tm00.bufr_d $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gmi1cr $job \ ${COMSP}gmi1cr.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_satwhr $job \ + ${COMSP}satwhr.tm00.bufr_d fi set +x @@ -1458,6 +1487,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=10 @@ -1558,6 +1588,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=11 @@ -1614,6 +1645,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=12 @@ -1692,6 +1724,65 @@ set -x EOF set -x +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_13; chmod +x thread_13 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_13 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +# UPRAIR requires early start, no need to NAP +#sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron +export STATUS=NO +export DUMP_NUMBER=13 + +#==================================================================== +# NOTES ABOUT THIS DUMP GROUP: +# (1) time window radius is -3.00 to +2.99 hours on all types +# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) +# +#-------------------------------------------------------------------------- +# Dump #13: UPRAIR: 5 subtype(s) +# -------------------- +# TOTAL NUMBER OF SUBTYPES = 5 +# +#==================================================================== + +DTIM_latest_uprair=${DTIM_latest_uprair:-"+2.99"} + +TIME_TRIM=${TIME_TRIM:-${TIME_TRIM13:-off}} + +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 uprair +error13=$? +echo "$error13" > $DATA/error13 + +if [ "$SENDDBN" = "YES" ]; then + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_uprair $job \ + ${COMSP}uprair.tm00.bufr_d +fi + +set +x +echo "********************************************************************" +echo Script thread_13 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/13.out 2>&1 +EOF +set -x + + #---------------------------------------------------------------- # Now launch the threads @@ -1705,8 +1796,15 @@ launcher=${launcher:-"cfp"} # if not "cfp", threads will be run serially. if [ "$launcher" = cfp ]; then > $DATA/poe.cmdfile + echo "Running threads in parallel IG2023" + myPDY=`date +\%Y\%m\%d\%H\%M\%S` + echo "DATE IG2023 start " $myPDY # To better take advantage of cfp, execute the longer running commands first. # Some reordering was done here based on recent sample runtimes. + + #[ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && echo ./thread_3 >> $DATA/poe.cmdfile + [ $DUMP_group3 = YES ] && echo ./thread_3 >> $DATA/poe.cmdfile # NAP_adpupa covers for ADPUPA_wait + [ $DUMP_group13 = YES ] && echo ./thread_13 >> $DATA/poe.cmdfile [ $DUMP_group7 = YES ] && echo ./thread_7 >> $DATA/poe.cmdfile # moved up [ $DUMP_group1 = YES ] && echo ./thread_1 >> $DATA/poe.cmdfile [ $DUMP_group5 = YES ] && echo ./thread_5 >> $DATA/poe.cmdfile # moved up @@ -1715,16 +1813,17 @@ if [ "$launcher" = cfp ]; then [ $DUMP_group11 = YES ] && echo ./thread_11 >> $DATA/poe.cmdfile # moved up [ $DUMP_group10 = YES ] && echo ./thread_10 >> $DATA/poe.cmdfile # moved up [ $DUMP_group2 = YES ] && echo ./thread_2 >> $DATA/poe.cmdfile - [ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && echo ./thread_3 >> $DATA/poe.cmdfile [ $DUMP_group4 = YES ] && echo ./thread_4 >> $DATA/poe.cmdfile [ $DUMP_group9 = YES ] && echo ./thread_9 >> $DATA/poe.cmdfile [ $DUMP_group12 = YES ] && echo ./thread_12 >> $DATA/poe.cmdfile - if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes - launcher_DUMP=${launcher_DUMP:-mpiexec} - $launcher_DUMP -np 14 --cpu-bind verbose,core cfp $DATA/poe.cmdfile 2>&1 + launcher_DUMP=${launcher_DUMP:-mpiexec} + NPROCS=${NPROCS:-14} # was 12 + $launcher_DUMP -np ${NPROCS} --cpu-bind verbose,core cfp $DATA/poe.cmdfile 2>&1 + #$launcher_DUMP -np 14 --cpu-bind core cfp $DATA/poe.cmdfile 2>&1 # 1) 3) + #$launcher_DUMP -np ${NPROCS} cfp $DATA/poe.cmdfile 2>&1 # 4) Carolyn Pasti suggestions errpoe=$? if [ $errpoe -ne 0 ]; then $DATA/err_exit "***FATAL: EXIT STATUS $errpoe RUNNING POE COMMAND FILE" @@ -1736,9 +1835,9 @@ if [ "$launcher" = cfp ]; then fi else echo "Running threads serially" + [ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && ./thread_3 [ $DUMP_group1 = YES ] && ./thread_1 [ $DUMP_group2 = YES ] && ./thread_2 - [ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && ./thread_3 [ $DUMP_group4 = YES ] && ./thread_4 [ $DUMP_group5 = YES ] && ./thread_5 [ $DUMP_group6 = YES ] && ./thread_6 @@ -1748,17 +1847,21 @@ else [ $DUMP_group10 = YES ] && ./thread_10 [ $DUMP_group11 = YES ] && ./thread_11 [ $DUMP_group12 = YES ] && ./thread_12 + [ $DUMP_group13 = YES ] && ./thread_13 # wait fi -# if ADPUPA_wait is YES, adpupa is dumped AFTER all other dump threads have -# run (normally done in real-time GFS runs to dump as late as possible in -# order to maximize data availability in GFS network, particularly DROPs) -# -------------------------------------------------------------------------- - -[ $DUMP_group3 = YES -a $ADPUPA_wait = YES ] && ./thread_3 +# long run times for uprair lead to use of NAP and NAP_adpupa variables (see code above) instead of this code +# +## if ADPUPA_wait is YES, adpupa and uprair are dumped AFTER all other dump +## threads have run (normally done in real-time GFS runs to dump as late as +## possible in order to maximize data availability in GFS network, +## particularly DROPs) +## -------------------------------------------------------------------------- +## +#[ $DUMP_group3 = YES -a $ADPUPA_wait = YES ] && ./thread_3 -cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.out $DATA/11.out $DATA/12.out +cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.out $DATA/11.out $DATA/12.out $DATA/13.out set +x echo " " @@ -1777,12 +1880,13 @@ set -x [ -s $DATA/error10 ] && err10=`cat $DATA/error10` [ -s $DATA/error11 ] && err11=`cat $DATA/error11` [ -s $DATA/error12 ] && err12=`cat $DATA/error12` +[ -s $DATA/error13 ] && err13=`cat $DATA/error13` #=============================================================================== export STATUS=YES -export DUMP_NUMBER=13 +export DUMP_NUMBER=14 $ushscript_dump/bufr_dump_obs.sh $dumptime 3.00 1 null # endif loop $PROCESS_DUMP @@ -1802,8 +1906,8 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o \ "$err4" -gt '5' -o "$err5" -gt '5' -o "$err6" -gt '5' -o \ "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' -o \ - "$err10" -gt '5' -o "$err11" -gt '5' -o "$err12" -gt '5' ]; then - for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 $err11 $err12 + "$err10" -gt '5' -o "$err11" -gt '5' -o "$err12" -gt '5' -o "$err13" -gt '5']; then + for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 $err11 $err12 $err13 do if [ "$n" -gt '5' ]; then if [ "$n" -ne '11' -a "$n" -ne '22' ]; then @@ -1814,7 +1918,7 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then echo echo " ###################################################### " echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7, $err8, $err9, $err10, $err11, $err12 " +$err5, $err6, $err7, $err8, $err9, $err10, $err11, $err12, $err13 " echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " echo " ###################################################### " echo @@ -1832,7 +1936,7 @@ echo echo echo " ###################################################### " echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7, $err8, $err9, $err10, $err11, $err12 " +$err5, $err6, $err7, $err8, $err9, $err10, $err11, $err12, $err13 " echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " echo " ###################################################### " echo diff --git a/scripts/exnam_dump.sh b/scripts/exnam_dump.sh index b172869..1760a59 100755 --- a/scripts/exnam_dump.sh +++ b/scripts/exnam_dump.sh @@ -31,6 +31,8 @@ echo " to match bufr_dumplist. Removed tideg from " echo " sfcshp dump group to make unique dump file. " echo " - Copy bufr_dumplist to COMOUT. " echo " Dec 09 2021 - Updated for use on WCOSS2 " +echo " Oct 17 2023 - Split msonet to msonet and msone1 (b255/xx030)" +echo " Mar 14 2024 - Separate msonet to own dump group " ############################################################################ set -xau @@ -46,13 +48,15 @@ set +u # Dump group #1 (non-pb) = 1bamua 1bmhs gpsro mtiasi esamua eshrs3 sevcsr atms # Dump group #2 (pb) = vadwnd satwnd # Dump group #3 (pb) = proflr rassda sfcshp tideg adpsfc ascatt -# Dump group #4 (pb) = msonet gpsipw +# Dump group #4 (pb) = gpsipw (moved msonet) # Dump group #5 (pb) = aircft aircar goesnd --> dump is GLOBAL # Dump group #6 (non-pb) = nexrad --> dump is GLOBAL # Dump group #7 (non-pb) = goesfv lgycld --> dump is GLOBAL # Dump group #8 (non-pb) = 1bhrs4 airsev osbuv8 esmhs ssmisu cris lghtng # Dump group #9 (pb) = adpupa -# Dump group #10 STATUS FILE +# Dump group #10 (pb)= msonet +# Dump group #11 (pb)= msone1 +# Dump group #12 STATUS FILE # ----------------------------------------------------------------------------- #VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV @@ -92,6 +96,8 @@ set -u DUMP_group7=${DUMP_group7:-"YES"} DUMP_group8=${DUMP_group8:-"YES"} DUMP_group9=${DUMP_group9:-"NO"} + DUMP_group10=${DUMP_group10:-"NO"} + DUMP_group11=${DUMP_group11:-"NO"} else dump_ind=DUMP DUMP_group1=${DUMP_group1:-"NO"} @@ -103,6 +109,8 @@ set -u DUMP_group7=${DUMP_group7:-"NO"} DUMP_group8=${DUMP_group8:-"NO"} DUMP_group9=${DUMP_group9:-"YES"} + DUMP_group10=${DUMP_group10:-"YES"} + DUMP_group11=${DUMP_group11:-"YES"} fi else dump_ind=DUMP @@ -115,6 +123,8 @@ else DUMP_group7=${DUMP_group7:-"YES"} DUMP_group8=${DUMP_group8:-"YES"} DUMP_group9=${DUMP_group9:-"YES"} + DUMP_group10=${DUMP_group10:-"YES"} + DUMP_group11=${DUMP_group11:-"YES"} fi if [ $tmmark = tm00 ]; then @@ -244,6 +254,8 @@ echo "=======> Dump group 6 (thread_6) not executed." > $DATA/6.out echo "=======> Dump group 7 (thread_7) not executed." > $DATA/7.out echo "=======> Dump group 8 (thread_8) not executed." > $DATA/8.out echo "=======> Dump group 9 (thread_9) not executed." > $DATA/9.out +echo "=======> Dump group 10 (thread_10) not executed." > $DATA/10.out +echo "=======> Dump group 11 (thread_11) not executed." > $DATA/11.out err1=0 err2=0 @@ -254,6 +266,8 @@ err6=0 err7=0 err8=0 err9=0 +err10=0 +err11=0 if [ "$PROCESS_DUMP" = 'YES' ]; then ####################################################################### @@ -540,8 +554,8 @@ export STATUS=NO export DUMP_NUMBER=4 #=========================================================================== -# Dump # 4 : MSONET, GPSIPW -- TOTAL NUMBER OF SUBTYPES = 31 -# (30) (1) +# Dump # 4 : [MSONET,] GPSIPW -- TOTAL NUMBER OF SUBTYPES = [31] 1 +# [(30)] (1) - moved msone* to separate groups! # for catch-up cycles tm06-01: # time window radius is (-0.50,+0.50) hours for MSONET # for on-time tm00 cycles: @@ -562,15 +576,7 @@ export DUMP_NUMBER=4 DTIM_earliest_gpsipw=${DTIM_earliest_gpsipw:-"-0.22"} DTIM_latest_gpsipw=${DTIM_latest_gpsipw:-"-0.12"} -if [ "$tmhr" = "00" ]; then - DTIM_earliest_msonet=${DTIM_earliest_msonet:-"-0.75"} - DTIM_latest_msonet=${DTIM_latest_msonet:-"+1.50"} -fi - -export SKIP_255031=YES # Skip for port to Dell since no new data allowed. -export SKIP_255101=YES # Also, b/c NAM is frozen; no new data. - -$ushscript_dump/bufr_dump_obs.sh $dumptime 0.5 1 msonet gpsipw +$ushscript_dump/bufr_dump_obs.sh $dumptime 0.5 1 gpsipw error4=$? echo "$error4" > $DATA/error4 @@ -632,7 +638,6 @@ fi if [ "$SENDDBN" = 'YES' ]; then $DBNROOT/bin/dbn_alert MODEL NAM_BUFR_gpsipw $job ${COMSP}gpsipw.$tmmark.bufr_d - $DBNROOT/bin/dbn_alert MODEL NAM_BUFR_msonet $job ${COMSP}msonet.$tmmark.bufr_d fi set +x @@ -1045,6 +1050,119 @@ set -x EOF set -x +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_10; chmod +x thread_10 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_10 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=10 + +#=========================================================================== +# Dump # 10 : MSONET -- TOTAL NUMBER OF SUBTYPES = 1 +# (1) +# for catch-up cycles tm06-01: +# time window radius is (-0.50,+0.50) hours for MSONET +# for on-time tm00 cycles: +# time window radius is (-0.75,+1.50) hours for MSONET +#=========================================================================== + +if [ "$tmhr" = "00" ]; then + DTIM_earliest_msonet=${DTIM_earliest_msonet:-"-0.75"} + DTIM_latest_msonet=${DTIM_latest_msonet:-"+1.50"} +fi + +export SKIP_255031=YES # Skip for port to Dell since no new data allowed. +export SKIP_255101=YES # Also, b/c NAM is frozen; no new data. + +$ushscript_dump/bufr_dump_obs.sh $dumptime 0.5 1 msonet +error10=$? +echo "$error10" > $DATA/error10 + +#if [ "$SENDDBN" = 'YES' ]; then +# $DBNROOT/bin/dbn_alert MODEL NAM_BUFR_msonet $job ${COMSP}msonet.$tmmark.bufr_d +#fi + +set +x +echo "********************************************************************" +echo Script thread_10 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/10.out 2>&1 +EOF +set -x + + +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_11; chmod +x thread_11 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_11 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=11 + +#=========================================================================== +# Dump # 11 : MSONE1 -- TOTAL NUMBER OF SUBTYPES = 1 +# (1) +# for catch-up cycles tm06-01: +# time window radius is (-0.50,+0.50) hours for MSONE1 +# for on-time tm00 cycles: +# time window radius is (-0.75,+1.50) hours for MSONE1 +#=========================================================================== + +if [ "$tmhr" = "00" ]; then + DTIM_earliest_msone1=${DTIM_earliest_msone1:-"-0.75"} + DTIM_latest_msone1=${DTIM_latest_msone1:-"+1.50"} +fi + +export SKIP_255031=YES # Skip for port to Dell since no new data allowed. +export SKIP_255101=YES # Also, b/c NAM is frozen; no new data. + +$ushscript_dump/bufr_dump_obs.sh $dumptime 0.5 1 msone1 +error11=$? +echo "$error11" > $DATA/error11 + +#if [ "$SENDDBN" = 'YES' ]; then +# $DBNROOT/bin/dbn_alert MODEL NAM_BUFR_msonet $job ${COMSP}msone1.$tmmark.bufr_d +#fi + +set +x +echo "********************************************************************" +echo Script thread_11 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/11.out 2>&1 +EOF +set -x + #---------------------------------------------------------------- # Now launch the threads @@ -1070,12 +1188,14 @@ if [ "$launcher" = cfp ]; then [ $DUMP_group6 = YES ] && echo ./thread_6 >> $DATA/poe.cmdfile [ $DUMP_group7 = YES ] && echo ./thread_7 >> $DATA/poe.cmdfile [ $DUMP_group9 = YES -a $ADPUPA_wait != YES ] && echo ./thread_9 >> $DATA/poe.cmdfile + [ $DUMP_group10 = YES ] && echo ./thread_10 >> $DATA/poe.cmdfile + [ $DUMP_group11 = YES ] && echo ./thread_11 >> $DATA/poe.cmdfile if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes # ?? launcher_DUMP=${launcher_DUMP:-mpiexec} - #$launcher_DUMP -np 7 --cpu-bind verbose,core cfp $DATA/poe.cmdfile - NPROCS=${NPROCS:-7} + #$launcher_DUMP -np 10 --cpu-bind verbose,core cfp $DATA/poe.cmdfile + NPROCS=${NPROCS:-11} $launcher_DUMP -np $NPROCS --cpu-bind verbose,core cfp $DATA/poe.cmdfile errpoe=$? if [ $errpoe -ne 0 ]; then @@ -1097,6 +1217,8 @@ else [ $DUMP_group7 = YES ] && ./thread_7 [ $DUMP_group8 = YES ] && ./thread_8 [ $DUMP_group9 = YES -a $ADPUPA_wait != YES ] && ./thread_9 + [ $DUMP_group10 = YES ] && ./thread_10 + [ $DUMP_group11 = YES ] && ./thread_11 fi # if ADPUPA_wait is YES, adpupa is dumped AFTER all other dump threads have @@ -1107,7 +1229,7 @@ fi [ $DUMP_group9 = YES -a $ADPUPA_wait = YES ] && ./thread_9 cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out \ - $DATA/7.out $DATA/8.out $DATA/9.out + $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.out $DATA/11.out set +x echo " " @@ -1123,12 +1245,12 @@ set -x [ -s $DATA/error7 ] && err7=`cat $DATA/error7` [ -s $DATA/error8 ] && err8=`cat $DATA/error8` [ -s $DATA/error9 ] && err9=`cat $DATA/error9` - - +[ -s $DATA/error10 ] && err10=`cat $DATA/error10` +[ -s $DATA/error11 ] && err11=`cat $DATA/error11` #=============================================================================== export STATUS=YES -export DUMP_NUMBER=10 +export DUMP_NUMBER=12 $ushscript_dump/bufr_dump_obs.sh $dumptime 3.00 1 null @@ -1146,8 +1268,9 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o \ "$err4" -gt '5' -o "$err5" -gt '5' -o "$err6" -gt '5' -o \ - "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' ]; then - for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 + "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' -o \ + "$err10" -gt '5' -o "$err11" -gt '5']; then + for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 $err11 do if [ "$n" -gt '5' ]; then if [ "$n" -ne '11' -a "$n" -ne '22' ]; then @@ -1158,7 +1281,7 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then echo echo " ###################################################### " echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7, $err8, $err9 " +$err5, $err6, $err7, $err8, $err9, $err10, $err11" echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " echo " ###################################################### " echo @@ -1176,7 +1299,7 @@ echo echo echo " ###################################################### " echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7, $err8, $err9 " +$err5, $err6, $err7, $err8, $err9, $err10, $err11" echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " echo " ###################################################### " echo @@ -1186,6 +1309,9 @@ $err5, $err6, $err7, $err8, $err9 " # endif loop $PROCESS_DUMP fi +# concatenate msonet and msone1, b/c prepobs only wants one file +cat ${COMSP}msone1.tm00.bufr_d >> ${COMSP}msonet.tm00.bufr_d + # # copy bufr_dumplist to $COMOUT per NCO SPA request # ------------------------------------------------- diff --git a/scripts/exrap_dump.sh b/scripts/exrap_dump.sh index 3099f93..9af99cb 100755 --- a/scripts/exrap_dump.sh +++ b/scripts/exrap_dump.sh @@ -54,6 +54,11 @@ echo " Dec 09 2021 - Updated to run on WCOSS2 " echo " Aug 10 2022 - Added subpfl,saldn to dump #1,snocvr to dump #3. " echo " added gmi1cr dump group #7 " echo " b005/xx081 added to satwnd " +echo " Sep 30 2022 - Enable dumping of UPRAIR data in group #2. " +echo " Oct 11 2023 - Split msonet to msonet and msone1, msone1=255.030 " +echo " concatenate msonet and msone1 right after dump " +echo " - Pull adpupa and uprair into own Dump group " +echo " Mar 14 2024 - Split gsrasr and gsrcsr to own dump hroups " ################################################################################ set -xau @@ -68,17 +73,20 @@ set +u # ------------------------------------------------------------------------ # Dump group #1 (non-pb) = 1bamua 1bmhs esamua esmhs atms mtiasi sevcsr # gpsro esiasi iasidb esatms atmsdb sevasr amsr2 -# subpfl saldrn -# Dump group #2 (pb) = vadwnd satwnd adpupa +# Dump group #2 (pb) = vadwnd satwnd # Dump group #3 (pb) = proflr rassda sfcshp adpsfc ascatt tideg snocvr -# Dump group #4 (pb) = msonet gpsipw +# subpfl saldrn +# Dump group #4 (pb) = msonet gpsipw # Dump group #5 (pb) = aircft aircar # Dump group #6 (non-pb) = nexrad # Dump group #7 (non-pb) = airsev 1bhrs4 eshrs3 lgycld ssmisu osbuv8 crsfdb # saphir gmi1cr -# Dump group #8 (non-pb) = gsrasr gsrcsr -# Dump group #9 (non-pb) = lghtng -# Dump group #10 STATUS FILE +# Dump group #8 (non-pb) = gsrasr [gsrcsr] +# Dump group #9 (non-pb) = lghtng + adpupa +# Dump group #10(pb) = msone1 # ONLY tank b255/xx030, the largest +# Dump group #11(pb) = adpupa uprair - adpupa +# Dump group #12 (non-pb)= gsrcsr +# Dump group #13 STATUS FILE # ------------------------------------------------------------------------ #VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV @@ -105,6 +113,9 @@ set -u DUMP_group7=${DUMP_group7:-"YES"} DUMP_group8=${DUMP_group8:-"YES"} DUMP_group9=${DUMP_group9:-"YES"} + DUMP_group10=${DUMP_group10:-"NO"} + DUMP_group11=${DUMP_group11:-"NO"} + DUMP_group12=${DUMP_group12:-"YES"} else dump_ind=DUMP DUMP_group1=${DUMP_group1:-"NO"} @@ -116,6 +127,9 @@ set -u DUMP_group7=${DUMP_group7:-"NO"} DUMP_group8=${DUMP_group8:-"NO"} DUMP_group9=${DUMP_group9:-"NO"} + DUMP_group10=${DUMP_group10:-"YES"} + DUMP_group11=${DUMP_group11:-"YES"} + DUMP_group12=${DUMP_group12:-"NO"} fi else dump_ind=DUMP @@ -128,6 +142,9 @@ else DUMP_group7=${DUMP_group7:-"YES"} DUMP_group8=${DUMP_group8:-"YES"} DUMP_group9=${DUMP_group9:-"YES"} + DUMP_group10=${DUMP_group10:-"YES"} + DUMP_group11=${DUMP_group11:-"YES"} + DUMP_group12=${DUMP_group12:-"YES"} fi # Oct 2019; disable -- not needed for HRRRv4 @@ -215,6 +232,8 @@ to ${COMSP}${i}" done fi # endif loop $PROCESS_GRIBFLDS +# NAP is introduced so that uprair can run early on his own +NAP=${NAP:-120} #b/c cron is moved to run 2min (120s) early echo "=======> Dump group 1 (thread_1) not executed." > $DATA/1.out echo "=======> Dump group 2 (thread_2) not executed." > $DATA/2.out @@ -225,7 +244,9 @@ echo "=======> Dump group 6 (thread_6) not executed." > $DATA/6.out echo "=======> Dump group 7 (thread_7) not executed." > $DATA/7.out echo "=======> Dump group 8 (thread_8) not executed." > $DATA/8.out echo "=======> Dump group 9 (thread_9) not executed." > $DATA/9.out - +echo "=======> Dump group 10 (thread_10) not executed." > $DATA/10.out +echo "=======> Dump group 11 (thread_11) not executed." > $DATA/11.out +echo "=======> Dump group 12 (thread_12) not executed." > $DATA/12.out err1=0 err2=0 err3=0 @@ -235,6 +256,36 @@ err6=0 err7=0 err8=0 err9=0 +err10=0 +err11=0 +err12=0 + +#restrict processing of unexpected big tanks +#this block appear in all /scripts/ex*_dump.sh proessing msonet and msone1 +TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb +TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb +TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb +if [ -s ${TANK}/${PDY}/b255/xx003 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then + export SKIP_255003=YES + msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov +fi +if [ -s ${TANK}/${PDY}/b255/xx004 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then + export SKIP_255004=YES + msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" + echo $msg | mail.py -s "$msg" +# echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov + fi +if [ -s ${TANK}/${PDY}/b255/xx030 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then + export SKIP_255030=YES + msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov +fi +#end of block + + if [ "$PROCESS_DUMP" = 'YES' ]; then ################################## @@ -263,6 +314,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=1 @@ -413,13 +465,14 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=2 #========================================================================== -# Dump # 2 : VADWND, SATWND, ADPUPA -# (2) (19) (6) -# -- TOTAL NUMBER OF SUBTYPES = 27 +# Dump # 2 : VADWND, SATWND +# (2) (19) +# -- TOTAL NUMBER OF SUBTYPES = 21 #========================================================================== # Skip all Indian satellite winds in SATWND (not in domain) @@ -436,14 +489,9 @@ For testing, skip in ecflow or obsproc_rap.ver file #export SKIP_005065=YES #export SKIP_005066=YES -# Add GOES-16/17 DMW data to SATWND +# Add GOES-16/17/18 DMW data to SATWND export ADD_satwnd="005030 005031 005032 005034 005039" -# Time window -1.00 to +1.00 hours for ADPUPA for full and partial cycle runs -# (note: time window increased over +/- 0.5 hr standard to get more data) -DTIM_earliest_adpupa=${DTIM_earliest_adpupa:-"-1.00"} -DTIM_latest_adpupa=${DTIM_latest_adpupa:-"+1.00"} - # Time window -1.50 to +1.49 hours for EUMETSAT SATWND for full and partial # cycle runs @@ -517,7 +565,7 @@ else fi $ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_2} 1 vadwnd \ - satwnd adpupa + satwnd error2=$? echo "$error2" > $DATA/error2 @@ -549,6 +597,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=3 @@ -642,6 +691,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=4 @@ -731,6 +781,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=5 @@ -818,6 +869,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=6 @@ -1042,6 +1094,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=7 @@ -1172,13 +1225,14 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=8 #=============================================================================== -# Dump # 8 : GSRASR, GSRCSR -# (1) (1) -# TOTAL NUMBER OF SUBTYPES = 2 +# Dump # 8 : GSRASR,[ GSRCSR - moved to own group ] +# (1) [ (1)] +# TOTAL NUMBER OF SUBTYPES = 1 [2] #=============================================================================== if [ "$RUN" = 'rap_p' ]; then @@ -1191,8 +1245,6 @@ if [ "$RUN" = 'rap_p' ]; then # Time window is -1.00 to +0.99 hours for GSRASR, GSRCSR DTIM_earliest_gsrasr=${DTIM_earliest_gsrasr:-"-1.00"} DTIM_latest_gsrasr=${DTIM_latest_gsrasr:-"+0.99"} - DTIM_earliest_gsrcsr=${DTIM_earliest_gsrcsr:-"-1.00"} - DTIM_latest_gsrcsr=${DTIM_latest_gsrcsr:-"+0.99"} else @@ -1204,12 +1256,10 @@ else # Time window is -2.00 to +1.99 hours for GSRASR, GSRCSR DTIM_earliest_gsrasr=${DTIM_earliest_gsrasr:-"-2.00"} DTIM_latest_gsrasr=${DTIM_latest_gsrasr:-"+1.99"} - DTIM_earliest_gsrcsr=${DTIM_earliest_gsrcsr:-"-2.00"} - DTIM_latest_gsrcsr=${DTIM_latest_gsrcsr:-"+1.99"} fi -$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_8} 1 gsrasr gsrcsr +$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_8} 1 gsrasr error8=$? echo "$error8" > $DATA/error8 @@ -1242,18 +1292,25 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=9 #========================================================================== -# Dump # 9 : LGHTNG -# TOTAL NUMBER OF SUBTYPES = 1 +# Dump # 9 : LGHTNG + ADPUPA +# TOTAL NUMBER OF SUBTYPES = 1 + 1 #========================================================================= - + # Time window -1.00 to +0.50 hours for LGHTNG for all cycle runs DTIM_earliest_lghtng=${DTIM_earliest_lghtng:-"-1.00"} DTIM_latest_lghtng=${DTIM_latest_lghtng:-"+0.50"} +# Time window -1.00 to +1.00 hours for ADPUPA/UPRAIR w/ full & partial cycle runs +# (note: time window increased over +/- 0.5 hr standard to get more data) + +DTIM_earliest_adpupa=${DTIM_earliest_adpupa:-"-1.00"} +DTIM_latest_adpupa=${DTIM_latest_adpupa:-"+1.00"} + if [ "$RUN" = 'rap_p' ]; then # ===> For RUN = rap_p -- partial cycle runs # ------------------------------------- @@ -1264,7 +1321,7 @@ else def_time_window_9=3.0 # default time window for dump 9 is -3.0 to +3.0 hours fi -$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_9} 1 lghtng +$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_9} 1 lghtng adpupa error9=$? echo "$error9" > $DATA/error9 @@ -1279,6 +1336,192 @@ set -x EOF set -x +### NEW GROUP MSONET IG #10 +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_10; chmod +x thread_10 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_10 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron +export STATUS=NO +export DUMP_NUMBER=10 + +#============================================================================ +# Dump # 10 : MSONE1 -- TOTAL NUMBER OF SUBTYPES = 1 +# (1) +#============================================================================ + +def_time_window_10=0.5 # default time window for dump 10 is -0.5 to +0.5 hours + +# Time window -0.50 to +0.50 hours for MSONET for full and partial cycle runs +# (default) + +$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_10} 1 msone1 +error10=$? +echo "$error10" > $DATA/error10 + +set +x +echo "********************************************************************" +echo Script thread_10 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/10.out 2>&1 +EOF +set -x +### NEW GROUP MSONET IG end #10 + +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_11; chmod +x thread_11 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_11 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +# UPRAIR need to start early +#sleep ${NAP} # to reverse 2min early start of jrap_dump in cron +export STATUS=NO +export DUMP_NUMBER=11 + +#========================================================================== +# Dump # 11 : ADPUPA minus UPRAIR +# (6) minus (5) +# -- TOTAL NUMBER OF SUBTYPES = 11-5 +#========================================================================== +# +if [ "$RUN" = 'rap_p' ]; then + +# ===> For RUN = rap_p -- partial cycle runs +# ------------------------------------- + + def_time_window_11=1.5 # default time window for dump 11 is -1.5 to +1.5 hours + +else + +# ===> For RUN = rap, rap_e -- full cycle runs (including early at 00/12z) +# ------------------------------------------------------------------- + + if [ $cyc -eq 00 -o $cyc -eq 12 ]; then + def_time_window_11=1.5 # default time window for dump 11 is -1.5 to +1.5 + # hours for 00 or 12z + else + def_time_window_11=2.5 # default time window for dump 11 is -2.5 to +2.5 + # hours for all other cycles + fi + +fi + +# Time window -1.00 to +1.00 hours for ADPUPA/UPRAIR w/ full & partial cycle runs +# (note: time window increased over +/- 0.5 hr standard to get more data) + +DTIM_earliest_uprair=${DTIM_earliest_uprair:-"-1.00"} +DTIM_latest_uprair=${DTIM_latest_uprair:-"+1.00"} + +$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_11} 1 uprair +error11=$? +echo "$error11" > $DATA/error11 + +set +x +echo "********************************************************************" +echo Script thread_11 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/11.out 2>&1 +EOF +set -x + + +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_12; chmod +x thread_12 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_12 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron +export STATUS=NO +export DUMP_NUMBER=12 + +#=============================================================================== +# Dump # 12 : GSRCSR +# (1) +# TOTAL NUMBER OF SUBTYPES = 1 +#=============================================================================== + +if [ "$RUN" = 'rap_p' ]; then + +# ===> For RUN = rap_p -- partial cycle runs +# ------------------------------------- + + def_time_window_12=1.0 # default time window for dump 12 is -1.0 to +1.0 hours + +# Time window is -1.00 to +0.99 hours for GSRASR, GSRCSR + DTIM_earliest_gsrcsr=${DTIM_earliest_gsrcsr:-"-1.00"} + DTIM_latest_gsrcsr=${DTIM_latest_gsrcsr:-"+0.99"} + +else + +# ===> For RUN = rap, rap_e -- full cycle runs (including early at 00/12z) +# ------------------------------------------------------------------- + + def_time_window_12=3.0 # default time window for dump 12 is -3.0 to +3.0 hours + +# Time window is -2.00 to +1.99 hours for GSRASR, GSRCSR + DTIM_earliest_gsrcsr=${DTIM_earliest_gsrcsr:-"-2.00"} + DTIM_latest_gsrcsr=${DTIM_latest_gsrcsr:-"+1.99"} + +fi + +$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_12} 1 gsrcsr +error12=$? +echo "$error12" > $DATA/error12 + +set +x +echo "********************************************************************" +echo Script thread_12 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/12.out 2>&1 +EOF +set -x + #---------------------------------------------------------------- # Now launch the threads @@ -1295,6 +1538,7 @@ if [ "$launcher" = cfp ]; then # To better take advantage of cfp, execute the longer running commands first. # Some reordering was done here based on recent sample runtimes. + [ $DUMP_group11 = YES ] && echo ./thread_11 >> $DATA/poe.cmdfile [ $DUMP_group9 = YES ] && echo ./thread_9 >> $DATA/poe.cmdfile # lghtng 1st [ $DUMP_group7 = YES ] && echo ./thread_7 >> $DATA/poe.cmdfile # moved up [ $DUMP_group1 = YES ] && echo ./thread_1 >> $DATA/poe.cmdfile @@ -1304,13 +1548,15 @@ if [ "$launcher" = cfp ]; then [ $DUMP_group5 = YES ] && echo ./thread_5 >> $DATA/poe.cmdfile # moved up [ $DUMP_group2 = YES ] && echo ./thread_2 >> $DATA/poe.cmdfile [ $DUMP_group3 = YES ] && echo ./thread_3 >> $DATA/poe.cmdfile - + [ $DUMP_group10 = YES ] && echo ./thread_10 >> $DATA/poe.cmdfile + #[ $DUMP_group11 = YES ] && echo ./thread_11 >> $DATA/poe.cmdfile + [ $DUMP_group12 = YES ] && echo ./thread_12 >> $DATA/poe.cmdfile if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes # ?? launcher_DUMP=${launcher_DUMP:-mpiexec} #$launcher_DUMP -np 3 --cpu-bind verbose,core cfp $DATA/poe.cmdfile - NPROCS=${NPROCS:-1} + NPROCS=${NPROCS:-13} $launcher_DUMP -np $NPROCS --cpu-bind verbose,core cfp $DATA/poe.cmdfile errpoe=$? if [ $errpoe -ne 0 ]; then @@ -1332,9 +1578,12 @@ else [ $DUMP_group7 = YES ] && ./thread_7 [ $DUMP_group8 = YES ] && ./thread_8 [ $DUMP_group9 = YES ] && ./thread_9 + [ $DUMP_group10 = YES ] && ./thread_10 + [ $DUMP_group11 = YES ] && ./thread_11 + [ $DUMP_group12 = YES ] && ./thread_12 fi -cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out $DATA/7.out $DATA/8.out $DATA/9.out +cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.out $DATA/11.out $DATA/12.out set +x echo " " @@ -1350,12 +1599,14 @@ set -x [ -s $DATA/error7 ] && err7=`cat $DATA/error7` [ -s $DATA/error8 ] && err8=`cat $DATA/error8` [ -s $DATA/error9 ] && err9=`cat $DATA/error9` - +[ -s $DATA/error10 ] && err10=`cat $DATA/error10` +[ -s $DATA/error11 ] && err11=`cat $DATA/error11` +[ -s $DATA/error12 ] && err12=`cat $DATA/error12` #=============================================================================== export STATUS=YES -export DUMP_NUMBER=10 +export DUMP_NUMBER=13 $ushscript_dump/bufr_dump_obs.sh $dumptime 3.00 1 null # endif loop $PROCESS_DUMP @@ -1373,8 +1624,9 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o \ "$err4" -gt '5' -o "$err5" -gt '5' -o "$err6" -gt '5' -o \ - "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' ]; then - for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 + "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' -o \ + "$err10" -gt '5' -o "$err11" -gt '5' -o "$err12" -gt '5' ]; then + for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 $err11 $err12 do if [ "$n" -gt '5' ]; then if [ "$n" -ne '11' -a "$n" -ne '22' ]; then @@ -1385,7 +1637,7 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then echo echo " ###################################################### " echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7 $err8 $err9 " +$err5, $err6, $err7, $err8, $err9, $err10, $err11, $err12" echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " echo " ###################################################### " echo @@ -1403,7 +1655,7 @@ echo echo echo " ###################################################### " echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7 $err8 $err9 " +$err5, $err6, $err7, $err8, $err9, $err10, $err11, $err12" echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " echo " ###################################################### " echo @@ -1413,6 +1665,10 @@ $err5, $err6, $err7 $err8 $err9 " # endif loop $PROCESS_DUMP fi +# concatenate msonet and msone1, b/c prepobs only wants one file +cat ${COMSP}msone1.tm00.bufr_d >> ${COMSP}msonet.tm00.bufr_d + + grep -q "004.004 in data group aircar for .............-.........\ .... HAS 0 REPORTS" ${COMSP}status.$tmmark.bufr_d err_grep1=$? @@ -1425,6 +1681,7 @@ if [ $err_grep1 -eq 0 -a $err_grep2 -eq 0 ]; then $DATA/postmsg "$jlogfile" "$msg" fi + if [ $SENDDBN = YES ]; then if [ -s ${COMSP}1bamua.tm00.bufr_d ]; then $DBNROOT/bin/dbn_alert MODEL ${RUN_uc}_BUFR_1bamua $job ${COMSP}1bamua.tm00.bufr_d @@ -1519,6 +1776,9 @@ if [ $SENDDBN = YES ]; then if [ -s ${COMSP}ssmisu.tm00.bufr_d ]; then $DBNROOT/bin/dbn_alert MODEL ${RUN_uc}_BUFR_ssmisu $job ${COMSP}ssmisu.tm00.bufr_d fi + if [ -s ${COMSP}uprair.tm00.bufr_d ]; then + $DBNROOT/bin/dbn_alert MODEL ${RUN_uc}_BUFR_uprair $job ${COMSP}uprair.tm00.bufr_d + fi fi # diff --git a/scripts/exrtma_dump.sh b/scripts/exrtma_dump.sh index a6489eb..fdbf1f3 100755 --- a/scripts/exrtma_dump.sh +++ b/scripts/exrtma_dump.sh @@ -28,6 +28,10 @@ echo " sfcshp dump group to make unique dump file. " echo " - Copy bufr_dumplist to COMOUT. " echo " Dec 15 2021 - set for use on WCOSS2. " echo " Jul 31 2022 - Subpfl,saldrn,snocvr & gmi1cr dump group added " +echo " Oct 12 2023 - Split msonet to msonet and msone1, where " +echo " msone1=255.030; concatenate msonet and msone1 " +echo " right after dump. Seperated satwnd to its own " +echo " dump group. " ##################################################################### set -x @@ -79,6 +83,35 @@ export COMSP=$COMOUT/$RUN.${cycle}. err1=0 err2=0 err3=0 +err4=0 +err5=0 + + +#restrict processing of unexpected big tanks +#this block appear in all /scripts/ex*_dump.sh proessing msonet and msone1 +TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb +TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb +TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb +if [ -s ${TANK}/${PDY}/b255/xx003 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then + export SKIP_255003=YES + msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov +fi +if [ -s ${TANK}/${PDY}/b255/xx004 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then + export SKIP_255004=YES + msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov + fi +if [ -s ${TANK}/${PDY}/b255/xx030 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then + export SKIP_255030=YES + msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov +fi +#end of block + if [ "$PROCESS_DUMP" = 'YES' ]; then ################################### @@ -111,9 +144,9 @@ export STATUS=NO export DUMP_NUMBER=1 #======================================================================== -# Dump # 1 : ASCATT, SATWND*, EFCLAM, SNOCVR, GMI1CR -- -# (1) (14) (1) (1) (1) -# TOTAL NUMB OF SUBTYPES = 18 +# Dump # 1 : ASCATT, EFCLAM, SNOCVR, GMI1CR -- +# (1) (1) (1) (1) +# TOTAL NUMB OF SUBTYPES = 14 # ===> Dumping of WNDSAT removed from here until new ingest feed is established # (had been dumped with a time window radius of -6.00 to 0.00 hours) # @@ -121,57 +154,9 @@ export DUMP_NUMBER=1 # monitors nc005090 and does not support nc005091). # # time window radius is -6.00 to 0.00 hours for ASCATT -# for SATWND subtypes 005/010, 005/011, 005/012 and 005/019 at all -# times: -# time window radius is -1.00 to -0.01 hours -# for SATWND subtypes 005/064, 005/065, 005/066, 005/067, 005/068 -# and 005/069 at all times: -# time window radius is -1.50 to +1.49 hours -# for all other SATWND subtypes: -# time window radius is +/- 2.5 hours -# for EFCLAM: -# time window radius is +/- 0.5 hours +# time window radius is +/- 0.5 hours for EFCLAM #======================================================================= -# Skip all Indian satellite winds in SATWND (not in domain) - -export SKIP_005021=YES -export SKIP_005022=YES -export SKIP_005023=YES - -# Skip legacy EUMETSAT AMV subsets; for testing skip in trigger or version file -#export SKIP_005064=YES -#export SKIP_005065=YES -#export SKIP_005066=YES - -# Skip VIIRS NPP/NOAA-20 IR long wave derived cld motion - -export SKIP_005090=YES # old sequence (discontinued) -export SKIP_005091=YES # new sequence (unsupported in GSI) - -DTIM_earliest_005010=-1.00 -DTIM_latest_005010=-0.01 -DTIM_earliest_005011=-1.00 -DTIM_latest_005011=-0.01 -DTIM_earliest_005012=-1.00 -DTIM_latest_005012=-0.01 -DTIM_earliest_005019=-1.00 -DTIM_latest_005019=-0.01 - -DTIM_earliest_005064=-1.50 -DTIM_latest_005064=+1.49 -DTIM_earliest_005065=-1.50 -DTIM_latest_005065=+1.49 -DTIM_earliest_005066=-1.50 -DTIM_latest_005066=+1.49 - -DTIM_earliest_005067=-1.50 -DTIM_latest_005067=+1.49 -DTIM_earliest_005068=-1.50 -DTIM_latest_005068=+1.49 -DTIM_earliest_005069=-1.50 -DTIM_latest_005069=+1.49 - DTIM_earliest_ascatt=-6.00 DTIM_latest_ascatt=0.00 @@ -186,12 +171,7 @@ DTIM_latest_snocvr=${DTIM_latest_snocvr:-"+2.00"} DTIM_earliest_gmi1cr=-1.50 DTIM_latest_gmi1cr=+1.49 -DTIM_earliest_005081=${DTIM_earliest_005081:-"-1.50"} -DTIM_latest_005081=${DTIM_latest_005081:-"+1.49"} -#DTIM_earliest_005081=-1.50 -#DTIM_latest_005081=+1.49 - -$ushscript_dump/bufr_dump_obs.sh $dumptime 2.5 1 ascatt satwnd efclam snocvr gmi1cr +$ushscript_dump/bufr_dump_obs.sh $dumptime 2.5 1 ascatt efclam snocvr gmi1cr error1=$? echo "$error1" > $DATA/error1 @@ -303,6 +283,141 @@ set -x EOF set -x + +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_4; chmod +x thread_4 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_4 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=4 + +#=========================================================================== +# Dump # 4 : SATWND -- TOTAL NUMBER OF SUBTYPES = 14 +# for SATWND subtypes 005/010, 005/011, 005/012 and 005/019 at all +# times: +# time window radius is -1.00 to -0.01 hours +# for SATWND subtypes 005/064, 005/065, 005/066, 005/067, 005/068 +# and 005/069 at all times: +# time window radius is -1.50 to +1.49 hours +# for all other SATWND subtypes: +# time window radius is +/- 2.5 hours +#=========================================================================== + +# Skip all Indian satellite winds in SATWND (not in domain) +export SKIP_005021=YES +export SKIP_005022=YES +export SKIP_005023=YES + +# Skip legacy EUMETSAT AMV subsets; for testing skip in trigger or version file +#export SKIP_005064=YES +#export SKIP_005065=YES +#export SKIP_005066=YES + +# Skip VIIRS NPP/NOAA-20 IR long wave derived cld motion +export SKIP_005090=YES # old sequence (discontinued) +export SKIP_005091=YES # new sequence (unsupported in GSI) + +DTIM_earliest_005010=-1.00 +DTIM_latest_005010=-0.01 +DTIM_earliest_005011=-1.00 +DTIM_latest_005011=-0.01 +DTIM_earliest_005012=-1.00 +DTIM_latest_005012=-0.01 +DTIM_earliest_005019=-1.00 +DTIM_latest_005019=-0.01 + +DTIM_earliest_005064=-1.50 +DTIM_latest_005064=+1.49 +DTIM_earliest_005065=-1.50 +DTIM_latest_005065=+1.49 +DTIM_earliest_005066=-1.50 +DTIM_latest_005066=+1.49 + +DTIM_earliest_005067=-1.50 +DTIM_latest_005067=+1.49 +DTIM_earliest_005068=-1.50 +DTIM_latest_005068=+1.49 +DTIM_earliest_005069=-1.50 +DTIM_latest_005069=+1.49 + +DTIM_earliest_005081=${DTIM_earliest_005081:-"-1.50"} +DTIM_latest_005081=${DTIM_latest_005081:-"+1.49"} +#DTIM_earliest_005081=-1.50 +#DTIM_latest_005081=+1.49 + +$ushscript_dump/bufr_dump_obs.sh $dumptime 2.5 1 satwnd +error4=$? +echo "$error4" > $DATA/error4 + +set +x +echo "********************************************************************" +echo Script thread_4 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/4.out 2>&1 +EOF +set -x + +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_5; chmod +x thread_5 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_5 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=5 + +#============================================================================ +# Dump # 5 : MSONE1 -- TOTAL NUMBER OF SUBTYPES = 1 +# (1) +#============================================================================ + +def_time_window_5=0.5 # default time window for dump 5 is -0.5 to +0.5 hours + +# Time window -0.50 to +0.50 hours for MSONET for full and partial cycle runs +# (default) + +$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_5} 1 msone1 +error5=$? +echo "$error5" > $DATA/error5 + +set +x +echo "********************************************************************" +echo Script thread_5 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/5.out 2>&1 +EOF +set -x + #---------------------------------------------------------------- # Now launch the threads @@ -320,11 +435,13 @@ if [ "$launcher" = cfp ]; then echo ./thread_3 >> $DATA/poe.cmdfile # moved up echo ./thread_1 >> $DATA/poe.cmdfile echo ./thread_2 >> $DATA/poe.cmdfile + echo ./thread_4 >> $DATA/poe.cmdfile + echo ./thread_5 >> $DATA/poe.cmdfile if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes # ?? launcher_DUMP=${launcher_DUMP:-mpiexec} - $launcher_DUMP -np 3 --cpu-bind verbose,core cfp $DATA/poe.cmdfile + $launcher_DUMP -np 5 --cpu-bind verbose,core cfp $DATA/poe.cmdfile errpoe=$? if [ $errpoe -ne 0 ]; then $DATA/err_exit "***FATAL: EXIT STATUS $errpoe RUNNING POE COMMAND FILE" @@ -339,10 +456,12 @@ else ./thread_1 ./thread_2 ./thread_3 + ./thread_4 + ./thread_5 # wait fi -cat $DATA/1.out $DATA/2.out $DATA/3.out +cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out set +x echo " " @@ -352,12 +471,13 @@ set -x err1=`cat $DATA/error1` err2=`cat $DATA/error2` err3=`cat $DATA/error3` - +err4=`cat $DATA/error4` +err5=`cat $DATA/error5` #================================================================ export STATUS=YES -export DUMP_NUMBER=4 +export DUMP_NUMBER=6 $ushscript_dump/bufr_dump_obs.sh $dumptime 3.00 1 null @@ -370,8 +490,9 @@ fi if [ "$PROCESS_DUMP" = 'YES' ]; then - if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' ] ; then - for n in $err1 $err2 $err3 + if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' \ + -o "$err4" -gt '5' -o "$err5" -gt '5' ] ; then + for n in $err1 $err2 $err3 $err4 $err5 do if [ "$n" -gt '5' ]; then if [ "$n" -ne '11' -a "$n" -ne '22' ]; then @@ -381,7 +502,7 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then set +x echo echo " ###################################################### " -echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3" +echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, $err5" echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " echo " ###################################################### " echo @@ -398,7 +519,7 @@ echo set +x echo echo " ###################################################### " - echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3" + echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, $err5" echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " echo " ###################################################### " echo @@ -412,6 +533,9 @@ if [ "$RUN" == "rtma_ru" ] && [ "${SENDDBN^^}" = YES ] && [ -s ${COMSP}satwnd.tm $DBNROOT/bin/dbn_alert MODEL RTMA_RU_BUFR_satwnd $job ${COMSP}satwnd.tm00.bufr_d fi +# concatenate msonet and msone1, b/c prepobs only wants one file +cat ${COMSP}msone1.tm00.bufr_d >> ${COMSP}msonet.tm00.bufr_d + # # copy bufr_dumplist to $COMOUT per NCO SPA request # ------------------------------------------------- diff --git a/scripts/exurma_dump.sh b/scripts/exurma_dump.sh index 44bdbfc..9a5f00a 100755 --- a/scripts/exurma_dump.sh +++ b/scripts/exurma_dump.sh @@ -30,6 +30,10 @@ echo " - Copy bufr_dumplist to COMOUT. " echo " Dec 15 2021 - set for use on WCOSS2. " echo " Aug 1 2022 - Added SUBPFL, SALDRN, SNOCVR, " echo " and GMI1CR types. " +echo " Oct 12 2023 - Split msonet to msonet and msone1, " +echo " where msone1=255.030; concatenate " +echo " msonet and msone1 right after dump. " +echo " Seperated satwnd to its own dump group." ##################################################################### set -x @@ -61,6 +65,33 @@ err1=0 err2=0 err3=0 err4=0 +err5=0 +err6=0 +#restrict processing of unexpected big tanks +#this block appear in all /scripts/ex*_dump.sh proessing msonet and msone1 +TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb +TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb +TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb +if [ -s ${TANK}/${PDY}/b255/xx003 ]&& [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then + export SKIP_255003=YES + msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov +fi +if [ -s ${TANK}/${PDY}/b255/xx004 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then + export SKIP_255004=YES + msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov + fi +if [ -s ${TANK}/${PDY}/b255/xx030 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then + export SKIP_255030=YES + msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov +fi +#end of block + if [ "$PROCESS_DUMP" = 'YES' ]; then ################################### @@ -93,76 +124,27 @@ export STATUS=NO export DUMP_NUMBER=1 #======================================================================== -# Dump # 1 : ASCATT, SATWND, EFCLAM, GMI1CR -# (1) (14) (1) (1) -# -- TOTAL NUMBER OF SUBTYPES = 17 +# Dump # 1 : ASCATT, EFCLAM, GMI1CR +# (1) (1) (1) +# -- TOTAL NUMBER OF SUBTYPES = 3 # ===> Dumping of WNDSAT removed from here until new ingest feed is established # (had been dumped with a time window radius of -6.00 to 0.00 hours) # # time window radius is -6.00 to 0.00 hours for ASCATT -# for SATWND subtypes 005/010, 005/011, 005/012 and 005/019 at all -# times: -# time window radius is -1.00 to -0.01 hours -# for SATWND subtypes 005/064, 005/065, 005/066, 005/067, 005/068, -# and 005/069 at all times: -# time window radius is -1.50 to +1.49 hours -# for all other SATWND subtypes: -# time window radius is +/- 2.5 hours -# for EFCLAM: -# time window radius is +/- 0.5 hours +# time window radius is +/- 0.5 hours for EFCLAM #======================================================================= -# Skip all Indian satellite winds in SATWND (not in domain) -export SKIP_005021=YES -export SKIP_005022=YES -export SKIP_005023=YES - -# Skip legacy Meteosat AMV subsets. Replaced by new BUFR sequence subsets Oct 2020. -# for testing skip in trigger or version file -#export SKIP_005064=YES -#export SKIP_005065=YES -#export SKIP_005066=YES - -DTIM_earliest_005010=-1.00 -DTIM_latest_005010=-0.01 -DTIM_earliest_005011=-1.00 -DTIM_latest_005011=-0.01 -DTIM_earliest_005012=-1.00 -DTIM_latest_005012=-0.01 -DTIM_earliest_005019=-1.00 -DTIM_latest_005019=-0.01 - -DTIM_earliest_005064=-1.50 -DTIM_latest_005064=+1.49 -DTIM_earliest_005065=-1.50 -DTIM_latest_005065=+1.49 -DTIM_earliest_005066=-1.50 -DTIM_latest_005066=+1.49 - -DTIM_earliest_005067=-1.50 -DTIM_latest_005067=+1.49 -DTIM_earliest_005068=-1.50 -DTIM_latest_005068=+1.49 -DTIM_earliest_005069=-1.50 -DTIM_latest_005069=+1.49 - DTIM_earliest_ascatt=-6.00 DTIM_latest_ascatt=0.00 DTIM_earliest_efclam=-0.50 DTIM_latest_efclam=+0.50 -#DTIM_earliest_infav1=-2.50 -#DTIM_latest_infav1=+2.49 -#DTIM_earliest_005081=${DTIM_earliest_005081:-"-3.00"} -#DTIM_latest_005081=${DTIM_latest_005081:-"+2.99"} -DTIM_earliest_005081=-3.00 -DTIM_latest_005081=+1.00 DTIM_earliest_gmi1cr=${DTIM_earliest_gmi1cr:-"-3.00"} DTIM_latest_gmi1cr=${DTIM_latest_gmi1cr:-"+2.99"} -$ushscript_dump/bufr_dump_obs.sh $dumptime 2.5 1 ascatt satwnd efclam gmi1cr +$ushscript_dump/bufr_dump_obs.sh $dumptime 2.5 1 ascatt efclam gmi1cr error1=$? echo "$error1" > $DATA/error1 @@ -311,6 +293,137 @@ set -x EOF set -x +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_5; chmod +x thread_5 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_5 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=5 + +#=========================================================================== +# Dump # 5 : SATWND -- TOTAL NUMBER OF SUBTYPES = 14 +# for SATWND subtypes 005/010, 005/011, 005/012 and 005/019 at all +# times: +# time window radius is -1.00 to -0.01 hours +# for SATWND subtypes 005/064, 005/065, 005/066, 005/067, 005/068, +# and 005/069 at all times: +# time window radius is -1.50 to +1.49 hours +# for all other SATWND subtypes: +# time window radius is +/- 2.5 hours +#=========================================================================== + +# Skip all Indian satellite winds in SATWND (not in domain) +export SKIP_005021=YES +export SKIP_005022=YES +export SKIP_005023=YES + +# Skip legacy Meteosat AMV subsets. Replaced by new BUFR sequence subsets Oct 2020. +# for testing skip in trigger or version file +#export SKIP_005064=YES +#export SKIP_005065=YES +#export SKIP_005066=YES + +DTIM_earliest_005010=-1.00 +DTIM_latest_005010=-0.01 +DTIM_earliest_005011=-1.00 +DTIM_latest_005011=-0.01 +DTIM_earliest_005012=-1.00 +DTIM_latest_005012=-0.01 +DTIM_earliest_005019=-1.00 +DTIM_latest_005019=-0.01 + +DTIM_earliest_005064=-1.50 +DTIM_latest_005064=+1.49 +DTIM_earliest_005065=-1.50 +DTIM_latest_005065=+1.49 +DTIM_earliest_005066=-1.50 +DTIM_latest_005066=+1.49 + +DTIM_earliest_005067=-1.50 +DTIM_latest_005067=+1.49 +DTIM_earliest_005068=-1.50 +DTIM_latest_005068=+1.49 +DTIM_earliest_005069=-1.50 +DTIM_latest_005069=+1.49 + +#DTIM_earliest_005081=${DTIM_earliest_005081:-"-3.00"} +#DTIM_latest_005081=${DTIM_latest_005081:-"+2.99"} +DTIM_earliest_005081=-3.00 +DTIM_latest_005081=+1.00 + +$ushscript_dump/bufr_dump_obs.sh $dumptime 2.5 1 satwnd +error5=$? +echo "$error5" > $DATA/error5 + +set +x +echo "********************************************************************" +echo Script thread_5 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/5.out 2>&1 +EOF +set -x + +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_6; chmod +x thread_6 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_6 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=6 + +#============================================================================ +# Dump # 6 : MSONE1 -- TOTAL NUMBER OF SUBTYPES = 1 +# (1) +#============================================================================ + +def_time_window_6=0.5 # default time window for dump 6 is -0.5 to +0.5 hours + +# Time window -0.50 to +0.50 hours for MSONET for full and partial cycle runs +# (default) + +$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_6} 1 msone1 +error6=$? +echo "$error6" > $DATA/error6 + +set +x +echo "********************************************************************" +echo Script thread_6 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/6.out 2>&1 +EOF +set -x + #---------------------------------------------------------------- # Now launch the threads @@ -329,11 +442,13 @@ if [ "$launcher" = cfp ]; then echo ./thread_1 >> $DATA/poe.cmdfile echo ./thread_2 >> $DATA/poe.cmdfile echo ./thread_4 >> $DATA/poe.cmdfile + echo ./thread_5 >> $DATA/poe.cmdfile + echo ./thread_6 >> $DATA/poe.cmdfile if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes # ?? launcher_DUMP=${launcher_DUMP:-mpiexec} - $launcher_DUMP -np 3 --cpu-bind verbose,core cfp $DATA/poe.cmdfile + $launcher_DUMP -np 6 --cpu-bind verbose,core cfp $DATA/poe.cmdfile errpoe=$? if [ $errpoe -ne 0 ]; then $DATA/err_exit "***FATAL: EXIT STATUS $errpoe RUNNING POE COMMAND FILE" @@ -349,10 +464,12 @@ else ./thread_2 ./thread_3 ./thread_4 + ./thread_5 + ./thread_6 fi -cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out +cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out set +x echo " " @@ -363,12 +480,14 @@ err1=`cat $DATA/error1` err2=`cat $DATA/error2` err3=`cat $DATA/error3` err4=`cat $DATA/error4` +err5=`cat $DATA/error5` +err6=`cat $DATA/error6` #================================================================ export STATUS=YES -export DUMP_NUMBER=5 +export DUMP_NUMBER=7 $ushscript_dump/bufr_dump_obs.sh $dumptime 3.00 1 null @@ -381,8 +500,9 @@ fi if [ "$PROCESS_DUMP" = 'YES' ]; then - if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o "$err4" -gt '5' ]; then - for n in $err1 $err2 $err3 $err4 + if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' \ + -o "$err4" -gt '5' -o "$err5" -gt '5' -o "$err6" -gt '5' ]; then + for n in $err1 $err2 $err3 $err4 $err5 $err6 do if [ "$n" -gt '5' ]; then if [ "$n" -ne '11' -a "$n" -ne '22' ]; then @@ -392,7 +512,7 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then set +x echo echo " ###################################################### " -echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4" +echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, $err5, $err6" echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " echo " ###################################################### " echo @@ -409,7 +529,7 @@ echo set +x echo echo " ###################################################### " - echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4" + echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, $err5, $err6" echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " echo " ###################################################### " echo @@ -419,6 +539,9 @@ echo # endif loop $PROCESS_DUMP fi +# concatenate msonet and msone1, b/c prepobs only wants one file +cat ${COMSP}msone1.tm00.bufr_d >> ${COMSP}msonet.tm00.bufr_d + # # copy bufr_dumplist to $COMOUT per NCO SPA request # ------------------------------------------------- diff --git a/sorc/bufr_remorest.fd/remorest.f b/sorc/bufr_remorest.fd/remorest.f index 7db0b28..252f782 100644 --- a/sorc/bufr_remorest.fd/remorest.f +++ b/sorc/bufr_remorest.fd/remorest.f @@ -5,25 +5,28 @@ C C ABSTRACT: THIS PROGRAM READS THROUGH AN INPUT BUFR FILE (NORMALLY C EITHER A PREPBUFR OR A DATA DUMP FILE) WHICH CAN CONTAIN A MIXTURE -C OF REPORTS WHICH ARE UNRESTRICTED OR RESTRICTED (W.R.T. -C REDISTRIBUTION OUTSIDE OF NCEP) AND EITHER WRITES OUT (TO AN -C OTHERWISE IDENTICAL BUFR FILE) ONLY THOSE REPORTS WHICH ARE -C UNRESTRICTED OR WRITES OUT ALL REPORTS BUT WITH MASKED REPORT ID's -C FOR THOSE REPORTS WHICH ARE RESTRICTED (WHAT IT DOES IS BASED ON -C NAMELIST SWITCHES). IT DETERMINES WHICH REPORTS ARE RESTRICTED -C BASED ON EITHER THE MESSAGE TYPE AND SUBTYPE (MAKING UP THE TABLE A -C ENTRY IN DATA DUMP FILES), THE PREPBUFR TABLE A ENTRY (PREPBUFR -C FILES) (IN EITHER CASE WHEN THE MESSAGE IS KNOWN TO CONTAIN EITHER -C ALL RESTRICTED OR ALL UNRESTRICTED REPORTS), THE DUMP REPORT TYPE -C (WHEN REPORTS IN A PARTICULAR PREPBUFR TABLE A ENTRY ARE MASKED) OR -C THE RESTRICTED FLAG (MNEMONIC "RSRD") WITHIN EACH REPORT IN A -C MESSAGE (WHEN THE MESSAGE MAY CONTAIN A MIXTURE OF RESTRICTED AND -C UNRESTRICTED REPORTS). (NOTE: THE CASE OF MASKING REPORT ID's IN -C DATA DUMP FILES CAN ONLY BE DONE CURRENTLY FOR TABLE A ENTRIES -C WHERE ALL REPORTS ARE CONSIDERED TO BE RESTRICTED. THE CASE OF -C MASKING REPORT ID's IN PREPBUFR FILES CAN ONLY BE DONE CURRENTLY -C FOR TABLE A ENTRIES WHERE ALL REPORTS WITH A PARTICULAR SET OF DUMP -C REPORT TYPES ARE CONSIDERED TO BE RESTRICTED.) +C OF REPORTS WHICH ARE UNRESTRICTED OR RESTRICTED FOR AT LEAST +C SOME PERIOD OF TIME (W.R.T. REDISTRIBUTION OUTSIDE OF NCEP) AND +C EITHER WRITES OUT (TO AN OTHERWISE IDENTICAL BUFR FILE) ONLY THOSE +C REPORTS WHICH ARE NON-RESTRICTED OR WRITES OUT ALL REPORTS BUT WITH +C MASKED REPORT ID's FOR THOSE REPORTS WHICH ARE RESTRICTED (WHAT IT +C DOES IS BASED ON NAMELIST SWITCHES). {NOTE: WHEN A RESTRICTED +C REPORT IS WRITTEN OUT WITH A MASKED REPORT ID, ITS RESTRICTION FLAG +C (MNEMONIC "RSRD") AND ITS NUMBER OF HOURS UNTIL THE RESTRICTION +C EXPIRES (MNEMONIC "EXPRSRD") ARE RE-SET TO MISSING SO THAT THE +C REPORT IS NO LONGER CONSIDERED TO BE RESTRICTED.} IT DETERMINES +C WHICH REPORTS ARE RESTRICTED BASED ON EITHER THE MESSAGE TYPE AND +C SUBTYPE (MAKING UP THE TABLE A ENTRY IN DATA DUMP FILES), THE +C PREPBUFR TABLE A ENTRY (PREPBUFR FILES) (IN EITHER CASE WHEN THE +C MESSAGE IS KNOWN TO CONTAIN EITHER ALL RESTRICTED OR ALL NON- +C RESTRICTED REPORTS), THE DUMP REPORT TYPE (WHEN REPORTS IN A +C PARTICULAR PREPBUFR TABLE A ENTRY ARE MASKED) OR THE REPORT'S +C VALUE FOR "RSRD") AND, IF "RSRD" IS SET, ITS VALUE FOR "EXPRSRD" +C WITHIN EACH REPORT IN A MESSAGE (WHEN THE MESSAGE MAY CONTAIN A +C MIXTURE OF RESTRICTED AND NON-RESTRICTED REPORTS). (NOTE: THE CASE +C OF MASKING REPORT ID's IN DATA DUMP FILES CAN ONLY BE DONE +C CURRENTLY FOR TABLE A ENTRIES WHERE ALL REPORTS ARE CONSIDERED TO +C BE RESTRICTED. C C PROGRAM HISTORY LOG: C 2003-07-14 D. A. KEYSER -- ORIGINAL AUTHOR @@ -101,6 +104,38 @@ C ID w/ MASKSTID. C BENEFIT: BUFR format restricted ships data can be properly C dispensed to public users. +C XXXX-XX-XX D. A. KEYSER -- All reports in message types in namelist +C "MSG_MIXED" are now also tested for their value for EXPRSRD +C (number of hours until the restriction expires) when their +C restriction flag (RSRD) is set - any reports with a non-missing +C EXPRSRD less than the difference in hours between the current +C UTC wall-clock date and the BUFR file center time ("DIFF_HR") +C minus 4 are now not considered restricted and are copied (prior +C to this, the value of EXPRSRD was ignored and all reports with +C RSRD set were restricted and skipped), "DIFF_HR" is a new +C imported script environment variable; Improved information that +C is printed out for each report that is either skipped or retained +C (latter is currently commented out); Improved information printed +C out at end summarizing counts of reports retained, skipped or +C masked +C XXXX-XX-XX D. A. KEYSER -- +C - For PREPBUFR files only, all reports in message types in +C namelist "MSG_MASK" (if their dump report type is listed in +C namelist "IMASK_T29") are now tested for their values for both +C RSRD and EXPRSRD and are only considered to be restricted if +C RSRD is set and EXPRSRD is .GE. the difference in hours between +C the current UTC wall-clock date and the BUFR file center time +C (read in via imported script environment variable "DIFF_HR") +C minus 4. (Prior to this RSRD and EXPRSRD were not examined, ALL +C reports from the message type having the listed dump report type +C were considered to be restricted. This is still the case for +C message types in namelist "MSG_MASK" for DUMP files.) +C - When a report coming out of "MSG_MASK" is deemed to be +C restricted (either PREPBUFR or DUMP) and its id is masked to be +C "MASKSTID", the report's value for RSRD is now re-set to MISSING +C when copied to non-restricted file (EXPRSRD is also set to +C MISSING, but it likely was already MISSING). +C - Improved documentation and printout. C C USAGE: C INPUT FILES: @@ -108,16 +143,17 @@ C UNIT 11 - INPUT BUFR FILENAME (IN CHARACTER) (USED ONLY FOR C DIAGNOSTIC PRINT INFO) C UNIT 21 - BUFR FILE (PREPBUFR OR DUMP) CONTAINING A MIXTURE OF -C RESTRICTED AND UNRESTRICTED REPORTS +C RESTRICTED (AT LEAST FOR SOME PERIOD FO TIME) AND +C NON-RESTRICTED REPORTS C C OUTPUT FILES: C UNIT 06 - STANDARD OUTPUT PRINT C UNIT 51 - BUFR FILE (PREPBUFR OR DUMP) CONTAINING EITHER ONLY -C UNRESTRICTED REPORTS OR UNRESTRICTED REPORTS AND -C RESTRICTED REPORTS WHOSE REPORT ID's HAVE BEEN MASKED -C {I.E., ALL OCCURRENCES OF ID IN A REPORT ARE -C UNILATERALLY CHANGED TO EITHER "MASKSTID" (WHERE THE -C ID IS STORED BY ITSELF) OR TO ALL "X"'s WHERE THE +C NON-RESTRICTED REPORTS OR NON-RESTRICTED REPORTS AND +C PREVIOUSLY RESTRICTED REPORTS WHOSE REPORT ID's HAVE +C BEEN MASKED {I.E., ALL OCCURRENCES OF ID IN A REPORT +C ARE UNILATERALLY CHANGED TO EITHER "MASKSTID" (WHERE +C THE ID IS STORED BY ITSELF) OR TO ALL "X"'s WHERE THE C NUMBER OF "X"'s CORRESPONDS TO THE THE NUMBER OF C CHARACTERS IN THE ORIGINAL REPORT ID (WHERE THE ID IS C EMBEDDED IN THE REPLICATED RAW REPORT BULLETIN HEADER @@ -126,6 +162,7 @@ C SUBPROGRAMS CALLED: (LIST ALL CALLED FROM ANYWHERE IN CODES) C C UNIQUE: - NONE +C SYSTEM: - GET_ENVIRONMENT_VARIABLE C LIBRARY: C W3LIB - W3TAGB W3TAGE ERREXIT C BUFR - DATELEN OPENBF IREADMG UFBCNT NMSUB @@ -139,35 +176,70 @@ C CONTENTS OF INPUT NAMELIST "SWITCHES": C MSG_RESTR - 20-WORD CHARACTER*8 ARRAY CONTAINING UP TO 20 BUFR C MESSAGE TABLE A ENTRIES FOR WHICH ALL REPORTS ARE -C RESTRICTED AND WILL BE REMOVED (These messages are -C skipped over without unpacking any reports) +C CONSIDERED TO BE RESTRICTED AND WILL ALWAYS BE +C REMOVED (These messages are skipped over without +C unpacking any reports) C MSG_MIXED - 20-WORD CHARACTER*8 ARRAY CONTAINING UP TO 20 BUFR -C MESSAGE TABLE A ENTRIES WHICH CONTAIN A MIXTURE OF -C RESTRICTED AND UNRESTRICTED REPORTS BASED ON BUFR -C MNEMONIC "RSRD" - ALL RESTRICTED REPORTS WILL BE -C REMOVED (These messages must be unpacked and every -C report must be checked to see if it is restricted - -C unrestricted reports are copied, restricted reports -C are skipped over) -C MSG_MASKA - 20-WORD CHARACTER*8 ARRAY CONTAINING UP TO 20 BUFR +C MESSAGE TABLE A ENTRIES WHICH MAY CONTAIN A MIXTURE +C OF REPORTS WITH AND WITHOUT THEIR RESTRICTION +C INDICATOR (BUFR MNEMONIC "RSRD") BEING SET. IF "RSRD" +C IS NOT SET -OR- IT IS SET AND THE TIME IN HOURS FOR +C THE EXPIRATION ON RESTRICTION (BUFR MNEMONIC +C "EXPRSRD") IS ALSO SET AND HAS A VALUE LESS THAN +C "DIFF_HR" (THE DIFFERENCE IN HOURS BETWEEN THE +C CURRENT UTC WALL-CLOCK DATE AND THE BUFR FILE CENTER +C TIME) MINUS 4, THE REPORT WILL BE RETAINED. +C OTHERWISE, IT WILL BE REMOVED. (These messages must +C be unpacked and values for "RSRD" and "EXPRSRD" must +C be checked for every report. If "EXPRSRD is missing, +C it is set to 99999999 hours essentially meaning the +C report is restricted for all time if "RSRD" is set.) +C MSG_MASKA - FOR PREPBUFR FILES: +C 20-WORD CHARACTER*8 ARRAY CONTAINING UP TO 20 BUFR +C MESSAGE TABLE A ENTRIES WHICH, IF THEIR DUMP REPORT +C TYPE IS ONE OF UP TO 10 POSSIBLE LISTED IN SWITCH +C IMASK_T29 (WHERE EACH LINE IN IMASK_T29 APPLIES TO +C THE TABLE A ENTRY IN THE SAME LINE NUMBER HERE), MAY +C CONTAIN A MIXTURE OF REPORTS WITH AND WITHOUT THEIR +C RESTRICTION INDICATOR (BUFR MNEMONIC "RSRD") BEING +C SET. IF "RSRD" IS NOT SET FOR A REPORT -OR- IT IS SET +C AND THE TIME IN HOURS FOR THE EXPIRATION ON +C RESTRICTION (BUFR MNEMONIC "EXPRSRD") IS ALSO SET AND +C HAS A VALUE LESS THAN "DIFF_HR" (THE DIFFERENCE IN +C HOURS BETWEEN THE CURRENT UTC WALL-CLOCK DATE AND THE +C PREPBUFR FILE CENTER TIME) MINUS 4, THE REPORT WILL +C BE COPIED WITHOUT ANY CHANGES. OTHERWISE, THE REPORT +C WILL NOT BE REMOVED, BUT ALL OCCURRENCES OF ITS ID +C WILL BE CHANGED TO "MASKSTID". IN ADDITION, ITS +C VALUES FOR "RSRD" AND "EXPRSRD" WILL BE RE-SET TO +C MISSING SO THAT THE REPORT WILL NO LONGER BE +C CONSIDERED AS RESTRICTED. REPORTS WITH A DUMP REPORT +C TYPE NOT LISTED IN SWITCH IMASK_T29 ARE CONSIDERED TO +C BE NON-RESTRICTED AND THEIR REPORT IDS ARE NOT +C CHANGED (MASKED OUT) WHEN COPIED. (These m essages +C must be unpacked and the values for "T29", "RSRD" and +C "EXPRSRD" must be checked for every report. If +C "EXPRSRD" is missing, it is set to 99999999 hours +C essentially meaning the report is restricted for all +C time if "RSRD" is set.) +C FOR DATA DUMP FILES: +C 20-WORD CHARACTER*8 ARRAY CONTAINING UP TO 20 BUFR C MESSAGE TABLE A ENTRIES FOR WHICH ALL REPORTS ARE -C RESTRICTED {BUT, FOR PREPBUFR FILES, ONLY IF THEIR -C DUMP REPORT TYPE IS ONE OF UP TO 10 POSSIBLE LISTED -C IN SWITCH IMASK_T29 (EACH LINE IN IMASK_T29 APPLIES -C TO THE TABLE A ENTRY IN THE SAME LINE NUMBER HERE); -C FOR DATA DUMP FILES, IMASK_T29 IS NOT USED, ALL -C REPORTS IN THE TABLE A ENTRIES HERE WILL BE -C RESTRICTED)} - THESE WILL NOT BE REMOVED, BUT ALL -C OCCURRENCES OF THEIR REPORT ID's WILL BE UNILATERALLY -C CHANGED TO EITHER "MASKSTID" (WHERE THE ID IS STORED -C BY ITSELF, PREPBUFR OR DUMP FILES) OR TO ALL "X"'s -C WHERE THE NUMBER OF "X"'s CORRESPONDS TO THE NUMBER -C OF CHARACTERS IN THE ORIGINAL REPORT ID (WHERE THE ID -C IS EMBEDDED IN THE REPLICATED RAW REPORT BULLETIN -C HEADER STRING, DUMP FILES ONLY) (These messages must -C be unpacked and every occurrence of every report's id -C must be changed to either "MASKSTID" or "X"'s before -C the report is copied) +C CONSIDERED TO BE RESTRICTED. THEY WILL NOT BE +C REMOVED, BUT ALL OCCURRENCES OF THEIR REPORT IDS WILL +C BE UNILATERALLY CHANGED TO EITHER "MASKSTID" (WHERE +C THE ID IS STORED BY ITSELF) OR TO ALL "X"'s WHERE THE +C NUMBER OF "X"'s CORRESPONDS TO THE NUMBER OF +C CHARACTERS IN THE ORIGINAL REPORT ID (WHERE THE ID IS +C EMBEDDED IN THE RAW REPORT BULLETIN HEADER STRING). +C IN ADDITION, THEIR VALUES FOR "RSRD" AND "EXPRSRD" +C WILL BE RE-SET TO MISSING SO THAT THE REPORTS WILL NO +C LONGER BE CONSIDERED AS RESTRICTED. (These messages +C must be unpacked and every occurrence of every +C report's id must be changed to either "MASKSTID" or +C "X"'s and every report's "RSRD" and "EXPRSRD" values +C must be changed to missing before the report is +C copied. Switch IMASK_T29 is not considered here.) C IMASK_T29 - (10,20) INTEGER ARRAY CONTAINING UP TO 10 POSSIBLE C DUMP REPORT TYPES (1ST DIMENSION) FOR THE UP TO 20 C POSSIBLE PREPBUFR TABLE A ENTRIES LISTED IN SWITCH @@ -178,7 +250,7 @@ C than one of MSG_RESTR, MSG_MIXED or MSG_MASKA. C Note 2: Any Table A entry not in either MSG_RESTR, MSG_MIXED or C MSG_MASKA is assumed to be a Table A entry for BUFR -C messages for which ALL reports are UNRESTRICTED (these +C messages for which ALL reports are NON-RESTRICTED (these C messages are copied intact, no reports are unpacked). C Note 3: Always fill in these arrays MSG_RESTR, MSG_MIXED and C MSG_MASKA beginning with word 1. If there are less than @@ -201,8 +273,8 @@ C Note 5: For PREPBUFR files, a value of "99999" in array IMASK_T29 C means not applicable whereas a value of "000" means C reports in all dump report types in the corresponding -C Table A entry in MSG_MASKA should be restricted (masked) -C {in this case IMASK_T29(1,x) should be set to 000 and +C Table A entry in MSG_MASKA should be considered {in this +C case IMASK_T29(1,x) should be set to 000 and C IMASK_T29(2:10,x) should be set to 99999 for all reports C in Table A entry MSG_MASKA(x) since they would all be C ignored - this is the default for all Table A entries @@ -211,21 +283,54 @@ C C LIST OF REPORT ID MNEMONICS IN EACH REPORT WHICH ARE CURRENTLY C MASKED WHEN TABLE A ENTRY IS FOUND IN MSG_MASKA (AND FOR PREPBUFR -C FILES DUMP REPORT TYPE MATCHES ONE OF THE TYPES IN IMASK_T29): +C FILES DUMP REPORT TYPE MATCHES ONE OF THE TYPES IN IMASK_T29 +C AND THE REPORT IS CONSIDERED TO BE RESTRICTED BASED ON ITS VALUES +C FOR "RSRD" and "EXPRSRD"): C C PREPBUFR file: "SID" - chgd to "MASKSTID" (all Tbl A entries) +C (PREPBUFR file "RSRD" and "EXPRSRD" also set to missing) C DUMP file: "RPID" - chgd to "MASKSTID" (all Tbl A entries) -C DUMP file: "SHPC8" - chgd to "MASKSTID" (Tbl A entry NC001001) +C DUMP file: "SHPC8" - chgd to "MASKSTID" (Tbl A entry NC001001 +C and NC001101) C DUMP file: "RRSTG" - chgd to "X" (where the number of "X"'s C corresponds to the the number of C characters in the original report id ) C (all applicable Tbl A entries) +C (DUMP file: "RSRD" and "EXPRSRD" also set to missing) C C Note: Currently for dump files, the only Table A entry where all C occurrences of report id in a report are known to be masked -C is NC001001. This code may have to be modified to add this -C ability to mask all occurrences of report id for other Table -C A entries. +C is NC001001 and NC001101. This code may have to be modified +C to add this ability to mask all occurrences of report id for +C other Table A entries. +C +C +C ONE SCRIPT ENVIRONMENT VARIABLE IS READ IN: +C DIFF_HR - The difference in hours between the current UTC +C wall-clock date and the BUFR file center time +C (should always be a positive number!). This +C is used (after subtracting 4 hours***) to +C determine if a BUFR subset that is marked as +C restricted (via mnemonic "RSRD") is past the +C expiration time of the restriction (mnemonic +C "EXPRSRD") and should thus not be filtered out. +C (Note: used only for BUFR DUMP and PREPBUFR +C subsets in message types listed in namelist +C switch MSG_MIXED and for PREPBUFR subsets in +C applicalbe dump types and message types listed +C in namelist switches IMASK_T29 and MSG_MASKA.) +C Defaults to ZERO if not found (i.e., not +C exported by the executing script). +C *** Four hours is subtracted from DIFF_HR prior to +C testing against the BUFR file center time in +C order to account for some reports having an obx +C time as much as 3-4 hours prior to the center +C time in either a dump or PEPBUFR file. This +C ensures that these reports are not inadvertently +C retained if the difference between the current +C wall-clock date and the BUFR file center time is +C very close to the time period of the restriction. +C DIFF_HR minus 4 can never be less than zero. C C C ATTRIBUTES: @@ -238,16 +343,34 @@ PROGRAM BUFR_REMOREST CHARACTER*2040 RAWRPT_STG CHARACTER*80 FILE + CHARACTER*8 DIFF_HR,PREPBUFR_MSGTYP(21),PRVSTG_prep,SPRVSTG_prep, + $ PRVSTG_dump,SPRVSTG_dump CHARACTER*8 SUBSET,SID,MSG_RESTR(20),MSG_MIXED(20),MSG_MASKA(20), $ RAWRPT(255),SID_orig - REAL*8 RID_8(6),GETBMISS,BMISS,RASTR_8(255) - INTEGER IMASK_T29(10,20) + REAL*8 RID_8(8),RASTR_8(255),LALOH_8(2),ACRN_8,ACID_8,SID_8, + $ RPID_8,PRV_prep_8(2),PRV1_dump_8(255), + $ PRV2_dump_8(255),BMISS,GETBMISS,rsrd_8(2) + INTEGER IMASK_T29(10,20),IRSUB_this_MR(20),IRSUB_this_MM(20), + $ IMSUB_this(10,20),IRSUB_this_sub_MR(20,0:255), + $ IRSUB_this_sub_MM(20,0:255),IUSUB_this(0:255,0:256) - EQUIVALENCE (RID_8(1),SID),(RAWRPT,RASTR_8) + EQUIVALENCE (RID_8(1),SID),(RAWRPT,RASTR_8), + $ (PRV_prep_8(1),PRVSTG_prep), + $ (PRV_prep_8(2),SPRVSTG_prep), + $ (PRV1_dump_8(1),PRVSTG_dump), + $ (PRV2_dump_8(1),SPRVSTG_dump) DATA LUBFI/21/,LUBFJ/51/,IREC/0/,IRSUB/0/,IMSUB/0/, - $ IUSUB/0/,ireco_last/0/ + $ IRSUB_this_MR/20*0/,IRSUB_this_MM/20*0/,IMSUB_this/200*0/, + $ IUSUB/0/,IRSUB_this_sub_MR/5120*0/,IRSUB_this_sub_MM/5120*0/, + $ IUSUB_this/65792*0/,ireco_last/0/ + DATA PREPBUFR_MSGTYP/'ADPUPA ','AIRCAR ','AIRCFT ','SATWND ', + $ 'PROFLR ','VADWND ','SATEMP ','ADPSFC ', + $ 'SFCSHP ','SFCBOG ','SPSSMI ','SYNDAT ', + $ 'ERS1DA ','GOESND ','QKSWND ','MSONET ', + $ 'GPSIPW ','RASSDA ','WDSATR ','ASCATW ', + $ 'unknown '/ NAMELIST/SWITCHES/MSG_RESTR,MSG_MIXED,MSG_MASKA,IMASK_T29 CALL W3TAGB('BUFR_REMOREST',2021,0175,0012,'NP22') @@ -261,6 +384,14 @@ PROGRAM BUFR_REMOREST $ 'REMOVES OR MASKS RESTRICTED REPORTS FROM A BUFR FILE'/42X, $ 'LAST REVISION 24 Jun 2021'//30X,'INPUT BUFR FILENAME IS: ',A) + IMASK_T29 = 99999 + IMASK_T29(1,:) = 000 + MSG_RESTR = ' ' + MSG_MIXED = ' ' + MSG_MASKA = ' ' + IDIFF_HR = 0 + READ(5,SWITCHES) + C Set BUFRLIB missing (BMISS) to 10E8_8 to avoid integer*4 overflows C ------------------------------------------------------------------ CALL SETBMISS(10E8_8) @@ -269,18 +400,18 @@ PROGRAM BUFR_REMOREST print'(" BUFRLIB value for missing is: ",G0)', bmiss print'(1X)' - IMASK_T29 = 99999 - IMASK_T29(1,:) = 000 - MSG_RESTR = ' ' - MSG_MIXED = ' ' - MSG_MASKA = ' ' - READ(5,SWITCHES) +C .... store rsrd_8 array as "almost" missing, will later encode back +C into output *.nr PRPEBUFR file for reports with id masked, +C overwriting original value (won't overwrite if exactly missing) +C --------------------------------------------------------------- + rsrd_8 = bmiss - 0.01_8 IF(MSG_RESTR(1).NE.' ') THEN PRINT 107 - 107 FORMAT(//' ANY BUFR MESSAGES READ IN WITH THE FOLLOWING TABLE A ', - $ 'ENTRIES ARE SKIPPED (WITHOUT UNPACKING) BECAUSE THEY CONTAIN ', - $ 'ONLY'/' RESTRICTED REPORTS ALL OF WHICH ARE TO BE REMOVED:'/) + 107 FORMAT(//' ALL BUFR MESSAGES READ IN WITH THE FOLLOWING TABLE A ', + $ 'ENTRIES ARE SKIPPED (WITHOUT UNPACKING) BECAUSE THEY'/ + $ ' CONTAIN ONLY RESTRICTED REPORTS (FOR SOME PERIOD OF TIME) ALL', + $ ' OF WHICH ARE TO BE REMOVED:'/) DO I = 1,20 IF(MSG_RESTR(I).EQ.' ') EXIT PRINT *, MSG_RESTR(I) @@ -288,32 +419,100 @@ PROGRAM BUFR_REMOREST END IF IF(MSG_MIXED(1).NE.' ') THEN PRINT 108 - 108 FORMAT(//' ANY BUFR MESSAGES READ IN WITH THE FOLLOWING TABLE A ', + 108 FORMAT(//' ALL BUFR MESSAGES READ IN WITH THE FOLLOWING TABLE A ', $ 'ENTRIES ARE UNPACKED REPORT BY REPORT BECAUSE THEY CAN'/ - $ ' CONTAIN A MIXTURE OF BOTH UNRESTRICTED AND RESTRICTED REPORTS', - $ ' - ALL RESTRICTED REPORTS ARE TO BE REMOVED:'/) + $ ' CONTAIN A MIXTURE OF BOTH NON-RESTRICTED AND RESTRICTED ', + $ 'REPORTS - ALL RESTRICTED REPORTS WITHIN THE TIME'/' PERIOD OF ', + $ 'THE RESTRICTION ARE TO BE REMOVED:'/) DO I = 1,20 IF(MSG_MIXED(I).EQ.' ') EXIT PRINT *, MSG_MIXED(I) ENDDO + CALL GET_ENVIRONMENT_VARIABLE('DIFF_HR',DIFF_HR) + READ(DIFF_HR,'(I8)',END=88,ERR=88) IDIFF_HR +cppppp +ccc print * +ccc print *, 'DIFF_HR, IDIFF_HR : ',DIFF_HR, IDIFF_HR +ccc print * +cppppp + GO TO 89 + 88 CONTINUE + PRINT 115 + 115 FORMAT(/'+++++BUFR_REMOREST: WARNING: ERROR OBTAINING IDIFF_HR -', + $ ' SET TO ZERO AND CONTINUE'/) + IDIFF_HR = 0 + 89 CONTINUE + IDIFF_HR_m4 = IDIFF_HR - 4 + IDIFF_HR_m4 = MAX(IDIFF_HR_m4,0) + PRINT 118, IDIFF_HR,IDIFF_HR_m4 + 118 FORMAT(/' ===> The difference between the current wall-clock ', + $ 'date and the BUFR file center time is',I8,' hours.'/6X, + $ 'Will consider the difference here to be only',I8,' hours when ', + $ 'comparing against the time period of the'/6X,'restriction ', + $ '(this takes into account that some reports may have obs times ', + $ 'as much as 3-4 hours prior to'/6X,'the BUFR file center time, ', + $ 'and so ensures that these reports are not inadvertently ', + $ 'retained if the'/6X,'difference between the current wall-clock', + $ ' date and the BUFR file center time is very close to the'/ + $ 6X,'the time period of the restriction).'/) END IF IF(MSG_MASKA(1).NE.' ') THEN - PRINT 1107 - 1107 FORMAT(//' ANY BUFR MESSAGES READ IN WITH THE FOLLOWING TABLE A ', + if(MSG_MASKA(1)(1:2).eq.'NC') then + +c .. DUMP file case +c -------------- + PRINT 1107 + 1107 FORMAT(//' ALL BUFR MESSAGES READ IN WITH THE FOLLOWING TABLE A ', + $ 'ENTRIES CONTAIN REPORTS ALL CONSIDERED TO BE RESTRICTED. THEY '/ + $ 'ARE'/' UNPACKED REPORT BY REPORT AND EACH REPORT''S ID (ALL ', + $ 'OCCURRENCES IN A REPORT) IS CHANGED TO "MASKSTID" OR "X"''S ', + $ '(MASKED)'/' AND IT''S VALUES FOR "RSRD" AND "EXPRSRD" ARE RE-', + $ 'SET TO MISSING PRIOR TO THEIR BEING WRITTEN BACK OUT:'/) + + else + +c .. PREPBUFR file case +c ------------------ + print 3108 + 3108 FORMAT(//' ALL BUFR MESSAGES READ IN WITH THE FOLLOWING TABLE A ', $ 'ENTRIES ARE UNPACKED REPORT BY REPORT BECAUSE THEY CAN'/ - $ ' CONTAIN EITHER ALL OR SOME RESTRICTED REPORTS WHOSE REPORT ', - $ 'ID''s (ALL OCCURRENCES IN A REPORT) ARE ALL CHANGED'/' TO ', - $ '"MASKSTID" OR "X"''S (MASKED) PRIOR TO THEIR BEING WRITTEN ', - $ 'BACK OUT:'/) + $ ' CONTAIN A MIXTURE OF BOTH NON-RESTRICTED AND RESTRICTED ', + $ 'REPORTS - THE ID''s FOR ALL RESTRICTED REPORTS WITHIN THE'/ + $ ' TIME PERIOD OF THE RESTRICTION ARE ALL CHANGED TO "MASKSTID" ', + $ '(MASKED) AND THEIR VALUES FOR "RSRD" AND "EXPRSRD"'/' ARE RE-', + $ 'SET TO MISSING PRIOR TO THEIR BEING WRITTEN BACK OUT:'/) + end if + DO I = 1,20 IF(MSG_MASKA(I).EQ.' ') EXIT PRINT *, MSG_MASKA(I) ENDDO + + if(MSG_MASKA(1)(1:2).ne.'NC') then + +c .. PREPBUFR file case +c ------------------ + CALL GET_ENVIRONMENT_VARIABLE('DIFF_HR',DIFF_HR) + READ(DIFF_HR,'(I8)',END=988,ERR=988) IDIFF_HR +cppppp +ccc print * +ccc print *, 'DIFF_HR, IDIFF_HR : ',DIFF_HR, IDIFF_HR +ccc print * +cppppp + GO TO 989 + 988 CONTINUE + PRINT 115 + IDIFF_HR = 0 + 989 CONTINUE + IDIFF_HR_m4 = IDIFF_HR - 4 + IDIFF_HR_m4 = MAX(IDIFF_HR_m4,0) + PRINT 118, IDIFF_HR,IDIFF_HR_m4 + end if END IF PRINT 109 109 FORMAT(//' ALL OTHER BUFR MESSAGES READ IN ARE COPIED INTACT ', - $ '(WITHOUT UNPACKING) BECAUSE THEY CONTAIN ONLY UNRESTRICTED ', + $ '(WITHOUT UNPACKING) BECAUSE THEY CONTAIN ONLY NON-RESTRICTED ', $ 'REPORTS'//) CALL DATELEN(10) @@ -403,57 +602,209 @@ PROGRAM BUFR_REMOREST CYCLE LOOP1 C********************************************************************** ELSE IF(SUBSET.EQ.MSG_RESTR(I) .OR. (MSG_RESTR(I)(6:8).EQ. - $ 'xxx'.AND.SUBSET(1:5).EQ.MSG_RESTR(I)(1:5))) THEN + $ 'xxx'.AND.SUBSET(1:5).EQ.MSG_RESTR(I)(1:5))) THEN PRINT 112 112 FORMAT(' #####>>>> ALL reports in this message are RESTRICTED ', $ 'and are REMOVED - do NOT copy this message to output BUFR file') + IRSUB_this_MR(I) = IRSUB_this_MR(I) + ISUB + IF(SUBSET(1:2).EQ.'NC'.AND.MSG_RESTR(I)(6:8).EQ.'xxx')THEN + READ(SUBSET(6:8),'(I3)') ISUBSET_678 + IF(ISUBSET_678.GE.0.AND.ISUBSET_678.LE.255) THEN + IRSUB_this_sub_MR(I,ISUBSET_678) = + $ IRSUB_this_sub_MR(I,ISUBSET_678) + ISUB + ELSE + PRINT 7115, ISUBSET_678,SUBSET + 7115 FORMAT(/'+++++BUFR_REMOREST: WARNING: INVALID BUFR MESSAGE ', + $ 'SUBTYPE READ IN: ',I5.3,', SUBSET = ',A,' CANNOT INCREMENT ', + $ 'RESTRICTED REPORT'/29X,'SKIPPED COUNTER FOR THIS SUBSET'/) + END IF + END IF IRSUB = IRSUB + ISUB CYCLE LOOP1 C********************************************************************** ELSE IF(SUBSET.EQ.MSG_MIXED(I) .OR. (MSG_MIXED(I)(6:8).EQ. - $ 'xxx'.AND.SUBSET(1:5).EQ.MSG_MIXED(I)(1:5))) THEN + $ 'xxx'.AND.SUBSET(1:5).EQ.MSG_MIXED(I)(1:5))) THEN PRINT 113 - 113 FORMAT(' #####>>>> This msg has mixture of restricted/non-', - $ 'restricted data, restricted data are REMOVED- unpk each rpt & ', - $ 'test on mnem. "RSRD"') + 113 FORMAT(' #####>>>> rpts in this msg mixed restr/non-restr- restr', + $ 'data w/i exp. time REMOVED- unpk each rpt & test mnems. ', + $ '"RSRD" & "EXPRSRD"') C READ A SUBSET (REPORT) IN MESSAGE C --------------------------------- LOOP1n2: DO WHILE(IREADSB(LUBFI).EQ.0) -C DECODE THE SUBSET (REPORT) LOOKING FOR RESTRICTED FLAG -C (mnemonic "RSRD") -C ------------------------------------------------------ +C DECODE THE SUBSET (REPORT) LOOKING FOR RESTRICTED FLAG (MNEMONIC +C "RSRD") AND TIME OF EXPIRATION ON RESTRICTION (MNEMONIC "EXPRSRD") +C (if "EXPRSRD" is missing set it to 99999999 hours essentially +C meaning the report is restricted for all time) +C ------------------------------------------------------------------- - CALL UFBINT(LUBFI,RID_8,6,1,NLV, - $ 'SID RPT YOB XOB TYP RSRD') - IF(RID_8(6).GT.0.AND.RID_8(6).LT.BMISS/2) THEN - IF(RID_8(5).LT.BMISS/2) THEN + CALL UFBINT(LUBFI,RID_8,7,1,NLV, + $ 'SID RPT YOB XOB TYP RSRD EXPRSRD') + IF(RID_8(6).GT.0.AND.IBFMS(RID_8(6)).EQ.0) THEN + IF(IBFMS(RID_8(7)).NE.0) RID_8(7) = 99999999. + IF(IDIFF_HR_m4.LE.RID_8(7)) THEN + IF(IBFMS(RID_8(5)).EQ.0) THEN C Normally for PREPBUFR files C --------------------------- - PRINT 104, SID,(RID_8(II),II=2,4), - $ (NINT(RID_8(II)),II=5,6) - 104 FORMAT(10X,'- Skip report ',A8,' at ',F6.2,' UTC, ',F6.2, - $ ' (N+/S-) LAT, ',F7.2,'(E) LON, RTYP= ',I3,', RSRD=',I5) - ELSE + IF(SUBSET.EQ.'MSONET ') THEN + CALL UFBINT(LUBFI,PRV_prep_8,2,1,NLV, + $ 'PRVSTG SPRVSTG') + PRINT 8104, SID,(RID_8(II),II=2,4), + $ (NINT(RID_8(II)),II=5,7),PRVSTG_prep, + $ SPRVSTG_prep + 8104 FORMAT(5X,'- **Skip ',A8,F7.2,'UTC',F7.2,'(N+/S-) LAT',F7.2,' E ', + $ 'LON, RTYP=',I3,', RSRD=',I5,' EXPRSRD=',I5,', PRVID=',A8, + $ ' SPRVID=',A8) + ELSE + PRINT 104, SID,(RID_8(II),II=2,4), + $ (NINT(RID_8(II)),II=5,7) + 104 FORMAT(5X,'- **Skip report ',A8,' at ',F6.2,' UTC, ',F6.2, + $ ' (N+/S-) LAT, ',F7.2,' E LON, RTYP= ',I3,', RSRD=',I5, + $ ' EXPRSRD=',I5) + END IF + ELSE C Normally for DATA DUMP files C ---------------------------- - CALL UFBINT(LUBFI,RID_8,5,1,NLV, - $ 'RPID HOUR MINU CLAT CLON') - PRINT 110, SID,(NINT(RID_8(II)),II=2,3), - $ (RID_8(II),II=4,5), NINT(RID_8(6)) - 110 FORMAT(10X,'- Skip report ',A8,' at ',2(I2.2),' UTC, ',F6.2, - $ ' (N+/S-) LAT, ',F7.2,'(E+/W-) LON, RSRD=',I5) + CALL UFBINT(LUBFI,RID_8,7,1,NLV, + $ 'RPID HOUR MINU CLAT CLON RSRD EXPRSRD') + IF(IBFMS(RID_8(4)).NE.0) THEN + CALL UFBINT(LUBFI,LALOH_8,2,1,NLV, + $ 'CLATH CLONH') + RID_8(4:5) = LALOH_8 + END IF + IF(IBFMS(RID_8(1)).NE.0) THEN + IF(SUBSET.EQ.'NC004004' .OR. + $ SUBSET.EQ.'NC004006' .OR. + $ SUBSET.EQ.'NC004009' .OR. + $ SUBSET.EQ.'NC004010' .OR. + $ SUBSET.EQ.'NC004011' .OR. + $ SUBSET.EQ.'NC004014') THEN + CALL UFBINT(LUBFI,ACRN_8,1,1,NLV,'ACRN') + RID_8(1) = ACRN_8 + ELSE IF(SUBSET.EQ.'NC004008' .OR. + $ SUBSET.EQ.'NC004012' .OR. + $ SUBSET.EQ.'NC004013') THEN + CALL UFBINT(LUBFI,ACID_8,1,1,NLV,'ACID') + RID_8(1) = ACID_8 + ELSE IF(SUBSET.EQ.'NC007001' .OR. + $ SUBSET.EQ.'NC007002') THEN + SID = ' ' + ELSE + SID = 'MISSING ' + END IF + END IF + IF(SUBSET(3:5).EQ.'255') THEN + CALL UFBINT(LUBFI,PRV1_dump_8,1,255,NLV, + $ 'PRVSTG') + IF(NLV.LT.1) PRVSTG_dump = ' ' + CALL UFBINT(LUBFI,PRV2_dump_8,1,255,NLV, + $ 'SPRVSTG') + IF(NLV.LT.1) SPRVSTG_dump = ' ' + PRINT 8105, SID,(NINT(RID_8(II)),II=2,3), + $ (RID_8(II),II=4,5), NINT(RID_8(6)), + $ NINT(RID_8(7)),PRVSTG_dump,SPRVSTG_dump + 8105 FORMAT(5X,'- **Skip ',A8,' at ',2(I2.2),' UTC',F7.2,' (N+/S-) ', + $ 'LAT',F8.2,' (E+/W-) LON, RSRD=',I5,' EXPRSRD=',I5,', PRVID=',A8, + $ ' SPRVID=',A8) + ELSE + PRINT 110, SID,(NINT(RID_8(II)),II=2,3), + $ (RID_8(II),II=4,5), NINT(RID_8(6)), + $ NINT(RID_8(7)) + 110 FORMAT(5X,'- **Skip report ',A8,' at ',2(I2.2),' UTC, ',F6.2, + $ ' (N+/S-) LAT, ',F7.2,'(E+/W-) LON, RSRD=',I5,' EXPRSRD=',I5) + END IF + END IF + IRSUB_this_MM(I) = IRSUB_this_MM(I) + 1 + IF(SUBSET(1:2).EQ.'NC'.AND. + $ MSG_MIXED(I)(6:8).EQ.'xxx') THEN + READ(SUBSET(6:8),'(I3)') ISUBSET_678 + IF(ISUBSET_678.GE.0.AND.ISUBSET_678.LE.255) + $ THEN + IRSUB_this_sub_MM(I,ISUBSET_678) = + $ IRSUB_this_sub_MM(I,ISUBSET_678) + 1 + ELSE + PRINT 7115, ISUBSET_678,SUBSET + END IF + END IF + IRSUB = IRSUB + 1 + CYCLE LOOP1n2 END IF - IRSUB = IRSUB + 1 - CYCLE LOOP1n2 END IF +cppppp +ccc IF(IBFMS(RID_8(5)).EQ.0) THEN +ccc IF(SUBSET.EQ.'MSONET ') THEN +ccc CALL UFBINT(LUBFI,PRV_prep_8,2,1,NLV, +ccc $ 'PRVSTG SPRVSTG') +ccc PRINT 9104, SID,(RID_8(II),II=2,4), +ccc $ (NINT(RID_8(II)),II=5,7),PRVSTG_prep, +ccc $ SPRVSTG_prep + 9104 FORMAT(5X,'- Retain ',A8,F7.2,'UTC',F7.2,'(N+/S-) LAT',F7.2,' E ', + $ 'LON, RTYP=',I3,', RSRD=',I5,' EXPRSRD=',I5,', PRVID=',A8, + $ ' SPRVID=',A8) +ccc ELSE +ccc PRINT 214, SID,(RID_8(II),II=2,4), +ccc $ (NINT(RID_8(II)),II=5,7) + 214 FORMAT(5X,'- Retain report ',A8,' at ',F6.2,' UTC, ',F6.2, + $ ' (N+/S-) LAT, ',F7.2,' E LON, RTYP= ',I3,', RSRD=',I5, + $ ' EXPRSRD=',I5) +ccc END IF +ccc ELSE +ccc CALL UFBINT(LUBFI,RID_8,7,1,NLV, +ccc $ 'RPID HOUR MINU CLAT CLON RSRD EXPRSRD') +ccc IF(IBFMS(RID_8(4)).NE.0) THEN +ccc CALL UFBINT(LUBFI,LALOH_8,2,1,NLV,'CLATH CLONH') +ccc RID_8(4:5) = LALOH_8 +ccc END IF +ccc IF(IBFMS(RID_8(1)).NE.0) THEN +ccc IF(SUBSET.EQ.'NC004004' .OR. +ccc $ SUBSET.EQ.'NC004006' .OR. +ccc $ SUBSET.EQ.'NC004009' .OR. +ccc $ SUBSET.EQ.'NC004010' .OR. +ccc $ SUBSET.EQ.'NC004011' .OR. +ccc $ SUBSET.EQ.'NC004014') THEN +ccc CALL UFBINT(LUBFI,ACRN_8,1,1,NLV,'ACRN') +ccc RID_8(1) = ACRN_8 +ccc ELSE IF(SUBSET.EQ.'NC004008' .OR. +ccc $ SUBSET.EQ.'NC004012' .OR. +ccc $ SUBSET.EQ.'NC004013') THEN +ccc CALL UFBINT(LUBFI,ACID_8,1,1,NLV,'ACID') +ccc RID_8(1) = ACID_8 +ccc ELSE IF(SUBSET.EQ.'NC007001' .OR. +ccc $ SUBSET.EQ.'NC007002') THEN +ccc SID = ' ' +ccc ELSE +ccc SID = 'MISSING ' +ccc END IF +ccc END IF +ccc IF(SUBSET(3:5).EQ.'255') THEN +ccc CALL UFBINT(LUBFI,PRV1_dump_8,1,255,NLV, +ccc $ 'PRVSTG') +ccc IF(NLV.LT.1) PRVSTG_dump = ' ' +ccc CALL UFBINT(LUBFI,PRV2_dump_8,1,255,NLV, +ccc $ 'SPRVSTG') +ccc IF(NLV.LT.1) SPRVSTG_dump = ' ' +ccc PRINT 9105, SID,(NINT(RID_8(II)),II=2,3), +ccc $ (RID_8(II),II=4,5), NINT(RID_8(6)), +ccc $ NINT(RID_8(7)),PRVSTG_dump,SPRVSTG_dump + 9105 FORMAT(5X,'- Retain ',A8,' at ',2(I2.2),' UTC',F7.2,' (N+/S-) ', + $ 'LAT',F8.2,' (E+/W-) LON, RSRD=',I5,' EXPRSRD=',I5,', PRVID=',A8, + $ ' SPRVID=',A8) +ccc ELSE +ccc PRINT 215, SID,(NINT(RID_8(II)),II=2,3), +ccc $ (RID_8(II),II=4,5),NINT(RID_8(6)), +ccc $ NINT(RID_8(7)) + 215 FORMAT(5X,'- Retain report ',A8,' at ',2(I2.2),' UTC, ',F6.2, + $ ' (N+/S-) LAT, ',F7.2,'(E+/W-) LON, RSRD=',I5,' EXPRSRD=',I5) +ccc END IF +ccc END IF +cppppp CALL OPENMB(LUBFJ,SUBSET,IDATE) call ufbcnt(lubfj,ireco,isubo) if(ireco.ne.ireco_last) then @@ -463,26 +814,66 @@ PROGRAM BUFR_REMOREST ireco_last = ireco CALL UFBCPY(LUBFI,LUBFJ) CALL WRITSB(LUBFJ) + IF(SUBSET(1:2).EQ.'NC') THEN + READ(SUBSET(3:5),'(I3)') ISUBSET_345 + READ(SUBSET(6:8),'(I3)') ISUBSET_678 + ELSE + ISUBSET_678 = 256 + ISUBSET_345 = -99 + DO II = 1,20 + IF(SUBSET.EQ.PREPBUFR_MSGTYP(II)) THEN + ISUBSET_345 = II + EXIT + END IF + ENDDO + IF(ISUBSET_345.EQ.-99) ISUBSET_345 = 21 + END IF + IF(SUBSET(1:2).EQ.'NC' .AND. + $ ((ISUBSET_345.LT.0.OR.ISUBSET_345.GT.255) .OR. + $ (ISUBSET_678.LT.0.OR.ISUBSET_678.GT.255))) THEN + PRINT 7116, SUBSET + 7116 FORMAT(/'+++++BUFR_REMOREST: WARNING: INVALID BUFR MESSAGE ', + $ 'TYPE AND/OR SUBTYPE READ IN, SUBSET = ',A,' CANNOT INCREMENT' + $ /29X,'NON-RESTRICTED REPORT COPIED COUNTER FOR THIS SUBSET'/) + ELSE + IUSUB_this(ISUBSET_345,ISUBSET_678) = + $ IUSUB_this(ISUBSET_345,ISUBSET_678) + 1 + END IF IUSUB = IUSUB + 1 ENDDO LOOP1n2 CYCLE LOOP1 C********************************************************************** ELSE IF(SUBSET.EQ.MSG_MASKA(I) .OR. (MSG_MASKA(I)(6:8).EQ. - $ 'xxx'.AND.SUBSET(1:5).EQ.MSG_MASKA(I)(1:5))) THEN + $ 'xxx'.AND.SUBSET(1:5).EQ.MSG_MASKA(I)(1:5))) THEN IF(IMASK_T29(1,I).EQ.000) THEN - PRINT 1113 - 1113 FORMAT(' #####>>>> ALL rpts in msg RESTRICTED - unpk each rpt, ', - $ 'MASK all occurrences of id (chg to "MASKSTID" or "X"''s), copy', - $ ' to output file') + if(subset(1:2).eq.'NC') then + +c .. DUMP file case +c -------------- + PRINT 1113 + 1113 FORMAT(' ###>> ALL rpts in msg RESTRICTED - unpk each rpt, MASK ', + $ 'all id''s (chg to "MASKSTID" or "X"''s), set to non-restr, ', + $ ' copy to output file') + else + +c .. PREPBUFR file case +c ------------------ + print 1114 + end if ELSE + +c .. PREPBUFR file case +c ------------------ PRINT 1114 - 1114 FORMAT(' #####>>>> SOME or ALL rpts in msg RESTRICTED:') + 1114 FORMAT(' ###>> some rpts in msg may be RESTRICTED: if so & w/i ', + $ 'expir time, MASK id (chg to "MASKSTID"), set to non-restr, cpy', + $ ' to output file') DO J = 1,10 IF(IMASK_T29(J,I).NE.99999) PRINT 1115, $ IMASK_T29(J,I) - 1115 FORMAT(' -- ALL rpts in dump type ',I3,' RESTRICTED- unpk', - $ ' each rpt, MASK all occurrences of id (chg to "MASKSTID" or ', - $ '"X"s), copy to o-put file') + 1115 FORMAT(' -- rpts in dump type ',I3,' mixed restr./non-restr.- ', + $ 'for restr. rpts w/i expir. time, MASK id, set to non-restr & ', + $ 'copy to output file') ENDDO END IF @@ -502,9 +893,9 @@ PROGRAM BUFR_REMOREST end if ireco_last = ireco CALL UFBCPY(LUBFI,LUBFJ) - CALL UFBINT(LUBFI,RID_8,6,1,NLV, - $ 'SID RPT YOB XOB TYP T29') - IF(RID_8(5).LT.BMISS/2) THEN + CALL UFBINT(LUBFI,RID_8,8,1,NLV, + $ 'SID RPT YOB XOB TYP T29 RSRD EXPRSRD') + IF(IBFMS(RID_8(5)).EQ.0) THEN C Come here for PREPBUFR files (report id is in mnemonic "SID") C -- check for a match of dump report type @@ -514,46 +905,68 @@ PROGRAM BUFR_REMOREST IF(IMASK_T29(1,I).EQ.000.OR. $ IMASK_T29(J,I).EQ.NINT(RID_8(6))) THEN -C .... dump report types match - this report is restricted -C --------------------------------------------------- - - PRINT 1104, SID,(RID_8(II),II=2,4), - $ (NINT(RID_8(II)),II=5,6) - 1104 FORMAT(10X,'- Change id of report ',A8,' at ',F6.2,' UTC, ',F6.2, - $ ' (N+/S-) LAT, ',F7.2,'(E) LON, RTYP= ',I3,', DTYP=',I5, - $ ' TO "MASKSTID"') - SID = 'MASKSTID' +C .... dump report types match - look for restricted flag (mnemonic +C "RSRD") and time of expiration on restriction (mnemonic +C "EXPRSRD") (if "EXPRSRD" is missing set it to 99999999 hours +C essentially meaning the report is restricted for all time) +C ------------------------------------------------------------ + if(rid_8(7).gt.0.and.ibfms(rid_8(7)).EQ.0) + $ then + if(ibfms(rid_8(8)).ne.0) + $ rid_8(8) = 99999999. + if(IDIFF_HR_m4.le.rid_8(8)) then + PRINT 1104, SID,(RID_8(II),II=2,4), + $ (NINT(RID_8(II)),II=5,8) + 1104 FORMAT(5X,'- Chg ID of rpt ',A8,' at',F6.2,' UTC, ',F6.2, + $ ' LAT,',F7.2,' E LON, RTYP=',I3,', DTYP=',I5, + $ ', RSRD=',I5,', EXPRSRD=',I5,' TO "MASKSTID"') + SID = 'MASKSTID' + SID_8 = RID_8(1) + IMSUB_this(J,I) = IMSUB_this(J,I) + 1 C .... update report id to masked value - "MASKSTID" C --------------------------------------------- + CALL UFBINT(LUBFJ,SID_8,1,1,IRET,'SID') - CALL UFBINT(LUBFJ,RID_8(1),1,1,IRET,'SID') - EXIT +C .... re-set RSRD & EXPRSRD to "almost" missing so rpt is no longer +C restricted +C -----------------------r------------------------------------- + call ufbint(lubfj,rsrd_8,2,1,iret,'RSRD EXPRSRD') + EXIT + END IF + END IF END IF - END DO + ENDDO ELSE C Come here for DATA DUMP files where all reports are restricted C {report id is in mnemonic "RPID" (and for surface ship reports in -C Table A entry 'NC001001' also in mnemonic "SHPC8"), report id may -C also be embedded in replicated raw bulletin header string if it is -C present) +C Table A entry 'NC001001' and 'NC001101' also in mnemonic "SHPC8"), +C report id may also be embedded in replicated raw bulletin header +C string if it is present} C ------------------------------------------------------------------- IUPDATE_RAWRPT = 0 - CALL UFBINT(LUBFI,RID_8,6,1,NLV, - $ 'RPID HOUR MINU CLAT CLON RSRD') + CALL UFBINT(LUBFI,RID_8,7,1,NLV, + $ 'RPID HOUR MINU CLAT CLON RSRD EXPRSRD') PRINT 1110, SID,(NINT(RID_8(II)),II=2,3), - $ (RID_8(II),II=4,5), NINT(RID_8(6)) - 1110 FORMAT(' - Chg all occurrences of id of rpt ',A8,' at ',2(I2.2), - $ ' UTC, ',F6.2,' (N+/S-) LAT, ',F7.2,'(E+/W-) LON, RSRD=',I5, - $ ' TO "MASKSTID" OR "X"''s') + $ (RID_8(II),II=4,5), NINT(RID_8(6)),NINT(RID_8(7)) + 1110 FORMAT(' - Chg all instances of rpt id ',A8,' ',2(I2.2),' UTC, ', + $ F6.2,'(N+/S-) LAT ',F7.2,'(E+/W-) LON, RSRD=',I5,', EXPRSRD=',I5, + $ ' to MASKSTID or X''s') SID_orig = SID SID = 'MASKSTID' + RPID_8 = RID_8(1) + IMSUB_this(1,I) = IMSUB_this(1,I) + 1 C .... update report id in output file to masked value - "MASKSTID" C ------------------------------------------------------------ + CALL UFBINT(LUBFJ,RPID_8,1,1,IRET,'RPID') +C .... re-set RSRD & EXPRSRD to "almost" missing so rpt is no longer +C restricted +C -----------------------r------------------------------------- + call ufbint(lubfj,rsrd_8,2,1,iret,'RSRD EXPRSRD') CALL UFBINT(LUBFJ,RID_8(1),1,1,IRET,'RPID') IF(SUBSET.EQ.'NC001001'.or. $ subset.eq.'NC001101') THEN @@ -561,21 +974,18 @@ PROGRAM BUFR_REMOREST C .... update 2nd rpt id in ship rpts in output file to masked value C - "MASKSTID" C ------------------------------------------------------------- - - CALL UFBINT(LUBFJ,RID_8(1),1,1,IRET,'SHPC8') + CALL UFBINT(LUBFJ,RPID_8,1,1,IRET,'SHPC8') END IF C .... see if a replicated raw report bulletin header string is C present C -------------------------------------------------------- - CALL UFBINT(LUBFI,RASTR_8,1,255,NLV,'RRSTG') IST = 1 - IF(NLV.GT.0) THEN + IF(NLV.GT.0) THEN C .... it is, store entire string in character array RAWRPT_STG C -------------------------------------------------------- - DO II = 1,NLV IEN = IST+7 RAWRPT_STG(IST:IEN) = RAWRPT(II) @@ -589,9 +999,8 @@ PROGRAM BUFR_REMOREST C for later check to see if it is embedded one or more C times in raw report bulletin header string C ------------------------------------------------------- - DO II = 1,8 - IF(SID_orig(II:II).EQ.' ') THEN + IF(SID_orig(II:II).EQ.' ') THEN ICHAR_id = II - 1 EXIT END IF @@ -599,17 +1008,16 @@ PROGRAM BUFR_REMOREST cppppp ccc print *, 'original sid had ',ICHAR_id,' characters' cppppp - IF(ICHAR_id.GE.1) THEN + IF(ICHAR_id.GE.1) THEN C .... next parse through entire raw report bulletin header C string looking for one or more occurrences of report id C and "X" out the id C ------------------------------------------------------- - DO II = 1,IEN IF(II+ICHAR_id-1.GT.IEN) EXIT IF(RAWRPT_STG(II:II+ICHAR_id-1).EQ. - $ SID_orig(1:ICHAR_id)) THEN + $ SID_orig(1:ICHAR_id)) THEN cppppp ccc print *, 'Found a match to orig sid in RAWRPT_STG',' - in bytes ', ccc $ II,' to ',II+ICHAR_id-1,' - set to "X"' @@ -624,7 +1032,6 @@ PROGRAM BUFR_REMOREST C .... reconstruct replicated raw report bulletin header string, C but now with the id "X"'d out C --------------------------------------------------------- - IF(IUPDATE_RAWRPT.EQ.1) THEN IST = 1 DO II = 1,NLV @@ -636,7 +1043,6 @@ PROGRAM BUFR_REMOREST C .... finally, update replicated raw report bulletin header C string with the id "X"'d out in output file C ----------------------------------------------------- - CALL UFBINT(LUBFJ,RASTR_8,1,NLV,IRET, $ 'RRSTG') cppppp @@ -646,9 +1052,31 @@ PROGRAM BUFR_REMOREST END IF END IF END IF - IF(SID.EQ.'MASKSTID') THEN + IF(SID.EQ.'MASKSTID') THEN IMSUB = IMSUB + 1 ELSE + IF(SUBSET(1:2).EQ.'NC') THEN + READ(SUBSET(3:5),'(I3)') ISUBSET_345 + READ(SUBSET(6:8),'(I3)') ISUBSET_678 + ELSE + ISUBSET_678 = 256 + ISUBSET_345 = -99 + DO II = 1,20 + IF(SUBSET.EQ.PREPBUFR_MSGTYP(II)) THEN + ISUBSET_345 = II + EXIT + END IF + ENDDO + IF(ISUBSET_345.EQ.-99) ISUBSET_345 = 21 + END IF + IF(SUBSET(1:2).EQ.'NC' .AND. + $ ((ISUBSET_345.LT.0.OR.ISUBSET_345.GT.255) .OR. + $ (ISUBSET_678.LT.0.OR.ISUBSET_678.GT.255))) THEN + PRINT 7116, SUBSET + ELSE + IUSUB_this(ISUBSET_345,ISUBSET_678) = + $ IUSUB_this(ISUBSET_345,ISUBSET_678) + 1 + END IF IUSUB = IUSUB + 1 END IF CALL WRITSB(LUBFJ) @@ -658,8 +1086,30 @@ PROGRAM BUFR_REMOREST ENDDO LOOP1n1 C********************************************************************** PRINT 114 - 114 FORMAT(' ALL reports in this message are UNRESTRICTED - copy', + 114 FORMAT(' ALL reports in this message are NON-RESTRICTED - copy', $ ' this message, intact, to output BUFR file') + IF(SUBSET(1:2).EQ.'NC') THEN + READ(SUBSET(3:5),'(I3)') ISUBSET_345 + READ(SUBSET(6:8),'(I3)') ISUBSET_678 + ELSE + ISUBSET_678 = 256 + ISUBSET_345 = -99 + DO II = 1,20 + IF(SUBSET.EQ.PREPBUFR_MSGTYP(II)) THEN + ISUBSET_345 = II + EXIT + END IF + ENDDO + IF(ISUBSET_345.EQ.-99) ISUBSET_345 = 21 + END IF + IF(SUBSET(1:2).EQ.'NC' .AND. + $ ((ISUBSET_345.LT.0.OR.ISUBSET_345.GT.255) .OR. + $ (ISUBSET_678.LT.0.OR.ISUBSET_678.GT.255))) THEN + PRINT 7116, SUBSET + ELSE + IUSUB_this(ISUBSET_345,ISUBSET_678) = + $ IUSUB_this(ISUBSET_345,ISUBSET_678) + ISUB + END IF IUSUB = IUSUB + ISUB CALL CLOSMG(LUBFJ) CALL COPYMG(LUBFI,LUBFJ) @@ -673,12 +1123,75 @@ PROGRAM BUFR_REMOREST CALL CLOSBF(LUBFI) CALL CLOSBF(LUBFJ) - PRINT 106, IRSUB,IMSUB,IUSUB - 106 FORMAT(//'==> A TOTAL OF',I11,' RESTRICTED REPORTS WERE SKIPPED' - $ /'==> A TOTAL OF',I11,' RESTRICTED REPORTS WERE MASKED', - $ ' (ALL OCCURRENCES OF REPORT ID) THEN COPIED' - $ /'==> A TOTAL OF',I11,' UNRESTRICTED REPORTS WERE COPIED'/ - $ /'PROGRAM COMPLETED SUCCESSFULLY'/) + PRINT 106, IRSUB + 106 FORMAT(//'==> A TOTAL OF',I11,' RESTRICTED REPORTS WERE ', + $ 'SKIPPED') + DO I = 1,20 + IF(IRSUB_this_MR(I).GT.0) THEN + PRINT 1106, IRSUB_this_MR(I),MSG_RESTR(I) + DO J = 0,255 + IF(IRSUB_this_sub_MR(I,J).GT.0) THEN + PRINT 2106, IRSUB_this_sub_MR(I,J),MSG_RESTR(I)(1:5),J + END IF + ENDDO + END IF + ENDDO + DO I = 1,20 + IF(IRSUB_this_MM(I).GT.0) THEN + PRINT 1106, IRSUB_this_MM(I),MSG_MIXED(I) + DO J = 0,255 + IF(IRSUB_this_sub_MM(I,J).GT.0) THEN + PRINT 2106, IRSUB_this_sub_MM(I,J),MSG_MIXED(I)(1:5),J + END IF + ENDDO + END IF + ENDDO + 1106 FORMAT(16X,'--> ',I11,' reports from message type ',A) + 2106 FORMAT(21X,'--> ',I11,' reports from message type ',A,I3.3) + + PRINT 116, IMSUB + 116 FORMAT(//'==> A TOTAL OF',I11,' RESTRICTED REPORTS WERE ', + $ 'MASKED AND THEN COPIED') + DO I = 1,20 + DO J = 1,10 + IF(IMSUB_this(J,I).GT.0) THEN + IF(IMASK_T29(J,I).GT.0) THEN + PRINT 1108,IMSUB_this(J,I),MSG_MASKA(I),IMASK_T29(J,I) + ELSE + PRINT 1106,IMSUB_this(J,I),MSG_MASKA(I) + END IF + END IF + ENDDO + ENDDO + 1108 FORMAT(16X,'--> ',I11,' reports from message type ',A, + $ ', DUMP report type ',I5) + + PRINT 117, IUSUB + 117 FORMAT(//'==> A TOTAL OF',I11,' NON-RESTRICTED REPORTS WERE ', + $ 'COPIED'/) + IPRINT_FLAG = 0 + DO I = 1,21 + IF(IUSUB_this(I,256).GT.0) THEN + PRINT 2108, IUSUB_this(I,256),PREPBUFR_MSGTYP(I) + IPRINT_FLAG = 1 + END IF + ENDDO + 2108 FORMAT(16X,'--> ',I11,' reports from message type ',A) + + IF(IPRINT_FLAG.EQ.0) THEN + DO I = 0,255 + DO J = 0,255 + IF(IUSUB_this(I,J).GT.0) THEN + PRINT 2107, IUSUB_this(I,J),I,J + END IF + ENDDO + ENDDO + END IF + 2107 FORMAT(16X,'--> ',I11,' reports from message type NC',I3.3,I3.3) + + PRINT 3117 + 3117 FORMAT(//'PROGRAM COMPLETED SUCCESSFULLY'/) + CALL W3TAGE('BUFR_REMOREST') diff --git a/triggers/cycqsub b/triggers/cycqsub index 75b4973..5d249a4 100755 --- a/triggers/cycqsub +++ b/triggers/cycqsub @@ -15,11 +15,7 @@ else fi # cd to destination output dir for logfiles to be written there w/ pid -<<<<<<< HEAD -outdir=/lfs/h2/emc/ptmp/$USER/CRON/$desc/output -======= outdir=/lfs/h2/emc/stmp/$USER/CRON/$desc/output ->>>>>>> 21d7d9a2d2983684f4403c7ee9023a38c152c3c3 echo $outdir mkdir -p $outdir cd $outdir diff --git a/triggers/jglobal_dump.wc2.pbs b/triggers/jglobal_dump.wc2.pbs index f41ba96..9b79e75 100755 --- a/triggers/jglobal_dump.wc2.pbs +++ b/triggers/jglobal_dump.wc2.pbs @@ -93,7 +93,7 @@ export COMOUT_ROOT=${DATAROOT}/CRON/${DESC}/com export COMPONENT=atmos export DEBUG_LEVEL=3 -export LOUD=ON +export LOUD=on export KEEPDATA=YES diff --git a/ush/bufr_avgdata.sh b/ush/bufr_avgdata.sh index 339fc12..f827a7e 100755 --- a/ush/bufr_avgdata.sh +++ b/ush/bufr_avgdata.sh @@ -338,7 +338,7 @@ for cycle_avg in $cycles; do > accum.obs.counts.${cycle_avg} for dtg in "${arr30d[@]}"; do set +x - grep -h -e " REPORTS" -e "^EOF @@@" \ + grep --text -h -e " REPORTS" -e "^EOF @@@" \ $OBCNTarch/${net}.${dtg}/*${cycle_avg}.status.*bufr_d | \ grep -e " in data group " -e "^EOF @@@" | grep -e " HAS" -e "^EOF @@@" | \ grep -v -e "Domain" >> delete @@ -354,7 +354,7 @@ for cycle_avg in $cycles; do done done -grep -e "^_" -e "^:" $DUMPLIST | grep -Fe "#>" > dumplist +grep --text -e "^_" -e "^:" $DUMPLIST | grep -Fe "#>" > dumplist # Generate a new average file diff --git a/ush/bufr_datacount.sh b/ush/bufr_datacount.sh index 4902aa0..012f9e2 100755 --- a/ush/bufr_datacount.sh +++ b/ush/bufr_datacount.sh @@ -105,7 +105,7 @@ echo $RETC > $COMOUT/${RUN}.${cycle}.dump_alert_flag.$tmmark # and 12-month ago 30-day avg files # ------------------------------------------------------------------------ -grep -e "^_" -e "^:" $LIST_CNT | grep -Fe "#>" > dumplist +grep --text -e "^_" -e "^:" $LIST_CNT | grep -Fe "#>" > dumplist if [ -s $COMIN/${RUN}.${cycle}.status.tm00.bufr_d ]; then status_bufr_d=$COMIN/${RUN}.${cycle}.status.tm00.bufr_d @@ -114,7 +114,7 @@ elif [ -s $COMOUT/${RUN}.${cycle}.status.tm00.bufr_d ]; then else status_bufr_d=/dev/null fi -grep -Fe " REPORTS" $status_bufr_d | grep -Fe " in data group " | \ +grep --text -Fe " REPORTS" $status_bufr_d | grep -Fe " in data group " | \ grep -Fe " HAS" | grep -v -Fe "Domain" > delete awk -F" HAS" '{print$2}' delete | awk -F" REPORTS" '{print$1}' | \ sed "s/[^0-9]/ /g" > delete.r @@ -126,7 +126,7 @@ rm delete delete.l delete.r > obcount_30davg.${NET}.current if [ -s $AVGDarch_IN/obcount_30davg.${NET}.current ]; then - grep "^# " $AVGDarch_IN/obcount_30davg.${NET}.current | \ + grep --text "^# " $AVGDarch_IN/obcount_30davg.${NET}.current | \ sed "s/# ......./ /g" | sed "s/[^0-9]/ /g" \ > obcount_30davg.${NET}.current fi @@ -146,7 +146,7 @@ for months_ago in 3 6 9 12; do > obcount_30davg.${NET}.${months_ago}months_ago if [ -s $AVGDarch_IN/obcount_30davg.${NET}.${arch_year}${arch_month} ]; then - grep "^# " $AVGDarch_IN/obcount_30davg.${NET}.${arch_year}${arch_month} | \ + grep --text "^# " $AVGDarch_IN/obcount_30davg.${NET}.${arch_year}${arch_month} | \ sed "s/# ......./ /g" | sed "s/[^0-9]/ /g" \ > obcount_30davg.${NET}.${months_ago}months_ago fi diff --git a/ush/build.sh b/ush/build.sh index 328dd51..d6add26 100755 --- a/ush/build.sh +++ b/ush/build.sh @@ -16,7 +16,7 @@ INSTALL_PREFIX=${INSTALL_PREFIX:-"$pkg_root/install"} BUILD_4NCO=${BUILD_4NCO:-"YES"} target=$(echo $INSTALL_TARGET | tr [:upper:] [:lower:]) -if [[ "$target" =~ ^(wcoss2|hera|orion)$ ]]; then +if [[ "$target" =~ ^(wcoss2|hera|orion|jet)$ ]]; then source $pkg_root/versions/build.ver set +x module use $pkg_root/modulefiles diff --git a/versions/run.ver b/versions/run.ver index cba0d08..6d70d84 100644 --- a/versions/run.ver +++ b/versions/run.ver @@ -6,20 +6,20 @@ export libjpeg_ver=9c export grib_util_ver=1.2.2 export netcdf_ver=4.7.4 export bufr_dump_ver=1.2.0 -export prepobs_ver=1.2.0 +export prepobs_ver=1.1.0 export intel_ver=19.1.3.304 -export craype_ver=2.7.13 #2.7.8 -export cray_mpich_ver=8.1.12 #8.1.7 -export cray_pals_ver=1.1.3 #1.0.12 +export craype_ver=2.7.13 +export cray_mpich_ver=8.1.12 +export cray_pals_ver=1.1.3 export cfp_ver=2.0.4 -export gfs_ver=v16.3 # make it v16.3 when implementinig in sync with GFSv16.3 +export gfs_ver=v16.3 #export nam_ver=v4.2 # secure turn off SYNDAT for nam export dictionaries_ver=v3.4.0 export seaice_analysis_ver=v4.5 #engice, seaice.*5min.grb file export cdas_ver=v1.2 -export nsst_ver=v1.2 #rtgsst file +export nsst_ver=v1.2 #rtgsst file export PROCESS_MASTER_SHIP_STNLST=NO #for monthly gdas count (run on 2nd of Month) export BACK=off