From 854e2b54f89490dfbfbce8d48fb6de0d2bbbe3df Mon Sep 17 00:00:00 2001 From: Shelley Melchior Date: Fri, 8 Apr 2022 01:34:34 +0000 Subject: [PATCH 01/53] First of many updates to remorest.f to address github issue 33. --- sorc/bufr_remorest.fd/remorest.f | 45 +++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/sorc/bufr_remorest.fd/remorest.f b/sorc/bufr_remorest.fd/remorest.f index 7db0b28..0f45001 100644 --- a/sorc/bufr_remorest.fd/remorest.f +++ b/sorc/bufr_remorest.fd/remorest.f @@ -101,6 +101,38 @@ C ID w/ MASKSTID. C BENEFIT: BUFR format restricted ships data can be properly C dispensed to public users. +C XXXX-XX-XX D. A. KEYSER -- All reports in message types in namelist +C "MSG_MIXED" are now also tested for their value for EXPRSRD +C (number of hours until the restriction expires) when their +C restriction flag (RSRD) is set - any reports with a non-missing +C EXPRSRD less than the difference in hours between the current +C UTC wall-clock date and the BUFR file center time ("DIFF_HR") +C minus 4 are now not considered restricted and are copied (prior +C to this, the value of EXPRSRD was ignored and all reports with +C RSRD set were restricted and skipped), "DIFF_HR" is a new +C imported script environment variable; Improved information that +C is printed out for each report that is either skipped or retained +C (latter is currently commented out); Improved information printed +C out at end summarizing counts of reports retained, skipped or +C masked +C XXXX-XX-XX D. A. KEYSER -- +C - For PREPBUFR files only, all reports in message types in +C namelist "MSG_MASK" (if their dump report type is listed in +C namelist "IMASK_T29") are now tested for their values for both +C RSRD and EXPRSRD and are only considered to be restricted if +C RSRD is set and EXPRSRD is .GE. the difference in hours between +C the current UTC wall-clock date and the BUFR file center time +C (read in via imported script environment variable "DIFF_HR") +C minus 4. (Prior to this RSRD and EXPRSRD were not examined, ALL +C reports from the message type having the listed dump report type +C were considered to be restricted. This is still the case for +C message types in namelist "MSG_MASK" for DUMP files.) +C - When a report coming out of "MSG_MASK" is deemed to be +C restricted (either PREPBUFR or DUMP) and its id is masked to be +C "MASKSTID", the report's value for RSRD is now re-set to MISSING +C when copied to non-restricted file (EXPRSRD is also set to +C MISSING, but it likely was already MISSING). +C - Improved documentation and printout. C C USAGE: C INPUT FILES: @@ -108,16 +140,16 @@ C UNIT 11 - INPUT BUFR FILENAME (IN CHARACTER) (USED ONLY FOR C DIAGNOSTIC PRINT INFO) C UNIT 21 - BUFR FILE (PREPBUFR OR DUMP) CONTAINING A MIXTURE OF -C RESTRICTED AND UNRESTRICTED REPORTS +C RESTRICTED (AT LEAST FOR SOME PERIOD FO TIME) AND +C NON-RESTRICTED REPORTS C C OUTPUT FILES: C UNIT 06 - STANDARD OUTPUT PRINT C UNIT 51 - BUFR FILE (PREPBUFR OR DUMP) CONTAINING EITHER ONLY -C UNRESTRICTED REPORTS OR UNRESTRICTED REPORTS AND -C RESTRICTED REPORTS WHOSE REPORT ID's HAVE BEEN MASKED -C {I.E., ALL OCCURRENCES OF ID IN A REPORT ARE -C UNILATERALLY CHANGED TO EITHER "MASKSTID" (WHERE THE -C ID IS STORED BY ITSELF) OR TO ALL "X"'s WHERE THE +C NON-RESTRICTED REPORTS WHOSE REPORT ID's HAVE +C BEEN MASKED {I.E., ALL OCCURRENCES OF ID IN A REPORT +C ARE UNILATERALLY CHANGED TO EITHER "MASKSTID" (WHERE +C THE ID IS STORED BY ITSELF) OR TO ALL "X"'s WHERE THE C NUMBER OF "X"'s CORRESPONDS TO THE THE NUMBER OF C CHARACTERS IN THE ORIGINAL REPORT ID (WHERE THE ID IS C EMBEDDED IN THE REPLICATED RAW REPORT BULLETIN HEADER @@ -126,6 +158,7 @@ C SUBPROGRAMS CALLED: (LIST ALL CALLED FROM ANYWHERE IN CODES) C C UNIQUE: - NONE +C SYSTEM: - GET_ENVIRONMENT_VARIABLE C LIBRARY: C W3LIB - W3TAGB W3TAGE ERREXIT C BUFR - DATELEN OPENBF IREADMG UFBCNT NMSUB From 035b87d623fb4b311ed4aedea5c11bd2a42e0f29 Mon Sep 17 00:00:00 2001 From: Shelley Melchior Date: Wed, 13 Apr 2022 21:27:04 +0000 Subject: [PATCH 02/53] Continued updates to remorest.f to address github issue 33. --- sorc/bufr_remorest.fd/remorest.f | 211 ++++++++++++++++++++++--------- 1 file changed, 154 insertions(+), 57 deletions(-) diff --git a/sorc/bufr_remorest.fd/remorest.f b/sorc/bufr_remorest.fd/remorest.f index 0f45001..975efe6 100644 --- a/sorc/bufr_remorest.fd/remorest.f +++ b/sorc/bufr_remorest.fd/remorest.f @@ -5,25 +5,28 @@ C C ABSTRACT: THIS PROGRAM READS THROUGH AN INPUT BUFR FILE (NORMALLY C EITHER A PREPBUFR OR A DATA DUMP FILE) WHICH CAN CONTAIN A MIXTURE -C OF REPORTS WHICH ARE UNRESTRICTED OR RESTRICTED (W.R.T. -C REDISTRIBUTION OUTSIDE OF NCEP) AND EITHER WRITES OUT (TO AN -C OTHERWISE IDENTICAL BUFR FILE) ONLY THOSE REPORTS WHICH ARE -C UNRESTRICTED OR WRITES OUT ALL REPORTS BUT WITH MASKED REPORT ID's -C FOR THOSE REPORTS WHICH ARE RESTRICTED (WHAT IT DOES IS BASED ON -C NAMELIST SWITCHES). IT DETERMINES WHICH REPORTS ARE RESTRICTED -C BASED ON EITHER THE MESSAGE TYPE AND SUBTYPE (MAKING UP THE TABLE A -C ENTRY IN DATA DUMP FILES), THE PREPBUFR TABLE A ENTRY (PREPBUFR -C FILES) (IN EITHER CASE WHEN THE MESSAGE IS KNOWN TO CONTAIN EITHER -C ALL RESTRICTED OR ALL UNRESTRICTED REPORTS), THE DUMP REPORT TYPE -C (WHEN REPORTS IN A PARTICULAR PREPBUFR TABLE A ENTRY ARE MASKED) OR -C THE RESTRICTED FLAG (MNEMONIC "RSRD") WITHIN EACH REPORT IN A -C MESSAGE (WHEN THE MESSAGE MAY CONTAIN A MIXTURE OF RESTRICTED AND -C UNRESTRICTED REPORTS). (NOTE: THE CASE OF MASKING REPORT ID's IN -C DATA DUMP FILES CAN ONLY BE DONE CURRENTLY FOR TABLE A ENTRIES -C WHERE ALL REPORTS ARE CONSIDERED TO BE RESTRICTED. THE CASE OF -C MASKING REPORT ID's IN PREPBUFR FILES CAN ONLY BE DONE CURRENTLY -C FOR TABLE A ENTRIES WHERE ALL REPORTS WITH A PARTICULAR SET OF DUMP -C REPORT TYPES ARE CONSIDERED TO BE RESTRICTED.) +C OF REPORTS WHICH ARE UNRESTRICTED OR RESTRICTED FOR AT LEAST +C SOME PERIOD OF TIME (W.R.T. REDISTRIBUTION OUTSIDE OF NCEP) AND +C EITHER WRITES OUT (TO AN OTHERWISE IDENTICAL BUFR FILE) ONLY THOSE +C REPORTS WHICH ARE NON-RESTRICTED OR WRITES OUT ALL REPORTS BUT WITH +C MASKED REPORT ID's FOR THOSE REPORTS WHICH ARE RESTRICTED (WHAT IT +C DOES IS BASED ON NAMELIST SWITCHES). {NOTE: WHEN A RESTRICTED +C REPORT IS WRITTEN OUT WITH A MASKED REPORT ID, ITS RESTRICTION FLAG +C (MNEMONIC "RSRD") AND ITS NUMBER OF HOURS UNTIL THE RESTRICTION +C EXPIRES (MNEMONIC "EXPRSRD") ARE RE-SET TO MISSING SO THAT THE +C REPORT IS NO LONGER CONSIDERED TO BE RESTRICTED.} IT DETERMINES +C WHICH REPORTS ARE RESTRICTED BASED ON EITHER THE MESSAGE TYPE AND +C SUBTYPE (MAKING UP THE TABLE A ENTRY IN DATA DUMP FILES), THE +C PREPBUFR TABLE A ENTRY (PREPBUFR FILES) (IN EITHER CASE WHEN THE +C MESSAGE IS KNOWN TO CONTAIN EITHER ALL RESTRICTED OR ALL NON- +C RESTRICTED REPORTS), THE DUMP REPORT TYPE (WHEN REPORTS IN A +C PARTICULAR PREPBUFR TABLE A ENTRY ARE MASKED) OR THE REPORT'S +C VALUE FOR "RSRD") AND, IF "RSRD" IS SET, ITS VALUE FOR "EXPRSRD" +C WITHIN EACH REPORT IN A MESSAGE (WHEN THE MESSAGE MAY CONTAIN A +C MIXTURE OF RESTRICTED AND NON-RESTRICTED REPORTS). (NOTE: THE CASE +C OF MASKING REPORT ID's IN DATA DUMP FILES CAN ONLY BE DONE +C CURRENTLY FOR TABLE A ENTRIES WHERE ALL REPORTS ARE CONSIDERED TO +C BE RESTRICTED. C C PROGRAM HISTORY LOG: C 2003-07-14 D. A. KEYSER -- ORIGINAL AUTHOR @@ -146,7 +149,8 @@ C OUTPUT FILES: C UNIT 06 - STANDARD OUTPUT PRINT C UNIT 51 - BUFR FILE (PREPBUFR OR DUMP) CONTAINING EITHER ONLY -C NON-RESTRICTED REPORTS WHOSE REPORT ID's HAVE +C NON-RESTRICTED REPORTS OR NON-RESTRICTED REPORTS AND +C PREVIOUSLY RESTRICTED REPORTS WHOSE REPORT ID's HAVE C BEEN MASKED {I.E., ALL OCCURRENCES OF ID IN A REPORT C ARE UNILATERALLY CHANGED TO EITHER "MASKSTID" (WHERE C THE ID IS STORED BY ITSELF) OR TO ALL "X"'s WHERE THE @@ -172,35 +176,70 @@ C CONTENTS OF INPUT NAMELIST "SWITCHES": C MSG_RESTR - 20-WORD CHARACTER*8 ARRAY CONTAINING UP TO 20 BUFR C MESSAGE TABLE A ENTRIES FOR WHICH ALL REPORTS ARE -C RESTRICTED AND WILL BE REMOVED (These messages are -C skipped over without unpacking any reports) +C CONSIDERED TO BE RESTRICTED AND WILL ALWAYS BE +C REMOVED (These messages are skipped over without +C unpacking any reports) C MSG_MIXED - 20-WORD CHARACTER*8 ARRAY CONTAINING UP TO 20 BUFR -C MESSAGE TABLE A ENTRIES WHICH CONTAIN A MIXTURE OF -C RESTRICTED AND UNRESTRICTED REPORTS BASED ON BUFR -C MNEMONIC "RSRD" - ALL RESTRICTED REPORTS WILL BE -C REMOVED (These messages must be unpacked and every -C report must be checked to see if it is restricted - -C unrestricted reports are copied, restricted reports -C are skipped over) -C MSG_MASKA - 20-WORD CHARACTER*8 ARRAY CONTAINING UP TO 20 BUFR +C MESSAGE TABLE A ENTRIES WHICH MAY CONTAIN A MIXTURE +C OF REPORTS WITH AND WITHOUT THEIR RESTRICTION +C INDICATOR (BUFR MNEMONIC "RSRD") BEING SET. IF "RSRD" +C IS NOT SET -OR- IT IS SET AND THE TIME IN HOURS FOR +C THE EXPIRATION ON RESTRICTION (BUFR MNEMONIC +C "EXPRSRD") IS ALSO SET AND HAS A VALUE LESS THAN +C "DIFF_HR" (THE DIFFERENCE IN HOURS BETWEEN THE +C CURRENT UTC WALL-CLOCK DATE AND THE BUFR FILE CENTER +C TIME) MINUS 4, THE REPORT WILL BE RETAINED. +C OTHERWISE, IT WILL BE REMOVED. (These messages must +C be unpacked and values for "RSRD" and "EXPRSRD" must +C be checked for every report. If "EXPRSRD is missing, +C it is set to 99999999 hours essentially meaning the +C report is restricted for all time if "RSRD" is set.) +C MSG_MASKA - FOR PREPBUFR FILES: +C 20-WORD CHARACTER*8 ARRAY CONTAINING UP TO 20 BUFR +C MESSAGE TABLE A ENTRIES WHICH, IF THEIR DUMP REPORT +C TYPE IS ONE OF UP TO 10 POSSIBLE LISTED IN SWITCH +C IMASK_T29 (WHERE EACH LINE IN IMASK_T29 APPLIES TO +C THE TABLE A ENTRY IN THE SAME LINE NUMBER HERE), MAY +C CONTAIN A MIXTURE OF REPORTS WITH AND WITHOUT THEIR +C RESTRICTION INDICATOR (BUFR MNEMONIC "RSRD") BEING +C SET. IF "RSRD" IS NOT SET FOR A REPORT -OR- IT IS SET +C AND THE TIME IN HOURS FOR THE EXPIRATION ON +C RESTRICTION (BUFR MNEMONIC "EXPRSRD") IS ALSO SET AND +C HAS A VALUE LESS THAN "DIFF_HR" (THE DIFFERENCE IN +C HOURS BETWEEN THE CURRENT UTC WALL-CLOCK DATE AND THE +C PREPBUFR FILE CENTER TIME) MINUS 4, THE REPORT WILL +C BE COPIED WITHOUT ANY CHANGES. OTHERWISE, THE REPORT +C WILL NOT BE REMOVED, BUT ALL OCCURRENCES OF ITS ID +C WILL BE CHANGED TO "MASKSTID". IN ADDITION, ITS +C VALUES FOR "RSRD" AND "EXPRSRD" WILL BE RE-SET TO +C MISSING SO THAT THE REPORT WILL NO LONGER BE +C CONSIDERED AS RESTRICTED. REPORTS WITH A DUMP REPORT +C TYPE NOT LISTED IN SWITCH IMASK_T29 ARE CONSIDERED TO +C BE NON-RESTRICTED AND THEIR REPORT IDS ARE NOT +C CHANGED (MASKED OUT) WHEN COPIED. (These m essages +C must be unpacked and the values for "T29", "RSRD" and +C "EXPRSRD" must be checked for every report. If +C "EXPRSRD" is missing, it is set to 99999999 hours +C essentially meaning the report is restricted for all +C time if "RSRD" is set.) +C FOR DATA DUMP FILES: +C 20-WORD CHARACTER*8 ARRAY CONTAINING UP TO 20 BUFR C MESSAGE TABLE A ENTRIES FOR WHICH ALL REPORTS ARE -C RESTRICTED {BUT, FOR PREPBUFR FILES, ONLY IF THEIR -C DUMP REPORT TYPE IS ONE OF UP TO 10 POSSIBLE LISTED -C IN SWITCH IMASK_T29 (EACH LINE IN IMASK_T29 APPLIES -C TO THE TABLE A ENTRY IN THE SAME LINE NUMBER HERE); -C FOR DATA DUMP FILES, IMASK_T29 IS NOT USED, ALL -C REPORTS IN THE TABLE A ENTRIES HERE WILL BE -C RESTRICTED)} - THESE WILL NOT BE REMOVED, BUT ALL -C OCCURRENCES OF THEIR REPORT ID's WILL BE UNILATERALLY -C CHANGED TO EITHER "MASKSTID" (WHERE THE ID IS STORED -C BY ITSELF, PREPBUFR OR DUMP FILES) OR TO ALL "X"'s -C WHERE THE NUMBER OF "X"'s CORRESPONDS TO THE NUMBER -C OF CHARACTERS IN THE ORIGINAL REPORT ID (WHERE THE ID -C IS EMBEDDED IN THE REPLICATED RAW REPORT BULLETIN -C HEADER STRING, DUMP FILES ONLY) (These messages must -C be unpacked and every occurrence of every report's id -C must be changed to either "MASKSTID" or "X"'s before -C the report is copied) +C CONSIDERED TO BE RESTRICTED. THEY WILL NOT BE +C REMOVED, BUT ALL OCCURRENCES OF THEIR REPORT IDS WILL +C BE UNILATERALLY CHANGED TO EITHER "MASKSTID" (WHERE +C THE ID IS STORED BY ITSELF) OR TO ALL "X"'s WHERE THE +C NUMBER OF "X"'s CORRESPONDS TO THE NUMBER OF +C CHARACTERS IN THE ORIGINAL REPORT ID (WHERE THE ID IS +C EMBEDDED IN THE RAW REPORT BULLETIN HEADER STRING). +C IN ADDITION, THEIR VALUES FOR "RSRD" AND "EXPRSRD" +C WILL BE RE-SET TO MISSING SO THAT THE REPORTS WILL NO +C LONGER BE CONSIDERED AS RESTRICTED. (These messages +C must be unpacked and every occurrence of every +C report's id must be changed to either "MASKSTID" or +C "X"'s and every report's "RSRD" and "EXPRSRD" values +C must be changed to missing before the report is +C copied. Switch IMASK_T29 is not considered here.) C IMASK_T29 - (10,20) INTEGER ARRAY CONTAINING UP TO 10 POSSIBLE C DUMP REPORT TYPES (1ST DIMENSION) FOR THE UP TO 20 C POSSIBLE PREPBUFR TABLE A ENTRIES LISTED IN SWITCH @@ -211,7 +250,7 @@ C than one of MSG_RESTR, MSG_MIXED or MSG_MASKA. C Note 2: Any Table A entry not in either MSG_RESTR, MSG_MIXED or C MSG_MASKA is assumed to be a Table A entry for BUFR -C messages for which ALL reports are UNRESTRICTED (these +C messages for which ALL reports are NON-RESTRICTED (these C messages are copied intact, no reports are unpacked). C Note 3: Always fill in these arrays MSG_RESTR, MSG_MIXED and C MSG_MASKA beginning with word 1. If there are less than @@ -234,8 +273,8 @@ C Note 5: For PREPBUFR files, a value of "99999" in array IMASK_T29 C means not applicable whereas a value of "000" means C reports in all dump report types in the corresponding -C Table A entry in MSG_MASKA should be restricted (masked) -C {in this case IMASK_T29(1,x) should be set to 000 and +C Table A entry in MSG_MASKA should be considered {in this +C case IMASK_T29(1,x) should be set to 000 and C IMASK_T29(2:10,x) should be set to 99999 for all reports C in Table A entry MSG_MASKA(x) since they would all be C ignored - this is the default for all Table A entries @@ -244,15 +283,19 @@ C C LIST OF REPORT ID MNEMONICS IN EACH REPORT WHICH ARE CURRENTLY C MASKED WHEN TABLE A ENTRY IS FOUND IN MSG_MASKA (AND FOR PREPBUFR -C FILES DUMP REPORT TYPE MATCHES ONE OF THE TYPES IN IMASK_T29): +C FILES DUMP REPORT TYPE MATCHES ONE OF THE TYPES IN IMASK_T29 +C AND THE REPORT IS CONSIDERED TO BE RESTRICTED BASED ON ITS VALUES +C FOR "RSRD" and "EXPRSRD"): C C PREPBUFR file: "SID" - chgd to "MASKSTID" (all Tbl A entries) +C (PREPBUFR file "RSRD" and "EXPRSRD" also set to missing) C DUMP file: "RPID" - chgd to "MASKSTID" (all Tbl A entries) C DUMP file: "SHPC8" - chgd to "MASKSTID" (Tbl A entry NC001001) C DUMP file: "RRSTG" - chgd to "X" (where the number of "X"'s C corresponds to the the number of C characters in the original report id ) C (all applicable Tbl A entries) +C (DUMP file: "RSRD" and "EXPRSRD" also set to missing) C C Note: Currently for dump files, the only Table A entry where all C occurrences of report id in a report are known to be masked @@ -261,6 +304,34 @@ C A entries. C C +C ONE SCRIPT ENVIRONMENT VARIABLE IS READ IN: +C DIFF_HR - The difference in hours between the current UTC +C wall-clock date and the BUFR file center time +C (should always be a positive number!). This +C is used (after subtracting 4 hours***) to +C determine if a BUFR subset that is marked as +C restricted (via mnemonic "RSRD") is past the +C expiration time of the restriction (mnemonic +C "EXPRSRD") and should thus not be filtered out. +C (Note: used only for BUFR DUMP and PREPBUFR +C subsets in message types listed in namelist +C switch MSG_MIXED and for PREPBUFR subsets in +C applicalbe dump types and message types listed +C in namelist switches IMASK_T29 and MSG_MASKA.) +C Defaults to ZERO if not found (i.e., not +C exported by the executing script). +C *** Four hours is subtracted from DIFF_HR prior to +C testing against the BUFR file center time in +C order to account for some reports having an obx +C time as much as 3-4 hours prior to the center +C time in either a dump or PEPBUFR file. This +C ensures that these reports are not inadvertently +C retained if the difference between the current +C wall-clock date and the BUFR file center time is +C very close to the time period of the restriction. +C DIFF_HR minus 4 can never be less than zero. +C +C C ATTRIBUTES: C LANGUAGE: FORTRAN 90 C MACHINE: NCEP WCOSS (iDataPlex and Cray-XC40) @@ -271,16 +342,34 @@ PROGRAM BUFR_REMOREST CHARACTER*2040 RAWRPT_STG CHARACTER*80 FILE + CHARACTER*8 DIFF_HR,PREPBUFR_MSGTYP(21),PRVSTG_prep,SPRVSTG_prep, + $ PRVSTG_dump,SPRVSTG_dump CHARACTER*8 SUBSET,SID,MSG_RESTR(20),MSG_MIXED(20),MSG_MASKA(20), $ RAWRPT(255),SID_orig - REAL*8 RID_8(6),GETBMISS,BMISS,RASTR_8(255) - INTEGER IMASK_T29(10,20) - - EQUIVALENCE (RID_8(1),SID),(RAWRPT,RASTR_8) + REAL*8 RID_8(8),RASTR_8(255),LALOH_8(2),ACRN_8,ACID_8,SID_8, + $ RPID_8,PRV_prep_8(2),PRV1_dump_8(255), + $ PRV2_dump_8(255),BMISS,GETBMISS,rsrd_8(2) + INTEGER IMASK_T29(10,20),IRSUB_this_MR(20),IRSUB_this_MM(20), + $ IMSUB_this(10,20),IRSUB_this_sub_MR(20,0:255), + $ IRSUB_this_sub_MM(20,0:255),IUSUB_this(0:255,0:256) + + EQUIVALENCE (RID_8(1),SID),(RAWRPT,RASTR_8), + $ (PRV_prep_8(1),PRVSTG_prep), + $ (PRV_prep_8(2),SPRVSTG_prep), + $ (PRV1_dump_8(1),PRVSTG_dump), + $ (PRV2_dump_8(1),SPRVSTG_dump) DATA LUBFI/21/,LUBFJ/51/,IREC/0/,IRSUB/0/,IMSUB/0/, - $ IUSUB/0/,ireco_last/0/ - + $ IRSUB_this_MR/20*0/,IRSUB_this_MM/20*0/,IMSUB_this/200*0/, + $ IUSUB/0/,IRSUB_this_sub_MR/5120*0/,IRSUB_this_sub_MM/5120*0/, + $ IUSUB_this/65792*0/,ireco_last/0/ + + DATA PREPBUFR_MSGTYP/'ADPUPA ','AIRCAR ','AIRCFT ','SATWND ', + $ 'PROFLR ','VADWND ','SATEMP ','ADPSFC ', + $ 'SFCSHP ','SFCBOG ','SPSSMI ','SYNDAT ', + $ 'ERS1DA ','GOESND ','QKSWND ','MSONET ', + $ 'GPSIPW ','RASSDA ','WDSATR ','ASCATW ', + $ 'unknown '/ NAMELIST/SWITCHES/MSG_RESTR,MSG_MIXED,MSG_MASKA,IMASK_T29 CALL W3TAGB('BUFR_REMOREST',2021,0175,0012,'NP22') @@ -294,6 +383,14 @@ PROGRAM BUFR_REMOREST $ 'REMOVES OR MASKS RESTRICTED REPORTS FROM A BUFR FILE'/42X, $ 'LAST REVISION 24 Jun 2021'//30X,'INPUT BUFR FILENAME IS: ',A) + IMASK_T29 = 99999 + IMASK_T29(1,:) = 000 + MSG_RESTR = ' ' + MSG_MIXED = ' ' + MSG_MASKA = ' ' + IDIFF_HR = 0 + READ(5,SWITCHES) + C Set BUFRLIB missing (BMISS) to 10E8_8 to avoid integer*4 overflows C ------------------------------------------------------------------ CALL SETBMISS(10E8_8) From 3a876b7198852a5cba34c38486048fb625a4107c Mon Sep 17 00:00:00 2001 From: Shelley Melchior Date: Wed, 13 Apr 2022 21:31:53 +0000 Subject: [PATCH 03/53] Squashed commit of the following: commit ebb6c6856c118705303594641f3fc9b8a07494cd Author: Cory Martin Date: Tue Apr 12 15:15:47 2022 -0400 Changes needed to build on Hera (#32) commit 0075564dc3e9d67b3369138cd9a4f2d704872ad9 Author: Shelley Melchior <53050457+ShelleyMelchior-NOAA@users.noreply.github.com> Date: Sat Apr 9 17:07:10 2022 -0400 Incorporating changes made by NCO SPA following: (#35) restart of cactus and changing output data directory for mods --- jobs/JOBSPROC_CDAS_DUMP | 2 +- jobs/JOBSPROC_GLOBAL_PREP | 4 ++-- jobs/JOBSPROC_MODS | 11 +++++++---- modulefiles/obsproc_hera.lua | 2 ++ ush/bufr_prepmods.sh | 3 ++- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/jobs/JOBSPROC_CDAS_DUMP b/jobs/JOBSPROC_CDAS_DUMP index f3d49cc..92e49ce 100755 --- a/jobs/JOBSPROC_CDAS_DUMP +++ b/jobs/JOBSPROC_CDAS_DUMP @@ -161,7 +161,7 @@ export FIXbufr=${FIXbufr:-$HOMEbufr_dump/fix} # TANK is root directory path to observational database used in dumpjb # and engice, sst & snowdepth grib files used in model script # -------------------------------------------------------------------- -export DCOMROOT=${DCOMROOT:-/lfs/h1/ops/${envir}/dcom} +export DCOMROOT=${DCOMROOT:-/lfs/h1/ops/prod/dcom} export TANK=${TANK:-${DCOMROOT}} # directory paths to engice, sst & snowdepth grib files used in model script diff --git a/jobs/JOBSPROC_GLOBAL_PREP b/jobs/JOBSPROC_GLOBAL_PREP index 65a6b97..e1ba8b7 100755 --- a/jobs/JOBSPROC_GLOBAL_PREP +++ b/jobs/JOBSPROC_GLOBAL_PREP @@ -156,8 +156,8 @@ export DICTPREP=${DICTPREP:-${OPSROOT}/decoders/decod_shared/dictionaries_${dict # prepobsp_prevents.sh. # --------------------------------------------------------------------------- #export HOMEgfs=${HOMEgfs:-${PACKAGEROOT:?}/gfs.${gfs_ver:?"###FATAL ERROR \"gfs_ver\" is not set"}} -#export HOMEgfs=${HOMEgfs:-$(compath.py ${envir}/gfs/${gfs_ver})} -export HOMEgfs=${HOMEgfs:-$HOMEobsproc} #Temporary place to let job run. change it to above layer when gfs run in realtime +export HOMEgfs=${HOMEgfs:-$(compath.py gfs/${gfs_ver})} +#export HOMEgfs=${HOMEgfs:-$HOMEobsproc} #Temporary place to let job run. change it to above layer when gfs run in realtime ######################################################################### # Add some prod utilities to working directory diff --git a/jobs/JOBSPROC_MODS b/jobs/JOBSPROC_MODS index ae64b17..9f2d52d 100755 --- a/jobs/JOBSPROC_MODS +++ b/jobs/JOBSPROC_MODS @@ -169,15 +169,17 @@ cp $UTILROOT/ush/setpdy.sh .; chmod +x setpdy.sh ############################## # Run setpdy and initialize PDY variables ############################## -setpdy.sh +setpdy.sh 9 . PDY +export YYYY=`echo $PDYm9 | cut -c1-4` #mods doing $PDYm9 data + ################################################## # Define COMOUT directory (no COMIN for this job) ################################################## -#export COMOUT=${COMOUT:-$(compath.py -o ${obsNET}/$obsproc_ver/${NET}.${PDY})} -export COMOUT=${COMOUT:-$(compath.py -o ${obsNET}/$obsproc_ver/${NET}.${PDY})} +export COMOUT=${COMOUT:-$(compath.py -o ${obsNET}/$obsproc_ver/${NET}.${PDY})} +export MODSDIR=${MODSDIR:-$(compath.py -o ${obsNET}/$obsproc_ver)/mods.$YYYY} mkdir -m 775 -p $COMOUT ############################################## @@ -187,7 +189,8 @@ mkdir -m 775 -p $COMOUT export DCOMROOT=${DCOMROOT:-/lfs/h1/ops/${envir}/dcom} echo $DCOMROOT -export MODSDIR=${MODSDIR:-${DCOMROOT}/mods} +#export MODSDIR=${MODSDIR:-${DCOMROOT}/mods} +#export MODSDIR=${MODSDIR:-${COM_OUT}/mods.$YYYY} env ; echo "JMODS: end of env listing" ; echo diff --git a/modulefiles/obsproc_hera.lua b/modulefiles/obsproc_hera.lua index 6eb61ea..e3be419 100644 --- a/modulefiles/obsproc_hera.lua +++ b/modulefiles/obsproc_hera.lua @@ -12,4 +12,6 @@ load("hpc-impi/2018.0.4") -- Load common modules for this package load("obsproc_common") +setenv("FC", "mpiifort") + whatis("Description: obsproc build environment") diff --git a/ush/bufr_prepmods.sh b/ush/bufr_prepmods.sh index 8251860..ec5e7ec 100755 --- a/ush/bufr_prepmods.sh +++ b/ush/bufr_prepmods.sh @@ -141,7 +141,8 @@ for dtyp in $dtypes ; do # make and/or move to directory for this data for this day # -------------------------------------------------------- - DBASE=$MODSDIR/$year/$mnth/$dtyp + #DBASE=$MODSDIR/$year/$mnth/$dtyp + DBASE=$MODSDIR/$mnth/$dtyp mkdir -p $DBASE if [ ".$dtyp" = '.ships' -o \ From 56a049385772e4d7aa940af685bc6df0291d7ac3 Mon Sep 17 00:00:00 2001 From: Shelley Melchior Date: Wed, 20 Apr 2022 00:00:21 +0000 Subject: [PATCH 04/53] Continued updates to remorest.f to address github issue 33. --- sorc/bufr_remorest.fd/remorest.f | 238 +++++++++++++++++++++++++------ 1 file changed, 195 insertions(+), 43 deletions(-) diff --git a/sorc/bufr_remorest.fd/remorest.f b/sorc/bufr_remorest.fd/remorest.f index 975efe6..1e026f1 100644 --- a/sorc/bufr_remorest.fd/remorest.f +++ b/sorc/bufr_remorest.fd/remorest.f @@ -399,18 +399,18 @@ PROGRAM BUFR_REMOREST print'(" BUFRLIB value for missing is: ",G0)', bmiss print'(1X)' - IMASK_T29 = 99999 - IMASK_T29(1,:) = 000 - MSG_RESTR = ' ' - MSG_MIXED = ' ' - MSG_MASKA = ' ' - READ(5,SWITCHES) +C .... store rsrd_8 array as "almost" missing, will later encode back +C into output *.nr PRPEBUFR file for reports with id masked, +C overwriting original value (won't overwrite if exactly missing) +C --------------------------------------------------------------- + rsrd_8 = bmiss - 0.01_8 IF(MSG_RESTR(1).NE.' ') THEN PRINT 107 - 107 FORMAT(//' ANY BUFR MESSAGES READ IN WITH THE FOLLOWING TABLE A ', - $ 'ENTRIES ARE SKIPPED (WITHOUT UNPACKING) BECAUSE THEY CONTAIN ', - $ 'ONLY'/' RESTRICTED REPORTS ALL OF WHICH ARE TO BE REMOVED:'/) + 107 FORMAT(//' ALL BUFR MESSAGES READ IN WITH THE FOLLOWING TABLE A ', + $ 'ENTRIES ARE SKIPPED (WITHOUT UNPACKING) BECAUSE THEY'/ + $ ' CONTAIN ONLY RESTRICTED REPORTS (FOR SOME PERIOD OF TIME) ALL', + $ ' OF WHICH ARE TO BE REMOVED:'/) DO I = 1,20 IF(MSG_RESTR(I).EQ.' ') EXIT PRINT *, MSG_RESTR(I) @@ -418,32 +418,100 @@ PROGRAM BUFR_REMOREST END IF IF(MSG_MIXED(1).NE.' ') THEN PRINT 108 - 108 FORMAT(//' ANY BUFR MESSAGES READ IN WITH THE FOLLOWING TABLE A ', + 108 FORMAT(//' ALL BUFR MESSAGES READ IN WITH THE FOLLOWING TABLE A ', $ 'ENTRIES ARE UNPACKED REPORT BY REPORT BECAUSE THEY CAN'/ - $ ' CONTAIN A MIXTURE OF BOTH UNRESTRICTED AND RESTRICTED REPORTS', - $ ' - ALL RESTRICTED REPORTS ARE TO BE REMOVED:'/) + $ ' CONTAIN A MIXTURE OF BOTH NON-RESTRICTED AND RESTRICTED ', + $ 'REPORTS - ALL RESTRICTED REPORTS WITHIN THE TIME'/' PERIOD OF ', + $ 'THE RESTRICTION ARE TO BE REMOVED:'/) DO I = 1,20 IF(MSG_MIXED(I).EQ.' ') EXIT PRINT *, MSG_MIXED(I) ENDDO + CALL GET_ENVIRONMENT_VARIABLE('DIFF_HR',DIFF_HR) + READ(DIFF_HR,'(I8)',END=88,ERR=88) IDIFF_HR +cppppp +ccc print * +ccc print *, 'DIFF_HR, IDIFF_HR : ',DIFF_HR, IDIFF_HR +ccc print * +cppppp + GO TO 89 + 88 CONTINUE + PRINT 115 + 115 FORMAT(/'+++++BUFR_REMOREST: WARNING: ERROR OBTAINING IDIFF_HR -', + $ ' SET TO ZERO AND CONTINUE'/) + IDIFF_HR = 0 + 89 CONTINUE + IDIFF_HR_m4 = IDIFF_HR - 4 + IDIFF_HR_m4 = MAX(IDIFF_HR_m4,0) + PRINT 118, IDIFF_HR,IDIFF_HR_m4 + 118 FORMAT(/' ===> The difference between the current wall-clock ', + $ 'date and the BUFR file center time is',I8,' hours.'/6X, + $ 'Will consider the difference here to be only',I8,' hours when ', + $ 'comparing against the time period of the'/6X,'restriction ', + $ '(this takes into account that some reports may have obs times ', + $ 'as much as 3-4 hours prior to'/6X,'the BUFR file center time, ', + $ 'and so ensures that these reports are not inadvertently ', + $ 'retained if the'/6X,'difference between the current wall-clock', + $ ' date and the BUFR file center time is very close to the'/ + $ 6X,'the time period of the restriction).'/) END IF IF(MSG_MASKA(1).NE.' ') THEN - PRINT 1107 - 1107 FORMAT(//' ANY BUFR MESSAGES READ IN WITH THE FOLLOWING TABLE A ', + if(MSG_MASKA(1)(1:2).eq.'NC') then + +c .. DUMP file case +c -------------- + PRINT 1107 + 1107 FORMAT(//' ALL BUFR MESSAGES READ IN WITH THE FOLLOWING TABLE A ', + $ 'ENTRIES CONTAIN REPORTS ALL CONSIDERED TO BE RESTRICTED. THEY '/ + $ 'ARE'/' UNPACKED REPORT BY REPORT AND EACH REPORT''S ID (ALL ', + $ 'OCCURRENCES IN A REPORT) IS CHANGED TO "MASKSTID" OR "X"''S ', + $ '(MASKED)'/' AND IT''S VALUES FOR "RSRD" AND "EXPRSRD" ARE RE-', + $ 'SET TO MISSING PRIOR TO THEIR BEING WRITTEN BACK OUT:'/) + + else + +c .. PREPBUFR file case +c ------------------ + print 3108 + 3108 FORMAT(//' ALL BUFR MESSAGES READ IN WITH THE FOLLOWING TABLE A ', $ 'ENTRIES ARE UNPACKED REPORT BY REPORT BECAUSE THEY CAN'/ - $ ' CONTAIN EITHER ALL OR SOME RESTRICTED REPORTS WHOSE REPORT ', - $ 'ID''s (ALL OCCURRENCES IN A REPORT) ARE ALL CHANGED'/' TO ', - $ '"MASKSTID" OR "X"''S (MASKED) PRIOR TO THEIR BEING WRITTEN ', - $ 'BACK OUT:'/) + $ ' CONTAIN A MIXTURE OF BOTH NON-RESTRICTED AND RESTRICTED ', + $ 'REPORTS - THE ID''s FOR ALL RESTRICTED REPORTS WITHIN THE'/ + $ ' TIME PERIOD OF THE RESTRICTION ARE ALL CHANGED TO "MASKSTID" ', + $ '(MASKED) AND THEIR VALUES FOR "RSRD" AND "EXPRSRD"'/' ARE RE-', + $ 'SET TO MISSING PRIOR TO THEIR BEING WRITTEN BACK OUT:'/) + end if + DO I = 1,20 IF(MSG_MASKA(I).EQ.' ') EXIT PRINT *, MSG_MASKA(I) ENDDO + + if(MSG_MASKA(1)(1:2).ne.'NC') then + +c .. PREPBUFR file case +c ------------------ + CALL GET_ENVIRONMENT_VARIABLE('DIFF_HR',DIFF_HR) + READ(DIFF_HR,'(I8)',END=988,ERR=988) IDIFF_HR +cppppp +ccc print * +ccc print *, 'DIFF_HR, IDIFF_HR : ',DIFF_HR, IDIFF_HR +ccc print * +cppppp + GO TO 989 + 988 CONTINUE + PRINT 115 + IDIFF_HR = 0 + 989 CONTINUE + IDIFF_HR_m4 = IDIFF_HR - 4 + IDIFF_HR_m4 = MAX(IDIFF_HR_m4,0) + PRINT 118, IDIFF_HR,IDIFF_HR_m4 + end if END IF PRINT 109 109 FORMAT(//' ALL OTHER BUFR MESSAGES READ IN ARE COPIED INTACT ', - $ '(WITHOUT UNPACKING) BECAUSE THEY CONTAIN ONLY UNRESTRICTED ', + $ '(WITHOUT UNPACKING) BECAUSE THEY CONTAIN ONLY NON-RESTRICTED ', $ 'REPORTS'//) CALL DATELEN(10) @@ -533,55 +601,139 @@ PROGRAM BUFR_REMOREST CYCLE LOOP1 C********************************************************************** ELSE IF(SUBSET.EQ.MSG_RESTR(I) .OR. (MSG_RESTR(I)(6:8).EQ. - $ 'xxx'.AND.SUBSET(1:5).EQ.MSG_RESTR(I)(1:5))) THEN + $ 'xxx'.AND.SUBSET(1:5).EQ.MSG_RESTR(I)(1:5))) THEN PRINT 112 112 FORMAT(' #####>>>> ALL reports in this message are RESTRICTED ', $ 'and are REMOVED - do NOT copy this message to output BUFR file') + IRSUB_this_MR(I) = IRSUB_this_MR(I) + ISUB + IF(SUBSET(1:2).EQ.'NC'.AND.MSG_RESTR(I)(6:8).EQ.'xxx')THEN + READ(SUBSET(6:8),'(I3)') ISUBSET_678 + IF(ISUBSET_678.GE.0.AND.ISUBSET_678.LE.255) THEN + IRSUB_this_sub_MR(I,ISUBSET_678) = + $ IRSUB_this_sub_MR(I,ISUBSET_678) + ISUB + ELSE + PRINT 7115, ISUBSET_678,SUBSET + 7115 FORMAT(/'+++++BUFR_REMOREST: WARNING: INVALID BUFR MESSAGE ', + $ 'SUBTYPE READ IN: ',I5.3,', SUBSET = ',A,' CANNOT INCREMENT ', + $ 'RESTRICTED REPORT'/29X,'SKIPPED COUNTER FOR THIS SUBSET'/) + END IF + END IF IRSUB = IRSUB + ISUB CYCLE LOOP1 C********************************************************************** ELSE IF(SUBSET.EQ.MSG_MIXED(I) .OR. (MSG_MIXED(I)(6:8).EQ. - $ 'xxx'.AND.SUBSET(1:5).EQ.MSG_MIXED(I)(1:5))) THEN + $ 'xxx'.AND.SUBSET(1:5).EQ.MSG_MIXED(I)(1:5))) THEN PRINT 113 - 113 FORMAT(' #####>>>> This msg has mixture of restricted/non-', - $ 'restricted data, restricted data are REMOVED- unpk each rpt & ', - $ 'test on mnem. "RSRD"') + 113 FORMAT(' #####>>>> rpts in this msg mixed restr/non-restr- restr', + $ 'data w/i exp. time REMOVED- unpk each rpt & test mnems. ', + $ '"RSRD" & "EXPRSRD"') C READ A SUBSET (REPORT) IN MESSAGE C --------------------------------- LOOP1n2: DO WHILE(IREADSB(LUBFI).EQ.0) -C DECODE THE SUBSET (REPORT) LOOKING FOR RESTRICTED FLAG -C (mnemonic "RSRD") -C ------------------------------------------------------ +C DECODE THE SUBSET (REPORT) LOOKING FOR RESTRICTED FLAG (MNEMONIC +C "RSRD") AND TIME OF EXPIRATION ON RESTRICTION (MNEMONIC "EXPRSRD") +C (if "EXPRSRD" is missing set it to 99999999 hours essentially +C meaning the report is restricted for all time) +C ------------------------------------------------------------------- CALL UFBINT(LUBFI,RID_8,6,1,NLV, - $ 'SID RPT YOB XOB TYP RSRD') - IF(RID_8(6).GT.0.AND.RID_8(6).LT.BMISS/2) THEN - IF(RID_8(5).LT.BMISS/2) THEN + $ 'SID RPT YOB XOB TYP RSRD EXPRSRD') + IF(RID_8(6).GT.0.AND.IBFMS(RID_8(6)).EQ.0) THEN + IF(IBFMS(RID_8(7)).NE.0) RID_8(7) = 99999999. + IF(IDIFF_HR_m4.LE.RID_8(7)) THEN + IF(IBFMS(RID_8(5)).EQ.0) THEN C Normally for PREPBUFR files C --------------------------- - PRINT 104, SID,(RID_8(II),II=2,4), - $ (NINT(RID_8(II)),II=5,6) - 104 FORMAT(10X,'- Skip report ',A8,' at ',F6.2,' UTC, ',F6.2, - $ ' (N+/S-) LAT, ',F7.2,'(E) LON, RTYP= ',I3,', RSRD=',I5) - ELSE + IF(SUBSET.EQ.'MSONET ') THEN + CALL UFBINT(LUBFI,PRV_prep_8,2,1,NLV, + $ 'PRVSTG SPRVSTG') + PRINT 8104, SID,(RID_8(II),II=2,4), + $ (NINT(RID_8(II)),II=5,7),PRVSTG_prep, + $ SPRVSTG_prep + 8104 FORMAT(5X,'- **Skip ',A8,F7.2,'UTC',F7.2,'(N+/S-) LAT',F7.2,' E ', + $ 'LON, RTYP=',I3,', RSRD=',I5,' EXPRSRD=',I5,', PRVID=',A8, + $ ' SPRVID=',A8) + ELSE + PRINT 104, SID,(RID_8(II),II=2,4), + $ (NINT(RID_8(II)),II=5,7) + 104 FORMAT(5X,'- **Skip report ',A8,' at ',F6.2,' UTC, ',F6.2, + $ ' (N+/S-) LAT, ',F7.2,' E LON, RTYP= ',I3,', RSRD=',I5, + $ ' EXPRSRD=',I5) + END IF + ELSE C Normally for DATA DUMP files C ---------------------------- - CALL UFBINT(LUBFI,RID_8,5,1,NLV, - $ 'RPID HOUR MINU CLAT CLON') - PRINT 110, SID,(NINT(RID_8(II)),II=2,3), - $ (RID_8(II),II=4,5), NINT(RID_8(6)) - 110 FORMAT(10X,'- Skip report ',A8,' at ',2(I2.2),' UTC, ',F6.2, - $ ' (N+/S-) LAT, ',F7.2,'(E+/W-) LON, RSRD=',I5) + CALL UFBINT(LUBFI,RID_8,7,1,NLV, + $ 'RPID HOUR MINU CLAT CLON RSRD EXPRSRD') + IF(IBFMS(RID_8(4)).NE.0) THEN + CALL UFBINT(LUBFI,LALOH_8,2,1,NLV, + $ 'CLATH CLONH') + RID_8(4:5) = LALOH_8 + END IF + IF(IBFMS(RID_8(1)).NE.0) THEN + IF(SUBSET.EQ.'NC004004' .OR. + $ SUBSET.EQ.'NC004006' .OR. + $ SUBSET.EQ.'NC004009' .OR. + $ SUBSET.EQ.'NC004010' .OR. + $ SUBSET.EQ.'NC004011' .OR. + $ SUBSET.EQ.'NC004014') THEN + CALL UFBINT(LUBFI,ACRN_8,1,1,NLV,'ACRN') + RID_8(1) = ACRN_8 + ELSE IF(SUBSET.EQ.'NC004008' .OR. + $ SUBSET.EQ.'NC004012' .OR. + $ SUBSET.EQ.'NC004013') THEN + CALL UFBINT(LUBFI,ACID_8,1,1,NLV,'ACID') + RID_8(1) = ACID_8 + ELSE IF(SUBSET.EQ.'NC007001' .OR. + $ SUBSET.EQ.'NC007002') THEN + SID = ' ' + ELSE + SID = 'MISSING ' + END IF + END IF + IF(SUBSET(3:5).EQ.'255') THEN + CALL UFBINT(LUBFI,PRV1_dump_8,1,255,NLV, + $ 'PRVSTG') + IF(NLV.LT.1) PRVSTG_dump = ' ' + CALL UFBINT(LUBFI,PRV2_dump_8,1,255,NLV, + $ 'SPRVSTG') + IF(NLV.LT.1) SPRVSTG_dump = ' ' + PRINT 8105, SID,(NINT(RID_8(II)),II=2,3), + $ (RID_8(II),II=4,5), NINT(RID_8(6)), + $ NINT(RID_8(7)),PRVSTG_dump,SPRVSTG_dump + 8105 FORMAT(5X,'- **Skip ',A8,' at ',2(I2.2),' UTC',F7.2,' (N+/S-) ', + $ 'LAT',F8.2,' (E+/W-) LON, RSRD=',I5,' EXPRSRD=',I5,', PRVID=',A8, + $ ' SPRVID=',A8) + ELSE + PRINT 110, SID,(NINT(RID_8(II)),II=2,3), + $ (RID_8(II),II=4,5), NINT(RID_8(6)), + $ NINT(RID_8(7)) + 110 FORMAT(5X,'- **Skip report ',A8,' at ',2(I2.2),' UTC, ',F6.2, + $ ' (N+/S-) LAT, ',F7.2,'(E+/W-) LON, RSRD=',I5,' EXPRSRD=',I5) + END IF + END IF + IRSUB_this_MM(I) = IRSUB_this_MM(I) + 1 + IF(SUBSET(1:2).EQ.'NC'.AND. + $ MSG_MIXED(I)(6:8).EQ.'xxx') THEN + READ(SUBSET(6:8),'(I3)') ISUBSET_678 + IF(ISUBSET_678.GE.0.AND.ISUBSET_678.LE.255) + $ THEN + IRSUB_this_sub_MM(I,ISUBSET_678) = + $ IRSUB_this_sub_MM(I,ISUBSET_678) + 1 + ELSE + PRINT 7115, ISUBSET_678,SUBSET + END IF + END IF + IRSUB = IRSUB + 1 + CYCLE LOOP1n2 END IF - IRSUB = IRSUB + 1 - CYCLE LOOP1n2 END IF CALL OPENMB(LUBFJ,SUBSET,IDATE) From 08a8805bbdcc34afe43e461c7d128f6c4b93a864 Mon Sep 17 00:00:00 2001 From: Shelley Melchior Date: Fri, 22 Apr 2022 01:31:54 +0000 Subject: [PATCH 05/53] Completed updates to remorest.f to address github issue 33. --- sorc/bufr_remorest.fd/remorest.f | 326 ++++++++++++++++++++++++++----- 1 file changed, 278 insertions(+), 48 deletions(-) diff --git a/sorc/bufr_remorest.fd/remorest.f b/sorc/bufr_remorest.fd/remorest.f index 1e026f1..bec5a0b 100644 --- a/sorc/bufr_remorest.fd/remorest.f +++ b/sorc/bufr_remorest.fd/remorest.f @@ -736,6 +736,74 @@ PROGRAM BUFR_REMOREST END IF END IF +cppppp +ccc IF(IBFMS(RID_8(5)).EQ.0) THEN +ccc IF(SUBSET.EQ.'MSONET ') THEN +ccc CALL UFBINT(LUBFI,PRV_prep_8,2,1,NLV, +ccc $ 'PRVSTG SPRVSTG') +ccc PRINT 9104, SID,(RID_8(II),II=2,4), +ccc $ (NINT(RID_8(II)),II=5,7),PRVSTG_prep, +ccc $ SPRVSTG_prep + 9104 FORMAT(5X,'- Retain ',A8,F7.2,'UTC',F7.2,'(N+/S-) LAT',F7.2,' E ', + $ 'LON, RTYP=',I3,', RSRD=',I5,' EXPRSRD=',I5,', PRVID=',A8, + $ ' SPRVID=',A8) +ccc ELSE +ccc PRINT 214, SID,(RID_8(II),II=2,4), +ccc $ (NINT(RID_8(II)),II=5,7) + 214 FORMAT(5X,'- Retain report ',A8,' at ',F6.2,' UTC, ',F6.2, + $ ' (N+/S-) LAT, ',F7.2,' E LON, RTYP= ',I3,', RSRD=',I5, + $ ' EXPRSRD=',I5) +ccc END IF +ccc ELSE +ccc CALL UFBINT(LUBFI,RID_8,7,1,NLV, +ccc $ 'RPID HOUR MINU CLAT CLON RSRD EXPRSRD') +ccc IF(IBFMS(RID_8(4)).NE.0) THEN +ccc CALL UFBINT(LUBFI,LALOH_8,2,1,NLV,'CLATH CLONH') +ccc RID_8(4:5) = LALOH_8 +ccc END IF +ccc IF(IBFMS(RID_8(1)).NE.0) THEN +ccc IF(SUBSET.EQ.'NC004004' .OR. +ccc $ SUBSET.EQ.'NC004006' .OR. +ccc $ SUBSET.EQ.'NC004009' .OR. +ccc $ SUBSET.EQ.'NC004010' .OR. +ccc $ SUBSET.EQ.'NC004011' .OR. +ccc $ SUBSET.EQ.'NC004014') THEN +ccc CALL UFBINT(LUBFI,ACRN_8,1,1,NLV,'ACRN') +ccc RID_8(1) = ACRN_8 +ccc ELSE IF(SUBSET.EQ.'NC004008' .OR. +ccc $ SUBSET.EQ.'NC004012' .OR. +ccc $ SUBSET.EQ.'NC004013') THEN +ccc CALL UFBINT(LUBFI,ACID_8,1,1,NLV,'ACID') +ccc RID_8(1) = ACID_8 +ccc ELSE IF(SUBSET.EQ.'NC007001' .OR. +ccc $ SUBSET.EQ.'NC007002') THEN +ccc SID = ' ' +ccc ELSE +ccc SID = 'MISSING ' +ccc END IF +ccc END IF +ccc IF(SUBSET(3:5).EQ.'255') THEN +ccc CALL UFBINT(LUBFI,PRV1_dump_8,1,255,NLV, +ccc $ 'PRVSTG') +ccc IF(NLV.LT.1) PRVSTG_dump = ' ' +ccc CALL UFBINT(LUBFI,PRV2_dump_8,1,255,NLV, +ccc $ 'SPRVSTG') +ccc IF(NLV.LT.1) SPRVSTG_dump = ' ' +ccc PRINT 9105, SID,(NINT(RID_8(II)),II=2,3), +ccc $ (RID_8(II),II=4,5), NINT(RID_8(6)), +ccc $ NINT(RID_8(7)),PRVSTG_dump,SPRVSTG_dump + 9105 FORMAT(5X,'- Retain ',A8,' at ',2(I2.2),' UTC',F7.2,' (N+/S-) ', + $ 'LAT',F8.2,' (E+/W-) LON, RSRD=',I5,' EXPRSRD=',I5,', PRVID=',A8, + $ ' SPRVID=',A8) +ccc ELSE +ccc PRINT 215, SID,(NINT(RID_8(II)),II=2,3), +ccc $ (RID_8(II),II=4,5),NINT(RID_8(6)), +ccc $ NINT(RID_8(7)) + 215 FORMAT(5X,'- Retain report ',A8,' at ',2(I2.2),' UTC, ',F6.2, + $ ' (N+/S-) LAT, ',F7.2,'(E+/W-) LON, RSRD=',I5,' EXPRSRD=',I5) +ccc END IF +ccc END IF +cppppp CALL OPENMB(LUBFJ,SUBSET,IDATE) call ufbcnt(lubfj,ireco,isubo) if(ireco.ne.ireco_last) then @@ -745,26 +813,66 @@ PROGRAM BUFR_REMOREST ireco_last = ireco CALL UFBCPY(LUBFI,LUBFJ) CALL WRITSB(LUBFJ) + IF(SUBSET(1:2).EQ.'NC') THEN + READ(SUBSET(3:5),'(I3)') ISUBSET_345 + READ(SUBSET(6:8),'(I3)') ISUBSET_678 + ELSE + ISUBSET_678 = 256 + ISUBSET_345 = -99 + DO II = 1,20 + IF(SUBSET.EQ.PREPBUFR_MSGTYP(II)) THEN + ISUBSET_345 = II + EXIT + END IF + ENDDO + IF(ISUBSET_345.EQ.-99) ISUBSET_345 = 21 + END IF + IF(SUBSET(1:2).EQ.'NC' .AND. + $ ((ISUBSET_345.LT.0.OR.ISUBSET_345.GT.255) .OR. + $ (ISUBSET_678.LT.0.OR.ISUBSET_678.GT.255))) THEN + PRINT 7116, SUBSET + 7116 FORMAT(/'+++++BUFR_REMOREST: WARNING: INVALID BUFR MESSAGE ', + $ 'TYPE AND/OR SUBTYPE READ IN, SUBSET = ',A,' CANNOT INCREMENT' + $ /29X,'NON-RESTRICTED REPORT COPIED COUNTER FOR THIS SUBSET'/) + ELSE + IUSUB_this(ISUBSET_345,ISUBSET_678) = + IUSUB_this(ISUBSET_345,ISUBSET_678) + 1 + END IF IUSUB = IUSUB + 1 ENDDO LOOP1n2 CYCLE LOOP1 C********************************************************************** ELSE IF(SUBSET.EQ.MSG_MASKA(I) .OR. (MSG_MASKA(I)(6:8).EQ. - $ 'xxx'.AND.SUBSET(1:5).EQ.MSG_MASKA(I)(1:5))) THEN + $ 'xxx'.AND.SUBSET(1:5).EQ.MSG_MASKA(I)(1:5))) THEN IF(IMASK_T29(1,I).EQ.000) THEN - PRINT 1113 - 1113 FORMAT(' #####>>>> ALL rpts in msg RESTRICTED - unpk each rpt, ', - $ 'MASK all occurrences of id (chg to "MASKSTID" or "X"''s), copy', - $ ' to output file') + if(subset(1:2).eq.'NC') then + +c .. DUMP file case +c -------------- + PRINT 1113 + 1113 FORMAT(' ###>> ALL rpts in msg RESTRICTED - unpk each rpt, MASK ', + $ 'all id''s (chg to "MASKSTID" or "X"''s), set to non-restr, ', + $ ' copy to output file') + else + +c .. PREPBUFR file case +c ------------------ + print 1114 + end if ELSE + +c .. PREPBUFR file case +c ------------------ PRINT 1114 - 1114 FORMAT(' #####>>>> SOME or ALL rpts in msg RESTRICTED:') + 1114 FORMAT(' ###>> some rpts in msg may be RESTRICTED: if so & w/i ', + $ 'expir time, MASK id (chg to "MASKSTID"), set to non-restr, cpy', + $ ' to output file') DO J = 1,10 IF(IMASK_T29(J,I).NE.99999) PRINT 1115, $ IMASK_T29(J,I) - 1115 FORMAT(' -- ALL rpts in dump type ',I3,' RESTRICTED- unpk', - $ ' each rpt, MASK all occurrences of id (chg to "MASKSTID" or ', - $ '"X"s), copy to o-put file') + 1115 FORMAT(' -- rpts in dump type ',I3,' mixed restr./non-restr.- ', + $ 'for restr. rpts w/i expir. time, MASK id, set to non-restr & ', + $ 'copy to output file') ENDDO END IF @@ -785,8 +893,8 @@ PROGRAM BUFR_REMOREST ireco_last = ireco CALL UFBCPY(LUBFI,LUBFJ) CALL UFBINT(LUBFI,RID_8,6,1,NLV, - $ 'SID RPT YOB XOB TYP T29') - IF(RID_8(5).LT.BMISS/2) THEN + $ 'SID RPT YOB XOB TYP T29 RSRD EXPRSRD') + IF(IBFMS(RID_8(5)).EQ.0) THEN C Come here for PREPBUFR files (report id is in mnemonic "SID") C -- check for a match of dump report type @@ -796,23 +904,38 @@ PROGRAM BUFR_REMOREST IF(IMASK_T29(1,I).EQ.000.OR. $ IMASK_T29(J,I).EQ.NINT(RID_8(6))) THEN -C .... dump report types match - this report is restricted -C --------------------------------------------------- - - PRINT 1104, SID,(RID_8(II),II=2,4), - $ (NINT(RID_8(II)),II=5,6) - 1104 FORMAT(10X,'- Change id of report ',A8,' at ',F6.2,' UTC, ',F6.2, - $ ' (N+/S-) LAT, ',F7.2,'(E) LON, RTYP= ',I3,', DTYP=',I5, - $ ' TO "MASKSTID"') - SID = 'MASKSTID' +C .... dump report types match - look for restricted flag (mnemonic +C "RSRD") and time of expiration on restriction (mnemonic +C "EXPRSRD") (if "EXPRSRD" is missing set it to 99999999 hours +C essentially meaning the report is restricted for all time) +C ------------------------------------------------------------ + if(rid_8(7).gt.0.and.ibfms(rid_8(7)).EQ.0) + $ then + if(ibfms(rid_8(8)).ne.0) + $ rid_8(8) = 99999999. + if(IDIFF_HR_m4.le.rid_8(8)) then + PRINT 1104, SID,(RID_8(II),II=2,4), + $ (NINT(RID_8(II)),II=5,8) + 1104 FORMAT(5X,'- Chg ID of rpt ',A8,' at',F6.2,' UTC, ',F6.2, + $ ' LAT,',F7.2,' E LON, RTYP=',I3,', DTYP=',I5, + $ ', RSRD=',I5,', EXPRSRD=',I5,' TO "MASKSTID"') + SID = 'MASKSTID' + SID_8 = RID_8(1) + IMSUB_this(J,I) = IMSUB_this(J,I) + 1 C .... update report id to masked value - "MASKSTID" C --------------------------------------------- + CALL UFBINT(LUBFJ,SID_8,1,1,IRET,'SID') - CALL UFBINT(LUBFJ,RID_8(1),1,1,IRET,'SID') - EXIT +C .... re-set RSRD & EXPRSRD to "almost" missing so rpt is no longer +C restricted +C -----------------------r------------------------------------- + call ufbint(lubfj,rsrd_8,2,1,iret,'RSRD EXPRSRD') + EXIT + END IF + END IF END IF - END DO + ENDDO ELSE C Come here for DATA DUMP files where all reports are restricted @@ -823,19 +946,26 @@ PROGRAM BUFR_REMOREST C ------------------------------------------------------------------- IUPDATE_RAWRPT = 0 - CALL UFBINT(LUBFI,RID_8,6,1,NLV, + CALL UFBINT(LUBFI,RID_8,7,1,NLV, $ 'RPID HOUR MINU CLAT CLON RSRD') PRINT 1110, SID,(NINT(RID_8(II)),II=2,3), - $ (RID_8(II),II=4,5), NINT(RID_8(6)) - 1110 FORMAT(' - Chg all occurrences of id of rpt ',A8,' at ',2(I2.2), - $ ' UTC, ',F6.2,' (N+/S-) LAT, ',F7.2,'(E+/W-) LON, RSRD=',I5, - $ ' TO "MASKSTID" OR "X"''s') + $ (RID_8(II),II=4,5), NINT(RID_8(6)),NINT(RID_8(7)) + 1110 FORMAT(' - Chg all instances of rpt id ',A8,' ',2(I2.2),' UTC, ', + $ F6.2,'(N+/S-) LAT ',F7.2,'(E+/W-) LON, RSRD=',I5,', EXPRSRD=',I5, + $ ' to MASKSTID or X''s') SID_orig = SID SID = 'MASKSTID' + RPID_8 = RID_8(1) + IMSUB_this(1,I) = IMSUB_this(1,I) + 1 C .... update report id in output file to masked value - "MASKSTID" C ------------------------------------------------------------ + CALL UFBINT(LUBFJ,RPID_8,1,1,IRET,'RPID') +C .... re-set RSRD & EXPRSRD to "almost" missing so rpt is no longer +C restricted +C -----------------------r------------------------------------- + call ufbint(lubfj,rsrd_8,2,1,iret,'RSRD EXPRSRD') CALL UFBINT(LUBFJ,RID_8(1),1,1,IRET,'RPID') IF(SUBSET.EQ.'NC001001'.or. $ subset.eq.'NC001101') THEN @@ -843,21 +973,18 @@ PROGRAM BUFR_REMOREST C .... update 2nd rpt id in ship rpts in output file to masked value C - "MASKSTID" C ------------------------------------------------------------- - - CALL UFBINT(LUBFJ,RID_8(1),1,1,IRET,'SHPC8') + CALL UFBINT(LUBFJ,RPID_8,1,1,IRET,'SHPC8') END IF C .... see if a replicated raw report bulletin header string is C present C -------------------------------------------------------- - CALL UFBINT(LUBFI,RASTR_8,1,255,NLV,'RRSTG') IST = 1 - IF(NLV.GT.0) THEN + IF(NLV.GT.0) THEN C .... it is, store entire string in character array RAWRPT_STG C -------------------------------------------------------- - DO II = 1,NLV IEN = IST+7 RAWRPT_STG(IST:IEN) = RAWRPT(II) @@ -871,9 +998,8 @@ PROGRAM BUFR_REMOREST C for later check to see if it is embedded one or more C times in raw report bulletin header string C ------------------------------------------------------- - DO II = 1,8 - IF(SID_orig(II:II).EQ.' ') THEN + IF(SID_orig(II:II).EQ.' ') THEN ICHAR_id = II - 1 EXIT END IF @@ -881,17 +1007,16 @@ PROGRAM BUFR_REMOREST cppppp ccc print *, 'original sid had ',ICHAR_id,' characters' cppppp - IF(ICHAR_id.GE.1) THEN + IF(ICHAR_id.GE.1) THEN C .... next parse through entire raw report bulletin header C string looking for one or more occurrences of report id C and "X" out the id C ------------------------------------------------------- - DO II = 1,IEN IF(II+ICHAR_id-1.GT.IEN) EXIT IF(RAWRPT_STG(II:II+ICHAR_id-1).EQ. - $ SID_orig(1:ICHAR_id)) THEN + $ SID_orig(1:ICHAR_id)) THEN cppppp ccc print *, 'Found a match to orig sid in RAWRPT_STG',' - in bytes ', ccc $ II,' to ',II+ICHAR_id-1,' - set to "X"' @@ -906,7 +1031,6 @@ PROGRAM BUFR_REMOREST C .... reconstruct replicated raw report bulletin header string, C but now with the id "X"'d out C --------------------------------------------------------- - IF(IUPDATE_RAWRPT.EQ.1) THEN IST = 1 DO II = 1,NLV @@ -918,7 +1042,6 @@ PROGRAM BUFR_REMOREST C .... finally, update replicated raw report bulletin header C string with the id "X"'d out in output file C ----------------------------------------------------- - CALL UFBINT(LUBFJ,RASTR_8,1,NLV,IRET, $ 'RRSTG') cppppp @@ -928,9 +1051,31 @@ PROGRAM BUFR_REMOREST END IF END IF END IF - IF(SID.EQ.'MASKSTID') THEN + IF(SID.EQ.'MASKSTID') THEN IMSUB = IMSUB + 1 ELSE + IF(SUBSET(1:2).EQ.'NC') THEN + READ(SUBSET(3:5),'(I3)') ISUBSET_345 + READ(SUBSET(6:8),'(I3)') ISUBSET_678 + ELSE + ISUBSET_678 = 256 + ISUBSET_345 = -99 + DO II = 1,20 + IF(SUBSET.EQ.PREPBUFR_MSGTYP(II)) THEN + ISUBSET_345 = II + EXIT + END IF + ENDDO + IF(ISUBSET_345.EQ.-99) ISUBSET_345 = 21 + END IF + IF(SUBSET(1:2).EQ.'NC' .AND. + $ ((ISUBSET_345.LT.0.OR.ISUBSET_345.GT.255) .OR. + $ (ISUBSET_678.LT.0.OR.ISUBSET_678.GT.255))) THEN + PRINT 7116, SUBSET + ELSE + IUSUB_this(ISUBSET_345,ISUBSET_678) = + $ IUSUB_this(ISUBSET_345,ISUBSET_678) + 1 + END IF IUSUB = IUSUB + 1 END IF CALL WRITSB(LUBFJ) @@ -940,8 +1085,30 @@ PROGRAM BUFR_REMOREST ENDDO LOOP1n1 C********************************************************************** PRINT 114 - 114 FORMAT(' ALL reports in this message are UNRESTRICTED - copy', + 114 FORMAT(' ALL reports in this message are NON-RESTRICTED - copy', $ ' this message, intact, to output BUFR file') + IF(SUBSET(1:2).EQ.'NC') THEN + READ(SUBSET(3:5),'(I3)') ISUBSET_345 + READ(SUBSET(6:8),'(I3)') ISUBSET_678 + ELSE + ISUBSET_678 = 256 + ISUBSET_345 = -99 + DO II = 1,20 + IF(SUBSET.EQ.PREPBUFR_MSGTYP(II)) THEN + ISUBSET_345 = II + EXIT + END IF + ENDDO + IF(ISUBSET_345.EQ.-99) ISUBSET_345 = 21 + END IF + IF(SUBSET(1:2).EQ.'NC' .AND. + $ ((ISUBSET_345.LT.0.OR.ISUBSET_345.GT.255) .OR. + $ (ISUBSET_678.LT.0.OR.ISUBSET_678.GT.255))) THEN + PRINT 7116, SUBSET + ELSE + IUSUB_this(ISUBSET_345,ISUBSET_678) = + $ IUSUB_this(ISUBSET_345,ISUBSET_678) + ISUB + END IF IUSUB = IUSUB + ISUB CALL CLOSMG(LUBFJ) CALL COPYMG(LUBFI,LUBFJ) @@ -955,12 +1122,75 @@ PROGRAM BUFR_REMOREST CALL CLOSBF(LUBFI) CALL CLOSBF(LUBFJ) - PRINT 106, IRSUB,IMSUB,IUSUB - 106 FORMAT(//'==> A TOTAL OF',I11,' RESTRICTED REPORTS WERE SKIPPED' - $ /'==> A TOTAL OF',I11,' RESTRICTED REPORTS WERE MASKED', - $ ' (ALL OCCURRENCES OF REPORT ID) THEN COPIED' - $ /'==> A TOTAL OF',I11,' UNRESTRICTED REPORTS WERE COPIED'/ - $ /'PROGRAM COMPLETED SUCCESSFULLY'/) + PRINT 106, IRSUB + 106 FORMAT(//'==> A TOTAL OF',I11,' RESTRICTED REPORTS WERE ', + $ 'SKIPPED') + DO I = 1,20 + IF(IRSUB_this_MR(I).GT.0) THEN + PRINT 1106, IRSUB_this_MR(I),MSG_RESTR(I) + DO J = 0,255 + IF(IRSUB_this_sub_MR(I,J).GT.0) THEN + PRINT 2106, IRSUB_this_sub_MR(I,J),MSG_RESTR(I)(1:5),J + END IF + ENDDO + END IF + ENDDO + DO I = 1,20 + IF(IRSUB_this_MM(I).GT.0) THEN + PRINT 1106, IRSUB_this_MM(I),MSG_MIXED(I) + DO J = 0,255 + IF(IRSUB_this_sub_MM(I,J).GT.0) THEN + PRINT 2106, IRSUB_this_sub_MM(I,J),MSG_MIXED(I)(1:5),J + END IF + ENDDO + END IF + ENDDO + 1106 FORMAT(16X,'--> ',I11,' reports from message type ',A) + 2106 FORMAT(21X,'--> ',I11,' reports from message type ',A,I3.3) + + PRINT 116, IMSUB + 116 FORMAT(//'==> A TOTAL OF',I11,' RESTRICTED REPORTS WERE ', + $ 'MASKED AND THEN COPIED') + DO I = 1,20 + DO J = 1,10 + IF(IMSUB_this(J,I).GT.0) THEN + IF(IMASK_T29(J,I).GT.0) THEN + PRINT 1108,IMSUB_this(J,I),MSG_MASKA(I),IMASK_T29(J,I) + ELSE + PRINT 1106,IMSUB_this(J,I),MSG_MASKA(I) + END IF + END IF + ENDDO + ENDDO + 1108 FORMAT(16X,'--> ',I11,' reports from message type ',A, + $ ', DUMP report type ',I5) + + PRINT 117, IUSUB + 117 FORMAT(//'==> A TOTAL OF',I11,' NON-RESTRICTED REPORTS WERE ', + $ 'COPIED'/) + IPRINT_FLAG = 0 + DO I = 1,21 + IF(IUSUB_this(I,256).GT.0) THEN + PRINT 2108, IUSUB_this(I,256),PREPBUFR_MSGTYP(I) + IPRINT_FLAG = 1 + END IF + ENDDO + 2108 FORMAT(16X,'--> ',I11,' reports from message type ',A) + + IF(IPRINT_FLAG.EQ.0) THEN + DO I = 0,255 + DO J = 0,255 + IF(IUSUB_this(I,J).GT.0) THEN + PRINT 2107, IUSUB_this(I,J),I,J + END IF + ENDDO + ENDDO + END IF + 2107 FORMAT(16X,'--> ',I11,' reports from message type NC',I3.3,I3.3) + + PRINT 3117 + 3117 FORMAT(//'PROGRAM COMPLETED SUCCESSFULLY'/) + CALL W3TAGE('BUFR_REMOREST') From b4a1e07e72e0d561155ceee75b0bf564e9779efb Mon Sep 17 00:00:00 2001 From: Shelley Melchior Date: Fri, 22 Apr 2022 01:54:13 +0000 Subject: [PATCH 06/53] Fixed typo that was found upon trying to compile the code. --- sorc/bufr_remorest.fd/remorest.f | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/bufr_remorest.fd/remorest.f b/sorc/bufr_remorest.fd/remorest.f index bec5a0b..3f84cf9 100644 --- a/sorc/bufr_remorest.fd/remorest.f +++ b/sorc/bufr_remorest.fd/remorest.f @@ -836,7 +836,7 @@ PROGRAM BUFR_REMOREST $ /29X,'NON-RESTRICTED REPORT COPIED COUNTER FOR THIS SUBSET'/) ELSE IUSUB_this(ISUBSET_345,ISUBSET_678) = - IUSUB_this(ISUBSET_345,ISUBSET_678) + 1 + $ IUSUB_this(ISUBSET_345,ISUBSET_678) + 1 END IF IUSUB = IUSUB + 1 ENDDO LOOP1n2 From 43caee0af0121ffa034f4cdb154ed2db8bad8a03 Mon Sep 17 00:00:00 2001 From: Shelley Melchior Date: Fri, 22 Apr 2022 19:07:32 +0000 Subject: [PATCH 07/53] Began updates to exdump_post.sh to address github issue 33. --- scripts/exdump_post.sh | 149 +++++++++++++++++++++++++++-------------- 1 file changed, 99 insertions(+), 50 deletions(-) diff --git a/scripts/exdump_post.sh b/scripts/exdump_post.sh index c014a24..d296ea3 100755 --- a/scripts/exdump_post.sh +++ b/scripts/exdump_post.sh @@ -4,11 +4,14 @@ echo "----------------------------------------------------------" echo "exdump_post.sh version $obsproc_ver - If requested: " echo " 1) Generates combined dump STATUS file " echo " 2) Prepares data counts for the SDM " -echo " 3) Removes or masks restricted data from dump files" -echo " 4) Converts dump files to unblocked format " +echo " 3) Removes or masks restricted data from today's " +echo " dump files " +echo " 4) Removes restricted data from 2-day old aircar " +echo " and aircft dump files " +echo " 5) Converts dump files to unblocked format " echo " (meaningless on WCOSS) " -echo " 5) Lists the contents of dump files " -echo " 6) Updates dump data count average tables " +echo " 6) Lists the contents of dump files " +echo " 7) Updates dump data count average tables " echo "----------------------------------------------------------" ##################################################################### # @@ -59,6 +62,8 @@ echo "----------------------------------------------------------" # whose reports are retained and instead station IDs # are masked. # 09 Dec 2021 Esposito - Updated for use on WCOSS2. +# ?? ??? 202? Keyser - Added new "remorest" processing to remove restricted +# data from 2-day old "aircar" and "aircft" dump files. ##################################################################### # NOTE: NET is changed to gdas in the parent Job script for the gdas RUN @@ -81,6 +86,7 @@ SITE=${SITE:-""} cat break > $pgmout +PROCESS_REMOREST_dm2=${PROCESS_REMOREST_dm2:-NO} MPMD=${MPMD:-YES} CHGRP_RSTPROD=${CHGRP_RSTPROD:-YES} UPDATE_AVERAGE_FILE=${UPDATE_AVERAGE_FILE:-YES} @@ -204,25 +210,60 @@ fi if [ "$PROCESS_REMOREST" = 'YES' ]; then +######################################################################## +# Remove or Mask Restricted Data from today's BUFR Data Dump Files # +######################################################################## + +# For dump files "adpsfc", "aircar", "aircft", "msonet", "lghtng" and "sfcshp" +# for today (this network, run and cycle): +# If either the restriction indicator (mnemonic RSRD) is set for a report, or +# the report is in a Table A entry pre-determined to contain only restricted +# reports, then the report is always restricted {regardless of the time in +# hours for the expiration on restriction (mnemonic EXPRSRD)}. +# ---> Since we are running in near-realtime, there is no need to test on +# the value of EXPRSRD. This will always be > current time difference. +# ---> For all Table A entries listed in parm card namelist switch +# "MSG_RESTR" (see below): +# All reports are considered to be restricted by definition of this +# switch. They will be skipped by program BUFR_REMOREST. +# ---> For all Table A entries listed in parm card namelist switch +# "MSG_MIXED" (see below): +# RSRD may not be set for all reports. Those with RSRD set are +# considered to be restricted regardless of EXPRSRD because +# "DIFF_HR" (the difference in hours between the current UTC wall- +# clock date and the dump file center date),is exported as 0 into +# program BUFR_REMOREST (EXPRSRD will always be > actual DIFF_HR +# since we are running in near-realtime). They will be skipped by +# program BUFR_REMOREST. Those reports that do not have RSRD set +# are considered to be non-restricted and will be retained by +# program BUFR_REMOREST. +# ---> For all Table A entries listed in parm card namelist switch +# "MSG_MASKA" (see below): +# All reports are considered to be restricted by definition of this +# switch. Reports are not removed but all occurrences of their +# report ids are unilaterally masked out by program BUFR_REMOREST. +# In addition, program BUFR_REMOREST re-sets their values for RSRD +# and EXPRSRD to missing so that the reports are no longer +# considered to be restricted. +# ----------------------------------------------------------------------------- + msg="REMOVE OR MASK RESTRICTED DATA FROM $tmmark_uc $net_uc DATA DUMPS \ CENTERED ON $dumptime" $DATA/postmsg "$jlogfile" "$msg" -######################################################################## -# Remove or Mask Restricted Data from BUFR Data Dump Files # -######################################################################## + export DIFF_HR=0 cat <<\EOFparm > bufr_remorest.datadump.parm ========================================================================= - Cards for DUMP Version of BUFR_REMOREST -- Version 18 November 2008 + Cards for DUMP Version of BUFR_REMOREST -- Version 18 November 202? &SWITCHES - MSG_RESTR = 'NC000000', ! These are the Table A Entries for - 'NC000100', ! BUFR messages for which ALL reports - 'NC004003', ! are RESTRICTED and will be REMOVED. - 'NC004004', ! (up to 20) - 'NC004006', + MSG_RESTR = 'NC000000', ! These are the Table A Entries for BUFR messages + 'NC000100', ! for which ALL reports are considered to be + 'NC004003', ! restricted. All reports will be REMOVED by + 'NC004004', ! program BUFR_REMOREST. + 'NC004006', ! (up to 20 Tabel A entries) 'NC004007', 'NC004008', 'NC004009', @@ -234,48 +275,56 @@ cat <<\EOFparm > bufr_remorest.datadump.parm 'NC007001', 'NC007002', 'NC021242', - 'NC255xxx', - MSG_MIXED = 'NC012004', ! These are the Table A Entries for - 'NC003010', ! BUFR messages which contain a MIXTURE - ' ', ! of restricted and unrestricted - ' ', ! reports (based on mnemonic "RSRD"). All - ' ', ! restricted reports will be REMOVED. - ' ', ! (up to 20) + 'NC255xxx', ! move this to MSG_MIXED when we know for sure that RSRD values are correct for each provider/sub-provider + MSG_MIXED = 'NC012004', ! These are the Table A Entries for BUFR messages + 'NC003010', ! which may contain a MIXTURE of reports with and + ' ', ! without mnemonic "RSRD" being set. If "RSRD" is + ' ', ! not set -or- it is set and mnemonic "EXPRSRD" + ' ', ! is also set and has a value less than "DIFF_HR" + ' ', ! (the difference in hrs between the current UTC + ' ', ! wall-clock date and the dump file center date) + ' ', ! minus 4, the report will be RETAINED by program + ' ', ! BUFR_REMOREST. Otherwise, it will be REMOVED by + ' ', ! program BUFR_REMOREST. + ' ', ! (up to 20 Tabel A entries) ' ', - MSG_MASKA = 'NC001001', ! These are the Table A Entries for - 'NC001101' ! BUFR messages for which ALL reports - ! are RESTRICTED. They will not be removed, - ! but all occurrences of their report ids will be - ! unilaterally changed to either "MASKSTID" - ! (where the id is stored by itself) or to all - ! "X"'s where the number of "X"'s corresponds to - ! the number of characters in the original report - ! id (where the id is embedded in the raw report - ! bulletin header string) - ! (up to 20) + MSG_MASKA = 'NC001001', ! These are the Table A Entries for BUFR messages + 'NC001101' ! for which ALL reports are considered to be + ! restricted. They will not be removed, but all + ! occurrences of their report ids will be + ! unilaterally CHANGED, by program BUFR_REMOREST, + ! to either "MASKSTID" (where the id is stored by + ! itself) or to all "X"'s where the number of + ! "X"'s corresponds to the number of characters + ! in the original report id (where the id is + ! embedded in the raw report bulletin header + ! string). In addition, program BUFR_REMOREST re- + ! sets their values for "RSRD" and "EXPRSRD" to + ! missing so that the reports are no longer + ! considered to be restricted. + ! (up to 20 Table A entries) / - Note 1: A particular Table A entry should NEVER appear in more than one - of MSG_RESTR, MSG_MIXED or MSG_MASKA. + Note 1: A particular Table A entry should NEVER appear in more than one of + MSG_RESTR, MSG_MIXED or MSG_MASKA. Note 2: Any Table A entry not in either MSG_RESTR, MSG_MIXED or MSG_MASKA - is assumed to be a Table A entry for BUFR messages for which - ALL reports are UNRESTRICTED (these messages are copied - intact, no reports are unpacked). - Note 3: Always fill in these arrays beginning with word 1. If - there are less than 20 words filled in an array, either set - the extra words to " " (8 blank characters) or do not - specify them here (they default to " "). - Note 4: For data dump Table A entries in the form "NCtttsss", - where "ttt" is the BUFR message type and "sss" is the - BUFR message subtype, if the last three characters (the - subtype) is specified as 'xxx', then ALL BUFR messages - of that type are either treated as having all restricted - data all which is to be removed (if in MSG_RESTR), mixed data - some of which is to be removed (if in MSG_MIXED) or all restricted - data all of which is to have its report id masked (if in - MSG_MASKA), regardless of the message subtype. (For example, - if MSG_RESTR(1)='NC255xxx', then ALL mesonet BUFR messages are + is assumed to be a Table A entry for BUFR messages for which ALL + reports are NON-RESTRICTED (these messages are copied intact, no + reports are unpacked). + Note 3: Always fill in these arrays beginning with word 1. If there are + less than 20 words filled in an array, either set the extra words + to " " (8 blank characters) or do not specify them here + (they default to " "). + Note 4: For data dump Table A entries in the form "NCtttsss", where "ttt" + is the BUFR message type and "sss" is the BUFR message subtype, if + the last three characters (the subtype) is specified as 'xxx', then + ALL BUFR messages of that type are either treated as having all + restricted data all which is to be removed (if in MSG_RESTR), mixed + data some of which is to be removed (if in MSG_MIXED) or all + restricted data all of which is to have its report id masked (if in + MSG_MASKA), regardless of the message subtype. (For example, if + MSG_RESTR(1)='NC255xxx', then ALL mesonet BUFR messages are considered to have all restricted data and are all removed regardless of their subtype.) From f35b91f77d8d754f2cc2d738357117eb151517e6 Mon Sep 17 00:00:00 2001 From: Shelley Melchior Date: Mon, 25 Apr 2022 17:08:39 +0000 Subject: [PATCH 08/53] Completed updates to exdump_post.sh to address github issue 33. --- scripts/exdump_post.sh | 165 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 148 insertions(+), 17 deletions(-) diff --git a/scripts/exdump_post.sh b/scripts/exdump_post.sh index d296ea3..5a2cd6f 100755 --- a/scripts/exdump_post.sh +++ b/scripts/exdump_post.sh @@ -331,8 +331,8 @@ cat <<\EOFparm > bufr_remorest.datadump.parm ========================================================================= EOFparm -REMX=${REMX:-$EXECobsproc/bufr_remorest} -REMC=${REMC:-bufr_remorest.datadump.parm} + REMX=${REMX:-$EXECobsproc/bufr_remorest} + REMC=${REMC:-bufr_remorest.datadump.parm} for file in adpsfc aircar aircft msonet sfcshp lghtng gpsipw saphir gpsro do @@ -345,11 +345,11 @@ REMC=${REMC:-bufr_remorest.datadump.parm} rc=$? if [ $rc -gt 0 ] ; then [ $rc -gt $retcode ] && retcode=$rc - msg="**WARNING: ERROR generating unrestricted $file BUFR file \ + msg="**WARNING: ERROR generating non-restricted $file BUFR file \ (rc = $rc)" $DATA/postmsg "$jlogfile" "$msg" else - msg="Successful generation of unrestricted $file BUFR file" + msg="Successful generation of non-restricted $file BUFR file" $DATA/postmsg "$jlogfile" "$msg" cp $filestem $COMOUT/$filestem.nr chmod 664 $COMOUT/$filestem.nr @@ -368,9 +368,112 @@ REMC=${REMC:-bufr_remorest.datadump.parm} done -# endif loop $PROCESS_REMOREST -fi +fi # endif loop $PROCESS_REMOREST + +if [ $PROCESS_REMOREST_dm2 = YES ]; then + +######################################################################## +# Remove Restriction on Data in 2-day old "aircar" and "aircft" # +# BUFR Data Dump files # +######################################################################## + +# For dump files "aircar" and "aircft" for 2-days ago (this network, run and +# cycle): +# ---> For all Table A entries listed in parm card namelist switch +# "MSG_MIXED" (see below): +# If the restriction indicator (mnemonic RSRD) is set for a report, +# then its value for time in hours for the expiration on restriction +# (mnemonic EXPRSRD) is examined. If "DIFF_HR" (the difference in +# hours between the current UTC wall-clock date and the dump file +# center date, exported into program BUFR_REMOREST) minus 4 is less +# than or equal to EXPRSRD, then the report is considered to be +# restricted and will be skipped by program BUFR_REMOREST. +# Otherwise, the report is considered to be non-restricted and will +# be retained by program BUFR_REMOREST. +# ---> Since we are running 2-days late, we want to test on the value of +# EXPRSRD (which currently should be set to 48 hours for any reports in +# these dumps which have RSRD set). This will allow non-rstprod users +# to have access to full "aircar" and "aircft" dumps after EXPRSRD + 4 +# (52) hours. +# ----------------------------------------------------------------------------- + + dumptimeM2=`$NDATE -$tmhr $PDYm2$cyc` + + msg="REMOVE OR MASK RESTRICTED DATA FROM $tmmark_uc $net_uc DATA DUMPS \ +CENTERED ON $dumptimeM2 (2-days ago)" + $DATA/postmsg "$jlogfile" "$msg" + + ymdh=$(date -u +'%Y%m%d%H') + export DIFF_HR=`$NHOUR $ymdh $dumptimeM2` +# current_date dump_date + + msg="Any reports with EXPRSRD less than `expr $DIFF_HR - 4 ` hrs will now \ +be retained" + $DATA/postmsg "$jlogfile" "$msg" + +cat <<\EOF_EXPRSRDparm > bufr_remorest.datadump_EXPRSRD.parm +========================================================================= + + Cards for DUMP Version of BUFR_REMOREST -- Version 1.1.0 (09 September 2015) + (documentation in bufr_remorest.datadump.parm above for $PROCESS_REMOREST + also applies here) + + &SWITCHES + MSG_RESTR = ' ', + MSG_MIXED = 'NC004003', + 'NC004004', + 'NC004006', + 'NC004007', + 'NC004008', + 'NC004009', + 'NC004010', + 'NC004011', + 'NC004012', + 'NC004013', + 'NC004014', + 'NC004015', + 'NC004103', + MSG_MASKA = ' ' + + / + +========================================================================= +EOF_EXPRSRDparm + + REMX=${REMX:-$EXECobsproc_shared_bufr_remorest/bufr_remorest} + REMC=${REMC_EXPRSRD:-bufr_remorest.datadump_EXPRSRD.parm} + + for file in aircar aircft + do + filestem=$RUN.$cycle.$file.$tmmark.bufr_d + [ -f $COMINm2/$filestem ] || continue + + cp $COMINm2/$filestem $filestem + + $USHobsproc/bufr_remorest.sh $filestem + rc=$? + if [ $rc -gt 0 ] ; then + [ $rc -gt $retcode ] && retcode=$rc + msg="**WARNING: ERROR generating non-restricted $file BUFR file \ +from 2-days ago (rc = $rc) -- existing file made 2-days ago is not overwritten" + $DATA/postmsg "$jlogfile" "$msg" + else + msg="Successful generation of non-restricted $file BUFR file from \ +2-days ago -- overwrite existing file made 2-days ago" + $DATA/postmsg "$jlogfile" "$msg" + cp $filestem $COMOUTm2/$filestem.nr + chmod 664 $COMOUTm2/$filestem.nr + if [ $SENDDBN = "YES" ] ; then + NETUP=`echo $RUN | tr {a-z} {A-Z}` # can this be net_uc? + $DBNROOT/bin/dbn_alert MODEL ${NETUP}_BUFR_${file}_nr $job \ + $COMOUTm2/$filestem.nr + fi + fi + + done + +fi # endif loop $PROCESS_REMOREST_dm2 if [ "$PROCESS_UNBLKBUFR" = 'YES' ]; then @@ -381,23 +484,43 @@ $dumptime" ######################################################################## # Unblock BUFR Data Dump Files # +# {Note: will also unblock non-restricted aircar and aircft dump files # +# from 2-days ago if they were overwritten in REMOREST above # +# (when $PROCESS_REMOREST_dm2 = YES)} # # ---> ON WCOSS dump files are already unblocked, so this whole set of # # processing can hopefully be removed someday!! # ######################################################################## + aircraft_nr_dm2="" + [ $PROCESS_REMOREST_dm2 = YES ] && \ + aircraft_nr_dm2="aircar_nr_dm2 aircft_nr_dm2" + for file in adpsfc adpupa aircar aircft satwnd sfcshp spssmi proflr \ vadwnd goesnd erscat sfcbog erswnd ssmip ssmipn ssmit \ atovs qkscat qkswnd trmm sptrmm geoimr 1bamua 1bamub \ 1bhrs3 1bmhs 1bhrs4 airs airswm amsre gpsipw msonet \ rassda nexrad gpsro airsev goesfv wndsat wdsatr osbuv8 \ ascatt ascatw mtiasi avcsam avcspm gome lghtng omi \ - esamua esamub eshrs3 esmhs ssmisu sevcsr lgycld efclam + esamua esamub eshrs3 esmhs ssmisu sevcsr lgycld efclam \ + $aircraft_nr_dm2 # --> don't add any new dumps here since files are already unblocked # on WCOSS!! do + file_orig=$file + if [ $PROCESS_REMOREST_dm2 = YES ]; then + if [ $file = aircar_nr_dm2 -o $file = aircft_nr_dm2 ]; then + file=`echo $file | cut -d"_" -f1` + COMIN_save=$COMIN + COMIN=$COMINm2 + COMOUT_save=$COMOUT + COMOUT=$COMOUTm2 + fi + fi for qual in "" ".nr" do + [ \( $file_orig = aircar_nr_dm2 -o $file_orig = aircft_nr_dm2 \) -a \ + "$qual" != .nr ] && continue qualdbn=$qual COMIN_here=$COMIN if [ "$qual" = .nr ]; then @@ -421,7 +544,12 @@ $dumptime" $file.unblock$qual (rc = $rc)" $DATA/postmsg "$jlogfile" "$msg" else - msg="$file$qual BUFR file SUCCESSFULLY copied to $file.unblock$qual" + if [ $file_orig = aircar_nr_dm2 -o $file_orig = aircft_nr_dm2 ];then + msg="$file$qual BUFR file from 2-days ago SUCCESSFULLY copied \ +to $file.unblock$qual -- overwrite existing file made 2-days ago" + else + msg="$file$qual BUFR file SUCCESSFULLY copied to $file.unblock$qual" + fi $DATA/postmsg "$jlogfile" "$msg" cp $file.unblock$qual $COMOUT/$filestem.unblok$qual chmod 664 $COMOUT/$filestem.unblok$qual @@ -469,11 +597,16 @@ a null file is copied in its place" fi done + if [ $PROCESS_REMOREST_dm2 = YES ]; then + if [ $file_orig = aircar_nr_dm2 -o $file_orig = aircft_nr_dm2 ]; then + COMIN=$COMIN_save + COMOUT=$COMOUT_save + fi + fi done -# endif loop $PROCESS_UNBLKBUFR -fi +fi # endif loop $PROCESS_UNBLKBUFR @@ -496,9 +629,9 @@ $dumptime" EOFlistdumps cat <<\EOFthread > $DATA/thread -#VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV -# Make herefile "thread" which will run in background shells to list dumps -#VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV +#VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV +# Herefile "thread" which can run in parallel (via MPMD methods) to list dumps +#VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV { echo set -x mkdir -p $DATA/thread${1} @@ -650,8 +783,7 @@ fi [ $err -gt $retcode ] && retcode=$err done -# endif loop $PROCESS_LISTERS -fi +fi # endif loop $PROCESS_LISTERS @@ -739,8 +871,7 @@ by choice" fi done -# endif loop $PROCESS_AVGTABLES -fi +fi # endif loop $PROCESS_AVGTABLES ####################### From 060555750a4a91196002dabda147006d79465ace Mon Sep 17 00:00:00 2001 From: Shelley Melchior Date: Mon, 25 Apr 2022 17:44:50 +0000 Subject: [PATCH 09/53] Completed updates to JOBSPROC_GLOBAL_DUMP_POST and JOBSPROC_GLOBAL_PREP_POST to address github issue 33. --- jobs/JOBSPROC_GLOBAL_DUMP_POST | 10 +++++++++- jobs/JOBSPROC_GLOBAL_PREP_POST | 8 ++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/jobs/JOBSPROC_GLOBAL_DUMP_POST b/jobs/JOBSPROC_GLOBAL_DUMP_POST index 3fe7b27..a3d2ae7 100755 --- a/jobs/JOBSPROC_GLOBAL_DUMP_POST +++ b/jobs/JOBSPROC_GLOBAL_DUMP_POST @@ -157,16 +157,24 @@ if [[ "$RUN_ENVIR" == nco ]]; then export COMIN=${COMIN:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT})} export COMOUT=${COMOUT:-$(compath.py -o ${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT})} mkdir -m 775 -p $COMOUT +# The following are used by $PROCESS_REMOREST_dm2 when it is YES +# -------------------------------------------------------------- + export COMINm2=${COMINm2:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT})} + export COMOUTm2=${COMOUTm2:-$(compath.py -o ${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT})} else export COMIN=${COMIN:-${COMIN_ROOT:?}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT}} - # COMOUT_ROOT for developers defaults to unique $DATA/com directory as root to +# COMOUT_ROOT for developers defaults to unique $DATA/com directory as root to # prevent job from writing into a non-unique directory that might already have # output in it from a previous checkout run for the same cycle # ----------------------------------------------------------------------------- export COMOUT_ROOT=${COMOUT_ROOT:-${DATA}/com} export COMOUT=${COMOUT:-${COMOUT_ROOT}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT}} mkdir -m 755 -p $COMOUT +# The following are used by $PROCESS_REMOREST_dm2 when it is YES +# -------------------------------------------------------------- + export COMINm2=${COMINm2:-${COMIN_ROOT:?}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT}} + export COMOUTm2=${COMOUTm2:-${COMOUT_ROOT}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT}} fi # comin and comin_m1 are paths to dump status files for bufr_avgdata.sh diff --git a/jobs/JOBSPROC_GLOBAL_PREP_POST b/jobs/JOBSPROC_GLOBAL_PREP_POST index fe0c885..d62a9fe 100755 --- a/jobs/JOBSPROC_GLOBAL_PREP_POST +++ b/jobs/JOBSPROC_GLOBAL_PREP_POST @@ -151,6 +151,10 @@ if [[ "$RUN_ENVIR" == nco ]]; then export COMIN=${COMIN:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT})} export COMOUT=${COMOUT:-$(compath.py -o ${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT})} mkdir -m 775 -p $COMOUT +# The following are used by $PROCESS_REMOREST_dm2 when it is YES +# -------------------------------------------------------------- + export COMINm2=${COMINm2:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT})} + export COMOUTm2=${COMOUTm2:-$(compath.py -o ${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT})} else export COMIN=${COMIN:-${COMIN_ROOT:?}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT}} @@ -161,6 +165,10 @@ else export COMOUT_ROOT=${COMOUT_ROOT:-${DATA}/com} export COMOUT=${COMOUT:-${COMOUT_ROOT}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT}} mkdir -m 755 -p $COMOUT +# The following are used by $PROCESS_REMOREST_dm2 when it is YES +# -------------------------------------------------------------- + export COMINm2=${COMINm2:-${COMIN_ROOT:?}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT}} + export COMOUTm2=${COMOUTm2:-${COMOUT_ROOT}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT}} fi # The following are used by $PROCESS_TIMETWINS in the GDAS when it is YES From b5cb5b1cc626c6fd0734d10caa734fc0b27223b3 Mon Sep 17 00:00:00 2001 From: Shelley Melchior Date: Wed, 18 May 2022 18:46:52 +0000 Subject: [PATCH 10/53] Minor tweaks to get the process running on cactus. --- jobs/JOBSPROC_GLOBAL_DUMP_POST | 1 + sorc/bufr_remorest.fd/remorest.f | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/jobs/JOBSPROC_GLOBAL_DUMP_POST b/jobs/JOBSPROC_GLOBAL_DUMP_POST index a3d2ae7..9399bae 100755 --- a/jobs/JOBSPROC_GLOBAL_DUMP_POST +++ b/jobs/JOBSPROC_GLOBAL_DUMP_POST @@ -175,6 +175,7 @@ else # -------------------------------------------------------------- export COMINm2=${COMINm2:-${COMIN_ROOT:?}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT}} export COMOUTm2=${COMOUTm2:-${COMOUT_ROOT}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT}} + mkdir -m 755 -p $COMOUTm2 fi # comin and comin_m1 are paths to dump status files for bufr_avgdata.sh diff --git a/sorc/bufr_remorest.fd/remorest.f b/sorc/bufr_remorest.fd/remorest.f index 3f84cf9..252f782 100644 --- a/sorc/bufr_remorest.fd/remorest.f +++ b/sorc/bufr_remorest.fd/remorest.f @@ -290,7 +290,8 @@ C PREPBUFR file: "SID" - chgd to "MASKSTID" (all Tbl A entries) C (PREPBUFR file "RSRD" and "EXPRSRD" also set to missing) C DUMP file: "RPID" - chgd to "MASKSTID" (all Tbl A entries) -C DUMP file: "SHPC8" - chgd to "MASKSTID" (Tbl A entry NC001001) +C DUMP file: "SHPC8" - chgd to "MASKSTID" (Tbl A entry NC001001 +C and NC001101) C DUMP file: "RRSTG" - chgd to "X" (where the number of "X"'s C corresponds to the the number of C characters in the original report id ) @@ -299,9 +300,9 @@ C C Note: Currently for dump files, the only Table A entry where all C occurrences of report id in a report are known to be masked -C is NC001001. This code may have to be modified to add this -C ability to mask all occurrences of report id for other Table -C A entries. +C is NC001001 and NC001101. This code may have to be modified +C to add this ability to mask all occurrences of report id for +C other Table A entries. C C C ONE SCRIPT ENVIRONMENT VARIABLE IS READ IN: @@ -639,7 +640,7 @@ PROGRAM BUFR_REMOREST C meaning the report is restricted for all time) C ------------------------------------------------------------------- - CALL UFBINT(LUBFI,RID_8,6,1,NLV, + CALL UFBINT(LUBFI,RID_8,7,1,NLV, $ 'SID RPT YOB XOB TYP RSRD EXPRSRD') IF(RID_8(6).GT.0.AND.IBFMS(RID_8(6)).EQ.0) THEN IF(IBFMS(RID_8(7)).NE.0) RID_8(7) = 99999999. @@ -892,7 +893,7 @@ PROGRAM BUFR_REMOREST end if ireco_last = ireco CALL UFBCPY(LUBFI,LUBFJ) - CALL UFBINT(LUBFI,RID_8,6,1,NLV, + CALL UFBINT(LUBFI,RID_8,8,1,NLV, $ 'SID RPT YOB XOB TYP T29 RSRD EXPRSRD') IF(IBFMS(RID_8(5)).EQ.0) THEN @@ -940,14 +941,14 @@ PROGRAM BUFR_REMOREST C Come here for DATA DUMP files where all reports are restricted C {report id is in mnemonic "RPID" (and for surface ship reports in -C Table A entry 'NC001001' also in mnemonic "SHPC8"), report id may -C also be embedded in replicated raw bulletin header string if it is -C present) +C Table A entry 'NC001001' and 'NC001101' also in mnemonic "SHPC8"), +C report id may also be embedded in replicated raw bulletin header +C string if it is present} C ------------------------------------------------------------------- IUPDATE_RAWRPT = 0 CALL UFBINT(LUBFI,RID_8,7,1,NLV, - $ 'RPID HOUR MINU CLAT CLON RSRD') + $ 'RPID HOUR MINU CLAT CLON RSRD EXPRSRD') PRINT 1110, SID,(NINT(RID_8(II)),II=2,3), $ (RID_8(II),II=4,5), NINT(RID_8(6)),NINT(RID_8(7)) 1110 FORMAT(' - Chg all instances of rpt id ',A8,' ',2(I2.2),' UTC, ', From d33bf6cdfd72df56857c2fb970b6e9d15da510f7 Mon Sep 17 00:00:00 2001 From: Shelley Melchior Date: Tue, 24 May 2022 01:32:59 +0000 Subject: [PATCH 11/53] Updated jobs/JOBSPROC_GLOBAL_PREP_POST to add line to create COMOUTm2 dir. --- jobs/JOBSPROC_GLOBAL_PREP_POST | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/JOBSPROC_GLOBAL_PREP_POST b/jobs/JOBSPROC_GLOBAL_PREP_POST index d62a9fe..2493b68 100755 --- a/jobs/JOBSPROC_GLOBAL_PREP_POST +++ b/jobs/JOBSPROC_GLOBAL_PREP_POST @@ -169,6 +169,7 @@ else # -------------------------------------------------------------- export COMINm2=${COMINm2:-${COMIN_ROOT:?}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT}} export COMOUTm2=${COMOUTm2:-${COMOUT_ROOT}/${obsNET}/${obsproc_ver}/${RUN}.${PDYm2}/${cyc}/${COMPONENT}} + mkdir -m 755 -p $COMOUTm2 fi # The following are used by $PROCESS_TIMETWINS in the GDAS when it is YES From cbc15ed57880a93833fb4e8fd7be712332a36096 Mon Sep 17 00:00:00 2001 From: Shelley Melchior Date: Tue, 7 Jun 2022 15:24:45 +0000 Subject: [PATCH 12/53] Squashed commit of the following: commit e3793621ea375479ccd39029f05c746afb404448 Author: Shelley Melchior Date: Thu Jun 2 14:38:13 2022 +0000 Squashed commit of the following: commit 21d7d9a2d2983684f4403c7ee9023a38c152c3c3 Author: Shelley Melchior Date: Thu Jun 2 13:30:52 2022 +0000 Squashed commit of the following: commit daff5af2cdfaf849f42d42a8b90de9235738f444 Author: Shelley Melchior Date: Fri May 27 18:05:05 2022 +0000 Added section to cd to logfile output dir prior to process submission. commit 30c01f9bd251b31d0116b996ca6f115ee7a43e71 Author: iliana Genkova <59100254+ilianagenkova@users.noreply.github.com> Date: Tue May 24 11:37:06 2022 -0500 Set obsproc_ver=v1.0 and HOMEgfs=$COMROOT/gfs/v16.2 (#38) * Set obsproc+ver=v1.0 and HOMEgfs=$COMROOT/gfs/v16.2 * Introduce 3-digit obsproc_ver_pckg and use with $PACKAGEROOT commit ebb6c6856c118705303594641f3fc9b8a07494cd Author: Cory Martin Date: Tue Apr 12 15:15:47 2022 -0400 Changes needed to build on Hera (#32) commit 0075564dc3e9d67b3369138cd9a4f2d704872ad9 Author: Shelley Melchior <53050457+ShelleyMelchior-NOAA@users.noreply.github.com> Date: Sat Apr 9 17:07:10 2022 -0400 Incorporating changes made by NCO SPA following: (#35) restart of cactus and changing output data directory for mods commit 59596a970403d0d248216cbdaa13121b115f9ee0 Author: Shelley Melchior Date: Thu Apr 7 21:27:40 2022 +0000 Removing top level README.md. This file now resides in docs/. commit 735f0092614d8ade4a8fd3194c9d5c2a88a9bfdc Merge: bd4211a 8f28c00 Author: Iliana Genkova Date: Thu Apr 7 20:51:34 2022 +0000 Merge branch 'release/obsproc.v1.0.0' into develop commit 8f28c00f18caf79a4f55e6bf04e9d07237ff604b Merge: f01f691 bd4211a Author: Iliana Genkova Date: Thu Apr 7 20:49:01 2022 +0000 Removed build.sh (now in /build-obsproc) and ush/prepobs_makeprepbufr.sh(now in module prepobs) Merge branch 'develop' into release/obsproc.v1.0.0 commit bd4211a592431b8cc90eeb174e74180b69c55c00 Author: iliana Genkova <59100254+ilianagenkova@users.noreply.github.com> Date: Mon Jan 3 19:23:04 2022 -0600 Adopt Jack Woolen's (CFS) faster prepobs_makeprepbufr.sh: (#19) * Adopt Jack Woolen's (CFS) faster prepobs_makeprepbufr.sh: -use cfp with mpiexe -introduce FORT5 -move ksh interpreter * Clean up interpreter commit 309371d52eef400859b9bf5ecaaf2f7271d8ef05 Author: Shelley Melchior <53050457+ShelleyMelchior-NOAA@users.noreply.github.com> Date: Mon Jan 3 18:25:21 2022 -0500 created README.md Created README.md to better explain installation instructions, modified to adopt NCO's working practices. commit c5342ebb3ced1db0d611dfbdb0a67ee44080e1f0 Author: Rahul Mahajan Date: Mon Jan 3 17:57:29 2022 -0500 adopt for current NCO working practices per discussion w/ StevenEarle et al (#18) commit e63a78c89826acddb1f8bdab2462b124229f4917 Author: Shelley Melchior Date: Wed Jun 1 16:21:51 2022 +0000 Updated to correctly locate tcvitals file. commit 0bbb542101afcec5935047406fa0993dc9bfb305 Author: Shelley Melchior Date: Fri Apr 8 20:31:15 2022 +0000 Incorporating changes made by NCO SPA following: restart of cactus and changing output data directory for mods commit daff5af2cdfaf849f42d42a8b90de9235738f444 Author: Shelley Melchior Date: Fri May 27 18:05:05 2022 +0000 Added section to cd to logfile output dir prior to process submission. commit 30c01f9bd251b31d0116b996ca6f115ee7a43e71 Author: iliana Genkova <59100254+ilianagenkova@users.noreply.github.com> Date: Tue May 24 11:37:06 2022 -0500 Set obsproc_ver=v1.0 and HOMEgfs=$COMROOT/gfs/v16.2 (#38) * Set obsproc+ver=v1.0 and HOMEgfs=$COMROOT/gfs/v16.2 * Introduce 3-digit obsproc_ver_pckg and use with $PACKAGEROOT commit ebb6c6856c118705303594641f3fc9b8a07494cd Author: Cory Martin Date: Tue Apr 12 15:15:47 2022 -0400 Changes needed to build on Hera (#32) commit 0075564dc3e9d67b3369138cd9a4f2d704872ad9 Author: Shelley Melchior <53050457+ShelleyMelchior-NOAA@users.noreply.github.com> Date: Sat Apr 9 17:07:10 2022 -0400 Incorporating changes made by NCO SPA following: (#35) restart of cactus and changing output data directory for mods --- jobs/JOBSPROC_GLOBAL_PREP | 4 ++-- scripts/exglobal_makeprepbufr.sh | 1 + triggers/cycqsub | 8 ++++++++ triggers/jairnow_dump.wc2.pbs | 8 +++++--- triggers/jcdas_dump.wc2.pbs | 7 ++++--- triggers/jcdas_dump_post.wc2.pbs | 5 +++-- triggers/jcdas_prep1.wc2.pbs | 7 ++++--- triggers/jcdas_prep1_post.wc2.pbs | 7 ++++--- triggers/jcdas_prep2.wc2.pbs | 7 ++++--- triggers/jcdas_prep2_post.wc2.pbs | 7 ++++--- triggers/jdump_alert.wc2.pbs | 3 ++- triggers/jdump_monitor.wc2.pbs | 7 ++++--- triggers/jglobal_dump.wc2.pbs | 7 ++++--- triggers/jglobal_dump_post.wc2.pbs | 5 +++-- triggers/jglobal_prep.wc2.pbs | 9 +++++---- triggers/jglobal_prep_post.wc2.pbs | 7 ++++--- triggers/jmods.wc2.pbs | 7 ++++--- triggers/jnam_dump.wc2.pbs | 7 ++++--- triggers/jnam_dump2.wc2.pbs | 7 ++++--- triggers/jnam_dump_post.wc2.pbs | 7 ++++--- triggers/jnam_prep.wc2.pbs | 7 ++++--- triggers/jnam_prep_post.wc2.pbs | 7 ++++--- triggers/jrap_dump.wc2.pbs | 7 ++++--- triggers/jrap_dump_post.wc2.pbs | 7 ++++--- triggers/jrap_prep.wc2.pbs | 7 ++++--- triggers/jrap_prep_post.wc2.pbs | 7 ++++--- triggers/jrtma_dump.wc2.pbs | 7 ++++--- triggers/jrtma_dump_post.wc2.pbs | 7 ++++--- triggers/jrtma_prep.wc2.pbs | 7 ++++--- triggers/jrtma_prep_post.wc2.pbs | 7 ++++--- triggers/jurma_dump.wc2.pbs | 7 ++++--- triggers/jurma_dump_post.wc2.pbs | 7 ++++--- triggers/jurma_prep.wc2.pbs | 7 ++++--- triggers/jurma_prep_post.wc2.pbs | 7 ++++--- 34 files changed, 133 insertions(+), 92 deletions(-) diff --git a/jobs/JOBSPROC_GLOBAL_PREP b/jobs/JOBSPROC_GLOBAL_PREP index e1ba8b7..d0275b7 100755 --- a/jobs/JOBSPROC_GLOBAL_PREP +++ b/jobs/JOBSPROC_GLOBAL_PREP @@ -157,7 +157,6 @@ export DICTPREP=${DICTPREP:-${OPSROOT}/decoders/decod_shared/dictionaries_${dict # --------------------------------------------------------------------------- #export HOMEgfs=${HOMEgfs:-${PACKAGEROOT:?}/gfs.${gfs_ver:?"###FATAL ERROR \"gfs_ver\" is not set"}} export HOMEgfs=${HOMEgfs:-$(compath.py gfs/${gfs_ver})} -#export HOMEgfs=${HOMEgfs:-$HOMEobsproc} #Temporary place to let job run. change it to above layer when gfs run in realtime ######################################################################### # Add some prod utilities to working directory @@ -184,11 +183,12 @@ export COMPONENT=${COMPONENT:-atmos} if [[ "$RUN_ENVIR" == nco ]]; then export COMIN=${COMIN:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT})} - export COMINtcvital=${COMINtcvital:-$(compath.py ${envir}/${NET}/${gfs_ver}/${RUN}.${PDY})/${cyc}/${COMPONENT}/${RUN}.${cycle}.} + export COMINtcvital=${COMINtcvital:-$(compath.py ${envir}/${NET}/${gfs_ver}/${RUN}.${PDY})/${cyc}/${COMPONENT}} export COMOUT=${COMOUT:-$(compath.py -o ${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT})} mkdir -m 775 -p $COMOUT else export COMIN=${COMIN:-${COMIN_ROOT:?}/${obsNET}/${obsproc_ver}/${RUN}.${PDY}/${cyc}/${COMPONENT}} + export COMINtcvital=${COMINtcvital:-$(compath.py ${envir}/${NET}/${gfs_ver}/${RUN}.${PDY})/${cyc}/${COMPONENT}} # COMOUT_ROOT for developers defaults to unique $DATA/com directory as root to # prevent job from writing into a non-unique directory that might already have diff --git a/scripts/exglobal_makeprepbufr.sh b/scripts/exglobal_makeprepbufr.sh index 7cb82f0..e0fde93 100755 --- a/scripts/exglobal_makeprepbufr.sh +++ b/scripts/exglobal_makeprepbufr.sh @@ -40,6 +40,7 @@ CHGRP_RSTPROD=${CHGRP_RSTPROD:-YES} MAKE_NSSTBUFR=${MAKE_NSSTBUFR:-YES} export COMSP=${COMSP:-$COMIN/${RUN}.${cycle}.} +export COMSPtcvital=${COMSPtcvital:-$COMINtcvital/${RUN}.${cycle}.} export tstsp=${tstsp:-/tmp/null/} if [ "$DO_QC" = 'YES' -a "$CQCBUFR" = 'YES' -a -n "$COM1" -a -n "$CQCC" ]; then diff --git a/triggers/cycqsub b/triggers/cycqsub index 37a8064..5d249a4 100755 --- a/triggers/cycqsub +++ b/triggers/cycqsub @@ -14,4 +14,12 @@ else qsub_command=qsub fi +# cd to destination output dir for logfiles to be written there w/ pid +outdir=/lfs/h2/emc/stmp/$USER/CRON/$desc/output +echo $outdir +mkdir -p $outdir +cd $outdir +echo `pwd` + sed "s/%HHMM%/`date -u +\%H\%M`/;s/%CC%/$cyc/;s/%CCM%/$cycM/;s/%JTYP%/$jtyp/;s/%YYYYMMDD%/`date -u +\%Y\%m\%d`/;s/%PDY%/$PDY/;s/%_PDY%/$_PDY/;s/%TMMARK%/$tmmark/;s/%DESC%/$desc/;s/%_DESC%/$_desc/" $pbs_file | $qsub_command + diff --git a/triggers/jairnow_dump.wc2.pbs b/triggers/jairnow_dump.wc2.pbs index 374925b..04c2963 100755 --- a/triggers/jairnow_dump.wc2.pbs +++ b/triggers/jairnow_dump.wc2.pbs @@ -32,11 +32,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 + export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jcdas_dump.wc2.pbs b/triggers/jcdas_dump.wc2.pbs index 9792a78..17d6925 100644 --- a/triggers/jcdas_dump.wc2.pbs +++ b/triggers/jcdas_dump.wc2.pbs @@ -30,12 +30,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export bufr_dump_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=$HOMEobsproc/versions/run.ver diff --git a/triggers/jcdas_dump_post.wc2.pbs b/triggers/jcdas_dump_post.wc2.pbs index 8f11cb5..50cd26e 100644 --- a/triggers/jcdas_dump_post.wc2.pbs +++ b/triggers/jcdas_dump_post.wc2.pbs @@ -31,11 +31,12 @@ PDYm1=$(date -d "${PDY} 1 day ago" +%Y%m%d) userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export bufr_dump_ver=1.0.0 export obsNET=obsproc -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para #export HOMEobsproc=${PACKAGEROOT}/${obsNET}.${obsproc_ver} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local diff --git a/triggers/jcdas_prep1.wc2.pbs b/triggers/jcdas_prep1.wc2.pbs index 205a99c..c75fea3 100644 --- a/triggers/jcdas_prep1.wc2.pbs +++ b/triggers/jcdas_prep1.wc2.pbs @@ -31,12 +31,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export prepobs_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jcdas_prep1_post.wc2.pbs b/triggers/jcdas_prep1_post.wc2.pbs index 56e4d98..aca3094 100644 --- a/triggers/jcdas_prep1_post.wc2.pbs +++ b/triggers/jcdas_prep1_post.wc2.pbs @@ -30,12 +30,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export obsNET=obsproc export prepobs_ver=1.0.0 PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jcdas_prep2.wc2.pbs b/triggers/jcdas_prep2.wc2.pbs index 9359910..8c5c763 100644 --- a/triggers/jcdas_prep2.wc2.pbs +++ b/triggers/jcdas_prep2.wc2.pbs @@ -36,11 +36,12 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER export obsNET=obsproc -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export prepobs_ver=1.0.0 PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jcdas_prep2_post.wc2.pbs b/triggers/jcdas_prep2_post.wc2.pbs index da7fe71..7f288a7 100644 --- a/triggers/jcdas_prep2_post.wc2.pbs +++ b/triggers/jcdas_prep2_post.wc2.pbs @@ -32,11 +32,12 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER export obsNET=obsproc -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export prepobs_ver=1.0.0 PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jdump_alert.wc2.pbs b/triggers/jdump_alert.wc2.pbs index 5cc8037..5514d90 100755 --- a/triggers/jdump_alert.wc2.pbs +++ b/triggers/jdump_alert.wc2.pbs @@ -35,7 +35,8 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages #export HOMEobsproc=${userROOT}/obsproc-cmake/$obsproc_ver/jdumpalert # local diff --git a/triggers/jdump_monitor.wc2.pbs b/triggers/jdump_monitor.wc2.pbs index 98367d8..fd16b20 100755 --- a/triggers/jdump_monitor.wc2.pbs +++ b/triggers/jdump_monitor.wc2.pbs @@ -29,12 +29,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export bufr_dump_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jglobal_dump.wc2.pbs b/triggers/jglobal_dump.wc2.pbs index ddf461d..755985f 100755 --- a/triggers/jglobal_dump.wc2.pbs +++ b/triggers/jglobal_dump.wc2.pbs @@ -32,12 +32,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export bufr_dump_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=$HOMEobsproc/versions/run.ver diff --git a/triggers/jglobal_dump_post.wc2.pbs b/triggers/jglobal_dump_post.wc2.pbs index 50554cd..74a0d77 100755 --- a/triggers/jglobal_dump_post.wc2.pbs +++ b/triggers/jglobal_dump_post.wc2.pbs @@ -34,10 +34,11 @@ PDYm1=$(date -d "${PDY} 1 day ago" +%Y%m%d) userROOT=/lfs/h2/emc/obsproc/noscrub/$USER export obsNET=obsproc -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export bufr_dump_ver=1.0.0 -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para #export HOMEobsproc=${PACKAGEROOT}/${obsNET}.${obsproc_ver} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local diff --git a/triggers/jglobal_prep.wc2.pbs b/triggers/jglobal_prep.wc2.pbs index a8faa27..c0e2da0 100755 --- a/triggers/jglobal_prep.wc2.pbs +++ b/triggers/jglobal_prep.wc2.pbs @@ -31,11 +31,12 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER export obsNET=obsproc -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export prepobs_ver=1.0.0 PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver @@ -78,7 +79,7 @@ export SENDDBN=NO # developer export DATAROOT=/lfs/h2/emc/stmp/$USER export jlogfile=/lfs/h2/emc/stmp/$USER/${JTYP}.$PDY.jlogfile -export HOMEgfs=$PACKAGEROOT/gfs.v16.2.0 +export HOMEgfs=$COMROOT/gfs/v16.2 export COMIN_ROOT=/lfs/h2/emc/stmp/$USER/CRON/${DESC}/com #export COMINgdas=/lfs/h1/ops/canned/com/gfs/v16.2/${JTYP}.${PDY}/${cyc}/${COMPONENT} diff --git a/triggers/jglobal_prep_post.wc2.pbs b/triggers/jglobal_prep_post.wc2.pbs index d39093e..9f2e991 100755 --- a/triggers/jglobal_prep_post.wc2.pbs +++ b/triggers/jglobal_prep_post.wc2.pbs @@ -31,11 +31,12 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER export obsNET=obsproc -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export prepobs_ver=1.0.0 PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jmods.wc2.pbs b/triggers/jmods.wc2.pbs index 4e9b904..34e0710 100644 --- a/triggers/jmods.wc2.pbs +++ b/triggers/jmods.wc2.pbs @@ -28,12 +28,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export bufr_dump_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jnam_dump.wc2.pbs b/triggers/jnam_dump.wc2.pbs index 370f89f..554b5a5 100755 --- a/triggers/jnam_dump.wc2.pbs +++ b/triggers/jnam_dump.wc2.pbs @@ -32,12 +32,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export bufr_dump_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=$HOMEobsproc/versions/run.ver diff --git a/triggers/jnam_dump2.wc2.pbs b/triggers/jnam_dump2.wc2.pbs index 9a54d96..f755c16 100755 --- a/triggers/jnam_dump2.wc2.pbs +++ b/triggers/jnam_dump2.wc2.pbs @@ -32,12 +32,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export bufr_dump_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=$HOMEobsproc/versions/run.ver diff --git a/triggers/jnam_dump_post.wc2.pbs b/triggers/jnam_dump_post.wc2.pbs index b8696ca..44a935e 100755 --- a/triggers/jnam_dump_post.wc2.pbs +++ b/triggers/jnam_dump_post.wc2.pbs @@ -32,12 +32,13 @@ PDYm1=$(date -d "${PDY} 1 day ago" +%Y%m%d) userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export bufr_dump_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=$HOMEobsproc/versions/run.ver diff --git a/triggers/jnam_prep.wc2.pbs b/triggers/jnam_prep.wc2.pbs index 16000d3..14c819f 100755 --- a/triggers/jnam_prep.wc2.pbs +++ b/triggers/jnam_prep.wc2.pbs @@ -35,12 +35,13 @@ export job=nam_prep_${tmmark}_${cyc} userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export prepobs_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jnam_prep_post.wc2.pbs b/triggers/jnam_prep_post.wc2.pbs index 49a30bb..c5c5a01 100755 --- a/triggers/jnam_prep_post.wc2.pbs +++ b/triggers/jnam_prep_post.wc2.pbs @@ -32,12 +32,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export prepobs_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jrap_dump.wc2.pbs b/triggers/jrap_dump.wc2.pbs index 91a67dc..8d1d1c3 100755 --- a/triggers/jrap_dump.wc2.pbs +++ b/triggers/jrap_dump.wc2.pbs @@ -38,12 +38,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export bufr_dump_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=$HOMEobsproc/versions/run.ver diff --git a/triggers/jrap_dump_post.wc2.pbs b/triggers/jrap_dump_post.wc2.pbs index 9ef29b5..2beeaf2 100755 --- a/triggers/jrap_dump_post.wc2.pbs +++ b/triggers/jrap_dump_post.wc2.pbs @@ -40,12 +40,13 @@ PDYm1=$(date -d "${PDY} 1 day ago" +%Y%m%d) userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export bufr_dump_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=$HOMEobsproc/versions/run.ver diff --git a/triggers/jrap_prep.wc2.pbs b/triggers/jrap_prep.wc2.pbs index 001ae68..ee71937 100755 --- a/triggers/jrap_prep.wc2.pbs +++ b/triggers/jrap_prep.wc2.pbs @@ -38,12 +38,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export prepobs_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jrap_prep_post.wc2.pbs b/triggers/jrap_prep_post.wc2.pbs index 4da008d..3878ab2 100755 --- a/triggers/jrap_prep_post.wc2.pbs +++ b/triggers/jrap_prep_post.wc2.pbs @@ -36,12 +36,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export prepobs_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jrtma_dump.wc2.pbs b/triggers/jrtma_dump.wc2.pbs index 6d4fc28..a7fb552 100644 --- a/triggers/jrtma_dump.wc2.pbs +++ b/triggers/jrtma_dump.wc2.pbs @@ -35,12 +35,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export bufr_dump_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jrtma_dump_post.wc2.pbs b/triggers/jrtma_dump_post.wc2.pbs index 9fba447..bf5ff18 100644 --- a/triggers/jrtma_dump_post.wc2.pbs +++ b/triggers/jrtma_dump_post.wc2.pbs @@ -33,12 +33,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 #export bufr_dump_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jrtma_prep.wc2.pbs b/triggers/jrtma_prep.wc2.pbs index 5e66d29..7398d26 100644 --- a/triggers/jrtma_prep.wc2.pbs +++ b/triggers/jrtma_prep.wc2.pbs @@ -36,12 +36,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export prepobs_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jrtma_prep_post.wc2.pbs b/triggers/jrtma_prep_post.wc2.pbs index 2bc550e..3d9dc7c 100644 --- a/triggers/jrtma_prep_post.wc2.pbs +++ b/triggers/jrtma_prep_post.wc2.pbs @@ -36,12 +36,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export prepobs_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jurma_dump.wc2.pbs b/triggers/jurma_dump.wc2.pbs index beef274..2a3d10f 100644 --- a/triggers/jurma_dump.wc2.pbs +++ b/triggers/jurma_dump.wc2.pbs @@ -28,12 +28,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export bufr_dump_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jurma_dump_post.wc2.pbs b/triggers/jurma_dump_post.wc2.pbs index 4d6ce4d..8a2c7c8 100644 --- a/triggers/jurma_dump_post.wc2.pbs +++ b/triggers/jurma_dump_post.wc2.pbs @@ -30,11 +30,12 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jurma_prep.wc2.pbs b/triggers/jurma_prep.wc2.pbs index 96d464a..46f753c 100644 --- a/triggers/jurma_prep.wc2.pbs +++ b/triggers/jurma_prep.wc2.pbs @@ -31,12 +31,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export prepobs_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver diff --git a/triggers/jurma_prep_post.wc2.pbs b/triggers/jurma_prep_post.wc2.pbs index b91ce4e..3c524df 100644 --- a/triggers/jurma_prep_post.wc2.pbs +++ b/triggers/jurma_prep_post.wc2.pbs @@ -31,12 +31,13 @@ export PDY=%PDY% userROOT=/lfs/h2/emc/obsproc/noscrub/$USER -export obsproc_ver=v1.0.0 +export obsproc_ver=v1.0 +export obsproc_ver_pckg=v1.0.0 export prepobs_ver=1.0.0 export obsNET=obsproc PACKAGEROOTpara=/lfs/h1/ops/para/packages -#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver} # NCO para -#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver} # NCO prod +#export HOMEobsproc=${PACKAGEROOTpara}/obsproc.${obsproc_ver_pckg} # NCO para +#export HOMEobsproc=${PACKAGEROOT}/obsproc.${obsproc_ver_pckg} # NCO prod export HOMEobsproc=${userROOT}/install/obsproc-rmprep # local VERSION_FILE=${HOMEobsproc}/versions/run.ver From 625bf97e367f542cab52dbaa9bc1dbeb7bd24e2a Mon Sep 17 00:00:00 2001 From: Shelley Melchior Date: Thu, 9 Jun 2022 17:09:03 +0000 Subject: [PATCH 13/53] Squashed commit of the following: commit dd5ec6e5a7caf6064a45a115a83a9f254d90f795 Author: Shelley Melchior Date: Thu Jun 9 17:01:24 2022 +0000 Added export before definition of envir. commit 5609b1ce0c429fe9e2016ca23e3e2572056ae295 Author: AshleyStanfield-NOAA <92459593+AshleyStanfield-NOAA@users.noreply.github.com> Date: Wed Jun 8 21:52:00 2022 -0400 Obsproc 'Module Load' order modifications (#39) * Modified order of modules, specifically bufr-dump and prepobs. * More shifts and changes. * More adjustements, shifted added lines slightly, fixed bugs. * Added in module unload phases. commit 21d7d9a2d2983684f4403c7ee9023a38c152c3c3 Author: Shelley Melchior Date: Thu Jun 2 13:30:52 2022 +0000 Squashed commit of the following: commit daff5af2cdfaf849f42d42a8b90de9235738f444 Author: Shelley Melchior Date: Fri May 27 18:05:05 2022 +0000 Added section to cd to logfile output dir prior to process submission. commit 30c01f9bd251b31d0116b996ca6f115ee7a43e71 Author: iliana Genkova <59100254+ilianagenkova@users.noreply.github.com> Date: Tue May 24 11:37:06 2022 -0500 Set obsproc_ver=v1.0 and HOMEgfs=$COMROOT/gfs/v16.2 (#38) * Set obsproc+ver=v1.0 and HOMEgfs=$COMROOT/gfs/v16.2 * Introduce 3-digit obsproc_ver_pckg and use with $PACKAGEROOT commit ebb6c6856c118705303594641f3fc9b8a07494cd Author: Cory Martin Date: Tue Apr 12 15:15:47 2022 -0400 Changes needed to build on Hera (#32) commit 0075564dc3e9d67b3369138cd9a4f2d704872ad9 Author: Shelley Melchior <53050457+ShelleyMelchior-NOAA@users.noreply.github.com> Date: Sat Apr 9 17:07:10 2022 -0400 Incorporating changes made by NCO SPA following: (#35) restart of cactus and changing output data directory for mods commit 59596a970403d0d248216cbdaa13121b115f9ee0 Author: Shelley Melchior Date: Thu Apr 7 21:27:40 2022 +0000 Removing top level README.md. This file now resides in docs/. commit 735f0092614d8ade4a8fd3194c9d5c2a88a9bfdc Merge: bd4211a 8f28c00 Author: Iliana Genkova Date: Thu Apr 7 20:51:34 2022 +0000 Merge branch 'release/obsproc.v1.0.0' into develop commit 8f28c00f18caf79a4f55e6bf04e9d07237ff604b Merge: f01f691 bd4211a Author: Iliana Genkova Date: Thu Apr 7 20:49:01 2022 +0000 Removed build.sh (now in /build-obsproc) and ush/prepobs_makeprepbufr.sh(now in module prepobs) Merge branch 'develop' into release/obsproc.v1.0.0 commit bd4211a592431b8cc90eeb174e74180b69c55c00 Author: iliana Genkova <59100254+ilianagenkova@users.noreply.github.com> Date: Mon Jan 3 19:23:04 2022 -0600 Adopt Jack Woolen's (CFS) faster prepobs_makeprepbufr.sh: (#19) * Adopt Jack Woolen's (CFS) faster prepobs_makeprepbufr.sh: -use cfp with mpiexe -introduce FORT5 -move ksh interpreter * Clean up interpreter commit 309371d52eef400859b9bf5ecaaf2f7271d8ef05 Author: Shelley Melchior <53050457+ShelleyMelchior-NOAA@users.noreply.github.com> Date: Mon Jan 3 18:25:21 2022 -0500 created README.md Created README.md to better explain installation instructions, modified to adopt NCO's working practices. commit c5342ebb3ced1db0d611dfbdb0a67ee44080e1f0 Author: Rahul Mahajan Date: Mon Jan 3 17:57:29 2022 -0500 adopt for current NCO working practices per discussion w/ StevenEarle et al (#18) commit e63a78c89826acddb1f8bdab2462b124229f4917 Author: Shelley Melchior Date: Wed Jun 1 16:21:51 2022 +0000 Updated to correctly locate tcvitals file. commit 0bbb542101afcec5935047406fa0993dc9bfb305 Author: Shelley Melchior Date: Fri Apr 8 20:31:15 2022 +0000 Incorporating changes made by NCO SPA following: restart of cactus and changing output data directory for mods --- triggers/jairnow_dump.wc2.pbs | 5 ++++- triggers/jcdas_dump.wc2.pbs | 4 +++- triggers/jcdas_dump_post.wc2.pbs | 4 +++- triggers/jcdas_prep1.wc2.pbs | 12 ++++++++---- triggers/jcdas_prep1_post.wc2.pbs | 12 ++++++++---- triggers/jcdas_prep2.wc2.pbs | 12 ++++++++---- triggers/jcdas_prep2_post.wc2.pbs | 12 ++++++++---- triggers/jdump_monitor.wc2.pbs | 4 +++- triggers/jglobal_dump.wc2.pbs | 4 +++- triggers/jglobal_dump_post.wc2.pbs | 4 +++- triggers/jglobal_prep.wc2.pbs | 12 ++++++++---- triggers/jglobal_prep_post.wc2.pbs | 12 ++++++++---- triggers/jmods.wc2.pbs | 4 +++- triggers/jnam_dump.wc2.pbs | 4 +++- triggers/jnam_dump2.wc2.pbs | 4 +++- triggers/jnam_dump_post.wc2.pbs | 4 +++- triggers/jnam_prep.wc2.pbs | 12 ++++++++---- triggers/jnam_prep_post.wc2.pbs | 12 ++++++++---- triggers/jrap_dump.wc2.pbs | 4 +++- triggers/jrap_dump_post.wc2.pbs | 4 +++- triggers/jrap_prep.wc2.pbs | 12 ++++++++---- triggers/jrap_prep_post.wc2.pbs | 12 ++++++++---- triggers/jrtma_dump.wc2.pbs | 4 +++- triggers/jrtma_dump_post.wc2.pbs | 4 +++- triggers/jrtma_prep.wc2.pbs | 12 ++++++++---- triggers/jrtma_prep_post.wc2.pbs | 12 ++++++++---- triggers/jurma_dump.wc2.pbs | 4 +++- triggers/jurma_dump_post.wc2.pbs | 6 ++++-- triggers/jurma_prep.wc2.pbs | 12 ++++++++---- triggers/jurma_prep_post.wc2.pbs | 12 ++++++++---- 30 files changed, 162 insertions(+), 73 deletions(-) diff --git a/triggers/jairnow_dump.wc2.pbs b/triggers/jairnow_dump.wc2.pbs index 04c2963..3ccf6b9 100755 --- a/triggers/jairnow_dump.wc2.pbs +++ b/triggers/jairnow_dump.wc2.pbs @@ -52,12 +52,15 @@ fi # Load the modules specified in $VERSION_FILE module load grib_util/${grib_util_ver} module load netcdf/${netcdf_ver} -module load bufr_dump/${bufr_dump_ver} module load intel/${intel_ver} module load craype/${craype_ver} module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} +# use local bufr_dump installation +#module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles +module load bufr_dump/${bufr_dump_ver} # Be sure the modules are loaded if [[ $(echo $LOADEDMODULES | egrep -c "(^|:)grib_util/") -eq 0 ]]; then echo "grib_util is not loaded!"; fi diff --git a/triggers/jcdas_dump.wc2.pbs b/triggers/jcdas_dump.wc2.pbs index 17d6925..61909c5 100644 --- a/triggers/jcdas_dump.wc2.pbs +++ b/triggers/jcdas_dump.wc2.pbs @@ -57,8 +57,10 @@ module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} # use para installation -module unload bufr_dump #module use /apps/ops/para/nco/modulefiles/compiler/intel/19.1.3.304 +# use local bufr_dump installation +module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles module use ${userROOT}/install/bufr-dump-rel.1.0.0/modulefiles module load bufr_dump/${bufr_dump_ver} diff --git a/triggers/jcdas_dump_post.wc2.pbs b/triggers/jcdas_dump_post.wc2.pbs index 50cd26e..ea7c207 100644 --- a/triggers/jcdas_dump_post.wc2.pbs +++ b/triggers/jcdas_dump_post.wc2.pbs @@ -58,8 +58,10 @@ module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} # use para installation -module unload bufr_dump #module use /apps/ops/para/nco/modulefiles/compiler/intel/19.1.3.304 +# use local bufr_dump installation +module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles module use ${userROOT}/install/bufr-dump-rel.1.0.0/modulefiles module load bufr_dump/${bufr_dump_ver} diff --git a/triggers/jcdas_prep1.wc2.pbs b/triggers/jcdas_prep1.wc2.pbs index c75fea3..a68c07e 100644 --- a/triggers/jcdas_prep1.wc2.pbs +++ b/triggers/jcdas_prep1.wc2.pbs @@ -51,15 +51,19 @@ fi #Load the modules module load grib_util/${grib_util_ver} module load netcdf/${netcdf_ver} -module load bufr_dump/${bufr_dump_ver} -# use local prepobs installation -#module use ${userROOT}/install/prepobs/modulefiles -module load prepobs/${prepobs_ver} module load intel/${intel_ver} module load craype/${craype_ver} module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} +# use local bufr_dump installation +#module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles +module load bufr_dump/${bufr_dump_ver} +# use local prepobs installation +#module unload prepobs +#module use ${userROOT}/install/prepobs/modulefiles +module load prepobs/${prepobs_ver} #Check if they exist if [[ $(echo $LOADEDMODULES | egrep -c "(^|:)grib_util/") -eq 0 ]]; then echo "grib_util is not loaded!"; fi diff --git a/triggers/jcdas_prep1_post.wc2.pbs b/triggers/jcdas_prep1_post.wc2.pbs index aca3094..7cfc195 100644 --- a/triggers/jcdas_prep1_post.wc2.pbs +++ b/triggers/jcdas_prep1_post.wc2.pbs @@ -50,15 +50,19 @@ fi #Load the modules module load grib_util/${grib_util_ver} module load netcdf/${netcdf_ver} -module load bufr_dump/${bufr_dump_ver} -# use local prepobs installation -#module use ${userROOT}/install/prepobs/modulefiles -module load prepobs/${prepobs_ver} module load intel/${intel_ver} module load craype/${craype_ver} module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} +# use local bufr_dump installation +#module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles +module load bufr_dump/${bufr_dump_ver} +# use local prepobs installation +#module unload prepobs +#module use ${userROOT}/install/prepobs/modulefiles +module load prepobs/${prepobs_ver} #Check if they exist if [[ $(echo $LOADEDMODULES | egrep -c "(^|:)grib_util/") -eq 0 ]]; then echo "grib_util is not loaded!"; fi diff --git a/triggers/jcdas_prep2.wc2.pbs b/triggers/jcdas_prep2.wc2.pbs index 8c5c763..7497ef9 100644 --- a/triggers/jcdas_prep2.wc2.pbs +++ b/triggers/jcdas_prep2.wc2.pbs @@ -55,15 +55,19 @@ fi #Load the modules module load grib_util/${grib_util_ver} module load netcdf/${netcdf_ver} -module load bufr_dump/${bufr_dump_ver} -# use local prepobs installation -#module use ${userROOT}/install/prepobs/modulefiles -module load prepobs/${prepobs_ver} module load intel/${intel_ver} module load craype/${craype_ver} module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} +# use local bufr_dump installation +#module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles +module load bufr_dump/${bufr_dump_ver} +# use local prepobs installation +#module unload prepobs +#module use ${userROOT}/install/prepobs/modulefiles +module load prepobs/${prepobs_ver} #Check if they exist if [[ $(echo $LOADEDMODULES | egrep -c "(^|:)grib_util/") -eq 0 ]]; then echo "grib_util is not loaded!"; fi diff --git a/triggers/jcdas_prep2_post.wc2.pbs b/triggers/jcdas_prep2_post.wc2.pbs index 7f288a7..7f6dc54 100644 --- a/triggers/jcdas_prep2_post.wc2.pbs +++ b/triggers/jcdas_prep2_post.wc2.pbs @@ -51,15 +51,19 @@ fi #Load the modules module load grib_util/${grib_util_ver} module load netcdf/${netcdf_ver} -module load bufr_dump/${bufr_dump_ver} -# use local prepobs installation -#module use ${userROOT}/install/prepobs/modulefiles -module load prepobs/${prepobs_ver} module load intel/${intel_ver} module load craype/${craype_ver} module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} +# use local bufr_dump installation +#module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles +module load bufr_dump/${bufr_dump_ver} +# use local prepobs installation +#module unload prepobs +#module use ${userROOT}/install/prepobs/modulefiles +module load prepobs/${prepobs_ver} #Check if they exist if [[ $(echo $LOADEDMODULES | egrep -c "(^|:)grib_util/") -eq 0 ]]; then echo "grib_util is not loaded!"; fi diff --git a/triggers/jdump_monitor.wc2.pbs b/triggers/jdump_monitor.wc2.pbs index fd16b20..54eee39 100755 --- a/triggers/jdump_monitor.wc2.pbs +++ b/triggers/jdump_monitor.wc2.pbs @@ -55,8 +55,10 @@ module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} # use para installation -module unload bufr_dump #module use /apps/ops/para/nco/modulefiles/compiler/intel/19.1.3.304 +# use local bufr_dump installation +module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles module use ${userROOT}/install/bufr-dump-rel.1.0.0/modulefiles module load bufr_dump/${bufr_dump_ver} diff --git a/triggers/jglobal_dump.wc2.pbs b/triggers/jglobal_dump.wc2.pbs index 755985f..de50aca 100755 --- a/triggers/jglobal_dump.wc2.pbs +++ b/triggers/jglobal_dump.wc2.pbs @@ -59,8 +59,10 @@ module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} # use para installation -module unload bufr_dump #module use /apps/ops/para/nco/modulefiles/compiler/intel/19.1.3.304 +# use local bufr_dump installation +module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles module use ${userROOT}/install/bufr-dump-rel.1.0.0/modulefiles module load bufr_dump/${bufr_dump_ver} diff --git a/triggers/jglobal_dump_post.wc2.pbs b/triggers/jglobal_dump_post.wc2.pbs index 74a0d77..ce8cdb3 100755 --- a/triggers/jglobal_dump_post.wc2.pbs +++ b/triggers/jglobal_dump_post.wc2.pbs @@ -59,8 +59,10 @@ module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} # use para installation -module unload bufr_dump #module use /apps/ops/para/nco/modulefiles/compiler/intel/19.1.3.304 +# use local bufr_dump installation +module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles module use ${userROOT}/install/bufr-dump-rel.1.0.0/modulefiles module load bufr_dump/${bufr_dump_ver} diff --git a/triggers/jglobal_prep.wc2.pbs b/triggers/jglobal_prep.wc2.pbs index c0e2da0..33d402b 100755 --- a/triggers/jglobal_prep.wc2.pbs +++ b/triggers/jglobal_prep.wc2.pbs @@ -50,15 +50,19 @@ fi #Load the modules module load grib_util/${grib_util_ver} module load netcdf/${netcdf_ver} -module load bufr_dump/${bufr_dump_ver} -# use local prepobs installation -#module use ${userROOT}/install/prepobs/modulefiles -module load prepobs/${prepobs_ver} module load intel/${intel_ver} module load craype/${craype_ver} module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} +# use local bufr_dump installation +#module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles +module load bufr_dump/${bufr_dump_ver} +# use local prepobs installation +#module unload prepobs +#module use ${userROOT}/install/prepobs/modulefiles +module load prepobs/${prepobs_ver} #Check if they exist if [[ $(echo $LOADEDMODULES | egrep -c "(^|:)grib_util/") -eq 0 ]]; then echo "grib_util is not loaded!"; fi diff --git a/triggers/jglobal_prep_post.wc2.pbs b/triggers/jglobal_prep_post.wc2.pbs index 9f2e991..da538d1 100755 --- a/triggers/jglobal_prep_post.wc2.pbs +++ b/triggers/jglobal_prep_post.wc2.pbs @@ -50,15 +50,19 @@ fi #Load the modules module load grib_util/${grib_util_ver} module load netcdf/${netcdf_ver} -module load bufr_dump/${bufr_dump_ver} -# use local prepobs installation -#module use ${userROOT}/install/prepobs/modulefiles -module load prepobs/${prepobs_ver} module load intel/${intel_ver} module load craype/${craype_ver} module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} +# use local bufr_dump installation +#module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles +module load bufr_dump/${bufr_dump_ver} +# use local prepobs installation +#module unload prepobs +#module use ${userROOT}/install/prepobs/modulefiles +module load prepobs/${prepobs_ver} #Check if they exist if [[ $(echo $LOADEDMODULES | egrep -c "(^|:)grib_util/") -eq 0 ]]; then echo "grib_util is not loaded!"; fi diff --git a/triggers/jmods.wc2.pbs b/triggers/jmods.wc2.pbs index 34e0710..22adc07 100644 --- a/triggers/jmods.wc2.pbs +++ b/triggers/jmods.wc2.pbs @@ -54,8 +54,10 @@ module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} # use para installation -module unload bufr_dump #module use /apps/ops/para/nco/modulefiles/compiler/intel/19.1.3.304 +# use local bufr_dump installation +module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles module use ${userROOT}/install/bufr-dump/modulefiles module load bufr_dump/${bufr_dump_ver} diff --git a/triggers/jnam_dump.wc2.pbs b/triggers/jnam_dump.wc2.pbs index 554b5a5..708b342 100755 --- a/triggers/jnam_dump.wc2.pbs +++ b/triggers/jnam_dump.wc2.pbs @@ -59,8 +59,10 @@ module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} # use para installation -module unload bufr_dump #module use /apps/ops/para/nco/modulefiles/compiler/intel/19.1.3.304 +# use local bufr_dump installation +module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles module use ${userROOT}/install/bufr-dump-rel.1.0.0/modulefiles module load bufr_dump/${bufr_dump_ver} diff --git a/triggers/jnam_dump2.wc2.pbs b/triggers/jnam_dump2.wc2.pbs index f755c16..e4b85e9 100755 --- a/triggers/jnam_dump2.wc2.pbs +++ b/triggers/jnam_dump2.wc2.pbs @@ -59,8 +59,10 @@ module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} # use para installation -module unload bufr_dump #module use /apps/ops/para/nco/modulefiles/compiler/intel/19.1.3.304 +# use local bufr_dump installation +module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles module use ${userROOT}/install/bufr-dump-rel.1.0.0/modulefiles module load bufr_dump/${bufr_dump_ver} diff --git a/triggers/jnam_dump_post.wc2.pbs b/triggers/jnam_dump_post.wc2.pbs index 44a935e..e2287e6 100755 --- a/triggers/jnam_dump_post.wc2.pbs +++ b/triggers/jnam_dump_post.wc2.pbs @@ -59,8 +59,10 @@ module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} # use para installation -module unload bufr_dump #module use /apps/ops/para/nco/modulefiles/compiler/intel/19.1.3.304 +# use local bufr_dump installation +module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles module use ${userROOT}/install/bufr-dump-rel.1.0.0/modulefiles module load bufr_dump/${bufr_dump_ver} diff --git a/triggers/jnam_prep.wc2.pbs b/triggers/jnam_prep.wc2.pbs index 14c819f..5c26231 100755 --- a/triggers/jnam_prep.wc2.pbs +++ b/triggers/jnam_prep.wc2.pbs @@ -55,15 +55,19 @@ fi #Load the modules module load grib_util/${grib_util_ver} module load netcdf/${netcdf_ver} -module load bufr_dump/${bufr_dump_ver} -# use local prepobs installation -#module use ${userROOT}/install/prepobs/modulefiles -module load prepobs/${prepobs_ver} module load intel/${intel_ver} module load craype/${craype_ver} module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} +# use local bufr_dump installation +#module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles +module load bufr_dump/${bufr_dump_ver} +# use local prepobs installation +#module unload prepobs +#module use ${userROOT}/install/prepobs/modulefiles +module load prepobs/${prepobs_ver} #Check if they exist if [[ $(echo $LOADEDMODULES | egrep -c "(^|:)grib_util/") -eq 0 ]]; then echo "grib_util is not loaded!"; fi diff --git a/triggers/jnam_prep_post.wc2.pbs b/triggers/jnam_prep_post.wc2.pbs index c5c5a01..2af5959 100755 --- a/triggers/jnam_prep_post.wc2.pbs +++ b/triggers/jnam_prep_post.wc2.pbs @@ -52,15 +52,19 @@ fi #Load the modules module load grib_util/${grib_util_ver} module load netcdf/${netcdf_ver} -module load bufr_dump/${bufr_dump_ver} -# use local prepobs installation -#module use ${userROOT}/install/prepobs/modulefiles -module load prepobs/${prepobs_ver} module load intel/${intel_ver} module load craype/${craype_ver} module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} +# use local bufr_dump installation +#module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles +module load bufr_dump/${bufr_dump_ver} +# use local prepobs installation +#module unload prepobs +#module use ${userROOT}/install/prepobs/modulefiles +module load prepobs/${prepobs_ver} #Check if they exist if [[ $(echo $LOADEDMODULES | egrep -c "(^|:)grib_util/") -eq 0 ]]; then echo "grib_util is not loaded!"; fi diff --git a/triggers/jrap_dump.wc2.pbs b/triggers/jrap_dump.wc2.pbs index 8d1d1c3..9b302c6 100755 --- a/triggers/jrap_dump.wc2.pbs +++ b/triggers/jrap_dump.wc2.pbs @@ -65,8 +65,10 @@ module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} # use para installation -module unload bufr_dump #module use /apps/ops/para/nco/modulefiles/compiler/intel/19.1.3.304 +# use local bufr_dump installation +module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles module use ${userROOT}/install/bufr-dump-rel.1.0.0/modulefiles module load bufr_dump/${bufr_dump_ver} diff --git a/triggers/jrap_dump_post.wc2.pbs b/triggers/jrap_dump_post.wc2.pbs index 2beeaf2..ed6326e 100755 --- a/triggers/jrap_dump_post.wc2.pbs +++ b/triggers/jrap_dump_post.wc2.pbs @@ -67,8 +67,10 @@ module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} # use para installation -module unload bufr_dump #module use /apps/ops/para/nco/modulefiles/compiler/intel/19.1.3.304 +# use local bufr_dump installation +module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles module use ${userROOT}/install/bufr-dump-rel.1.0.0/modulefiles module load bufr_dump/${bufr_dump_ver} diff --git a/triggers/jrap_prep.wc2.pbs b/triggers/jrap_prep.wc2.pbs index ee71937..a507c98 100755 --- a/triggers/jrap_prep.wc2.pbs +++ b/triggers/jrap_prep.wc2.pbs @@ -58,15 +58,19 @@ fi #Load the modules module load grib_util/${grib_util_ver} module load netcdf/${netcdf_ver} -module load bufr_dump/${bufr_dump_ver} -# use local prepobs installation -#module use ${userROOT}/install/prepobs/modulefiles -module load prepobs/${prepobs_ver} module load intel/${intel_ver} module load craype/${craype_ver} module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} +# use local bufr_dump installation +#module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles +module load bufr_dump/${bufr_dump_ver} +# use local prepobs installation +#module unload prepobs +#module use ${userROOT}/install/prepobs/modulefiles +module load prepobs/${prepobs_ver} #Check if they exist if [[ $(echo $LOADEDMODULES | egrep -c "(^|:)grib_util/") -eq 0 ]]; then echo "grib_util is not loaded!"; fi diff --git a/triggers/jrap_prep_post.wc2.pbs b/triggers/jrap_prep_post.wc2.pbs index 3878ab2..5a9fca1 100755 --- a/triggers/jrap_prep_post.wc2.pbs +++ b/triggers/jrap_prep_post.wc2.pbs @@ -56,15 +56,19 @@ fi #Load the modules module load grib_util/${grib_util_ver} module load netcdf/${netcdf_ver} -module load bufr_dump/${bufr_dump_ver} -# use local prepobs installation -#module use ${userROOT}/install/prepobs/modulefiles -module load prepobs/${prepobs_ver} module load intel/${intel_ver} module load craype/${craype_ver} module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} +# use local bufr_dump installation +#module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles +module load bufr_dump/${bufr_dump_ver} +# use local prepobs installation +#module unload prepobs +#module use ${userROOT}/install/prepobs/modulefiles +module load prepobs/${prepobs_ver} #Check if they exist if [[ $(echo $LOADEDMODULES | egrep -c "(^|:)grib_util/") -eq 0 ]]; then echo "grib_util is not loaded!"; fi diff --git a/triggers/jrtma_dump.wc2.pbs b/triggers/jrtma_dump.wc2.pbs index a7fb552..15184ac 100644 --- a/triggers/jrtma_dump.wc2.pbs +++ b/triggers/jrtma_dump.wc2.pbs @@ -62,8 +62,10 @@ module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} # use para installation -module unload bufr_dump #module use /apps/ops/para/nco/modulefiles/compiler/intel/19.1.3.304 +# use local bufr_dump installation +module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles module use ${userROOT}/install/bufr-dump-rel.1.0.0/modulefiles module load bufr_dump/${bufr_dump_ver} diff --git a/triggers/jrtma_dump_post.wc2.pbs b/triggers/jrtma_dump_post.wc2.pbs index bf5ff18..e3fdc68 100644 --- a/triggers/jrtma_dump_post.wc2.pbs +++ b/triggers/jrtma_dump_post.wc2.pbs @@ -59,8 +59,10 @@ module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} # use para installation -module unload bufr_dump #module use /apps/ops/para/nco/modulefiles/compiler/intel/19.1.3.304 +# use local bufr_dump installation +module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles module use ${userROOT}/install/bufr-dump-rel.1.0.0/modulefiles module load bufr_dump/${bufr_dump_ver} diff --git a/triggers/jrtma_prep.wc2.pbs b/triggers/jrtma_prep.wc2.pbs index 7398d26..56617a2 100644 --- a/triggers/jrtma_prep.wc2.pbs +++ b/triggers/jrtma_prep.wc2.pbs @@ -56,15 +56,19 @@ fi #Load the modules module load grib_util/${grib_util_ver} module load netcdf/${netcdf_ver} -module load bufr_dump/${bufr_dump_ver} -# use local prepobs installation -#module use ${userROOT}/install/prepobs/modulefiles -module load prepobs/${prepobs_ver} module load intel/${intel_ver} module load craype/${craype_ver} module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} +# use local bufr_dump installation +#module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles +module load bufr_dump/${bufr_dump_ver} +# use local prepobs installation +#module unload prepobs +#module use ${userROOT}/install/prepobs/modulefiles +module load prepobs/${prepobs_ver} #Check if they exist if [[ $(echo $LOADEDMODULES | egrep -c "(^|:)grib_util/") -eq 0 ]]; then echo "grib_util is not loaded!"; fi diff --git a/triggers/jrtma_prep_post.wc2.pbs b/triggers/jrtma_prep_post.wc2.pbs index 3d9dc7c..6075e52 100644 --- a/triggers/jrtma_prep_post.wc2.pbs +++ b/triggers/jrtma_prep_post.wc2.pbs @@ -56,15 +56,19 @@ fi #Load the modules module load grib_util/${grib_util_ver} module load netcdf/${netcdf_ver} -module load bufr_dump/${bufr_dump_ver} -# use local prepobs installation -#module use ${userROOT}/install/prepobs/modulefiles -module load prepobs/${prepobs_ver} module load intel/${intel_ver} module load craype/${craype_ver} module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} +# use local bufr_dump installation +#module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles +module load bufr_dump/${bufr_dump_ver} +# use local prepobs installation +#module unload prepobs +#module use ${userROOT}/install/prepobs/modulefiles +module load prepobs/${prepobs_ver} #Check if they exist if [[ $(echo $LOADEDMODULES | egrep -c "(^|:)grib_util/") -eq 0 ]]; then echo "grib_util is not loaded!"; fi diff --git a/triggers/jurma_dump.wc2.pbs b/triggers/jurma_dump.wc2.pbs index 2a3d10f..8ed9f44 100644 --- a/triggers/jurma_dump.wc2.pbs +++ b/triggers/jurma_dump.wc2.pbs @@ -55,8 +55,10 @@ module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} # use para installation -module unload bufr_dump #module use /apps/ops/para/nco/modulefiles/compiler/intel/19.1.3.304 +# use local bufr_dump installation +module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles module use ${userROOT}/install/bufr-dump-rel.1.0.0/modulefiles module load bufr_dump/${bufr_dump_ver} diff --git a/triggers/jurma_dump_post.wc2.pbs b/triggers/jurma_dump_post.wc2.pbs index 8a2c7c8..ee21940 100644 --- a/triggers/jurma_dump_post.wc2.pbs +++ b/triggers/jurma_dump_post.wc2.pbs @@ -21,7 +21,7 @@ set -xu -envir=prod +export envir=prod export cyc=%CC% DESC=%DESC% JTYP=%JTYP% @@ -55,8 +55,10 @@ module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} # use para installation -module unload bufr_dump #module use /apps/ops/para/nco/modulefiles/compiler/intel/19.1.3.304 +# use local bufr_dump installation +module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles module use ${userROOT}/install/bufr-dump-rel.1.0.0/modulefiles module load bufr_dump/${bufr_dump_ver} diff --git a/triggers/jurma_prep.wc2.pbs b/triggers/jurma_prep.wc2.pbs index 46f753c..7293e55 100644 --- a/triggers/jurma_prep.wc2.pbs +++ b/triggers/jurma_prep.wc2.pbs @@ -51,15 +51,19 @@ fi #Load the modules module load grib_util/${grib_util_ver} module load netcdf/${netcdf_ver} -module load bufr_dump/${bufr_dump_ver} -# use local prepobs installation -#module use ${userROOT}/install/prepobs/modulefiles -module load prepobs/${prepobs_ver} module load intel/${intel_ver} module load craype/${craype_ver} module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} +# use local bufr_dump installation +#module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles +module load bufr_dump/${bufr_dump_ver} +# use local prepobs installation +#module unload prepobs +#module use ${userROOT}/install/prepobs/modulefiles +module load prepobs/${prepobs_ver} #Check if they exist if [[ $(echo $LOADEDMODULES | egrep -c "(^|:)grib_util/") -eq 0 ]]; then echo "grib_util is not loaded!"; fi diff --git a/triggers/jurma_prep_post.wc2.pbs b/triggers/jurma_prep_post.wc2.pbs index 3c524df..501345f 100644 --- a/triggers/jurma_prep_post.wc2.pbs +++ b/triggers/jurma_prep_post.wc2.pbs @@ -51,15 +51,19 @@ fi #Load the modules module load grib_util/${grib_util_ver} module load netcdf/${netcdf_ver} -module load bufr_dump/${bufr_dump_ver} -# use local prepobs installation -#module use ${userROOT}/install/prepobs/modulefiles -module load prepobs/${prepobs_ver} module load intel/${intel_ver} module load craype/${craype_ver} module load cray-mpich/${cray_mpich_ver} module load cray-pals/${cray_pals_ver} module load cfp/${cfp_ver} +# use local bufr_dump installation +#module unload bufr_dump +#module use ${userROOT}/install/bufr_dump/modulefiles +module load bufr_dump/${bufr_dump_ver} +# use local prepobs installation +#module unload prepobs +#module use ${userROOT}/install/prepobs/modulefiles +module load prepobs/${prepobs_ver} #Check if they exist if [[ $(echo $LOADEDMODULES | egrep -c "(^|:)grib_util/") -eq 0 ]]; then echo "grib_util is not loaded!"; fi From 96d9c5aff35cc90173a123c7a48025d9ebda8840 Mon Sep 17 00:00:00 2001 From: christopher hill Date: Thu, 30 Jun 2022 18:43:59 +0000 Subject: [PATCH 14/53] BUFR sonde data dump processing is enabled through control scripts for GFS/GDAS, CDAS, and RAP. --- scripts/excdas_dump.sh | 9 ++++++--- scripts/exglobal_dump.sh | 18 ++++++++++++------ scripts/exrap_dump.sh | 17 ++++++++++------- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/scripts/excdas_dump.sh b/scripts/excdas_dump.sh index f0899e7..00f2a23 100755 --- a/scripts/excdas_dump.sh +++ b/scripts/excdas_dump.sh @@ -38,6 +38,7 @@ echo " file. " echo " - Copy bufr_dumplist to COMOUT. " echo " Mar 08 2022 - Enable the dumping of 002017 in vadwnd dump " echo " group. " +echo " ??? ?? 2022 - Enable dumping of UPRAIR data in group #3. " ########################################################################### set -xau @@ -59,7 +60,7 @@ set +u # sfcshp tideg atovs adpsfc # # Dump group #3 (pb, TIME_TRIM = OFF) = -# adpupa gpsipw ascatt +# adpupa uprair gpsipw ascatt # # Dump group #4 (pb, TIME_TRIM = default = ON) = # aircar aircft proflr vadwnd rassda @@ -495,22 +496,24 @@ export DUMP_NUMBER=3 # #-------------------------------------------------------------------------- # Dump #3: ADPUPA: 6 subtype(s) +# UPRAIR: 5 subtype(s) # GPSIPW: 1 subtype(s) # ASCATT: 1 subtype(s) # --------------------- -# TOTAL NUMBER OF SUBTYPES = 8 +# TOTAL NUMBER OF SUBTYPES = 13 # #==================================================================== # other dump types # ---------------- DTIM_latest_adpupa=${DTIM_latest_adpupa:-"+2.99"} +DTIM_latest_uprair=${DTIM_latest_uprair:-"+2.99"} DTIM_latest_gpsipw=${DTIM_latest_gpsipw:-"+2.99"} DTIM_latest_ascatt=${DTIM_latest_ascatt:-"+2.99"} TIME_TRIM=off -$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 adpupa gpsipw ascatt +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 adpupa uprair gpsipw ascatt error3=$? echo "$error3" > $DATA/error3 diff --git a/scripts/exglobal_dump.sh b/scripts/exglobal_dump.sh index 1822747..0db6216 100755 --- a/scripts/exglobal_dump.sh +++ b/scripts/exglobal_dump.sh @@ -78,6 +78,7 @@ echo " - Copy bufr_dumplist to COMOUT. " echo " Dec 16 2021 - modified to work on WCOSS2 " echo " Mar 09 2022 - Enable the dumping of 002017 in vadwnd dump " echo " group. " +echo " ??? ?? 2022 - Enable dumping of UPRAIR data in group #3. " ############################################################################# # NOTE: NET is changed to gdas in the parent Job script for the gdas RUN @@ -114,7 +115,7 @@ set +u # * - for GDAS only # # Dump group #3 (pb, TIME_TRIM defaults to OFF) = -# adpupa +# adpupa uprair # # Dump group #4 (pb, TIME_TRIM defaults to ON) = # aircar aircft proflr vadwnd rassda gpsipw hdob @@ -837,22 +838,26 @@ export DUMP_NUMBER=3 # #-------------------------------------------------------------------------- # Dump #3: ADPUPA: 6 subtype(s) +# UPRAIR: 5 subtype(s) # -------------------- -# TOTAL NUMBER OF SUBTYPES = 6 +# TOTAL NUMBER OF SUBTYPES = 11 # #==================================================================== DTIM_latest_adpupa=${DTIM_latest_adpupa:-"+2.99"} +DTIM_latest_uprair=${DTIM_latest_uprair:-"+2.99"} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM3:-off}} -$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 adpupa +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 adpupa uprair error3=$? echo "$error3" > $DATA/error3 if [ "$SENDDBN" = "YES" ]; then $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_adpupa $job \ ${COMSP}adpupa.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_uprair $job \ + ${COMSP}uprair.tm00.bufr_d fi set +x @@ -1667,9 +1672,10 @@ else # wait fi -# if ADPUPA_wait is YES, adpupa is dumped AFTER all other dump threads have -# run (normally done in real-time GFS runs to dump as late as possible in -# order to maximize data availability in GFS network, particularly DROPs) +# if ADPUPA_wait is YES, adpupa and uprair are dumped AFTER all other dump +# threads have run (normally done in real-time GFS runs to dump as late as +# possible in order to maximize data availability in GFS network, +# particularly DROPs) # -------------------------------------------------------------------------- [ $DUMP_group3 = YES -a $ADPUPA_wait = YES ] && ./thread_3 diff --git a/scripts/exrap_dump.sh b/scripts/exrap_dump.sh index 4ed8e35..9497a3f 100755 --- a/scripts/exrap_dump.sh +++ b/scripts/exrap_dump.sh @@ -51,6 +51,7 @@ echo " match bufr_dumplist. Removed tideg from sfcshp " echo " dump group to make individual dump file. " echo " - Copy bufr_dumplist to COMOUT. " echo " Dec 09 2021 - Updated to run on WCOSS2 " +echo " ??? ?? 2022 - Enable dumping of UPRAIR data in group #2. " ################################################################################ set -xau @@ -65,7 +66,7 @@ set +u # ------------------------------------------------------------------------ # Dump group #1 (non-pb) = 1bamua 1bmhs esamua esmhs atms mtiasi sevcsr # gpsro esiasi iasidb esatms atmsdb sevasr amsr2 -# Dump group #2 (pb) = vadwnd satwnd adpupa +# Dump group #2 (pb) = vadwnd satwnd adpupa uprair # Dump group #3 (pb) = proflr rassda sfcshp adpsfc ascatt tideg # Dump group #4 (pb) = msonet gpsipw # Dump group #5 (pb) = aircft aircar @@ -400,9 +401,9 @@ export STATUS=NO export DUMP_NUMBER=2 #========================================================================== -# Dump # 2 : VADWND, SATWND, ADPUPA -# (2) (19) (6) -# -- TOTAL NUMBER OF SUBTYPES = 27 +# Dump # 2 : VADWND, SATWND, ADPUPA, UPRAIR +# (2) (19) (6) (5) +# -- TOTAL NUMBER OF SUBTYPES = 32 #========================================================================== # Skip all Indian satellite winds in SATWND (not in domain) @@ -419,13 +420,15 @@ For testing, skip in ecflow or obsproc_rap.ver file #export SKIP_005065=YES #export SKIP_005066=YES -# Add GOES-16/17 DMW data to SATWND +# Add GOES-16/17/18 DMW data to SATWND export ADD_satwnd="005030 005031 005032 005034 005039" -# Time window -1.00 to +1.00 hours for ADPUPA for full and partial cycle runs +# Time window -1.00 to +1.00 hours for ADPUPA/UPRAIR w/ full & partial cycle runs # (note: time window increased over +/- 0.5 hr standard to get more data) DTIM_earliest_adpupa=${DTIM_earliest_adpupa:-"-1.00"} DTIM_latest_adpupa=${DTIM_latest_adpupa:-"+1.00"} +DTIM_earliest_uprair=${DTIM_earliest_uprair:-"-1.00"} +DTIM_latest_uprair=${DTIM_latest_uprair:-"+1.00"} # Time window -1.50 to +1.49 hours for EUMETSAT SATWND for full and partial # cycle runs @@ -498,7 +501,7 @@ else fi $ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_2} 1 vadwnd \ - satwnd adpupa + satwnd adpupa uprair error2=$? echo "$error2" > $DATA/error2 From 7347d7b2dc1d6b4e94a4e9281938d2cf0799de7f Mon Sep 17 00:00:00 2001 From: christopher hill Date: Tue, 4 Oct 2022 03:59:33 +0000 Subject: [PATCH 15/53] The GLOBAL_PREP job script and GLOBAL_DUMP trigger script are modified to allow processing of uprair data. --- jobs/JOBSPROC_GLOBAL_PREP | 2 +- triggers/jglobal_dump.wc2.pbs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jobs/JOBSPROC_GLOBAL_PREP b/jobs/JOBSPROC_GLOBAL_PREP index d0275b7..1ca00a3 100755 --- a/jobs/JOBSPROC_GLOBAL_PREP +++ b/jobs/JOBSPROC_GLOBAL_PREP @@ -237,7 +237,7 @@ export NSPLIT=${NSPLIT:-4} # BUFRLIST indicates the list of BUFR data dump file names to process -export BUFRLIST=${BUFRLIST:-"adpupa proflr aircar aircft satwnd adpsfc \ +export BUFRLIST=${BUFRLIST:-"uprair proflr aircar aircft satwnd adpupa adpsfc \ sfcshp vadwnd wdsatr ascatw rassda gpsipw"} export errPREPDATA_limit=4 # allow program PREPOBS_PREPDATA to continue at all diff --git a/triggers/jglobal_dump.wc2.pbs b/triggers/jglobal_dump.wc2.pbs index de50aca..d76c2d9 100755 --- a/triggers/jglobal_dump.wc2.pbs +++ b/triggers/jglobal_dump.wc2.pbs @@ -5,7 +5,7 @@ #PBS -q dev #PBS -l walltime=00:20:00 #PBS -A OBSPROC-DEV -#PBS -l place=vscatter,select=1:ncpus=14:mem=60gb +#PBS -l place=vscatter,select=1:ncpus=14:mem=66gb #PBS -l debug=true ############################################## @@ -93,7 +93,7 @@ export COMOUT_ROOT=${DATAROOT}/CRON/${DESC}/com export COMPONENT=atmos export DEBUG_LEVEL=3 -export LOUD=ON +export LOUD=on export KEEPDATA=YES From 541a0414ae66282f9ff31bda5be2cb36d4714979 Mon Sep 17 00:00:00 2001 From: iliana Genkova <59100254+ilianagenkova@users.noreply.github.com> Date: Sun, 30 Oct 2022 19:52:17 -0500 Subject: [PATCH 16/53] Update exrap_dump.sh --- scripts/exrap_dump.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scripts/exrap_dump.sh b/scripts/exrap_dump.sh index ad44a46..5516f90 100755 --- a/scripts/exrap_dump.sh +++ b/scripts/exrap_dump.sh @@ -69,14 +69,9 @@ set +u # ------------------------------------------------------------------------ # Dump group #1 (non-pb) = 1bamua 1bmhs esamua esmhs atms mtiasi sevcsr # gpsro esiasi iasidb esatms atmsdb sevasr amsr2 -<<<<<<< HEAD # Dump group #2 (pb) = vadwnd satwnd adpupa uprair -# Dump group #3 (pb) = proflr rassda sfcshp adpsfc ascatt tideg -======= -# subpfl saldrn -# Dump group #2 (pb) = vadwnd satwnd adpupa # Dump group #3 (pb) = proflr rassda sfcshp adpsfc ascatt tideg snocvr ->>>>>>> origin/release/obsproc.v1.1.0 +# subpfl saldrn # Dump group #4 (pb) = msonet gpsipw # Dump group #5 (pb) = aircft aircar # Dump group #6 (non-pb) = nexrad From a617893931964e2edb3b8bffee52b5d972d226d9 Mon Sep 17 00:00:00 2001 From: Shelley Melchior Date: Wed, 21 Dec 2022 20:41:10 +0000 Subject: [PATCH 17/53] Updating run.ver to update gfs_ver for 16.3. --- versions/run.ver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions/run.ver b/versions/run.ver index 1cc47d4..ed09d83 100644 --- a/versions/run.ver +++ b/versions/run.ver @@ -14,7 +14,7 @@ export cray_mpich_ver=8.1.12 #8.1.7 export cray_pals_ver=1.1.3 #1.0.12 export cfp_ver=2.0.4 -export gfs_ver=v16.2 # make it v16.3 when implementinig in sync with GFSv16.3 +export gfs_ver=v16.3 # make it v16.3 when implementinig in sync with GFSv16.3 #export nam_ver=v4.2 # secure turn off SYNDAT for nam export dictionaries_ver=v3.4.0 export seaice_analysis_ver=v4.5 #engice, seaice.*5min.grb file From 30a58f057934516d9d4288dfc483ed0ee6de1982 Mon Sep 17 00:00:00 2001 From: Shelley Melchior Date: Wed, 25 Jan 2023 22:21:18 +0000 Subject: [PATCH 18/53] Updating versions files to match production. --- versions/VERSION | 2 +- versions/run.ver | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/versions/VERSION b/versions/VERSION index 9084fa2..45a1b3f 100644 --- a/versions/VERSION +++ b/versions/VERSION @@ -1 +1 @@ -1.1.0 +1.1.2 diff --git a/versions/run.ver b/versions/run.ver index ed09d83..78803d0 100644 --- a/versions/run.ver +++ b/versions/run.ver @@ -5,7 +5,7 @@ export libjpeg_ver=9c export grib_util_ver=1.2.2 export netcdf_ver=4.7.4 -export bufr_dump_ver=1.1.0 +export bufr_dump_ver=1.1.2 export prepobs_ver=1.0.1 export intel_ver=19.1.3.304 From 5feff1d8b611a7b438a753f074bc59914eb28349 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Tue, 31 Jan 2023 20:53:55 +0000 Subject: [PATCH 19/53] Began changing versions --- CHANGES | 3 +++ versions/run.ver | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 CHANGES diff --git a/CHANGES b/CHANGES new file mode 100644 index 0000000..37a3015 --- /dev/null +++ b/CHANGES @@ -0,0 +1,3 @@ +-updated versions for obsproco, b-d, prepobs + + diff --git a/versions/run.ver b/versions/run.ver index cba0d08..00d2b9e 100644 --- a/versions/run.ver +++ b/versions/run.ver @@ -9,17 +9,17 @@ export bufr_dump_ver=1.2.0 export prepobs_ver=1.2.0 export intel_ver=19.1.3.304 -export craype_ver=2.7.13 #2.7.8 -export cray_mpich_ver=8.1.12 #8.1.7 -export cray_pals_ver=1.1.3 #1.0.12 +export craype_ver=2.7.13 +export cray_mpich_ver=8.1.12 +export cray_pals_ver=1.1.3 export cfp_ver=2.0.4 -export gfs_ver=v16.3 # make it v16.3 when implementinig in sync with GFSv16.3 +export gfs_ver=v16.3 #export nam_ver=v4.2 # secure turn off SYNDAT for nam export dictionaries_ver=v3.4.0 export seaice_analysis_ver=v4.5 #engice, seaice.*5min.grb file export cdas_ver=v1.2 -export nsst_ver=v1.2 #rtgsst file +export nsst_ver=v1.2 #rtgsst file export PROCESS_MASTER_SHIP_STNLST=NO #for monthly gdas count (run on 2nd of Month) export BACK=off From 237a46c4599c2ce5482f56556b1e37940e8a53d9 Mon Sep 17 00:00:00 2001 From: christopher hill Date: Wed, 1 Feb 2023 05:53:16 +0000 Subject: [PATCH 20/53] VLab issue 49636 - Update RAP dump script and the versions file. --- scripts/exrap_dump.sh | 7 +------ versions/run.ver | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/scripts/exrap_dump.sh b/scripts/exrap_dump.sh index ad44a46..36f2055 100755 --- a/scripts/exrap_dump.sh +++ b/scripts/exrap_dump.sh @@ -69,14 +69,9 @@ set +u # ------------------------------------------------------------------------ # Dump group #1 (non-pb) = 1bamua 1bmhs esamua esmhs atms mtiasi sevcsr # gpsro esiasi iasidb esatms atmsdb sevasr amsr2 -<<<<<<< HEAD -# Dump group #2 (pb) = vadwnd satwnd adpupa uprair -# Dump group #3 (pb) = proflr rassda sfcshp adpsfc ascatt tideg -======= # subpfl saldrn -# Dump group #2 (pb) = vadwnd satwnd adpupa +# Dump group #2 (pb) = vadwnd satwnd adpupa uprair # Dump group #3 (pb) = proflr rassda sfcshp adpsfc ascatt tideg snocvr ->>>>>>> origin/release/obsproc.v1.1.0 # Dump group #4 (pb) = msonet gpsipw # Dump group #5 (pb) = aircft aircar # Dump group #6 (non-pb) = nexrad diff --git a/versions/run.ver b/versions/run.ver index 1cc47d4..ed09d83 100644 --- a/versions/run.ver +++ b/versions/run.ver @@ -14,7 +14,7 @@ export cray_mpich_ver=8.1.12 #8.1.7 export cray_pals_ver=1.1.3 #1.0.12 export cfp_ver=2.0.4 -export gfs_ver=v16.2 # make it v16.3 when implementinig in sync with GFSv16.3 +export gfs_ver=v16.3 # make it v16.3 when implementinig in sync with GFSv16.3 #export nam_ver=v4.2 # secure turn off SYNDAT for nam export dictionaries_ver=v3.4.0 export seaice_analysis_ver=v4.5 #engice, seaice.*5min.grb file From 0dfb97f6f8a724bc473a18b47730c6b8ca2df393 Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Mon, 27 Feb 2023 16:54:36 -0500 Subject: [PATCH 21/53] Add Jet build support to obsproc (#79) - Add "jet" to target list in ush/build.sh. - Create obsproc_jet.lua modulefile. Refs #78 --- modulefiles/obsproc_jet.lua | 17 +++++++++++++++++ ush/build.sh | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 modulefiles/obsproc_jet.lua diff --git a/modulefiles/obsproc_jet.lua b/modulefiles/obsproc_jet.lua new file mode 100644 index 0000000..bab1f08 --- /dev/null +++ b/modulefiles/obsproc_jet.lua @@ -0,0 +1,17 @@ +help([[ +Load environment to build obsproc on Jet +]]) + +load("cmake/3.20.1") + +prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/role.epic/hpc-stack/libs/intel-18.0.5.274/modulefiles/stack") +load("hpc/1.2.0") +load("hpc-intel/18.0.5.274") +load("hpc-impi/2018.4.274") + +-- Load common modules for this package +load("obsproc_common") + +setenv("FC", "mpiifort") + +whatis("Description: obsproc build environment") diff --git a/ush/build.sh b/ush/build.sh index 328dd51..d6add26 100755 --- a/ush/build.sh +++ b/ush/build.sh @@ -16,7 +16,7 @@ INSTALL_PREFIX=${INSTALL_PREFIX:-"$pkg_root/install"} BUILD_4NCO=${BUILD_4NCO:-"YES"} target=$(echo $INSTALL_TARGET | tr [:upper:] [:lower:]) -if [[ "$target" =~ ^(wcoss2|hera|orion)$ ]]; then +if [[ "$target" =~ ^(wcoss2|hera|orion|jet)$ ]]; then source $pkg_root/versions/build.ver set +x module use $pkg_root/modulefiles From e5288581438adeeb875124e04bca016a1ed13d82 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Wed, 15 Mar 2023 21:13:41 +0000 Subject: [PATCH 22/53] changed VERSION --- versions/run.ver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions/run.ver b/versions/run.ver index 1cc47d4..ed09d83 100644 --- a/versions/run.ver +++ b/versions/run.ver @@ -14,7 +14,7 @@ export cray_mpich_ver=8.1.12 #8.1.7 export cray_pals_ver=1.1.3 #1.0.12 export cfp_ver=2.0.4 -export gfs_ver=v16.2 # make it v16.3 when implementinig in sync with GFSv16.3 +export gfs_ver=v16.3 # make it v16.3 when implementinig in sync with GFSv16.3 #export nam_ver=v4.2 # secure turn off SYNDAT for nam export dictionaries_ver=v3.4.0 export seaice_analysis_ver=v4.5 #engice, seaice.*5min.grb file From 9003ea463763039ba5d3bcb53115c501bdff01a9 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Wed, 15 Mar 2023 21:20:49 +0000 Subject: [PATCH 23/53] Set version to v1.1.1 --- versions/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions/VERSION b/versions/VERSION index 9084fa2..524cb55 100644 --- a/versions/VERSION +++ b/versions/VERSION @@ -1 +1 @@ -1.1.0 +1.1.1 From 32ea83a99b2c32b40fc7feb8b181286f85ae0546 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Mon, 20 Mar 2023 19:24:11 +0000 Subject: [PATCH 24/53] temporarily add CHANGES --- CHANGES | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 37a3015..3a25824 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,5 @@ --updated versions for obsproco, b-d, prepobs +-updated versions/* for obsproc +-merged in develop (which had Ashley's new global dumping, i.e. add new group to utilize all CPUs) + From f02131fe302612cdae141fd11ebdafde8216e6f3 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Tue, 21 Mar 2023 19:38:14 +0000 Subject: [PATCH 25/53] do not overwrite *.nr, make *.ur --- scripts/exdump_post.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/exdump_post.sh b/scripts/exdump_post.sh index 5a2cd6f..df1c0f1 100755 --- a/scripts/exdump_post.sh +++ b/scripts/exdump_post.sh @@ -353,14 +353,14 @@ EOFparm $DATA/postmsg "$jlogfile" "$msg" cp $filestem $COMOUT/$filestem.nr chmod 664 $COMOUT/$filestem.nr - if [ "$SENDDBN" = "YES" ] ; then + if [ "$SENDDBN" = "YES" ] ; then NETUP=`echo $RUN | tr {a-z} {A-Z}` if [[ $NETUP != 'GDAS' ]] || [[ $file != "saphir" ]]; then ### no alert gdas.tCCz.saphir.tm00.bufr_d.nr if [[ $NETUP != 'CDAS' ]] || [[ $file != "gpsro" ]]; then ### no alert cdas.tCCz.gpsro.tm00.bufr_d.nr if [[ $NETUP != 'RAP' ]] || [[ $file != "gpsro" ]]; then ### no alert rap.tCCz.gpsro.tm00.bufr_d.nr $DBNROOT/bin/dbn_alert MODEL ${NETUP}_BUFR_${file}_nr $job \ $COMOUT/$filestem.nr - fi + fi fi fi fi @@ -462,13 +462,16 @@ from 2-days ago (rc = $rc) -- existing file made 2-days ago is not overwritten" msg="Successful generation of non-restricted $file BUFR file from \ 2-days ago -- overwrite existing file made 2-days ago" $DATA/postmsg "$jlogfile" "$msg" - cp $filestem $COMOUTm2/$filestem.nr - chmod 664 $COMOUTm2/$filestem.nr - if [ $SENDDBN = "YES" ] ; then +# cp $filestem $COMOUTm2/$filestem.nr +# chmod 664 $COMOUTm2/$filestem.nr + cp $filestem $COMOUTm2/$filestem.ur #IG do not overwrite + chmod 664 $COMOUTm2/$filestem.ur + if [ $SENDDBN = "YES" ] ; then NETUP=`echo $RUN | tr {a-z} {A-Z}` # can this be net_uc? $DBNROOT/bin/dbn_alert MODEL ${NETUP}_BUFR_${file}_nr $job \ - $COMOUTm2/$filestem.nr - fi +# $COMOUTm2/$filestem.nr + $COMOUTm2/$filestem.ur #IG do not overwrite + fi fi done From 66c7dda7370041a46b8a4ae3ef501ebf5c361795 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Wed, 22 Mar 2023 20:46:03 +0000 Subject: [PATCH 26/53] Ensure no REMOREST_dm2 run on prod machine --- scripts/exdump_post.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/scripts/exdump_post.sh b/scripts/exdump_post.sh index df1c0f1..8d473b1 100755 --- a/scripts/exdump_post.sh +++ b/scripts/exdump_post.sh @@ -371,8 +371,12 @@ EOFparm fi # endif loop $PROCESS_REMOREST -if [ $PROCESS_REMOREST_dm2 = YES ]; then +# PROCESS_REMOREST_dm2 = YES runs only on demand, on dev machines +# Unrestrict 48h old aircraft obs(exc. TAMDAR), add *.ur file to $COMOUTm2 +dev_m=$(grep backup /lfs/h1/ops/prod/config/prodmachinefile | cut -d: -f2) +this_m=$(cat /etc/cluster_name) +if [ $PROCESS_REMOREST_dm2 = YES -a $this_m = $dev_m ]; then ######################################################################## # Remove Restriction on Data in 2-day old "aircar" and "aircft" # # BUFR Data Dump files # @@ -462,15 +466,12 @@ from 2-days ago (rc = $rc) -- existing file made 2-days ago is not overwritten" msg="Successful generation of non-restricted $file BUFR file from \ 2-days ago -- overwrite existing file made 2-days ago" $DATA/postmsg "$jlogfile" "$msg" -# cp $filestem $COMOUTm2/$filestem.nr -# chmod 664 $COMOUTm2/$filestem.nr - cp $filestem $COMOUTm2/$filestem.ur #IG do not overwrite + cp $filestem $COMOUTm2/$filestem.ur chmod 664 $COMOUTm2/$filestem.ur if [ $SENDDBN = "YES" ] ; then NETUP=`echo $RUN | tr {a-z} {A-Z}` # can this be net_uc? $DBNROOT/bin/dbn_alert MODEL ${NETUP}_BUFR_${file}_nr $job \ -# $COMOUTm2/$filestem.nr - $COMOUTm2/$filestem.ur #IG do not overwrite + $COMOUTm2/$filestem.ur fi fi @@ -495,7 +496,7 @@ $dumptime" ######################################################################## aircraft_nr_dm2="" - [ $PROCESS_REMOREST_dm2 = YES ] && \ + [ $PROCESS_REMOREST_dm2 = YES -a $this_m = $dev_m ] && \ aircraft_nr_dm2="aircar_nr_dm2 aircft_nr_dm2" for file in adpsfc adpupa aircar aircft satwnd sfcshp spssmi proflr \ @@ -511,7 +512,7 @@ $dumptime" do file_orig=$file - if [ $PROCESS_REMOREST_dm2 = YES ]; then + if [ $PROCESS_REMOREST_dm2 = YES -a $this_m = $dev_m ]; then if [ $file = aircar_nr_dm2 -o $file = aircft_nr_dm2 ]; then file=`echo $file | cut -d"_" -f1` COMIN_save=$COMIN @@ -600,7 +601,7 @@ a null file is copied in its place" fi done - if [ $PROCESS_REMOREST_dm2 = YES ]; then + if [ $PROCESS_REMOREST_dm2 = YES -a $this_m = $dev_m ]; then if [ $file_orig = aircar_nr_dm2 -o $file_orig = aircft_nr_dm2 ]; then COMIN=$COMIN_save COMOUT=$COMOUT_save From 2fe67931d43131d8af5a1827eec7208a3d3be443 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Wed, 22 Mar 2023 21:40:45 +0000 Subject: [PATCH 27/53] add satwhr (has a bug) --- fix_satwnd | 1918 ++++++++++++++++++++++++++++++++++++++ scripts/exglobal_dump.sh | 22 +- 2 files changed, 1933 insertions(+), 7 deletions(-) create mode 100644 fix_satwnd diff --git a/fix_satwnd b/fix_satwnd new file mode 100644 index 0000000..0bc6b5d --- /dev/null +++ b/fix_satwnd @@ -0,0 +1,1918 @@ +#!/bin/ksh #!/bin/ksh +############################################################################# ############################################################################# +echo "----------------------------------------------------------------------" echo "----------------------------------------------------------------------" +echo "exglobal_dump.sh - Global (GDAS, GFS) network data dump processing" echo "exglobal_dump.sh - Global (GDAS, GFS) network data dump processing" +echo "----------------------------------------------------------------------" echo "----------------------------------------------------------------------" +echo "History: Jan 18 2000 - Original script. " echo "History: Jan 18 2000 - Original script. " +echo " May 16 2007 - Added DBNet alerts for GFS products. " echo " May 16 2007 - Added DBNet alerts for GFS products. " +echo " Apr 2014 - Pick up grib files for planned GFS upgrade. " echo " Apr 2014 - Pick up grib files for planned GFS upgrade. " +echo " Oct 2014 - Remove attempts to dump obsolete sources. " echo " Oct 2014 - Remove attempts to dump obsolete sources. " +echo " Nov 2014 - Use parallel scripting to process dump groups. " echo " Nov 2014 - Use parallel scripting to process dump groups. " +echo " Widen hourly satwnd dump window for GFS/GDAS. " echo " Widen hourly satwnd dump window for GFS/GDAS. " +echo " Add new satwnd subtypes for GFS & GDAS. " echo " Add new satwnd subtypes for GFS & GDAS. " +echo " GFS/GDAS continue if surface file unavailable. " echo " GFS/GDAS continue if surface file unavailable. " +echo " Remove DBNet alerts for old surface files. " echo " Remove DBNet alerts for old surface files. " +echo " Dec 3 2014 - CDAS network, split off into its own script " echo " Dec 3 2014 - CDAS network, split off into its own script " +echo " excdas_dump.sh.ecf. This script now tailored " echo " excdas_dump.sh.ecf. This script now tailored " +echo " exclusively to GDAS and GFS. " echo " exclusively to GDAS and GFS. " +echo " Feb 2 2015 - Dump window for new satwnd type NC005090 set " echo " Feb 2 2015 - Dump window for new satwnd type NC005090 set " +echo " to 3.00 to +2.99 hours about center dump time. " echo " to 3.00 to +2.99 hours about center dump time. " +echo " Removed ADD_satwnd=\"005019 005080\" since " echo " Removed ADD_satwnd=\"005019 005080\" since " +echo " types are now part of "satwnd" dump group " echo " types are now part of "satwnd" dump group " +echo " mnemonic in bufr_dumplist. " echo " mnemonic in bufr_dumplist. " +echo " Aug 22 2016 - GSPIPW dump window reset for new data stream " echo " Aug 22 2016 - GSPIPW dump window reset for new data stream " +echo " (moved to dump group #4 where TIME_TRIM=on) " echo " (moved to dump group #4 where TIME_TRIM=on) " +echo " Jan 5 2017 - Dump new satellite data types. Reordered to " echo " Jan 5 2017 - Dump new satellite data types. Reordered to " +echo " improve run time with all the new data. " echo " improve run time with all the new data. " +echo " Feb 8 2017 - Update to run on Cray-XC40 or IBM iDataPlex " echo " Feb 8 2017 - Update to run on Cray-XC40 or IBM iDataPlex " +echo " Nov 13 2017 - Add dump of GOES-16 AMV's in tanks b005/xx030, " echo " Nov 13 2017 - Add dump of GOES-16 AMV's in tanks b005/xx030, " +echo " b005/xx031, b005/xx032, b005/xx034 and " echo " b005/xx031, b005/xx032, b005/xx034 and " +echo " b005/xx039. Set dump window to -3.00 to +2.99 " echo " b005/xx039. Set dump window to -3.00 to +2.99 " +echo " hours for these 5 new GOES-16 tanks. " echo " hours for these 5 new GOES-16 tanks. " +echo " Aug 10 2018 - Updated to run on Dell-p3 (as well as Cray-XC40" echo " Aug 10 2018 - Updated to run on Dell-p3 (as well as Cray-XC40" +echo " and IBM iDataPlex. " echo " and IBM iDataPlex. " +echo " Oct 24 2018 - Add dump of OMPS VSN8 nadir profile (NP) and " echo " Oct 24 2018 - Add dump of OMPS VSN8 nadir profile (NP) and " +echo " total column (TC), Indiasat AMV's in tanks " echo " total column (TC), Indiasat AMV's in tanks " +echo " b005/xx024 b005/xx025 b005/xx026. " echo " b005/xx024 b005/xx025 b005/xx026. " +echo " Mar 15 2019 - Add dumps of BUFR-feed drifting (NC001102) & " echo " Mar 15 2019 - Add dumps of BUFR-feed drifting (NC001102) & " +echo " moored (NC001203) buoys to DUMP group #2 " echo " moored (NC001203) buoys to DUMP group #2 " +echo " Aug 15 2019 - rm'd background threads; now run serially; " echo " Aug 15 2019 - rm'd background threads; now run serially; " +echo " added "./" on "thread_*", needed for Dell/ph3 " echo " added "./" on "thread_*", needed for Dell/ph3 " +echo " Disabled processing that copies sstoi files to " echo " Disabled processing that copies sstoi files to " +echo " COMSP. " echo " COMSP. " +echo " Oct 15 2019 - set DTIM_LATEST_00110[23]=+2.99, as in sfcshp " echo " Oct 15 2019 - set DTIM_LATEST_00110[23]=+2.99, as in sfcshp " +echo " Oct 26 2019 - set DTIM_EARLIEST_005091=-3.00 and " echo " Oct 26 2019 - set DTIM_EARLIEST_005091=-3.00 and " +echo " DTIM_LATEST_005091=+2.99. DTIM_* settings for " echo " DTIM_LATEST_005091=+2.99. DTIM_* settings for " +echo " 005090 are obsolete; added SKIP_005090=YES " echo " 005090 are obsolete; added SKIP_005090=YES " +echo " Jan 03 2020 - turn on gdas.tCCz.saphir.tm00.bufr_d alerts, " echo " Jan 03 2020 - turn on gdas.tCCz.saphir.tm00.bufr_d alerts, " +echo " per DF request " echo " per DF request " +echo " Apr 06 2020 - Update to remove crisdb and escris and replace " echo " Apr 06 2020 - Update to remove crisdb and escris and replace " +echo " with crsfdb and escrsf. On Apr 22, 2020, NSR " echo " with crsfdb and escrsf. On Apr 22, 2020, NSR " +echo " CrIS data from NPP is being replaced by FSR. " echo " CrIS data from NPP is being replaced by FSR. " +echo " The cris dump group is obsolete with the change" echo " The cris dump group is obsolete with the change" +echo " to FSR. " echo " to FSR. " +echo " Aug 20 2020 - Incremented subsets for the adpsfc and sfcshp " echo " Aug 20 2020 - Incremented subsets for the adpsfc and sfcshp " +echo " dump groups to match bufr_dumplist. " echo " dump groups to match bufr_dumplist. " +echo " Removed mbuoyb and dbuoyb from dump group #2. " echo " Removed mbuoyb and dbuoyb from dump group #2. " +echo " Removed obsolete dump groups escrsf and goesfv " echo " Removed obsolete dump groups escrsf and goesfv " +echo " from dump group #7. " echo " from dump group #7. " +echo " - In Dump group #8, disabled processing of legacy" echo " - In Dump group #8, disabled processing of legacy" +echo " EUMETSAT AMV subsets 005064, 005065, and " echo " EUMETSAT AMV subsets 005064, 005065, and " +echo " 005066. Added DTIM settings for new WMO BUFR " echo " 005066. Added DTIM settings for new WMO BUFR " +echo " format EUMETSAT AMV subsets 005067, 005068, and" echo " format EUMETSAT AMV subsets 005067, 005068, and" +echo " 005069. On Oct 6, 2020, EUMETSAT AMV format " echo " 005069. On Oct 6, 2020, EUMETSAT AMV format " +echo " changes to use new WMO BUFR sequence, 3-10-077." echo " changes to use new WMO BUFR sequence, 3-10-077." +echo " Oct 09 2020 - Update to dump gsrcsr, gsrasr, ompslp, ahicsr, " echo " Oct 09 2020 - Update to dump gsrcsr, gsrasr, ompslp, ahicsr, " +echo " sstvcw, sstvpw, leogeo, hdob " echo " sstvcw, sstvpw, leogeo, hdob " +echo " Update to remove obsolete GOES-15 data. " echo " Update to remove obsolete GOES-15 data. " +echo " Update to remove legacy VIIRS AMV data. " echo " Update to remove legacy VIIRS AMV data. " +echo " Feb 22 2021 - Disabled DBN alerts for gpsro dump files. " echo " Feb 22 2021 - Disabled DBN alerts for gpsro dump files. " +echo " These gpsro dump files have the potential to " echo " These gpsro dump files have the potential to " +echo " contain commercial data. The equivalent " echo " contain commercial data. The equivalent " +echo " non-restricted gpsro dump files are alerted " echo " non-restricted gpsro dump files are alerted " +echo " instead. " echo " instead. " +echo " Sep 21 2020 - Incremented subsets for the sfcshp dump groups " echo " Sep 21 2020 - Incremented subsets for the sfcshp dump groups " +echo " to match bufr_dumplist. Removed tideg from " echo " to match bufr_dumplist. Removed tideg from " +echo " sfcshp dump group to make individual dump file." echo " sfcshp dump group to make individual dump file." +echo " - Copy bufr_dumplist to COMOUT. " echo " - Copy bufr_dumplist to COMOUT. " +echo " Dec 16 2021 - modified to work on WCOSS2 " echo " Dec 16 2021 - modified to work on WCOSS2 " +echo " Mar 09 2022 - Enable the dumping of 002017 in vadwnd dump " echo " Mar 09 2022 - Enable the dumping of 002017 in vadwnd dump " +echo " group. " echo " group. " +echo " Aug 10 2022 - subpfl and saldrn added to dump group #1. " echo " Aug 10 2022 - subpfl and saldrn added to dump group #1. " +echo " gmi1cr added to dump group #9 " echo " gmi1cr added to dump group #9 " +echo " snocvr added to dump group #2 " echo " snocvr added to dump group #2 " +echo " b005/xx081 added to satwnd " echo " b005/xx081 added to satwnd " +echo " subpfl aadded to nsstbufr file " echo " subpfl aadded to nsstbufr file " +echo " DBN alerts are also enabled for subpfl,saldrn, " echo " DBN alerts are also enabled for subpfl,saldrn, " +echo " gmi1cr,and snocvr " echo " gmi1cr,and snocvr " + > echo " Oct 17 2022 - Split up groups 1 and 10 into a new group 12 " + > echo " for better optimization. " +############################################################################# ############################################################################# + +# NOTE: NET is changed to gdas in the parent Job script for the gdas RUN # NOTE: NET is changed to gdas in the parent Job script for the gdas RUN +# (was gfs - NET remains gfs for gfs RUN) # (was gfs - NET remains gfs for gfs RUN) +# ----------------------------------------------------------------------- # ----------------------------------------------------------------------- + +set -xau set -xau + +# function to highlight an echoed msg with surrounding hashed separator lines. # function to highlight an echoed msg with surrounding hashed separator lines. + echo_hashed_msg () { echo_hashed_msg () { + set +x set +x + msg=$* msg=$* + echo -e "\n ${msg//?/#}" echo -e "\n ${msg//?/#}" + echo " ${msg}" echo " ${msg}" + echo -e " ${msg//?/#}\n" echo -e " ${msg//?/#}\n" + set -x set -x + } } +# end of function setup # end of function setup +# # +# set some variables if they have not already been set # set some variables if they have not already been set + +set +u set +u + +# JOB_NUMBER = 1 indicates the prepbufr dump job. # JOB_NUMBER = 1 indicates the prepbufr dump job. +# JOB_NUMBER = 2 indicates the non-prepbufr dump job. # JOB_NUMBER = 2 indicates the non-prepbufr dump job. +# JOB_NUMBER not present indicates dump BOTH prepbufr and non-prepbufr data. # JOB_NUMBER not present indicates dump BOTH prepbufr and non-prepbufr data. +# ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- +# Dump group #1 (non-pb, TIME_TRIM defaults to OFF) = # Dump group #1 (non-pb, TIME_TRIM defaults to OFF) = +# avcsam eshrs3 ssmisu saphir atms 1bhrs4 sevcsr tesac mls | # avcsam eshrs3 ssmisu saphir 1bhrs4 sevcsr tesac mls +# esatms crisfs gsrcsr ahicsr sstvcw subpfl saldrn | # esatms gsrcsr ahicsr sstvcw subpfl saldrn +# # +# Dump group #2 (pb, TIME_TRIM defaults to OFF) = # Dump group #2 (pb, TIME_TRIM defaults to OFF) = +# sfcshp tideg atovs* adpsfc ascatt # sfcshp tideg atovs* adpsfc ascatt +# * - for GDAS only # * - for GDAS only +# # +# Dump group #3 (pb, TIME_TRIM defaults to OFF) = # Dump group #3 (pb, TIME_TRIM defaults to OFF) = +# adpupa # adpupa +# # +# Dump group #4 (pb, TIME_TRIM defaults to ON) = # Dump group #4 (pb, TIME_TRIM defaults to ON) = +# aircar aircft proflr vadwnd rassda gpsipw hdob # aircar aircft proflr vadwnd rassda gpsipw hdob +# # +# Dump group #5 (pb, TIME_TRIM defaults to OFF) = # Dump group #5 (pb, TIME_TRIM defaults to OFF) = +# msonet # msonet +# # +# Dump group #6 (non-pb, TIME_TRIM defaults to OFF) = # Dump group #6 (non-pb, TIME_TRIM defaults to OFF) = +# nexrad # nexrad +# # +# Dump group #7 (non-pb, TIME_TRIM defaults to OFF) = # Dump group #7 (non-pb, TIME_TRIM defaults to OFF) = +# avcspm esmhs 1bmhs airsev atmsdb gome omi trkob gpsro # avcspm esmhs 1bmhs airsev atmsdb gome omi trkob gpsro +# crisf4 # crisf4 +# # +# Dump group #8 (pb, TIME_TRIM defaults to ON) = # Dump group #8 (pb, TIME_TRIM defaults to ON) = +# satwnd # satwnd +# # +# Dump group #9 (non-pb, TIME_TRIM defaults to ON) = # Dump group #9 (non-pb, TIME_TRIM defaults to ON) = +# geoimr # geoimr +# # +# Dump group #10 (non-pb, TIME_TRIM defaults to OFF) = # Dump group #10 (non-pb, TIME_TRIM defaults to OFF) = +# esiasi mtiasi esamua crsfdb iasidb sevasr 1bamua bathy osbuv8 | # esiasi mtiasi esamua sevasr 1bamua bathy osbuv8 +# ompst8 ompsn8 gsrasr ompslp sstvpw # ompst8 ompsn8 gsrasr ompslp sstvpw +# # +# Dump group #11 (non-pb, TIME_TRIM defaults to OFF) = # Dump group #11 (non-pb, TIME_TRIM defaults to OFF) = +# amsr2 # amsr2 +# # +# Dump group #12 STATUS FILE | # Dump group #12 crisfs atms (previously group1) + > # crsfdb iasidb (previously group10) + > # + > # Dump group #13 STATUS FILE +# ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- + +#VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV #VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV +# The settings below are based on a future change when the DUMP job will dump # The settings below are based on a future change when the DUMP job will dump +# only types that go into PREPBUFR and the DUMP2 job will dump only types that # only types that go into PREPBUFR and the DUMP2 job will dump only types that +# do not go into PREPBUFR. This will speed up the DUMP + PREP processing. # do not go into PREPBUFR. This will speed up the DUMP + PREP processing. +# Although the logic is in place to now do this (see below), for now we will # Although the logic is in place to now do this (see below), for now we will +# continue to run only a DUMP job which will dump ALL types (no DUMP2 job) - # continue to run only a DUMP job which will dump ALL types (no DUMP2 job) - +# since JOB_NUMBER is not imported to this script, the logic below will dump # since JOB_NUMBER is not imported to this script, the logic below will dump +# all types ... # all types ... +# ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- +#^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +if [ -n "$JOB_NUMBER" ]; then if [ -n "$JOB_NUMBER" ]; then +set -u set -u + if [ $JOB_NUMBER = 2 ]; then if [ $JOB_NUMBER = 2 ]; then + dump_ind=DUMP2 dump_ind=DUMP2 + DUMP_group1=${DUMP_group1:-"YES"} DUMP_group1=${DUMP_group1:-"YES"} + DUMP_group2=${DUMP_group2:-"NO"} DUMP_group2=${DUMP_group2:-"NO"} + DUMP_group3=${DUMP_group3:-"NO"} DUMP_group3=${DUMP_group3:-"NO"} + DUMP_group4=${DUMP_group4:-"NO"} DUMP_group4=${DUMP_group4:-"NO"} + DUMP_group5=${DUMP_group5:-"NO"} DUMP_group5=${DUMP_group5:-"NO"} + DUMP_group6=${DUMP_group6:-"NO"} DUMP_group6=${DUMP_group6:-"NO"} + DUMP_group7=${DUMP_group7:-"YES"} DUMP_group7=${DUMP_group7:-"YES"} + DUMP_group8=${DUMP_group8:-"NO"} DUMP_group8=${DUMP_group8:-"NO"} + DUMP_group9=${DUMP_group9:-"YES"} DUMP_group9=${DUMP_group9:-"YES"} + DUMP_group10=${DUMP_group10:-"YES"} DUMP_group10=${DUMP_group10:-"YES"} + DUMP_group11=${DUMP_group11:-"YES"} DUMP_group11=${DUMP_group11:-"YES"} + > DUMP_group12=${DUMP_group12:-"YES"} + else else + dump_ind=DUMP dump_ind=DUMP + DUMP_group1=${DUMP_group1:-"NO"} DUMP_group1=${DUMP_group1:-"NO"} + DUMP_group2=${DUMP_group2:-"YES"} DUMP_group2=${DUMP_group2:-"YES"} + DUMP_group3=${DUMP_group3:-"YES"} DUMP_group3=${DUMP_group3:-"YES"} + DUMP_group4=${DUMP_group4:-"YES"} DUMP_group4=${DUMP_group4:-"YES"} + DUMP_group5=${DUMP_group5:-"NO"} DUMP_group5=${DUMP_group5:-"NO"} + DUMP_group6=${DUMP_group6:-"NO"} DUMP_group6=${DUMP_group6:-"NO"} + DUMP_group7=${DUMP_group7:-"NO"} DUMP_group7=${DUMP_group7:-"NO"} + DUMP_group8=${DUMP_group8:-"YES"} DUMP_group8=${DUMP_group8:-"YES"} + DUMP_group9=${DUMP_group9:-"NO"} DUMP_group9=${DUMP_group9:-"NO"} + DUMP_group10=${DUMP_group10:-"NO"} DUMP_group10=${DUMP_group10:-"NO"} + DUMP_group11=${DUMP_group11:-"NO"} DUMP_group11=${DUMP_group11:-"NO"} + > DUMP_group12=${DUMP_group12:-"NO"} + fi fi +else else + dump_ind=DUMP dump_ind=DUMP + DUMP_group1=${DUMP_group1:-"YES"} DUMP_group1=${DUMP_group1:-"YES"} + DUMP_group2=${DUMP_group2:-"YES"} DUMP_group2=${DUMP_group2:-"YES"} + DUMP_group3=${DUMP_group3:-"YES"} DUMP_group3=${DUMP_group3:-"YES"} + DUMP_group4=${DUMP_group4:-"YES"} DUMP_group4=${DUMP_group4:-"YES"} + DUMP_group5=${DUMP_group5:-"NO"} DUMP_group5=${DUMP_group5:-"NO"} + DUMP_group6=${DUMP_group6:-"NO"} DUMP_group6=${DUMP_group6:-"NO"} + DUMP_group7=${DUMP_group7:-"YES"} DUMP_group7=${DUMP_group7:-"YES"} + DUMP_group8=${DUMP_group8:-"YES"} DUMP_group8=${DUMP_group8:-"YES"} + DUMP_group9=${DUMP_group9:-"YES"} DUMP_group9=${DUMP_group9:-"YES"} + DUMP_group10=${DUMP_group10:-"YES"} DUMP_group10=${DUMP_group10:-"YES"} + DUMP_group11=${DUMP_group11:-"YES"} DUMP_group11=${DUMP_group11:-"YES"} + > DUMP_group12=${DUMP_group12:-"YES"} +fi fi + +if [ "$NET" = 'gfs' ]; then if [ "$NET" = 'gfs' ]; then + ADPUPA_wait=${ADPUPA_wait:-"YES"} ADPUPA_wait=${ADPUPA_wait:-"YES"} +########ADPUPA_wait=${ADPUPA_wait:-"NO"} # saves time if ADPUPA_wait=NO ########ADPUPA_wait=${ADPUPA_wait:-"NO"} # saves time if ADPUPA_wait=NO +else else + ADPUPA_wait=${ADPUPA_wait:-"NO"} ADPUPA_wait=${ADPUPA_wait:-"NO"} +fi fi + + +# send extra output of DUMP2 for monitoring purposes. # send extra output of DUMP2 for monitoring purposes. +set +u set +u +if [ -n "$JOB_NUMBER" ]; then if [ -n "$JOB_NUMBER" ]; then + [ $JOB_NUMBER = 2 ] && export PS4='$SECONDS + ' [ $JOB_NUMBER = 2 ] && export PS4='$SECONDS + ' +fi fi +set -u set -u + +# Make sure we are in the $DATA directory # Make sure we are in the $DATA directory +cd $DATA cd $DATA + +msg="HAS BEGUN on `hostname`" msg="HAS BEGUN on `hostname`" +$DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" + +cat break > $pgmout cat break > $pgmout + +export dumptime=`cut -c7-16 ncepdate` export dumptime=`cut -c7-16 ncepdate` +export cycp=`echo $dumptime|cut -c9-10` export cycp=`echo $dumptime|cut -c9-10` + +export NET_uc=$(echo $NET | tr [a-z] [A-Z]) export NET_uc=$(echo $NET | tr [a-z] [A-Z]) +export tmmark_uc=$(echo $tmmark | tr [a-z] [A-Z]) export tmmark_uc=$(echo $tmmark | tr [a-z] [A-Z]) + +msg="$NET_uc ANALYSIS TIME IS $PDY$cyc" msg="$NET_uc ANALYSIS TIME IS $PDY$cyc" +$DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" + +set +x set +x +echo echo +echo "CENTER DATA DUMP DATE-TIME FOR $tmmark_uc $NET_uc IS $dumptime" echo "CENTER DATA DUMP DATE-TIME FOR $tmmark_uc $NET_uc IS $dumptime" +echo echo +set -x set -x + +export COMSP=$COMOUT/$RUN.${cycle}. export COMSP=$COMOUT/$RUN.${cycle}. + +if [ "$PROCESS_GRIBFLDS" = 'YES' ]; then if [ "$PROCESS_GRIBFLDS" = 'YES' ]; then + +######################################################## ######################################################## +######################################################## ######################################################## +## The following files are not *required* but will still ## The following files are not *required* but will still +# be processed here for the near term (missing files # be processed here for the near term (missing files +# will not cause job to fail) # will not cause job to fail) +# # +# copy snogrb (0.5 deg) from $TANK_GRIBFLDS # copy snogrb (0.5 deg) from $TANK_GRIBFLDS +# copy snogrb_t574 from $TANK_GRIBFLDS # copy snogrb_t574 from $TANK_GRIBFLDS +# copy engicegrb from $COM_ENGICE # copy engicegrb from $COM_ENGICE +# copy sstgrb from $COM_SSTOI # copy sstgrb from $COM_SSTOI +# generate sstgrb index file # generate sstgrb index file +######################################################## ######################################################## +######################################################## ######################################################## + + # JY - 05/02: remove the dependency of snowdepth files created from isnowgrib job # JY - 05/02: remove the dependency of snowdepth files created from isnowgrib job + #snogrb=$TANK_GRIBFLDS/$PDY/wgrbbul/snowdepth.global.grb #snogrb=$TANK_GRIBFLDS/$PDY/wgrbbul/snowdepth.global.grb + #snoold=$TANK_GRIBFLDS/$PDYm1/wgrbbul/snowdepth.global.grb #snoold=$TANK_GRIBFLDS/$PDYm1/wgrbbul/snowdepth.global.grb + + #if [ -s $snogrb ]; then #if [ -s $snogrb ]; then + # cp $snogrb ${COMSP}snogrb # cp $snogrb ${COMSP}snogrb + # msg="todays 0.5 degree snow grib file located and copied to ${COMSP}snogrb" # msg="todays 0.5 degree snow grib file located and copied to ${COMSP}snogrb" + # $DATA/postmsg "$jlogfile" "$msg" # $DATA/postmsg "$jlogfile" "$msg" + #elif [ -s $snoold ]; then #elif [ -s $snoold ]; then + # cp $snoold ${COMSP}snogrb # cp $snoold ${COMSP}snogrb + # msg="**todays 0.5 degree snow grib file not located - copy 1-day old file" # msg="**todays 0.5 degree snow grib file not located - copy 1-day old file" + # $DATA/postmsg "$jlogfile" "$msg" # $DATA/postmsg "$jlogfile" "$msg" + #else #else + # set +x # set +x + # echo " " # echo " " + # echo " #####################################################" # echo " #####################################################" + # echo " cannot locate 0.5 degree snow grib file" # echo " cannot locate 0.5 degree snow grib file" + # echo " #####################################################" # echo " #####################################################" + # echo " " # echo " " + # set -x # set -x + # msg="***WARNING: CANNOT LOCATE 0.5 DEGREE SNOW GRIB FILE. Not critical." # msg="***WARNING: CANNOT LOCATE 0.5 DEGREE SNOW GRIB FILE. Not critical." + # $DATA/postmsg "$jlogfile" "$msg" # $DATA/postmsg "$jlogfile" "$msg" + #fi #fi + + #snogrb_t574=$TANK_GRIBFLDS/$PDY/wgrbbul/snowdepth.t574.grb #snogrb_t574=$TANK_GRIBFLDS/$PDY/wgrbbul/snowdepth.t574.grb + #snoold_t574=$TANK_GRIBFLDS/$PDYm1/wgrbbul/snowdepth.t574.grb #snoold_t574=$TANK_GRIBFLDS/$PDYm1/wgrbbul/snowdepth.t574.grb + + #if [ -s $snogrb_t574 ]; then #if [ -s $snogrb_t574 ]; then + # cp $snogrb_t574 ${COMSP}snogrb_t574 # cp $snogrb_t574 ${COMSP}snogrb_t574 + # msg="todays T574 snow grib file located and copied to ${COMSP}snogrb_t574" # msg="todays T574 snow grib file located and copied to ${COMSP}snogrb_t574" + # $DATA/postmsg "$jlogfile" "$msg" # $DATA/postmsg "$jlogfile" "$msg" + #elif [ -s $snoold_t574 ]; then #elif [ -s $snoold_t574 ]; then + # cp $snoold_t574 ${COMSP}snogrb_t574 # cp $snoold_t574 ${COMSP}snogrb_t574 + # msg="**todays T574 snow grib file not located - copy 1-day old file" # msg="**todays T574 snow grib file not located - copy 1-day old file" + # $DATA/postmsg "$jlogfile" "$msg" # $DATA/postmsg "$jlogfile" "$msg" + #else #else + # set +x # set +x + # echo " " # echo " " + # echo " ###############################################" # echo " ###############################################" + # echo " cannot locate T574 snow grib file" # echo " cannot locate T574 snow grib file" + # echo " ###############################################" # echo " ###############################################" + # echo " " # echo " " + # set -x # set -x + # msg="***WARNING: CANNOT LOCATE T574 SNOW GRIB FILE. Not critical." # msg="***WARNING: CANNOT LOCATE T574 SNOW GRIB FILE. Not critical." + # $DATA/postmsg "$jlogfile" "$msg" # $DATA/postmsg "$jlogfile" "$msg" + #fi #fi + + engicegrb=${COM_ENGICE}.$PDY/engice.t00z.grb engicegrb=${COM_ENGICE}.$PDY/engice.t00z.grb + engiceold=${COM_ENGICE}.$PDYm1/engice.t00z.grb engiceold=${COM_ENGICE}.$PDYm1/engice.t00z.grb + + if [ -s $engicegrb ]; then if [ -s $engicegrb ]; then + cp $engicegrb ${COMSP}engicegrb cp $engicegrb ${COMSP}engicegrb + msg="todays engice grib file located and copied to ${COMSP}engicegrb" msg="todays engice grib file located and copied to ${COMSP}engicegrb" + $DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" + elif [ -s $engiceold ]; then elif [ -s $engiceold ]; then + cp $engiceold ${COMSP}engicegrb cp $engiceold ${COMSP}engicegrb + msg="**todays engice grib file not located - copy 1-day old file" msg="**todays engice grib file not located - copy 1-day old file" + $DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" + else else + set +x set +x + echo " " echo " " + echo " ############################################" echo " ############################################" + echo " cannot locate engice grib file" echo " cannot locate engice grib file" + echo " ############################################" echo " ############################################" + echo " " echo " " + set -x set -x + msg="***WARNING: CANNOT LOCATE LOW RES ENGICE GRIB FILE. Not critical." msg="***WARNING: CANNOT LOCATE LOW RES ENGICE GRIB FILE. Not critical." + $DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" + fi fi + +# Disabled w/ GFSv15.2 b/c sstoi file no longer needed # Disabled w/ GFSv15.2 b/c sstoi file no longer needed +# sstgrb=${COM_SSTOI}.$PDY/sstoi_grb # sstgrb=${COM_SSTOI}.$PDY/sstoi_grb +# sstold=${COM_SSTOI}.$PDYm1/sstoi_grb # sstold=${COM_SSTOI}.$PDYm1/sstoi_grb + +# if [ -s $sstgrb ]; then # if [ -s $sstgrb ]; then +# cp $sstgrb ${COMSP}sstgrb # cp $sstgrb ${COMSP}sstgrb +# msg="todays lowres sst grib file located and copied to ${COMSP}sstgrb" # msg="todays lowres sst grib file located and copied to ${COMSP}sstgrb" +# $DATA/postmsg "$jlogfile" "$msg" # $DATA/postmsg "$jlogfile" "$msg" +# elif [ -s $sstold ]; then # elif [ -s $sstold ]; then +# cp $sstold ${COMSP}sstgrb # cp $sstold ${COMSP}sstgrb +# msg="**todays lowres sst grib file not located - copy 1-day old file" # msg="**todays lowres sst grib file not located - copy 1-day old file" +# $DATA/postmsg "$jlogfile" "$msg" # $DATA/postmsg "$jlogfile" "$msg" +# else # else +# set +x # set +x +# echo " " # echo " " +# echo " #########################################" # echo " #########################################" +# echo " cannot locate lowres sst grib file" # echo " cannot locate lowres sst grib file" +# echo " #########################################" # echo " #########################################" +# echo " " # echo " " +# set -x # set -x +# msg="***WARNING: CANNOT LOCATE LOW RES SST GRIB FILE. Not critical." # msg="***WARNING: CANNOT LOCATE LOW RES SST GRIB FILE. Not critical." +# $DATA/postmsg "$jlogfile" "$msg" # $DATA/postmsg "$jlogfile" "$msg" +# fi # fi + +# if [ -s ${COMSP}sstgrb ]; then # if [ -s ${COMSP}sstgrb ]; then +# rm errfile # rm errfile +# $GRBINDEX ${COMSP}sstgrb ${COMSP}sstgrb.index 2> errfile # $GRBINDEX ${COMSP}sstgrb ${COMSP}sstgrb.index 2> errfile +# errindx=$? # errindx=$? +# [ "$errindx" -ne '0' ] && cat errfile # [ "$errindx" -ne '0' ] && cat errfile +# rm errfile # rm errfile +# else # else +# echo_hashed_msg "cannot create grib index since sst file does not exist" # echo_hashed_msg "cannot create grib index since sst file does not exist" +# fi # fi + +# The following may no longer be needed, but leave them in place for now. # The following may no longer be needed, but leave them in place for now. +# Print msg in the rare case the grib2 files cannot be created. # Print msg in the rare case the grib2 files cannot be created. + if [ "$NET" = 'gdas' ]; then if [ "$NET" = 'gdas' ]; then + if [ -s ${COMSP}engicegrb ]; then if [ -s ${COMSP}engicegrb ]; then + $CNVGRIB -g12 -p40 ${COMSP}engicegrb ${COMSP}engicegrb.grib2 $CNVGRIB -g12 -p40 ${COMSP}engicegrb ${COMSP}engicegrb.grib2 + else else + echo_hashed_msg "Skip engicegrb.grib2 since grib1 file does not exist" echo_hashed_msg "Skip engicegrb.grib2 since grib1 file does not exist" + fi fi + # Disabled w/ GFSv15.2 b/c sstoi file no longer needed # Disabled w/ GFSv15.2 b/c sstoi file no longer needed + #if [ -s ${COMSP}sstgrb ]; then #if [ -s ${COMSP}sstgrb ]; then + # $CNVGRIB -g12 -p40 ${COMSP}sstgrb ${COMSP}sstgrb.grib2 # $CNVGRIB -g12 -p40 ${COMSP}sstgrb ${COMSP}sstgrb.grib2 + #else #else + # echo_hashed_msg "Skip sstgrb.grib2 since grib1 file does not exist" # echo_hashed_msg "Skip sstgrb.grib2 since grib1 file does not exist" + #fi #fi + #if [ -s ${COMSP}snogrb ]; then #if [ -s ${COMSP}snogrb ]; then + # $CNVGRIB -g12 -p40 ${COMSP}snogrb ${COMSP}snogrb.grib2 # $CNVGRIB -g12 -p40 ${COMSP}snogrb ${COMSP}snogrb.grib2 + #else #else + # echo_hashed_msg "Skip snogrb.grib2 since grib1 file does not exist" # echo_hashed_msg "Skip snogrb.grib2 since grib1 file does not exist" + #fi #fi + fi fi + + +###################################################################### ###################################################################### +###################################################################### ###################################################################### +# For the following, try as far as $ndaysback to find recent file. # # For the following, try as far as $ndaysback to find recent file. # +# Post warning if no file found for $ndaysback_warn or beyond. # # Post warning if no file found for $ndaysback_warn or beyond. # +# The job will continue if no suitable file is available. # # The job will continue if no suitable file is available. # +# ---------------------------------------------------------------- # # ---------------------------------------------------------------- # +# copy NPR.SNWN.SP.S1200.MESH16 from $TANK_GRIBFLDS # # copy NPR.SNWN.SP.S1200.MESH16 from $TANK_GRIBFLDS # +# copy NPR.SNWS.SP.S1200.MESH16 from $TANK_GRIBFLDS # # copy NPR.SNWS.SP.S1200.MESH16 from $TANK_GRIBFLDS # +# copy imssnow96.grb.grib2 from $TANK_GRIBFLDS # # copy imssnow96.grb.grib2 from $TANK_GRIBFLDS # +# copy seaice.t00z.5min.grb from $COM_ICE5MIN # # copy seaice.t00z.5min.grb from $COM_ICE5MIN # +# copy seaice.t00z.5min.grb.grib2 from $COM_ICE5MIN # # copy seaice.t00z.5min.grb.grib2 from $COM_ICE5MIN # +# copy rtgssthr_grb_0.083 from $COM_SSTRTG # # copy rtgssthr_grb_0.083 from $COM_SSTRTG # +# copy rtgssthr_grb_0.083.grib2 from $COM_SSTRTG # # copy rtgssthr_grb_0.083.grib2 from $COM_SSTRTG # +###################################################################### ###################################################################### +###################################################################### ###################################################################### + for gribfile in \ for gribfile in \ + NPR.SNWN.SP.S1200.MESH16 \ NPR.SNWN.SP.S1200.MESH16 \ + NPR.SNWS.SP.S1200.MESH16 \ NPR.SNWS.SP.S1200.MESH16 \ + imssnow96.grb.grib2 \ imssnow96.grb.grib2 \ + seaice.t00z.5min.grb \ seaice.t00z.5min.grb \ + seaice.t00z.5min.grb.grib2 \ seaice.t00z.5min.grb.grib2 \ + rtgssthr_grb_0.083 \ rtgssthr_grb_0.083 \ + rtgssthr_grb_0.083.grib2 rtgssthr_grb_0.083.grib2 + do do +# set the values specific to each file # set the values specific to each file + case $gribfile in case $gribfile in + NPR.SNWN.SP.S1200.MESH16 | NPR.SNWS.SP.S1200.MESH16 ) # AFWA snow NPR.SNWN.SP.S1200.MESH16 | NPR.SNWS.SP.S1200.MESH16 ) # AFWA snow + grib_source='$TANK_GRIBFLDS/$DDATE/wgrbbul'; grib_source='$TANK_GRIBFLDS/$DDATE/wgrbbul'; + target_filename=$gribfile.grb target_filename=$gribfile.grb + ndaysback=1; ndaysback=1; + ndaysback_warn=1;; ndaysback_warn=1;; + imssnow96.grb.grib2 ) # IMS snow imssnow96.grb.grib2 ) # IMS snow + grib_source='$TANK_GRIBFLDS/$DDATE/wgrbbul'; grib_source='$TANK_GRIBFLDS/$DDATE/wgrbbul'; + target_filename=imssnow96.grib2 target_filename=imssnow96.grib2 + ndaysback=1; ndaysback=1; + ndaysback_warn=1;; ndaysback_warn=1;; + seaice.t00z.5min.grb ) seaice.t00z.5min.grb ) + grib_source='${COM_ICE5MIN}.$DDATE'; grib_source='${COM_ICE5MIN}.$DDATE'; + target_filename=seaice.5min.grb target_filename=seaice.5min.grb + ndaysback=7; ndaysback=7; + ndaysback_warn=1;; ndaysback_warn=1;; + seaice.t00z.5min.grb.grib2 ) seaice.t00z.5min.grb.grib2 ) + grib_source='${COM_ICE5MIN}.$DDATE'; grib_source='${COM_ICE5MIN}.$DDATE'; + target_filename=seaice.5min.grib2 target_filename=seaice.5min.grib2 + ndaysback=7; ndaysback=7; + ndaysback_warn=1;; ndaysback_warn=1;; + rtgssthr_grb_0.083 ) rtgssthr_grb_0.083 ) + grib_source='${COM_SSTRTG}.$DDATE'; grib_source='${COM_SSTRTG}.$DDATE'; + target_filename=rtgssthr.grb target_filename=rtgssthr.grb + ndaysback=10; ndaysback=10; + ndaysback_warn=1;; ndaysback_warn=1;; + rtgssthr_grb_0.083.grib2 ) rtgssthr_grb_0.083.grib2 ) + grib_source='${COM_SSTRTG}.$DDATE'; grib_source='${COM_SSTRTG}.$DDATE'; + target_filename=rtgssthr.grib2 target_filename=rtgssthr.grib2 + ndaysback=10; ndaysback=10; + ndaysback_warn=1;; ndaysback_warn=1;; + *) *) + msg="***FATAL ERROR: unexpected grib field file $gribfile"; msg="***FATAL ERROR: unexpected grib field file $gribfile"; + echo_hashed_msg "$msg" echo_hashed_msg "$msg" + $DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" + $DATA/err_exit;; $DATA/err_exit;; + esac esac +# set up string of dates to check # set up string of dates to check + if [ $ndaysback -gt 0 ];then if [ $ndaysback -gt 0 ];then +set +x; echo -e "\n---> path to finddate.sh below is: `which finddate.sh`"; set -x set +x; echo -e "\n---> path to finddate.sh below is: `which finddate.sh`"; set -x + CHECK_DATES="$PDY $(finddate.sh $PDY s-$ndaysback)" CHECK_DATES="$PDY $(finddate.sh $PDY s-$ndaysback)" + else else + CHECK_DATES=$PDY CHECK_DATES=$PDY + fi fi + set +x; set +x; + echo -e "\nWill check as far back as ${CHECK_DATES##* } for $gribfile" echo -e "\nWill check as far back as ${CHECK_DATES##* } for $gribfile" + set -x set -x + ndtry=0 ndtry=0 + found=false found=false +# loop through dates to check for this file type # loop through dates to check for this file type + for DDATE in $CHECK_DATES;do for DDATE in $CHECK_DATES;do + ndtry=`expr $ndtry + 1` ndtry=`expr $ndtry + 1` + eval tryfile=$grib_source/$gribfile eval tryfile=$grib_source/$gribfile + if [ -s $tryfile ];then if [ -s $tryfile ];then + set +x; echo -e "\nPicking up file $tryfile\n"; set -x set +x; echo -e "\nPicking up file $tryfile\n"; set -x + cp $tryfile ${COMSP}$target_filename cp $tryfile ${COMSP}$target_filename + found=true found=true + break break + fi fi + if [ $DDATE -ne ${CHECK_DATES##* } ]; then if [ $DDATE -ne ${CHECK_DATES##* } ]; then + set +x;echo -e "\n$tryfile not available. Try previous day.\n" set +x;echo -e "\n$tryfile not available. Try previous day.\n" + set -x set -x + else else + set +x;echo -e "\n$tryfile not available.\n";set -x set +x;echo -e "\n$tryfile not available.\n";set -x + fi fi + if [ $ndtry -gt $ndaysback_warn ];then if [ $ndtry -gt $ndaysback_warn ];then + msg="***WARNING: INVESTIGATE UNEXPECTED ABSENCE OF $tryfile" msg="***WARNING: INVESTIGATE UNEXPECTED ABSENCE OF $tryfile" + echo_hashed_msg "$msg" echo_hashed_msg "$msg" + $DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" + fi fi + done done + if [ $found != true ]; then if [ $found != true ]; then + msg="***WARNING: NO USEFUL RECENT FILES FOUND FOR $gribfile!!!" msg="***WARNING: NO USEFUL RECENT FILES FOUND FOR $gribfile!!!" + echo_hashed_msg "$msg" echo_hashed_msg "$msg" + $DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" + fi fi + done done + if [ "$SENDECF" = "YES" ]; then if [ "$SENDECF" = "YES" ]; then + ecflow_client --event=release_sfcprep ecflow_client --event=release_sfcprep + fi fi + +# endif loop $PROCESS_GRIBFLDS # endif loop $PROCESS_GRIBFLDS +fi fi + + +echo "=======> Dump group 1 (thread_1) not executed." > $DATA/1.out echo "=======> Dump group 1 (thread_1) not executed." > $DATA/1.out +echo "=======> Dump group 2 (thread_2) not executed." > $DATA/2.out echo "=======> Dump group 2 (thread_2) not executed." > $DATA/2.out +echo "=======> Dump group 3 (thread_3) not executed." > $DATA/3.out echo "=======> Dump group 3 (thread_3) not executed." > $DATA/3.out +echo "=======> Dump group 4 (thread_4) not executed." > $DATA/4.out echo "=======> Dump group 4 (thread_4) not executed." > $DATA/4.out +echo "=======> Dump group 5 (thread_5) not executed." > $DATA/5.out echo "=======> Dump group 5 (thread_5) not executed." > $DATA/5.out +echo "=======> Dump group 6 (thread_6) not executed." > $DATA/6.out echo "=======> Dump group 6 (thread_6) not executed." > $DATA/6.out +echo "=======> Dump group 7 (thread_7) not executed." > $DATA/7.out echo "=======> Dump group 7 (thread_7) not executed." > $DATA/7.out +echo "=======> Dump group 8 (thread_8) not executed." > $DATA/8.out echo "=======> Dump group 8 (thread_8) not executed." > $DATA/8.out +echo "=======> Dump group 9 (thread_9) not executed." > $DATA/9.out echo "=======> Dump group 9 (thread_9) not executed." > $DATA/9.out +echo "=======> Dump group 10 (thread_10) not executed." > $DATA/10.out echo "=======> Dump group 10 (thread_10) not executed." > $DATA/10.out +echo "=======> Dump group 11 (thread_11) not executed." > $DATA/11.out echo "=======> Dump group 11 (thread_11) not executed." > $DATA/11.out + > echo "=======> Dump group 12 (thread_12) not executed." > $DATA/12.out + > + +err1=0 err1=0 +err2=0 err2=0 +err3=0 err3=0 +err4=0 err4=0 +err5=0 err5=0 +err6=0 err6=0 +err7=0 err7=0 +err8=0 err8=0 +err9=0 err9=0 +err10=0 err10=0 +err11=0 err11=0 + > err12=0 +if [ "$PROCESS_DUMP" = 'YES' ]; then if [ "$PROCESS_DUMP" = 'YES' ]; then + +#################################### #################################### +#################################### #################################### +# The data "dump" script for tm00 # The data "dump" script for tm00 +#################################### #################################### +#################################### #################################### + +msg="START THE $tmmark_uc $NET_uc DATA $dump_ind CENTERED ON $dumptime" msg="START THE $tmmark_uc $NET_uc DATA $dump_ind CENTERED ON $dumptime" +$DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" + +set +x set +x +#---------------------------------------------------------------- #---------------------------------------------------------------- +cat<<\EOF>thread_1; chmod +x thread_1 cat<<\EOF>thread_1; chmod +x thread_1 +set -uax set -uax + +cd $DATA cd $DATA + +{ echo { echo +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_1 echo Script thread_1 +echo Executing on node `hostname` echo Executing on node `hostname` +echo Starting time: `date -u` echo Starting time: `date -u` +echo "********************************************************************" echo "********************************************************************" +echo echo +set -x set -x + +export STATUS=NO export STATUS=NO +export DUMP_NUMBER=1 export DUMP_NUMBER=1 + +#========================================================================= #========================================================================= +# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: +# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types +# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) +# # +#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- +# Dump # 1 : AVCSAM: 1 subtype(s) # Dump # 1 : AVCSAM: 1 subtype(s) +# ESHRS3: 1 subtype(s) # ESHRS3: 1 subtype(s) +# SSMISU: 1 subtype(s) # SSMISU: 1 subtype(s) +# SAPHIR: 1 subtype(s) # SAPHIR: 1 subtype(s) +# ATMS: 1 subtype(s) (if present in past 10 days of tanks) < +# 1BHRS4: 1 subtype(s) # 1BHRS4: 1 subtype(s) +# SEVCSR: 1 subtype(s) # SEVCSR: 1 subtype(s) +# TESAC: 1 subtype(s) # TESAC: 1 subtype(s) +# MLS: 1 subtype(s) (if present in past 10 days of tanks) # MLS: 1 subtype(s) (if present in past 10 days of tanks) +# ESATMS: 1 subtype(s) (if present in past 10 days of tanks) # ESATMS: 1 subtype(s) (if present in past 10 days of tanks) +# CRISFS: 1 subtype(s) (if present in past 10 days of tanks) < +# GSRCSR: 1 subtype(s) # GSRCSR: 1 subtype(s) +# AHICSR: 1 subtype(s) # AHICSR: 1 subtype(s) +# SSTVCW: 1 subtype(s) # SSTVCW: 1 subtype(s) +# -------------------- # -------------------- +# TOTAL NUMBER OF SUBTYPES = 14 # TOTAL NUMBER OF SUBTYPES = 14 +# # +#========================================================================= #========================================================================= + +DTIM_latest_avcsam=${DTIM_latest_avcsam:-"+2.99"} DTIM_latest_avcsam=${DTIM_latest_avcsam:-"+2.99"} +DTIM_latest_eshrs3=${DTIM_latest_eshrs3:-"+2.99"} DTIM_latest_eshrs3=${DTIM_latest_eshrs3:-"+2.99"} +DTIM_latest_ssmisu=${DTIM_latest_ssmisu:-"+2.99"} DTIM_latest_ssmisu=${DTIM_latest_ssmisu:-"+2.99"} +#----------------------------------------------- < +DTIM_latest_saphir=${DTIM_latest_saphir:-"+2.99"} DTIM_latest_saphir=${DTIM_latest_saphir:-"+2.99"} +#----------------------------------------------- < +# check for atms tank presence in past 10 days < +atms="" < +err_check_tanks=0 < +sh $USHbufr_dump/check_tanks.sh atms < +err_check_tanks=$? < +if [ $err_check_tanks -eq 0 ];then < + atms=atms < + DTIM_latest_atms=${DTIM_latest_atms:-"+2.99"} < +fi < +#----------------------------------------------- < +DTIM_latest_saldrn=${DTIM_latest_saldrn:-"+2.99"} DTIM_latest_saldrn=${DTIM_latest_saldrn:-"+2.99"} +DTIM_latest_1bhrs4=${DTIM_latest_1bhrs4:-"+2.99"} DTIM_latest_1bhrs4=${DTIM_latest_1bhrs4:-"+2.99"} +DTIM_latest_sevcsr=${DTIM_latest_sevcsr:-"+2.99"} DTIM_latest_sevcsr=${DTIM_latest_sevcsr:-"+2.99"} +DTIM_latest_tesac=${DTIM_latest_tesac:-"+2.99"} DTIM_latest_tesac=${DTIM_latest_tesac:-"+2.99"} +#----------------------------------------------- #----------------------------------------------- +# check for mls tank presence in past 10 days # check for mls tank presence in past 10 days +mls="" mls="" +err_check_tanks=0 err_check_tanks=0 +sh $USHbufr_dump/check_tanks.sh mls sh $USHbufr_dump/check_tanks.sh mls +err_check_tanks=$? err_check_tanks=$? +if [ $err_check_tanks -eq 0 ];then if [ $err_check_tanks -eq 0 ];then + mls=mls mls=mls + DTIM_latest_mls=${DTIM_latest_mls:-"+2.99"} DTIM_latest_mls=${DTIM_latest_mls:-"+2.99"} +fi fi +#----------------------------------------------- #----------------------------------------------- +#----------------------------------------------- #----------------------------------------------- +# check for esatms tank presence in past 10 days # check for esatms tank presence in past 10 days +esatms="" esatms="" +err_check_tanks=0 err_check_tanks=0 +sh $USHbufr_dump/check_tanks.sh esatms sh $USHbufr_dump/check_tanks.sh esatms +err_check_tanks=$? err_check_tanks=$? +if [ $err_check_tanks -eq 0 ];then if [ $err_check_tanks -eq 0 ];then + esatms=esatms esatms=esatms + DTIM_latest_esatms=${DTIM_latest_esatms:-"+2.99"} DTIM_latest_esatms=${DTIM_latest_esatms:-"+2.99"} +fi fi +#----------------------------------------------- #----------------------------------------------- +#----------------------------------------------- < +# check for crisfs tank presence in past 10 days < +crisfs="" < +err_check_tanks=0 < +sh $USHbufr_dump/check_tanks.sh crisfs < +err_check_tanks=$? < +if [ $err_check_tanks -eq 0 ];then < + crisfs=crisfs < + DTIM_latest_crisfs=${DTIM_latest_crisfs:-"+2.99"} < +fi < +#----------------------------------------------- < +DTIM_latest_gsrcsr=${DTIM_latest_gsrcsr:-"+2.99"} DTIM_latest_gsrcsr=${DTIM_latest_gsrcsr:-"+2.99"} +DTIM_latest_ahicsr=${DTIM_latest_ahicsr:-"+2.99"} DTIM_latest_ahicsr=${DTIM_latest_ahicsr:-"+2.99"} +DTIM_latest_sstvcw=${DTIM_latest_sstvcw:-"+2.99"} DTIM_latest_sstvcw=${DTIM_latest_sstvcw:-"+2.99"} + +TIME_TRIM=${TIME_TRIM:-${TIME_TRIM1:-off}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM1:-off}} + +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 avcsam eshrs3 ssmisu \ $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 avcsam eshrs3 ssmisu \ + saphir $atms 1bhrs4 sevcsr tesac $mls $esatms $crisfs gsrcsr ahicsr sstvcw subpfl saldrn | saphir 1bhrs4 sevcsr tesac $mls $esatms gsrcsr ahicsr sstvcw subpfl saldrn +error1=$? error1=$? +echo "$error1" > $DATA/error1 echo "$error1" > $DATA/error1 + +if [ "$SENDDBN" = "YES" ]; then if [ "$SENDDBN" = "YES" ]; then + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_avcsam $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_avcsam $job \ + ${COMSP}avcsam.tm00.bufr_d ${COMSP}avcsam.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_eshrs3 $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_eshrs3 $job \ + ${COMSP}eshrs3.tm00.bufr_d ${COMSP}eshrs3.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ssmisu $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ssmisu $job \ + ${COMSP}ssmisu.tm00.bufr_d ${COMSP}ssmisu.tm00.bufr_d + if [ "${NET}" = "gdas" ]; then if [ "${NET}" = "gdas" ]; then + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_saphir $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_saphir $job \ + ${COMSP}saphir.tm00.bufr_d ### restricted, only GDAS, turn on 01/13/2020 ${COMSP}saphir.tm00.bufr_d ### restricted, only GDAS, turn on 01/13/2020 + fi fi + if [ "$atms" = atms ];then < + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_atms $job \ < + ${COMSP}atms.tm00.bufr_d < + fi < + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_1bhrs4 $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_1bhrs4 $job \ + ${COMSP}1bhrs4.tm00.bufr_d ${COMSP}1bhrs4.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sevcsr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sevcsr $job \ + ${COMSP}sevcsr.tm00.bufr_d ${COMSP}sevcsr.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_tesac $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_tesac $job \ + ${COMSP}tesac.tm00.bufr_d ${COMSP}tesac.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_saldrn $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_saldrn $job \ + ${COMSP}saldrn.tm00.bufr_d ${COMSP}saldrn.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_subpfl $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_subpfl $job \ + ${COMSP}subpfl.tm00.bufr_d ${COMSP}subpfl.tm00.bufr_d + if [ "$mls" = mls ];then if [ "$mls" = mls ];then + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_mls $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_mls $job \ + ${COMSP}mls.tm00.bufr_d ${COMSP}mls.tm00.bufr_d + fi fi + if [ "$esatms" = esatms ];then if [ "$esatms" = esatms ];then + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_esatms $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_esatms $job \ + ${COMSP}esatms.tm00.bufr_d ${COMSP}esatms.tm00.bufr_d + fi fi +####### ALERTS TURNED OFF UNTIL REQUESTED BY USER ######################### < +# if [ "$crisfs" = crisfs ];then < +# $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_crisfs $job \ < +# ${COMSP}crisfs.tm00.bufr_d < +# fi < +########################################################################### < + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gsrcsr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gsrcsr $job \ + ${COMSP}gsrcsr.tm00.bufr_d ${COMSP}gsrcsr.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ahicsr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ahicsr $job \ + ${COMSP}ahicsr.tm00.bufr_d ${COMSP}ahicsr.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sstvcw $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sstvcw $job \ + ${COMSP}sstvcw.tm00.bufr_d ${COMSP}sstvcw.tm00.bufr_d +fi fi + +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_1 echo Script thread_1 +echo Finished executing on node `hostname` echo Finished executing on node `hostname` +echo Ending time : `date -u` echo Ending time : `date -u` +echo "********************************************************************" echo "********************************************************************" +set -x set -x +} > $DATA/1.out 2>&1 } > $DATA/1.out 2>&1 +EOF EOF +set -x set -x + +set +x set +x +#---------------------------------------------------------------- #---------------------------------------------------------------- +cat<<\EOF>thread_2; chmod +x thread_2 cat<<\EOF>thread_2; chmod +x thread_2 +set -uax set -uax + +cd $DATA cd $DATA + +{ echo { echo +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_2 echo Script thread_2 +echo Executing on node `hostname` echo Executing on node `hostname` +echo Starting time: `date -u` echo Starting time: `date -u` +echo "********************************************************************" echo "********************************************************************" +echo echo +set -x set -x + +export STATUS=NO export STATUS=NO +export DUMP_NUMBER=2 export DUMP_NUMBER=2 + +#========================================================================== #========================================================================== +# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: +# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types +# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) +# (3) GDAS GSI doesn't use ATOVS, but NASA/GSFC is pulling them off our # (3) GDAS GSI doesn't use ATOVS, but NASA/GSFC is pulling them off our +# server, also CDAS may be running special tests using data from GDAS # server, also CDAS may be running special tests using data from GDAS +# cutoff time (ATOVS is not dumped in GFS) # cutoff time (ATOVS is not dumped in GFS) +# # +#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- +# GDAS: # GDAS: +# Dump # 2 : SFCSHP: 11 subtype(s) (added shipsb & shipub in dumplist) # Dump # 2 : SFCSHP: 11 subtype(s) (added shipsb & shipub in dumplist) +# TIDEG: 1 subtype(s) # TIDEG: 1 subtype(s) +# ATOVS: 1 subtype(s) # ATOVS: 1 subtype(s) +# ADPSFC: 7 subtype(s) # ADPSFC: 7 subtype(s) +# ASCATT: 1 subtype(s) # ASCATT: 1 subtype(s) +# xxxxxxxxx WNDSAT: 1 subtype(s) (if present in past 10 days of tanks) # xxxxxxxxx WNDSAT: 1 subtype(s) (if present in past 10 days of tanks) +# ===> Dumping of WNDSAT removed from here until new ingest feed is established # ===> Dumping of WNDSAT removed from here until new ingest feed is established +# (had been dumped with a time window radius of -3.00 to +2.99 hours) # (had been dumped with a time window radius of -3.00 to +2.99 hours) +# -------------------- # -------------------- +# TOTAL NUMBER OF SUBTYPES = 20 - 21 # TOTAL NUMBER OF SUBTYPES = 20 - 21 +# # +#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- +# GFS: # GFS: +# Dump # 2 : SFCSHP: 11 subtype(s) (added shipsb & shipub in dumplist) # Dump # 2 : SFCSHP: 11 subtype(s) (added shipsb & shipub in dumplist) +# TIDEG: 1 subtype(s) # TIDEG: 1 subtype(s) +# ADPSFC: 7 subtype(s) # ADPSFC: 7 subtype(s) +# ASCATT: 1 subtype(s) # ASCATT: 1 subtype(s) +# xxxxxxxxx WNDSAT: 1 subtype(s) (if present in past 10 days of tanks) # xxxxxxxxx WNDSAT: 1 subtype(s) (if present in past 10 days of tanks) +# ===> Dumping of WNDSAT removed from here until new ingest feed is established # ===> Dumping of WNDSAT removed from here until new ingest feed is established +# (had been dumped with a time window radius of -3.00 to +2.99 hours) # (had been dumped with a time window radius of -3.00 to +2.99 hours) +# -------------------- # -------------------- +# TOTAL NUMBER OF SUBTYPES = 19 - 20 # TOTAL NUMBER OF SUBTYPES = 19 - 20 +# # +#========================================================================== #========================================================================== +DTIM_latest_snocvr=${DTIM_latest_snocvr:-"+2.99"} DTIM_latest_snocvr=${DTIM_latest_snocvr:-"+2.99"} +DTIM_latest_sfcshp=${DTIM_latest_sfcshp:-"+2.99"} DTIM_latest_sfcshp=${DTIM_latest_sfcshp:-"+2.99"} +DTIM_latest_tideg=${DTIM_latest_tideg:-"+2.99"} DTIM_latest_tideg=${DTIM_latest_tideg:-"+2.99"} + +atovs="" atovs="" +if [ "$NET" = 'gdas' ]; then if [ "$NET" = 'gdas' ]; then + atovs=atovs atovs=atovs + DTIM_latest_atovs=${DTIM_latest_atovs:-"+2.99"} DTIM_latest_atovs=${DTIM_latest_atovs:-"+2.99"} +fi fi + +DTIM_latest_adpsfc=${DTIM_latest_adpsfc:-"+2.99"} DTIM_latest_adpsfc=${DTIM_latest_adpsfc:-"+2.99"} +DTIM_latest_ascatt=${DTIM_latest_ascatt:-"+2.99"} DTIM_latest_ascatt=${DTIM_latest_ascatt:-"+2.99"} +#----------------------------------------------- #----------------------------------------------- +# check for wndsat tank presence in past 10 days # check for wndsat tank presence in past 10 days +wndsat="" wndsat="" +err_check_tanks=0 err_check_tanks=0 +##########sh $USHbufr_dump/check_tanks.sh wndsat ##########sh $USHbufr_dump/check_tanks.sh wndsat +##########err_check_tanks=$? ##########err_check_tanks=$? +err_check_tanks=99 # comment out 2 lines above & add this line to ensure wndsat err_check_tanks=99 # comment out 2 lines above & add this line to ensure wndsat + # is not ever dumped # is not ever dumped +if [ $err_check_tanks -eq 0 ];then if [ $err_check_tanks -eq 0 ];then + wndsat=wndsat wndsat=wndsat + DTIM_latest_wndsat=${DTIM_latest_wndsat:-"+2.99"} DTIM_latest_wndsat=${DTIM_latest_wndsat:-"+2.99"} +fi fi +#----------------------------------------------- #----------------------------------------------- + +TIME_TRIM=${TIME_TRIM:-${TIME_TRIM2:-off}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM2:-off}} + +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 sfcshp tideg $atovs adpsfc snocvr ascatt $wndsat $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 sfcshp tideg $atovs adpsfc snocvr ascatt $wndsat +error2=$? error2=$? +echo "$error2" > $DATA/error2 echo "$error2" > $DATA/error2 + +if [ "$SENDDBN" = "YES" ]; then if [ "$SENDDBN" = "YES" ]; then + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sfcshp $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sfcshp $job \ + ${COMSP}sfcshp.tm00.bufr_d ${COMSP}sfcshp.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_tideg $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_tideg $job \ + ${COMSP}tideg.tm00.bufr_d ${COMSP}tideg.tm00.bufr_d + [ -f ${COMSP}atovs.tm00.bufr_d ] && \ [ -f ${COMSP}atovs.tm00.bufr_d ] && \ + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_atovs $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_atovs $job \ + ${COMSP}atovs.tm00.bufr_d ${COMSP}atovs.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_adpsfc $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_adpsfc $job \ + ${COMSP}adpsfc.tm00.bufr_d ${COMSP}adpsfc.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ascatt $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ascatt $job \ + ${COMSP}ascatt.tm00.bufr_d ${COMSP}ascatt.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_snocvr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_snocvr $job \ + ${COMSP}snocvr.tm00.bufr_d ${COMSP}snocvr.tm00.bufr_d + if [ "$NET" = 'gdas' ]; then if [ "$NET" = 'gdas' ]; then + ####### ALERT TURNED ON for GDAS only ######################## ####### ALERT TURNED ON for GDAS only ######################## + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ascatw $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ascatw $job \ + ${COMSP}ascatw.tm00.bufr_d ${COMSP}ascatw.tm00.bufr_d + fi fi + if [ "$wndsat" = wndsat ];then if [ "$wndsat" = wndsat ];then + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_wndsat $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_wndsat $job \ + ${COMSP}wndsat.tm00.bufr_d ${COMSP}wndsat.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_wdsatr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_wdsatr $job \ + ${COMSP}wdsatr.tm00.bufr_d ${COMSP}wdsatr.tm00.bufr_d + fi fi +fi fi + +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_2 echo Script thread_2 +echo Finished executing on node `hostname` echo Finished executing on node `hostname` +echo Ending time : `date -u` echo Ending time : `date -u` +echo "********************************************************************" echo "********************************************************************" +set -x set -x +} > $DATA/2.out 2>&1 } > $DATA/2.out 2>&1 +EOF EOF +set -x set -x + +set +x set +x +#---------------------------------------------------------------- #---------------------------------------------------------------- +cat<<\EOF>thread_3; chmod +x thread_3 cat<<\EOF>thread_3; chmod +x thread_3 +set -uax set -uax + +cd $DATA cd $DATA + +{ echo { echo +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_3 echo Script thread_3 +echo Executing on node `hostname` echo Executing on node `hostname` +echo Starting time: `date -u` echo Starting time: `date -u` +echo "********************************************************************" echo "********************************************************************" +echo echo +set -x set -x + +export STATUS=NO export STATUS=NO +export DUMP_NUMBER=3 export DUMP_NUMBER=3 + +#==================================================================== #==================================================================== +# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: +# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types +# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) +# # +#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- +# Dump #3: ADPUPA: 6 subtype(s) # Dump #3: ADPUPA: 6 subtype(s) +# -------------------- # -------------------- +# TOTAL NUMBER OF SUBTYPES = 6 # TOTAL NUMBER OF SUBTYPES = 6 +# # +#==================================================================== #==================================================================== + +DTIM_latest_adpupa=${DTIM_latest_adpupa:-"+2.99"} DTIM_latest_adpupa=${DTIM_latest_adpupa:-"+2.99"} + +TIME_TRIM=${TIME_TRIM:-${TIME_TRIM3:-off}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM3:-off}} + +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 adpupa $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 adpupa +error3=$? error3=$? +echo "$error3" > $DATA/error3 echo "$error3" > $DATA/error3 + +if [ "$SENDDBN" = "YES" ]; then if [ "$SENDDBN" = "YES" ]; then + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_adpupa $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_adpupa $job \ + ${COMSP}adpupa.tm00.bufr_d ${COMSP}adpupa.tm00.bufr_d +fi fi + +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_3 echo Script thread_3 +echo Finished executing on node `hostname` echo Finished executing on node `hostname` +echo Ending time : `date -u` echo Ending time : `date -u` +echo "********************************************************************" echo "********************************************************************" +set -x set -x +} > $DATA/3.out 2>&1 } > $DATA/3.out 2>&1 +EOF EOF +set -x set -x + +set +x set +x +#---------------------------------------------------------------- #---------------------------------------------------------------- +cat<<\EOF>thread_4; chmod +x thread_4 cat<<\EOF>thread_4; chmod +x thread_4 +set -uax set -uax + +cd $DATA cd $DATA + +{ echo { echo +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_4 echo Script thread_4 +echo Executing on node `hostname` echo Executing on node `hostname` +echo Starting time: `date -u` echo Starting time: `date -u` +echo "********************************************************************" echo "********************************************************************" +echo echo +set -x set -x + +export STATUS=NO export STATUS=NO +export DUMP_NUMBER=4 export DUMP_NUMBER=4 + +#======================================================================= #======================================================================= +# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: +# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types +# EXCEPT: AIRCFT where it is +/- 3.25 hours # EXCEPT: AIRCFT where it is +/- 3.25 hours +# AIRCAR where it is +/- 3.25 hours # AIRCAR where it is +/- 3.25 hours +# PROFLR where it is -4.00 to +3.99 hours # PROFLR where it is -4.00 to +3.99 hours +# GSPIPW where it is +/- 0.05 hours (+/- 3min) # GSPIPW where it is +/- 0.05 hours (+/- 3min) +# (2) TIME TRIMMING IS DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS DONE IN THIS DUMP (default, unless overridden) +# # +#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- +# Dump # 4 : AIRCAR: 2 subtype(s) # Dump # 4 : AIRCAR: 2 subtype(s) +# AIRCFT: 8 subtype(s) # AIRCFT: 8 subtype(s) +# PROFLR: 4 subtype(s) # PROFLR: 4 subtype(s) +# VADWND: 1 subtype(s) # VADWND: 1 subtype(s) +# RASSDA: 1 subtype(s) # RASSDA: 1 subtype(s) +# GPSIPW: 1 subtype(s) # GPSIPW: 1 subtype(s) +# HDOB : 1 subtype(s) # HDOB : 1 subtype(s) +# -------------------- # -------------------- +# TOTAL NUMBER OF SUBTYPES = 18 # TOTAL NUMBER OF SUBTYPES = 18 +# # +#======================================================================= #======================================================================= + +# Skip NeXRaD VAD WINDS FROM LEVEL 2 DECODER (not ready to be handled in GSI) (002017) # Skip NeXRaD VAD WINDS FROM LEVEL 2 DECODER (not ready to be handled in GSI) (002017) +# 3/9/2022 -- enable the dumping of 002017 in the vadwnd dump group. # 3/9/2022 -- enable the dumping of 002017 in the vadwnd dump group. +#export SKIP_002017=YES #export SKIP_002017=YES + +# Dump AIRCFT and AIRCAR with wide time window to improve PREPOBS_PREPACQC # Dump AIRCFT and AIRCAR with wide time window to improve PREPOBS_PREPACQC +# track-check performance # track-check performance +# (time window will be winnowed down to +/- 3.00 hours in output from # (time window will be winnowed down to +/- 3.00 hours in output from +# PREPOBS_PREPACQC) # PREPOBS_PREPACQC) + +# Dump PROFLR with wide time window to improve PREPOBS_PROFCQC performance # Dump PROFLR with wide time window to improve PREPOBS_PROFCQC performance +# (time window will be winnowed down in output from PREPOBS_PROFCQC, see # (time window will be winnowed down in output from PREPOBS_PROFCQC, see +# parm cards for output time window) # parm cards for output time window) + +# Dump GPSIPW with narrow (+/- 3-min) time window since new Ground Based # Dump GPSIPW with narrow (+/- 3-min) time window since new Ground Based +# GPS-IPW/ZTD (from U.S.-ENI and foreign GNSS providers) is currently limited # GPS-IPW/ZTD (from U.S.-ENI and foreign GNSS providers) is currently limited +# to obs only at cycle-time # to obs only at cycle-time + +DTIM_earliest_aircft=${DTIM_earliest_aircft:-"-3.25"} DTIM_earliest_aircft=${DTIM_earliest_aircft:-"-3.25"} +DTIM_latest_aircft=${DTIM_latest_aircft:-"+3.25"} DTIM_latest_aircft=${DTIM_latest_aircft:-"+3.25"} + +DTIM_earliest_aircar=${DTIM_earliest_aircar:-"-3.25"} DTIM_earliest_aircar=${DTIM_earliest_aircar:-"-3.25"} +DTIM_latest_aircar=${DTIM_latest_aircar:-"+3.25"} DTIM_latest_aircar=${DTIM_latest_aircar:-"+3.25"} + +DTIM_earliest_proflr=${DTIM_earliest_proflr:-"-4.00"} DTIM_earliest_proflr=${DTIM_earliest_proflr:-"-4.00"} +DTIM_latest_proflr=${DTIM_latest_proflr:-"+3.99"} DTIM_latest_proflr=${DTIM_latest_proflr:-"+3.99"} + +DTIM_latest_vadwnd=${DTIM_latest_vadwnd:-"+2.99"} DTIM_latest_vadwnd=${DTIM_latest_vadwnd:-"+2.99"} +DTIM_latest_rassda=${DTIM_latest_rassda:-"+2.99"} DTIM_latest_rassda=${DTIM_latest_rassda:-"+2.99"} + +DTIM_earliest_gpsipw=${DTIM_latest_gpsipw:-"-0.05"} DTIM_earliest_gpsipw=${DTIM_latest_gpsipw:-"-0.05"} +DTIM_latest_gpsipw=${DTIM_latest_gpsipw:-"+0.05"} DTIM_latest_gpsipw=${DTIM_latest_gpsipw:-"+0.05"} + +TIME_TRIM=${TIME_TRIM:-${TIME_TRIM4:-on}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM4:-on}} + +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 aircar aircft proflr vadwnd \ $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 aircar aircft proflr vadwnd \ + rassda gpsipw hdob rassda gpsipw hdob +error4=$? error4=$? +echo "$error4" > $DATA/error4 echo "$error4" > $DATA/error4 + +if [ "$SENDDBN" = "YES" ]; then if [ "$SENDDBN" = "YES" ]; then + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_aircar $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_aircar $job \ + ${COMSP}aircar.tm00.bufr_d ${COMSP}aircar.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_aircft $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_aircft $job \ + ${COMSP}aircft.tm00.bufr_d ${COMSP}aircft.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_proflr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_proflr $job \ + ${COMSP}proflr.tm00.bufr_d ${COMSP}proflr.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_vadwnd $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_vadwnd $job \ + ${COMSP}vadwnd.tm00.bufr_d ${COMSP}vadwnd.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_rassda $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_rassda $job \ + ${COMSP}rassda.tm00.bufr_d ${COMSP}rassda.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gpsipw $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gpsipw $job \ + ${COMSP}gpsipw.tm00.bufr_d ${COMSP}gpsipw.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_hdob $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_hdob $job \ + ${COMSP}hdob.tm00.bufr_d ${COMSP}hdob.tm00.bufr_d +fi fi + +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_4 echo Script thread_4 +echo Finished executing on node `hostname` echo Finished executing on node `hostname` +echo Ending time : `date -u` echo Ending time : `date -u` +echo "********************************************************************" echo "********************************************************************" +set -x set -x +} > $DATA/4.out 2>&1 } > $DATA/4.out 2>&1 +EOF EOF +set -x set -x + +set +x set +x +#---------------------------------------------------------------- #---------------------------------------------------------------- +cat<<\EOF>thread_5; chmod +x thread_5 cat<<\EOF>thread_5; chmod +x thread_5 +set -uax set -uax + +cd $DATA cd $DATA + +{ echo { echo +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_5 echo Script thread_5 +echo Executing on node `hostname` echo Executing on node `hostname` +echo Starting time: `date -u` echo Starting time: `date -u` +echo "********************************************************************" echo "********************************************************************" +echo echo +set -x set -x + +export STATUS=NO export STATUS=NO +export DUMP_NUMBER=5 export DUMP_NUMBER=5 + +#=================================================================== #=================================================================== +# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: +# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types +# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) +# # +#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- +# Currently not executed in GDAS or GFS: # Currently not executed in GDAS or GFS: +# Dump # 5 : MSONET: 30 subtype(s) # Dump # 5 : MSONET: 30 subtype(s) +# --------------------- # --------------------- +# TOTAL NUMBER OF SUBTYPES = 30 # TOTAL NUMBER OF SUBTYPES = 30 +# # +#=================================================================== #=================================================================== + +DTIM_latest_msonet=${DTIM_latest_msonet:-"+2.99"} DTIM_latest_msonet=${DTIM_latest_msonet:-"+2.99"} + +TIME_TRIM=${TIME_TRIM:-${TIME_TRIM5:-off}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM5:-off}} + +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 msonet $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 msonet +error5=$? error5=$? +echo "$error5" > $DATA/error5 echo "$error5" > $DATA/error5 + +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_5 echo Script thread_5 +echo Finished executing on node `hostname` echo Finished executing on node `hostname` +echo Ending time : `date -u` echo Ending time : `date -u` +echo "********************************************************************" echo "********************************************************************" +set -x set -x +} > $DATA/5.out 2>&1 } > $DATA/5.out 2>&1 +EOF EOF +set -x set -x + +set +x set +x +#---------------------------------------------------------------- #---------------------------------------------------------------- +cat<<\EOF>thread_6; chmod +x thread_6 cat<<\EOF>thread_6; chmod +x thread_6 +set -uax set -uax + +cd $DATA cd $DATA + +{ echo { echo +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_6 echo Script thread_6 +echo Executing on node `hostname` echo Executing on node `hostname` +echo Starting time: `date -u` echo Starting time: `date -u` +echo "********************************************************************" echo "********************************************************************" +echo echo +set -x set -x + +export STATUS=NO export STATUS=NO +export DUMP_NUMBER=6 export DUMP_NUMBER=6 + +#=================================================================== #=================================================================== +# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: +# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types +# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) +# # +#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- +# Currently not executed in GDAS or GFS: # Currently not executed in GDAS or GFS: +# Dump # 6 : NEXRAD: 8 subtype(s) # Dump # 6 : NEXRAD: 8 subtype(s) +# -------------------- # -------------------- +# TOTAL NUMBER OF SUBTYPES = 8 # TOTAL NUMBER OF SUBTYPES = 8 +# # +#=================================================================== #=================================================================== + +DTIM_latest_nexrad=${DTIM_latest_nexrad:-"+2.99"} DTIM_latest_nexrad=${DTIM_latest_nexrad:-"+2.99"} + +TIME_TRIM=${TIME_TRIM:-${TIME_TRIM6:-off}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM6:-off}} + +# NEXRAD tanks are hourly # NEXRAD tanks are hourly +# Process only those hourly tanks w/i requested dump center cycle time window # Process only those hourly tanks w/i requested dump center cycle time window + +SKIP_006010=YES # radial wind 00Z SKIP_006010=YES # radial wind 00Z +SKIP_006011=YES # radial wind 01Z SKIP_006011=YES # radial wind 01Z +SKIP_006012=YES # radial wind 02Z SKIP_006012=YES # radial wind 02Z +SKIP_006013=YES # radial wind 03Z SKIP_006013=YES # radial wind 03Z +SKIP_006014=YES # radial wind 04Z SKIP_006014=YES # radial wind 04Z +SKIP_006015=YES # radial wind 05Z SKIP_006015=YES # radial wind 05Z +SKIP_006016=YES # radial wind 06Z SKIP_006016=YES # radial wind 06Z +SKIP_006017=YES # radial wind 07Z SKIP_006017=YES # radial wind 07Z +SKIP_006018=YES # radial wind 08Z SKIP_006018=YES # radial wind 08Z +SKIP_006019=YES # radial wind 09Z SKIP_006019=YES # radial wind 09Z +SKIP_006020=YES # radial wind 10Z SKIP_006020=YES # radial wind 10Z +SKIP_006021=YES # radial wind 11Z SKIP_006021=YES # radial wind 11Z +SKIP_006022=YES # radial wind 12Z SKIP_006022=YES # radial wind 12Z +SKIP_006023=YES # radial wind 13Z SKIP_006023=YES # radial wind 13Z +SKIP_006024=YES # radial wind 14Z SKIP_006024=YES # radial wind 14Z +SKIP_006025=YES # radial wind 15Z SKIP_006025=YES # radial wind 15Z +SKIP_006026=YES # radial wind 16Z SKIP_006026=YES # radial wind 16Z +SKIP_006027=YES # radial wind 17Z SKIP_006027=YES # radial wind 17Z +SKIP_006028=YES # radial wind 18Z SKIP_006028=YES # radial wind 18Z +SKIP_006029=YES # radial wind 19Z SKIP_006029=YES # radial wind 19Z +SKIP_006030=YES # radial wind 20Z SKIP_006030=YES # radial wind 20Z +SKIP_006031=YES # radial wind 21Z SKIP_006031=YES # radial wind 21Z +SKIP_006032=YES # radial wind 22Z SKIP_006032=YES # radial wind 22Z +SKIP_006033=YES # radial wind 23Z SKIP_006033=YES # radial wind 23Z + +SKIP_006040=YES # reflectivity 00Z SKIP_006040=YES # reflectivity 00Z +SKIP_006041=YES # reflectivity 01Z SKIP_006041=YES # reflectivity 01Z +SKIP_006042=YES # reflectivity 02Z SKIP_006042=YES # reflectivity 02Z +SKIP_006043=YES # reflectivity 03Z SKIP_006043=YES # reflectivity 03Z +SKIP_006044=YES # reflectivity 04Z SKIP_006044=YES # reflectivity 04Z +SKIP_006045=YES # reflectivity 05Z SKIP_006045=YES # reflectivity 05Z +SKIP_006046=YES # reflectivity 06Z SKIP_006046=YES # reflectivity 06Z +SKIP_006047=YES # reflectivity 07Z SKIP_006047=YES # reflectivity 07Z +SKIP_006048=YES # reflectivity 08Z SKIP_006048=YES # reflectivity 08Z +SKIP_006049=YES # reflectivity 09Z SKIP_006049=YES # reflectivity 09Z +SKIP_006050=YES # reflectivity 10Z SKIP_006050=YES # reflectivity 10Z +SKIP_006051=YES # reflectivity 11Z SKIP_006051=YES # reflectivity 11Z +SKIP_006052=YES # reflectivity 12Z SKIP_006052=YES # reflectivity 12Z +SKIP_006053=YES # reflectivity 13Z SKIP_006053=YES # reflectivity 13Z +SKIP_006054=YES # reflectivity 14Z SKIP_006054=YES # reflectivity 14Z +SKIP_006055=YES # reflectivity 15Z SKIP_006055=YES # reflectivity 15Z +SKIP_006056=YES # reflectivity 16Z SKIP_006056=YES # reflectivity 16Z +SKIP_006057=YES # reflectivity 17Z SKIP_006057=YES # reflectivity 17Z +SKIP_006058=YES # reflectivity 18Z SKIP_006058=YES # reflectivity 18Z +SKIP_006059=YES # reflectivity 19Z SKIP_006059=YES # reflectivity 19Z +SKIP_006060=YES # reflectivity 20Z SKIP_006060=YES # reflectivity 20Z +SKIP_006061=YES # reflectivity 21Z SKIP_006061=YES # reflectivity 21Z +SKIP_006062=YES # reflectivity 22Z SKIP_006062=YES # reflectivity 22Z +SKIP_006063=YES # reflectivity 23Z SKIP_006063=YES # reflectivity 23Z + +if [ $cycp -eq 00 ]; then # (22.5 - 01.5 Z) if [ $cycp -eq 00 ]; then # (22.5 - 01.5 Z) + unset SKIP_006032 # radial wind 22Z unset SKIP_006032 # radial wind 22Z + unset SKIP_006033 # radial wind 23Z unset SKIP_006033 # radial wind 23Z + unset SKIP_006010 # radial wind 00Z unset SKIP_006010 # radial wind 00Z + unset SKIP_006011 # radial wind 01Z unset SKIP_006011 # radial wind 01Z + unset SKIP_006062 # reflectivity 22Z unset SKIP_006062 # reflectivity 22Z + unset SKIP_006063 # reflectivity 23Z unset SKIP_006063 # reflectivity 23Z + unset SKIP_006040 # reflectivity 00Z unset SKIP_006040 # reflectivity 00Z + unset SKIP_006041 # reflectivity 01Z unset SKIP_006041 # reflectivity 01Z +elif [ $cycp -eq 06 ]; then # (04.5 - 07.5 Z) elif [ $cycp -eq 06 ]; then # (04.5 - 07.5 Z) + unset SKIP_006014 # radial wind 04Z unset SKIP_006014 # radial wind 04Z + unset SKIP_006015 # radial wind 05Z unset SKIP_006015 # radial wind 05Z + unset SKIP_006016 # radial wind 06Z unset SKIP_006016 # radial wind 06Z + unset SKIP_006017 # radial wind 07Z unset SKIP_006017 # radial wind 07Z + unset SKIP_006044 # reflectivity 04Z unset SKIP_006044 # reflectivity 04Z + unset SKIP_006045 # reflectivity 05Z unset SKIP_006045 # reflectivity 05Z + unset SKIP_006046 # reflectivity 06Z unset SKIP_006046 # reflectivity 06Z + unset SKIP_006047 # reflectivity 07Z unset SKIP_006047 # reflectivity 07Z +elif [ $cycp -eq 12 ]; then # (10.5 - 13.5 Z) elif [ $cycp -eq 12 ]; then # (10.5 - 13.5 Z) + unset SKIP_006020 # radial wind 10Z unset SKIP_006020 # radial wind 10Z + unset SKIP_006021 # radial wind 11Z unset SKIP_006021 # radial wind 11Z + unset SKIP_006022 # radial wind 12Z unset SKIP_006022 # radial wind 12Z + unset SKIP_006023 # radial wind 13Z unset SKIP_006023 # radial wind 13Z + unset SKIP_006050 # reflectivity 10Z unset SKIP_006050 # reflectivity 10Z + unset SKIP_006051 # reflectivity 11Z unset SKIP_006051 # reflectivity 11Z + unset SKIP_006052 # reflectivity 12Z unset SKIP_006052 # reflectivity 12Z + unset SKIP_006053 # reflectivity 13Z unset SKIP_006053 # reflectivity 13Z +elif [ $cycp -eq 18 ]; then # (16.5 - 19.5 Z) elif [ $cycp -eq 18 ]; then # (16.5 - 19.5 Z) + unset SKIP_006026 # radial wind 16Z unset SKIP_006026 # radial wind 16Z + unset SKIP_006027 # radial wind 17Z unset SKIP_006027 # radial wind 17Z + unset SKIP_006028 # radial wind 18Z unset SKIP_006028 # radial wind 18Z + unset SKIP_006029 # radial wind 19Z unset SKIP_006029 # radial wind 19Z + unset SKIP_006056 # reflectivity 16Z unset SKIP_006056 # reflectivity 16Z + unset SKIP_006057 # reflectivity 17Z unset SKIP_006057 # reflectivity 17Z + unset SKIP_006058 # reflectivity 18Z unset SKIP_006058 # reflectivity 18Z + unset SKIP_006059 # reflectivity 19Z unset SKIP_006059 # reflectivity 19Z +fi fi + +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 nexrad $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 nexrad +error6=$? error6=$? +echo "$error6" > $DATA/error6 echo "$error6" > $DATA/error6 + +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_6 echo Script thread_6 +echo Finished executing on node `hostname` echo Finished executing on node `hostname` +echo Ending time : `date -u` echo Ending time : `date -u` +echo "********************************************************************" echo "********************************************************************" +set -x set -x +} > $DATA/6.out 2>&1 } > $DATA/6.out 2>&1 +EOF EOF +set -x set -x + +set +x set +x +#------------------------------------------------------------------------------ #------------------------------------------------------------------------------ +cat<<\EOF>thread_7; chmod +x thread_7 cat<<\EOF>thread_7; chmod +x thread_7 +set -uax set -uax + +cd $DATA cd $DATA + +{ echo { echo +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_7 echo Script thread_7 +echo Executing on node `hostname` echo Executing on node `hostname` +echo Starting time: `date -u` echo Starting time: `date -u` +echo "********************************************************************" echo "********************************************************************" +echo echo +set -x set -x + +export STATUS=NO export STATUS=NO +export DUMP_NUMBER=7 export DUMP_NUMBER=7 + +#========================================================================= #========================================================================= +# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: +# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types +# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) +# # +#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- +# Dump # 7 : AVCSPM: 1 subtype(s) # Dump # 7 : AVCSPM: 1 subtype(s) +# ESMHS: 1 subtype(s) # ESMHS: 1 subtype(s) +# 1BMHS: 1 subtype(s) # 1BMHS: 1 subtype(s) +# AIRSEV: 1 subtype(s) # AIRSEV: 1 subtype(s) +# ATMSDB: 1 subtype(s) # ATMSDB: 1 subtype(s) +# GOME: 1 subtype(s) # GOME: 1 subtype(s) +# OMI: 1 subtype(s) # OMI: 1 subtype(s) +# TRKOB: 1 subtype(s) # TRKOB: 1 subtype(s) +# GPSRO: 1 subtype(s) # GPSRO: 1 subtype(s) +# CRISF4: 1 subtype(s) (if present in past 10 days of tanks) # CRISF4: 1 subtype(s) (if present in past 10 days of tanks) +# -------------------- # -------------------- +# TOTAL NUMBER OF SUBTYPES = 10 # TOTAL NUMBER OF SUBTYPES = 10 +# # +#========================================================================= #========================================================================= + +DTIM_latest_avcspm=${DTIM_latest_avcspm:-"+2.99"} DTIM_latest_avcspm=${DTIM_latest_avcspm:-"+2.99"} +DTIM_latest_esmhs=${DTIM_latest_esmhs:-"+2.99"} DTIM_latest_esmhs=${DTIM_latest_esmhs:-"+2.99"} +DTIM_latest_1bmhs=${DTIM_latest_1bmhs:-"+2.99"} DTIM_latest_1bmhs=${DTIM_latest_1bmhs:-"+2.99"} +DTIM_latest_airsev=${DTIM_latest_airsev:-"+2.99"} DTIM_latest_airsev=${DTIM_latest_airsev:-"+2.99"} +DTIM_latest_atmsdb=${DTIM_latest_atmsdb:-"+2.99"} DTIM_latest_atmsdb=${DTIM_latest_atmsdb:-"+2.99"} +DTIM_latest_gome=${DTIM_latest_gome:-"+2.99"} DTIM_latest_gome=${DTIM_latest_gome:-"+2.99"} +DTIM_latest_omi=${DTIM_latest_omi:-"+2.99"} DTIM_latest_omi=${DTIM_latest_omi:-"+2.99"} +DTIM_latest_trkob=${DTIM_latest_trkob:-"+2.99"} DTIM_latest_trkob=${DTIM_latest_trkob:-"+2.99"} +DTIM_latest_gpsro=${DTIM_latest_gpsro:-"+2.99"} DTIM_latest_gpsro=${DTIM_latest_gpsro:-"+2.99"} +#----------------------------------------------- #----------------------------------------------- +# check for crisf4 tank presence in past 10 days # check for crisf4 tank presence in past 10 days +crisf4="" crisf4="" +err_check_tanks=0 err_check_tanks=0 +sh $USHbufr_dump/check_tanks.sh crisf4 sh $USHbufr_dump/check_tanks.sh crisf4 +err_check_tanks=$? err_check_tanks=$? +if [ $err_check_tanks -eq 0 ];then if [ $err_check_tanks -eq 0 ];then + crisf4=crisf4 crisf4=crisf4 + DTIM_latest_crisf4=${DTIM_latest_crisf4:-"+2.99"} DTIM_latest_crisf4=${DTIM_latest_crisf4:-"+2.99"} +fi fi +#----------------------------------------------- #----------------------------------------------- + +TIME_TRIM=${TIME_TRIM:-${TIME_TRIM7:-off}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM7:-off}} + +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 avcspm esmhs 1bmhs \ $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 avcspm esmhs 1bmhs \ + airsev atmsdb gome omi trkob gpsro $crisf4 airsev atmsdb gome omi trkob gpsro $crisf4 +error7=$? error7=$? +echo "$error7" > $DATA/error7 echo "$error7" > $DATA/error7 + +if [ "$SENDDBN" = "YES" ]; then if [ "$SENDDBN" = "YES" ]; then + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_avcspm $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_avcspm $job \ + ${COMSP}avcspm.tm00.bufr_d ${COMSP}avcspm.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_esmhs $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_esmhs $job \ + ${COMSP}esmhs.tm00.bufr_d ${COMSP}esmhs.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_1bmhs $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_1bmhs $job \ + ${COMSP}1bmhs.tm00.bufr_d ${COMSP}1bmhs.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_airsev $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_airsev $job \ + ${COMSP}airsev.tm00.bufr_d ${COMSP}airsev.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_atmsdb $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_atmsdb $job \ + ${COMSP}atmsdb.tm00.bufr_d ${COMSP}atmsdb.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gome $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gome $job \ + ${COMSP}gome.tm00.bufr_d ${COMSP}gome.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_omi $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_omi $job \ + ${COMSP}omi.tm00.bufr_d ${COMSP}omi.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_trkob $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_trkob $job \ + ${COMSP}trkob.tm00.bufr_d ${COMSP}trkob.tm00.bufr_d +# gpsro dump file has nr version which is alerted from # gpsro dump file has nr version which is alerted from +# exdump_post.sh.ecf # exdump_post.sh.ecf + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gpsro $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gpsro $job \ + ${COMSP}gpsro.tm00.bufr_d ${COMSP}gpsro.tm00.bufr_d + if [ "$crisf4" = crisf4 ];then if [ "$crisf4" = crisf4 ];then + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_crisf4 $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_crisf4 $job \ + ${COMSP}crisf4.tm00.bufr_d ${COMSP}crisf4.tm00.bufr_d + fi fi +fi fi + +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_7 echo Script thread_7 +echo Finished executing on node `hostname` echo Finished executing on node `hostname` +echo Ending time : `date -u` echo Ending time : `date -u` +echo "********************************************************************" echo "********************************************************************" +set -x set -x +} > $DATA/7.out 2>&1 } > $DATA/7.out 2>&1 +EOF EOF +set -x set -x + +set +x set +x +#---------------------------------------------------------------- #---------------------------------------------------------------- +cat<<\EOF>thread_8; chmod +x thread_8 cat<<\EOF>thread_8; chmod +x thread_8 +set -uax set -uax + +cd $DATA cd $DATA + +{ echo { echo +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_8 echo Script thread_8 +echo Executing on node `hostname` echo Executing on node `hostname` +echo Starting time: `date -u` echo Starting time: `date -u` +echo "********************************************************************" echo "********************************************************************" +echo echo +set -x set -x + +export STATUS=NO export STATUS=NO +export DUMP_NUMBER=8 export DUMP_NUMBER=8 + +#======================================================================= #======================================================================= +# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: +# (1) time window radius is +/- 1.5 hrs for all SATWND types # (1) time window radius is +/- 1.5 hrs for all SATWND types +# EXCEPT: SATWND subtypes 005/030, 005/031, 005/032, 005/034, 005/039, # EXCEPT: SATWND subtypes 005/030, 005/031, 005/032, 005/034, 005/039, +# 005/064, 005/065, 005/066, 005/067, 005/068, 005/069, # 005/064, 005/065, 005/066, 005/067, 005/068, 005/069, +# 005/070, 005/071, 005/072, 005/080 and 005/091 # 005/070, 005/071, 005/072, 005/080 and 005/091 +# where it is # where it is +# -3.00 to +2.99 hours. # -3.00 to +2.99 hours. +# (2) TIME TRIMMING IS DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS DONE IN THIS DUMP (default, unless overridden) +# # +#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- +# Dump # 8 : SATWND: 16 subtype(s) (bufr_dumplist.v2.3.0) # Dump # 8 : SATWND: 16 subtype(s) (bufr_dumplist.v2.3.0) +# --------------------- # --------------------- +# TOTAL NUMBER OF SUBTYPES = 25 # TOTAL NUMBER OF SUBTYPES = 25 +# # +#======================================================================= #======================================================================= + +ADD_satwnd="005024 005025 005026 005030 005031 005032 005034 005039 005072" ADD_satwnd="005024 005025 005026 005030 005031 005032 005034 005039 005072" + +# Skip old bufr METEOSAT AMVs; for testing skip in trigger or version file # Skip old bufr METEOSAT AMVs; for testing skip in trigger or version file +#export SKIP_005064=YES #export SKIP_005064=YES +#export SKIP_005065=YES #export SKIP_005065=YES +#export SKIP_005066=YES #export SKIP_005066=YES + +# satwnd types # satwnd types +# ------------ # ------------ +DTIM_earliest_005024=${DTIM_earliest_005024:-"-3.00"} DTIM_earliest_005024=${DTIM_earliest_005024:-"-3.00"} +DTIM_latest_005024=${DTIM_latest_005024:-"+2.99"} DTIM_latest_005024=${DTIM_latest_005024:-"+2.99"} +DTIM_earliest_005025=${DTIM_earliest_005025:-"-3.00"} DTIM_earliest_005025=${DTIM_earliest_005025:-"-3.00"} +DTIM_latest_005025=${DTIM_latest_005025:-"+2.99"} DTIM_latest_005025=${DTIM_latest_005025:-"+2.99"} +DTIM_earliest_005026=${DTIM_earliest_005026:-"-3.00"} DTIM_earliest_005026=${DTIM_earliest_005026:-"-3.00"} +DTIM_latest_005026=${DTIM_latest_005026:-"+2.99"} DTIM_latest_005026=${DTIM_latest_005026:-"+2.99"} +DTIM_earliest_005030=${DTIM_earliest_005030:-"-3.00"} DTIM_earliest_005030=${DTIM_earliest_005030:-"-3.00"} +DTIM_latest_005030=${DTIM_latest_005030:-"+2.99"} DTIM_latest_005030=${DTIM_latest_005030:-"+2.99"} +DTIM_earliest_005031=${DTIM_earliest_005031:-"-3.00"} DTIM_earliest_005031=${DTIM_earliest_005031:-"-3.00"} +DTIM_latest_005031=${DTIM_latest_005031:-"+2.99"} DTIM_latest_005031=${DTIM_latest_005031:-"+2.99"} +DTIM_earliest_005032=${DTIM_earliest_005032:-"-3.00"} DTIM_earliest_005032=${DTIM_earliest_005032:-"-3.00"} +DTIM_latest_005032=${DTIM_latest_005032:-"+2.99"} DTIM_latest_005032=${DTIM_latest_005032:-"+2.99"} +DTIM_earliest_005034=${DTIM_earliest_005034:-"-3.00"} DTIM_earliest_005034=${DTIM_earliest_005034:-"-3.00"} +DTIM_latest_005034=${DTIM_latest_005034:-"+2.99"} DTIM_latest_005034=${DTIM_latest_005034:-"+2.99"} +DTIM_earliest_005039=${DTIM_earliest_005039:-"-3.00"} DTIM_earliest_005039=${DTIM_earliest_005039:-"-3.00"} +DTIM_latest_005039=${DTIM_latest_005039:-"+2.99"} DTIM_latest_005039=${DTIM_latest_005039:-"+2.99"} +DTIM_earliest_005064=${DTIM_earliest_005064:-"-3.00"} DTIM_earliest_005064=${DTIM_earliest_005064:-"-3.00"} +DTIM_latest_005064=${DTIM_latest_005064:-"+2.99"} DTIM_latest_005064=${DTIM_latest_005064:-"+2.99"} +DTIM_earliest_005065=${DTIM_earliest_005065:-"-3.00"} DTIM_earliest_005065=${DTIM_earliest_005065:-"-3.00"} +DTIM_latest_005065=${DTIM_latest_005065:-"+2.99"} DTIM_latest_005065=${DTIM_latest_005065:-"+2.99"} +DTIM_earliest_005066=${DTIM_earliest_005066:-"-3.00"} DTIM_earliest_005066=${DTIM_earliest_005066:-"-3.00"} +DTIM_latest_005066=${DTIM_latest_005066:-"+2.99"} DTIM_latest_005066=${DTIM_latest_005066:-"+2.99"} +DTIM_earliest_005067=${DTIM_earliest_005067:-"-3.00"} DTIM_earliest_005067=${DTIM_earliest_005067:-"-3.00"} +DTIM_latest_005067=${DTIM_latest_005067:-"+2.99"} DTIM_latest_005067=${DTIM_latest_005067:-"+2.99"} +DTIM_earliest_005068=${DTIM_earliest_005068:-"-3.00"} DTIM_earliest_005068=${DTIM_earliest_005068:-"-3.00"} +DTIM_latest_005068=${DTIM_latest_005068:-"+2.99"} DTIM_latest_005068=${DTIM_latest_005068:-"+2.99"} +DTIM_earliest_005069=${DTIM_earliest_005069:-"-3.00"} DTIM_earliest_005069=${DTIM_earliest_005069:-"-3.00"} +DTIM_latest_005069=${DTIM_latest_005069:-"+2.99"} DTIM_latest_005069=${DTIM_latest_005069:-"+2.99"} +DTIM_earliest_005070=${DTIM_earliest_005070:-"-3.00"} DTIM_earliest_005070=${DTIM_earliest_005070:-"-3.00"} +DTIM_latest_005070=${DTIM_latest_005070:-"+2.99"} DTIM_latest_005070=${DTIM_latest_005070:-"+2.99"} +DTIM_earliest_005071=${DTIM_earliest_005071:-"-3.00"} DTIM_earliest_005071=${DTIM_earliest_005071:-"-3.00"} +DTIM_latest_005071=${DTIM_latest_005071:-"+2.99"} DTIM_latest_005071=${DTIM_latest_005071:-"+2.99"} +DTIM_earliest_005072=${DTIM_earliest_005072:-"-3.00"} DTIM_earliest_005072=${DTIM_earliest_005072:-"-3.00"} +DTIM_latest_005072=${DTIM_latest_005072:-"+2.99"} DTIM_latest_005072=${DTIM_latest_005072:-"+2.99"} +DTIM_earliest_005080=${DTIM_earliest_005080:-"-3.00"} DTIM_earliest_005080=${DTIM_earliest_005080:-"-3.00"} +DTIM_latest_005080=${DTIM_latest_005080:-"+2.99"} DTIM_latest_005080=${DTIM_latest_005080:-"+2.99"} +DTIM_earliest_005081=${DTIM_earliest_005081:-"-3.00"} DTIM_earliest_005081=${DTIM_earliest_005081:-"-3.00"} +DTIM_latest_005081=${DTIM_latest_005081:-"+2.99"} DTIM_latest_005081=${DTIM_latest_005081:-"+2.99"} +DTIM_earliest_005091=${DTIM_earliest_005091:-"-3.00"} DTIM_earliest_005091=${DTIM_earliest_005091:-"-3.00"} +DTIM_latest_005091=${DTIM_latest_005091:-"+2.99"} DTIM_latest_005091=${DTIM_latest_005091:-"+2.99"} + + + +TIME_TRIM=${TIME_TRIM:-${TIME_TRIM8:-on}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM8:-on}} + +$ushscript_dump/bufr_dump_obs.sh $dumptime 1.5 1 satwnd $ushscript_dump/bufr_dump_obs.sh $dumptime 1.5 1 satwnd +error8=$? error8=$? +echo "$error8" > $DATA/error8 echo "$error8" > $DATA/error8 + +if [ "$SENDDBN" = "YES" ]; then if [ "$SENDDBN" = "YES" ]; then + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_satwnd $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_satwnd $job \ + ${COMSP}satwnd.tm00.bufr_d ${COMSP}satwnd.tm00.bufr_d +fi fi + +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_8 echo Script thread_8 +echo Finished executing on node `hostname` echo Finished executing on node `hostname` +echo Ending time : `date -u` echo Ending time : `date -u` +echo "********************************************************************" echo "********************************************************************" +set -x set -x +} > $DATA/8.out 2>&1 } > $DATA/8.out 2>&1 +EOF EOF +set -x set -x + +set +x set +x +#---------------------------------------------------------------- #---------------------------------------------------------------- +cat<<\EOF>thread_9; chmod +x thread_9 cat<<\EOF>thread_9; chmod +x thread_9 +set -uax set -uax + +cd $DATA cd $DATA + +{ echo { echo +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_9 echo Script thread_9 +echo Executing on node `hostname` echo Executing on node `hostname` +echo Starting time: `date -u` echo Starting time: `date -u` +echo "********************************************************************" echo "********************************************************************" +echo echo +set -x set -x + +export STATUS=NO export STATUS=NO +export DUMP_NUMBER=9 export DUMP_NUMBER=9 + +#======================================================================= #======================================================================= +# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: +# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types +# EXCEPT: GEOIMR where it is -0.50 to +0.50 hour # EXCEPT: GEOIMR where it is -0.50 to +0.50 hour +# (2) TIME TRIMMING IS DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS DONE IN THIS DUMP (default, unless overridden) +# # +#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- +# Dump # 9 : GEOIMR: 1 subtype(s) # Dump # 9 : GEOIMR: 1 subtype(s) +# -------------------- # -------------------- +# TOTAL NUMBER OF SUBTYPES = 1 # TOTAL NUMBER OF SUBTYPES = 1 +# # +#======================================================================= #======================================================================= +DTIM_earliest_gmi1cr=${DTIM_earliest_gmi1cr:-"-3.00"} DTIM_earliest_gmi1cr=${DTIM_earliest_gmi1cr:-"-3.00"} +DTIM_latest_gmi1cr=${DTIM_latest_gmi1cr:-"+2.99"} DTIM_latest_gmi1cr=${DTIM_latest_gmi1cr:-"+2.99"} + +DTIM_earliest_geoimr=${DTIM_earliest_geoimr:-"-0.50"} DTIM_earliest_geoimr=${DTIM_earliest_geoimr:-"-0.50"} +DTIM_latest_geoimr=${DTIM_latest_geoimr:-"+0.50"} DTIM_latest_geoimr=${DTIM_latest_geoimr:-"+0.50"} + +TIME_TRIM=${TIME_TRIM:-${TIME_TRIM9:-on}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM9:-on}} + +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 geoimr gmi1cr $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 geoimr gmi1cr + +error9=$? error9=$? +echo "$error9" > $DATA/error9 echo "$error9" > $DATA/error9 + +if [ "$SENDDBN" = "YES" ]; then if [ "$SENDDBN" = "YES" ]; then + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_geoimr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_geoimr $job \ + ${COMSP}geoimr.tm00.bufr_d ${COMSP}geoimr.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gmi1cr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gmi1cr $job \ + ${COMSP}gmi1cr.tm00.bufr_d ${COMSP}gmi1cr.tm00.bufr_d +fi fi + +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_9 echo Script thread_9 +echo Finished executing on node `hostname` echo Finished executing on node `hostname` +echo Ending time : `date -u` echo Ending time : `date -u` +echo "********************************************************************" echo "********************************************************************" +set -x set -x +} > $DATA/9.out 2>&1 } > $DATA/9.out 2>&1 +EOF EOF +set -x set -x + +set +x set +x +#------------------------------------------------------------------------------ #------------------------------------------------------------------------------ +cat<<\EOF>thread_10; chmod +x thread_10 cat<<\EOF>thread_10; chmod +x thread_10 +set -uax set -uax + +cd $DATA cd $DATA + +{ echo { echo +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_10 echo Script thread_10 +echo Executing on node `hostname` echo Executing on node `hostname` +echo Starting time: `date -u` echo Starting time: `date -u` +echo "********************************************************************" echo "********************************************************************" +echo echo +set -x set -x + +export STATUS=NO export STATUS=NO +export DUMP_NUMBER=10 export DUMP_NUMBER=10 + +#========================================================================= #========================================================================= +# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: +# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types +# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) +# # +#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- +# Dump #10 : ESIASI: 1 subtype(s) # Dump #10 : ESIASI: 1 subtype(s) +# MTIASI: 1 subtype(s) # MTIASI: 1 subtype(s) +# ESAMUA: 1 subtype(s) # ESAMUA: 1 subtype(s) +# CRSFDB: 1 subtype(s) < +# IASIDB: 1 subtype(s) < +# SEVASR: 1 subtype(s) # SEVASR: 1 subtype(s) +# 1BAMUA: 1 subtype(s) # 1BAMUA: 1 subtype(s) +# BATHY: 1 subtype(s) # BATHY: 1 subtype(s) +# OSBUV8: 1 subtype(s) # OSBUV8: 1 subtype(s) +# OMPSN8: 1 subtype(s) # OMPSN8: 1 subtype(s) +# OMPST8: 1 subtype(s) # OMPST8: 1 subtype(s) +# GSRASR: 1 subtype(s) # GSRASR: 1 subtype(s) +# OMPSLP: 1 subtype(s) # OMPSLP: 1 subtype(s) +# -------------------- # -------------------- +# TOTAL NUMBER OF SUBTYPES = 13 # TOTAL NUMBER OF SUBTYPES = 13 +# # +#========================================================================= #========================================================================= + +DTIM_latest_esiasi=${DTIM_latest_esiasi:-"+2.99"} DTIM_latest_esiasi=${DTIM_latest_esiasi:-"+2.99"} +DTIM_latest_mtiasi=${DTIM_latest_mtiasi:-"+2.99"} DTIM_latest_mtiasi=${DTIM_latest_mtiasi:-"+2.99"} +DTIM_latest_esamua=${DTIM_latest_esamua:-"+2.99"} DTIM_latest_esamua=${DTIM_latest_esamua:-"+2.99"} +DTIM_latest_crsfdb=${DTIM_latest_crsfdb:-"+2.99"} < +DTIM_latest_iasidb=${DTIM_latest_iasidb:-"+2.99"} < +DTIM_latest_sevasr=${DTIM_latest_sevasr:-"+2.99"} DTIM_latest_sevasr=${DTIM_latest_sevasr:-"+2.99"} +DTIM_latest_1bamua=${DTIM_latest_1bamua:-"+2.99"} DTIM_latest_1bamua=${DTIM_latest_1bamua:-"+2.99"} +DTIM_latest_bathy=${DTIM_latest_bathy:-"+2.99"} DTIM_latest_bathy=${DTIM_latest_bathy:-"+2.99"} +DTIM_latest_osbuv8=${DTIM_latest_osbuv8:-"+2.99"} DTIM_latest_osbuv8=${DTIM_latest_osbuv8:-"+2.99"} +DTIM_latest_ompsn8=${DTIM_latest_ompsn8:-"+2.99"} DTIM_latest_ompsn8=${DTIM_latest_ompsn8:-"+2.99"} +DTIM_latest_ompst8=${DTIM_latest_ompst8:-"+2.99"} DTIM_latest_ompst8=${DTIM_latest_ompst8:-"+2.99"} +DTIM_latest_gsrasr=${DTIM_latest_gsrasr:-"+2.99"} DTIM_latest_gsrasr=${DTIM_latest_gsrasr:-"+2.99"} +DTIM_latest_ompslp=${DTIM_latest_ompslp:-"+2.99"} DTIM_latest_ompslp=${DTIM_latest_ompslp:-"+2.99"} +DTIM_latest_sstvpw=${DTIM_latest_sstvpw:-"+2.99"} DTIM_latest_sstvpw=${DTIM_latest_sstvpw:-"+2.99"} + +TIME_TRIM=${TIME_TRIM:-${TIME_TRIM10:-off}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM10:-off}} + +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 esiasi mtiasi esamua \ $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 esiasi mtiasi esamua \ + crsfdb iasidb sevasr 1bamua bathy osbuv8 ompsn8 ompst8 gsrasr ompslp \ | sevasr 1bamua bathy osbuv8 ompsn8 ompst8 gsrasr ompslp sstvpw + sstvpw < +error10=$? error10=$? +echo "$error10" > $DATA/error10 echo "$error10" > $DATA/error10 + +if [ "$SENDDBN" = "YES" ]; then if [ "$SENDDBN" = "YES" ]; then + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_esiasi $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_esiasi $job \ + ${COMSP}esiasi.tm00.bufr_d ${COMSP}esiasi.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_mtiasi $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_mtiasi $job \ + ${COMSP}mtiasi.tm00.bufr_d ${COMSP}mtiasi.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_esamua $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_esamua $job \ + ${COMSP}esamua.tm00.bufr_d ${COMSP}esamua.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_crsfdb $job \ < + ${COMSP}crsfdb.tm00.bufr_d < + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_iasidb $job \ < + ${COMSP}iasidb.tm00.bufr_d < + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sevasr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sevasr $job \ + ${COMSP}sevasr.tm00.bufr_d ${COMSP}sevasr.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_1bamua $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_1bamua $job \ + ${COMSP}1bamua.tm00.bufr_d ${COMSP}1bamua.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_bathy $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_bathy $job \ + ${COMSP}bathy.tm00.bufr_d ${COMSP}bathy.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_osbuv8 $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_osbuv8 $job \ + ${COMSP}osbuv8.tm00.bufr_d ${COMSP}osbuv8.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ompsn8 $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ompsn8 $job \ + ${COMSP}ompsn8.tm00.bufr_d ${COMSP}ompsn8.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ompst8 $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ompst8 $job \ + ${COMSP}ompst8.tm00.bufr_d ${COMSP}ompst8.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gsrasr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gsrasr $job \ + ${COMSP}gsrasr.tm00.bufr_d ${COMSP}gsrasr.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ompslp $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ompslp $job \ + ${COMSP}ompslp.tm00.bufr_d ${COMSP}ompslp.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sstvpw $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sstvpw $job \ + ${COMSP}sstvpw.tm00.bufr_d ${COMSP}sstvpw.tm00.bufr_d +fi fi + +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_10 echo Script thread_10 +echo Finished executing on node `hostname` echo Finished executing on node `hostname` +echo Ending time : `date -u` echo Ending time : `date -u` +echo "********************************************************************" echo "********************************************************************" +set -x set -x +} > $DATA/10.out 2>&1 } > $DATA/10.out 2>&1 +EOF EOF +set -x set -x + +set +x set +x +#------------------------------------------------------------------------------ #------------------------------------------------------------------------------ +cat<<\EOF>thread_11; chmod +x thread_11 cat<<\EOF>thread_11; chmod +x thread_11 +set -uax set -uax + +cd $DATA cd $DATA + +{ echo { echo +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_11 echo Script thread_11 +echo Executing on node `hostname` echo Executing on node `hostname` +echo Starting time: `date -u` echo Starting time: `date -u` +echo "********************************************************************" echo "********************************************************************" +echo echo +set -x set -x + +export STATUS=NO export STATUS=NO +export DUMP_NUMBER=11 export DUMP_NUMBER=11 + +#========================================================================= #========================================================================= +# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: +# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types +# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) +# # +#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- +# Dump #11 : AMSR2: 1 subtype(s) # Dump #11 : AMSR2: 1 subtype(s) +# -------------------- # -------------------- +# TOTAL NUMBER OF SUBTYPES = 1 # TOTAL NUMBER OF SUBTYPES = 1 +# # +#========================================================================= #========================================================================= + +DTIM_latest_amsr2=${DTIM_latest_amsr2:-"+2.99"} DTIM_latest_amsr2=${DTIM_latest_amsr2:-"+2.99"} + +TIME_TRIM=${TIME_TRIM:-${TIME_TRIM11:-off}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM11:-off}} + +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 amsr2 $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 amsr2 +error11=$? error11=$? +echo "$error11" > $DATA/error11 echo "$error11" > $DATA/error11 + +if [ "$SENDDBN" = "YES" ]; then if [ "$SENDDBN" = "YES" ]; then + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_amsr2 $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_amsr2 $job \ + ${COMSP}amsr2.tm00.bufr_d ${COMSP}amsr2.tm00.bufr_d +fi fi + +set +x set +x +echo "********************************************************************" echo "********************************************************************" +echo Script thread_11 echo Script thread_11 +echo Finished executing on node `hostname` echo Finished executing on node `hostname` +echo Ending time : `date -u` echo Ending time : `date -u` +echo "********************************************************************" echo "********************************************************************" +set -x set -x +} > $DATA/11.out 2>&1 } > $DATA/11.out 2>&1 +EOF EOF +set -x set -x + + > set +x + > #---------------------------------------------------------------- + > cat<<\EOF>thread_12; chmod +x thread_12 + > set -uax + > + > cd $DATA + > + > { echo + > set +x + > echo "********************************************************************" + > echo Script thread_12 + > echo Executing on node `hostname` + > echo Starting time: `date -u` + > echo "********************************************************************" + > echo + > set -x + > + > export STATUS=NO + > export DUMP_NUMBER=12 + > + > #========================================================================= + > # NOTES ABOUT THIS DUMP GROUP: + > # (1) time window radius is -3.00 to +2.99 hours on all types + > # (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) + > # + > #-------------------------------------------------------------------------- + > # Dump # 12 : ATMS: 1 subtype(s) (if present in past 10 days of tanks) + > # CRISFS: 1 subtype(s) (if present in past 10 days of tanks) + > # CRSFDB: 1 subtype(s) + > # IASIDB: 1 subtype(s) + > # -------------------- + > # TOTAL NUMBER OF SUBTYPES = 4 + > # + > #========================================================================= + > #----------------------------------------------- + > #----------------------------------------------- + > # check for atms tank presence in past 10 days + > atms="" + > err_check_tanks=0 + > sh $USHbufr_dump/check_tanks.sh atms + > err_check_tanks=$? + > if [ $err_check_tanks -eq 0 ];then + > atms=atms + > DTIM_latest_atms=${DTIM_latest_atms:-"+2.99"} + > fi + > #----------------------------------------------- + > #----------------------------------------------- + > # check for crisfs tank presence in past 10 days + > crisfs="" + > err_check_tanks=0 + > sh $USHbufr_dump/check_tanks.sh crisfs + > err_check_tanks=$? + > if [ $err_check_tanks -eq 0 ];then + > crisfs=crisfs + > DTIM_latest_crisfs=${DTIM_latest_crisfs:-"+2.99"} + > fi + > #----------------------------------------------- + > + > DTIM_latest_crsfdb=${DTIM_latest_crsfdb:-"+2.99"} + > DTIM_latest_iasidb=${DTIM_latest_iasidb:-"+2.99"} + > + > TIME_TRIM=${TIME_TRIM:-${TIME_TRIM1:-off}} + > + > $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 $atms $crisfs crsfdb iasidb + > error12=$? + > echo "$error12" > $DATA/error12 + > + > if [ "$SENDDBN" = "YES" ]; then + > if [ "$atms" = atms ];then + > $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_atms $job \ + > ${COMSP}atms.tm00.bufr_d + > fi + > ####### ALERTS TURNED OFF UNTIL REQUESTED BY USER ######################### + > # if [ "$crisfs" = crisfs ];then + > # $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_crisfs $job \ + > # ${COMSP}crisfs.tm00.bufr_d + > # fi + > ########################################################################### + > $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_crsfdb $job \ + > ${COMSP}crsfdb.tm00.bufr_d + > $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_iasidb $job \ + > ${COMSP}iasidb.tm00.bufr_d + > fi + > + > set +x + > echo "********************************************************************" + > echo Script thread_12 + > echo Finished executing on node `hostname` + > echo Ending time : `date -u` + > echo "********************************************************************" + > set -x + > } > $DATA/12.out 2>&1 + > EOF + > set -x + +#---------------------------------------------------------------- #---------------------------------------------------------------- +# Now launch the threads # Now launch the threads + +# determine local system name and type if available # determine local system name and type if available +# ------------------------------------------------- # ------------------------------------------------- + +SITE=${SITE:-""} SITE=${SITE:-""} + +set +u set +u +launcher=${launcher:-"cfp"} # if not "cfp", threads will be run serially. launcher=${launcher:-"cfp"} # if not "cfp", threads will be run serially. + +if [ "$launcher" = cfp ]; then if [ "$launcher" = cfp ]; then + > $DATA/poe.cmdfile > $DATA/poe.cmdfile +# To better take advantage of cfp, execute the longer running commands first. # To better take advantage of cfp, execute the longer running commands first. +# Some reordering was done here based on recent sample runtimes. # Some reordering was done here based on recent sample runtimes. + [ $DUMP_group7 = YES ] && echo ./thread_7 >> $DATA/poe.cmdfile # moved up [ $DUMP_group7 = YES ] && echo ./thread_7 >> $DATA/poe.cmdfile # moved up + [ $DUMP_group1 = YES ] && echo ./thread_1 >> $DATA/poe.cmdfile [ $DUMP_group1 = YES ] && echo ./thread_1 >> $DATA/poe.cmdfile + [ $DUMP_group5 = YES ] && echo ./thread_5 >> $DATA/poe.cmdfile # moved up [ $DUMP_group5 = YES ] && echo ./thread_5 >> $DATA/poe.cmdfile # moved up + [ $DUMP_group6 = YES ] && echo ./thread_6 >> $DATA/poe.cmdfile # moved up [ $DUMP_group6 = YES ] && echo ./thread_6 >> $DATA/poe.cmdfile # moved up + [ $DUMP_group8 = YES ] && echo ./thread_8 >> $DATA/poe.cmdfile # moved up [ $DUMP_group8 = YES ] && echo ./thread_8 >> $DATA/poe.cmdfile # moved up + [ $DUMP_group11 = YES ] && echo ./thread_11 >> $DATA/poe.cmdfile # moved up [ $DUMP_group11 = YES ] && echo ./thread_11 >> $DATA/poe.cmdfile # moved up + [ $DUMP_group10 = YES ] && echo ./thread_10 >> $DATA/poe.cmdfile # moved up [ $DUMP_group10 = YES ] && echo ./thread_10 >> $DATA/poe.cmdfile # moved up + [ $DUMP_group2 = YES ] && echo ./thread_2 >> $DATA/poe.cmdfile [ $DUMP_group2 = YES ] && echo ./thread_2 >> $DATA/poe.cmdfile + [ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && echo ./thread_3 >> $DATA/poe.cmdfile [ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && echo ./thread_3 >> $DATA/poe.cmdfile + [ $DUMP_group4 = YES ] && echo ./thread_4 >> $DATA/poe.cmdfile [ $DUMP_group4 = YES ] && echo ./thread_4 >> $DATA/poe.cmdfile + [ $DUMP_group9 = YES ] && echo ./thread_9 >> $DATA/poe.cmdfile [ $DUMP_group9 = YES ] && echo ./thread_9 >> $DATA/poe.cmdfile + > [ $DUMP_group12 = YES ] && echo ./thread_12 >> $DATA/poe.cmdfile + + + if [ -s $DATA/poe.cmdfile ]; then if [ -s $DATA/poe.cmdfile ]; then + export MP_CSS_INTERRUPT=yes export MP_CSS_INTERRUPT=yes + launcher_DUMP=${launcher_DUMP:-mpiexec} launcher_DUMP=${launcher_DUMP:-mpiexec} + $launcher_DUMP -np 14 --cpu-bind verbose,core cfp $DATA/poe.cmdfile 2>&1 $launcher_DUMP -np 14 --cpu-bind verbose,core cfp $DATA/poe.cmdfile 2>&1 + errpoe=$? errpoe=$? + if [ $errpoe -ne 0 ]; then if [ $errpoe -ne 0 ]; then + $DATA/err_exit "***FATAL: EXIT STATUS $errpoe RUNNING POE COMMAND FILE" $DATA/err_exit "***FATAL: EXIT STATUS $errpoe RUNNING POE COMMAND FILE" + fi fi + else else + echo echo + echo "==> There are no tasks in POE Command File - POE not run" echo "==> There are no tasks in POE Command File - POE not run" + echo echo + fi fi +else else + echo "Running threads serially" echo "Running threads serially" + [ $DUMP_group1 = YES ] && ./thread_1 [ $DUMP_group1 = YES ] && ./thread_1 + [ $DUMP_group2 = YES ] && ./thread_2 [ $DUMP_group2 = YES ] && ./thread_2 + [ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && ./thread_3 [ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && ./thread_3 + [ $DUMP_group4 = YES ] && ./thread_4 [ $DUMP_group4 = YES ] && ./thread_4 + [ $DUMP_group5 = YES ] && ./thread_5 [ $DUMP_group5 = YES ] && ./thread_5 + [ $DUMP_group6 = YES ] && ./thread_6 [ $DUMP_group6 = YES ] && ./thread_6 + [ $DUMP_group7 = YES ] && ./thread_7 [ $DUMP_group7 = YES ] && ./thread_7 + [ $DUMP_group8 = YES ] && ./thread_8 [ $DUMP_group8 = YES ] && ./thread_8 + [ $DUMP_group9 = YES ] && ./thread_9 [ $DUMP_group9 = YES ] && ./thread_9 + [ $DUMP_group10 = YES ] && ./thread_10 [ $DUMP_group10 = YES ] && ./thread_10 + [ $DUMP_group11 = YES ] && ./thread_11 [ $DUMP_group11 = YES ] && ./thread_11 + > [ $DUMP_group12 = YES ] && ./thread_12 +# wait # wait +fi fi + +# if ADPUPA_wait is YES, adpupa is dumped AFTER all other dump threads have # if ADPUPA_wait is YES, adpupa is dumped AFTER all other dump threads have +# run (normally done in real-time GFS runs to dump as late as possible in # run (normally done in real-time GFS runs to dump as late as possible in +# order to maximize data availability in GFS network, particularly DROPs) # order to maximize data availability in GFS network, particularly DROPs) +# -------------------------------------------------------------------------- # -------------------------------------------------------------------------- + +[ $DUMP_group3 = YES -a $ADPUPA_wait = YES ] && ./thread_3 [ $DUMP_group3 = YES -a $ADPUPA_wait = YES ] && ./thread_3 + +cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.o | cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.o + +set +x set +x +echo " " echo " " +echo " " echo " " +set -x set -x + +[ -s $DATA/error1 ] && err1=`cat $DATA/error1` [ -s $DATA/error1 ] && err1=`cat $DATA/error1` +[ -s $DATA/error2 ] && err2=`cat $DATA/error2` [ -s $DATA/error2 ] && err2=`cat $DATA/error2` +[ -s $DATA/error3 ] && err3=`cat $DATA/error3` [ -s $DATA/error3 ] && err3=`cat $DATA/error3` +[ -s $DATA/error4 ] && err4=`cat $DATA/error4` [ -s $DATA/error4 ] && err4=`cat $DATA/error4` +[ -s $DATA/error5 ] && err5=`cat $DATA/error5` [ -s $DATA/error5 ] && err5=`cat $DATA/error5` +[ -s $DATA/error6 ] && err6=`cat $DATA/error6` [ -s $DATA/error6 ] && err6=`cat $DATA/error6` +[ -s $DATA/error7 ] && err7=`cat $DATA/error7` [ -s $DATA/error7 ] && err7=`cat $DATA/error7` +[ -s $DATA/error8 ] && err8=`cat $DATA/error8` [ -s $DATA/error8 ] && err8=`cat $DATA/error8` +[ -s $DATA/error9 ] && err9=`cat $DATA/error9` [ -s $DATA/error9 ] && err9=`cat $DATA/error9` +[ -s $DATA/error10 ] && err10=`cat $DATA/error10` [ -s $DATA/error10 ] && err10=`cat $DATA/error10` +[ -s $DATA/error11 ] && err11=`cat $DATA/error11` [ -s $DATA/error11 ] && err11=`cat $DATA/error11` + > [ -s $DATA/error12 ] && err12=`cat $DATA/error12` + + +#=============================================================================== #=============================================================================== + +export STATUS=YES export STATUS=YES +export DUMP_NUMBER=12 | export DUMP_NUMBER=13 +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.00 1 null $ushscript_dump/bufr_dump_obs.sh $dumptime 3.00 1 null + +# endif loop $PROCESS_DUMP # endif loop $PROCESS_DUMP +fi fi + +echo " " >> $pgmout echo " " >> $pgmout +echo "##################################################################\ echo "##################################################################\ +####################" >> $pgmout ####################" >> $pgmout +echo " " >> $pgmout echo " " >> $pgmout + +#================================================================ #================================================================ +#================================================================ #================================================================ + + +if [ "$PROCESS_DUMP" = 'YES' ]; then if [ "$PROCESS_DUMP" = 'YES' ]; then + + if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o \ if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o \ + "$err4" -gt '5' -o "$err5" -gt '5' -o "$err6" -gt '5' -o \ "$err4" -gt '5' -o "$err5" -gt '5' -o "$err6" -gt '5' -o \ + "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' -o \ "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' -o \ + "$err10" -gt '5' -o "$err11" -gt '5' ]; then | "$err10" -gt '5' -o "$err11" -gt '5' -o "$err12" -gt '5' ]; then + for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 $err11 | for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 $err11 $err12 + do do + if [ "$n" -gt '5' ]; then if [ "$n" -gt '5' ]; then + if [ "$n" -ne '11' -a "$n" -ne '22' ]; then if [ "$n" -ne '11' -a "$n" -ne '22' ]; then + +## fatal error in dumping of BUFR obs. files ## fatal error in dumping of BUFR obs. files + + set +x set +x +echo echo +echo " ###################################################### " echo " ###################################################### " +echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ +$err5, $err6, $err7, $err8, $err9, $err10, $err11 " | $err5, $err6, $err7, $err8, $err9, $err10, $err11, $err12 " +echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " +echo " ###################################################### " echo " ###################################################### " +echo echo + set -x set -x + $DATA/err_exit $DATA/err_exit + exit 9 exit 9 + fi fi + fi fi + done done + +## a status code of 11 or 22 from dumping of BUFR obs. files ## a status code of 11 or 22 from dumping of BUFR obs. files +## is non-fatal but still worth noting ## is non-fatal but still worth noting + + set +x set +x + echo echo + echo " ###################################################### " echo " ###################################################### " + echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ +$err5, $err6, $err7, $err8, $err9, $err10, $err11 " | $err5, $err6, $err7, $err8, $err9, $err10, $err11, $err12 " + echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " + echo " ###################################################### " echo " ###################################################### " + echo echo + set -x set -x + fi fi + +# endif loop $PROCESS_DUMP # endif loop $PROCESS_DUMP +fi fi + +# # +# copy bufr_dumplist to $COMOUT per NCO SPA request # copy bufr_dumplist to $COMOUT per NCO SPA request +# ------------------------------------------------- # ------------------------------------------------- +echo "Copy bufr_dumplist to comout" echo "Copy bufr_dumplist to comout" +LIST_cp=$COMOUT/${RUN}.t${cyc}z.bufr_dumplist.${tmmark} LIST_cp=$COMOUT/${RUN}.t${cyc}z.bufr_dumplist.${tmmark} +cp ${FIXbufr_dump}/bufr_dumplist $LIST_cp cp ${FIXbufr_dump}/bufr_dumplist $LIST_cp +chmod 644 $LIST_cp chmod 644 $LIST_cp + +# GOOD RUN # GOOD RUN +set +x set +x +echo " " echo " " +echo " ****** PROCESSING COMPLETED NORMALLY" echo " ****** PROCESSING COMPLETED NORMALLY" +echo " ****** PROCESSING COMPLETED NORMALLY" echo " ****** PROCESSING COMPLETED NORMALLY" +echo " ****** PROCESSING COMPLETED NORMALLY" echo " ****** PROCESSING COMPLETED NORMALLY" +echo " ****** PROCESSING COMPLETED NORMALLY" echo " ****** PROCESSING COMPLETED NORMALLY" +echo " " echo " " +set -x set -x + + +# save standard output # save standard output +cat break $pgmout break > allout cat break $pgmout break > allout +cat allout cat allout +# rm allout # rm allout + +sleep 10 sleep 10 + +msg='ENDED NORMALLY.' msg='ENDED NORMALLY.' +$DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" + +################## END OF SCRIPT ####################### ################## END OF SCRIPT ####################### diff --git a/scripts/exglobal_dump.sh b/scripts/exglobal_dump.sh index e906d8e..d2ce901 100755 --- a/scripts/exglobal_dump.sh +++ b/scripts/exglobal_dump.sh @@ -119,7 +119,7 @@ set +u # esatms gsrcsr ahicsr sstvcw subpfl saldrn # # Dump group #2 (pb, TIME_TRIM defaults to OFF) = -# sfcshp tideg atovs* adpsfc ascatt +# sfcshp tideg atovs* adpsfc ascatt snocvr # * - for GDAS only # # Dump group #3 (pb, TIME_TRIM defaults to OFF) = @@ -142,8 +142,7 @@ set +u # satwnd # # Dump group #9 (non-pb, TIME_TRIM defaults to ON) = -# geoimr -# +# geoimr gmi1cr satwhr # Dump group #10 (non-pb, TIME_TRIM defaults to OFF) = # esiasi mtiasi esamua sevasr 1bamua bathy osbuv8 # ompst8 ompsn8 gsrasr ompslp sstvpw @@ -582,6 +581,8 @@ export DUMP_NUMBER=1 # GSRCSR: 1 subtype(s) # AHICSR: 1 subtype(s) # SSTVCW: 1 subtype(s) +# SUBPFL: 1 subtype(s) +# SALDRN: 1 subtype(s) # -------------------- # TOTAL NUMBER OF SUBTYPES = 14 # @@ -711,11 +712,12 @@ export DUMP_NUMBER=2 # ATOVS: 1 subtype(s) # ADPSFC: 7 subtype(s) # ASCATT: 1 subtype(s) +# SNOCVR: 1 subtype(s) # xxxxxxxxx WNDSAT: 1 subtype(s) (if present in past 10 days of tanks) # ===> Dumping of WNDSAT removed from here until new ingest feed is established # (had been dumped with a time window radius of -3.00 to +2.99 hours) # -------------------- -# TOTAL NUMBER OF SUBTYPES = 20 - 21 +# TOTAL NUMBER OF SUBTYPES = 21 - 22 # #-------------------------------------------------------------------------- # GFS: @@ -723,11 +725,12 @@ export DUMP_NUMBER=2 # TIDEG: 1 subtype(s) # ADPSFC: 7 subtype(s) # ASCATT: 1 subtype(s) +# SNOCVR: 1 subtype(s) # xxxxxxxxx WNDSAT: 1 subtype(s) (if present in past 10 days of tanks) # ===> Dumping of WNDSAT removed from here until new ingest feed is established # (had been dumped with a time window radius of -3.00 to +2.99 hours) # -------------------- -# TOTAL NUMBER OF SUBTYPES = 19 - 20 +# TOTAL NUMBER OF SUBTYPES = 21 - 22 # #========================================================================== DTIM_latest_snocvr=${DTIM_latest_snocvr:-"+2.99"} @@ -1406,19 +1409,24 @@ export DUMP_NUMBER=9 # #-------------------------------------------------------------------------- # Dump # 9 : GEOIMR: 1 subtype(s) +# GMI1CR: 1 subtype(s) +# SATWHR: 1 subtype(s) # -------------------- -# TOTAL NUMBER OF SUBTYPES = 1 +# TOTAL NUMBER OF SUBTYPES = 3 # #======================================================================= DTIM_earliest_gmi1cr=${DTIM_earliest_gmi1cr:-"-3.00"} DTIM_latest_gmi1cr=${DTIM_latest_gmi1cr:-"+2.99"} +DTIM_earliest_satwhr=${DTIM_earliest_satwhr:-"-3.00"} +DTIM_latest_satwhr=${DTIM_latest_satwhr:-"+2.99"} + DTIM_earliest_geoimr=${DTIM_earliest_geoimr:-"-0.50"} DTIM_latest_geoimr=${DTIM_latest_geoimr:-"+0.50"} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM9:-on}} -$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 geoimr gmi1cr +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 geoimr gmi1cr satwhr error9=$? echo "$error9" > $DATA/error9 From b1bd5b9320211d9884da944b066e983e65ef4305 Mon Sep 17 00:00:00 2001 From: PraveenKumar-NOAA <63739712+PraveenKumar-NOAA@users.noreply.github.com> Date: Tue, 18 Apr 2023 15:52:43 -0400 Subject: [PATCH 28/53] Updated exrtma_dump.sh to fix rtma_ru 0000Z cycle failure issue (#81) Co-authored-by: praveen kumar --- scripts/exrtma_dump.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/exrtma_dump.sh b/scripts/exrtma_dump.sh index a6489eb..bf774d1 100755 --- a/scripts/exrtma_dump.sh +++ b/scripts/exrtma_dump.sh @@ -243,9 +243,10 @@ DTIM_earliest_saldrn=-0.50 DTIM_latest_saldrn=+0.50 # for rtma_ru_0000 read only from previous day's tank -# Temporary bug fix +echo "DTIM_latest_000000=-0.01 changed to DTIM_latest_000000=0.5 by Praveen on 03/13/23" if [ $RUN = "rtma_ru" -a $cycle = "t0000z" ]; then - DTIM_latest_000000=-0.01 + DTIM_latest_000000=0.5 +# DTIM_latest_000000=-0.01 fi $ushscript_dump/bufr_dump_obs.sh $dumptime 0.5 1 sfcshp tideg adpsfc subpfl saldrn From 1e3982abeeccb963ac6266cbaee42c38fab955d6 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Thu, 6 Jul 2023 01:12:54 +0000 Subject: [PATCH 29/53] last testing --- jobs/JOBSPROC_RAP_PREP | 2 +- versions/run.ver | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/JOBSPROC_RAP_PREP b/jobs/JOBSPROC_RAP_PREP index fc0241c..f5cc510 100755 --- a/jobs/JOBSPROC_RAP_PREP +++ b/jobs/JOBSPROC_RAP_PREP @@ -225,7 +225,7 @@ export NSPLIT=${NSPLIT:-1} # BUFRLIST indicates the list of BUFR data dump file names to process -export BUFRLIST=${BUFRLIST:-"adpupa proflr aircar aircft satwnd adpsfc \ +export BUFRLIST=${BUFRLIST:-"uprair adpupa proflr aircar aircft satwnd adpsfc \ sfcshp vadwnd goesnd msonet gpsipw rassda ascatw wdsatr"} export errPREPDATA_limit=4 # allow program PREPOBS_PREPDATA to continue at all diff --git a/versions/run.ver b/versions/run.ver index 00d2b9e..6d70d84 100644 --- a/versions/run.ver +++ b/versions/run.ver @@ -6,7 +6,7 @@ export libjpeg_ver=9c export grib_util_ver=1.2.2 export netcdf_ver=4.7.4 export bufr_dump_ver=1.2.0 -export prepobs_ver=1.2.0 +export prepobs_ver=1.1.0 export intel_ver=19.1.3.304 export craype_ver=2.7.13 From cb1fb6837cc515002d7e4f760f442375bff13890 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Thu, 27 Jul 2023 23:49:49 +0000 Subject: [PATCH 30/53] Increase errPREP... from 0 to 4 --- jobs/JOBSPROC_CDAS_PREP1 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/jobs/JOBSPROC_CDAS_PREP1 b/jobs/JOBSPROC_CDAS_PREP1 index a4233e6..5072451 100755 --- a/jobs/JOBSPROC_CDAS_PREP1 +++ b/jobs/JOBSPROC_CDAS_PREP1 @@ -251,6 +251,13 @@ export NSPLIT=${NSPLIT:-12} export BUFRLIST=${BUFRLIST:-"adpupa proflr aircar aircft satwnd adpsfc \ sfcshp atovs"} +# I.G. 07/27/2023 +export errPREPDATA_limit=4 # allow program PREPOBS_PREPDATA to continue at all + # center hour (cycle) times if foreground exit + # status is 4 (i.e., when either either "ADPUPA" or + # "ADPSFC" dump is empty) + + # VARIABLES THAT CONTROL PROCESSING OF INDIVIDUAL PROGRAMS # -------------------------------------------------------- From 0dcb5b4fad224a68f8509633a2b331716031920e Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Thu, 27 Jul 2023 23:52:30 +0000 Subject: [PATCH 31/53] clean up --- CHANGES | 5 - fix_satwnd | 1918 ---------------------------------------------------- 2 files changed, 1923 deletions(-) delete mode 100644 CHANGES delete mode 100644 fix_satwnd diff --git a/CHANGES b/CHANGES deleted file mode 100644 index 3a25824..0000000 --- a/CHANGES +++ /dev/null @@ -1,5 +0,0 @@ --updated versions/* for obsproc --merged in develop (which had Ashley's new global dumping, i.e. add new group to utilize all CPUs) - - - diff --git a/fix_satwnd b/fix_satwnd deleted file mode 100644 index 0bc6b5d..0000000 --- a/fix_satwnd +++ /dev/null @@ -1,1918 +0,0 @@ -#!/bin/ksh #!/bin/ksh -############################################################################# ############################################################################# -echo "----------------------------------------------------------------------" echo "----------------------------------------------------------------------" -echo "exglobal_dump.sh - Global (GDAS, GFS) network data dump processing" echo "exglobal_dump.sh - Global (GDAS, GFS) network data dump processing" -echo "----------------------------------------------------------------------" echo "----------------------------------------------------------------------" -echo "History: Jan 18 2000 - Original script. " echo "History: Jan 18 2000 - Original script. " -echo " May 16 2007 - Added DBNet alerts for GFS products. " echo " May 16 2007 - Added DBNet alerts for GFS products. " -echo " Apr 2014 - Pick up grib files for planned GFS upgrade. " echo " Apr 2014 - Pick up grib files for planned GFS upgrade. " -echo " Oct 2014 - Remove attempts to dump obsolete sources. " echo " Oct 2014 - Remove attempts to dump obsolete sources. " -echo " Nov 2014 - Use parallel scripting to process dump groups. " echo " Nov 2014 - Use parallel scripting to process dump groups. " -echo " Widen hourly satwnd dump window for GFS/GDAS. " echo " Widen hourly satwnd dump window for GFS/GDAS. " -echo " Add new satwnd subtypes for GFS & GDAS. " echo " Add new satwnd subtypes for GFS & GDAS. " -echo " GFS/GDAS continue if surface file unavailable. " echo " GFS/GDAS continue if surface file unavailable. " -echo " Remove DBNet alerts for old surface files. " echo " Remove DBNet alerts for old surface files. " -echo " Dec 3 2014 - CDAS network, split off into its own script " echo " Dec 3 2014 - CDAS network, split off into its own script " -echo " excdas_dump.sh.ecf. This script now tailored " echo " excdas_dump.sh.ecf. This script now tailored " -echo " exclusively to GDAS and GFS. " echo " exclusively to GDAS and GFS. " -echo " Feb 2 2015 - Dump window for new satwnd type NC005090 set " echo " Feb 2 2015 - Dump window for new satwnd type NC005090 set " -echo " to 3.00 to +2.99 hours about center dump time. " echo " to 3.00 to +2.99 hours about center dump time. " -echo " Removed ADD_satwnd=\"005019 005080\" since " echo " Removed ADD_satwnd=\"005019 005080\" since " -echo " types are now part of "satwnd" dump group " echo " types are now part of "satwnd" dump group " -echo " mnemonic in bufr_dumplist. " echo " mnemonic in bufr_dumplist. " -echo " Aug 22 2016 - GSPIPW dump window reset for new data stream " echo " Aug 22 2016 - GSPIPW dump window reset for new data stream " -echo " (moved to dump group #4 where TIME_TRIM=on) " echo " (moved to dump group #4 where TIME_TRIM=on) " -echo " Jan 5 2017 - Dump new satellite data types. Reordered to " echo " Jan 5 2017 - Dump new satellite data types. Reordered to " -echo " improve run time with all the new data. " echo " improve run time with all the new data. " -echo " Feb 8 2017 - Update to run on Cray-XC40 or IBM iDataPlex " echo " Feb 8 2017 - Update to run on Cray-XC40 or IBM iDataPlex " -echo " Nov 13 2017 - Add dump of GOES-16 AMV's in tanks b005/xx030, " echo " Nov 13 2017 - Add dump of GOES-16 AMV's in tanks b005/xx030, " -echo " b005/xx031, b005/xx032, b005/xx034 and " echo " b005/xx031, b005/xx032, b005/xx034 and " -echo " b005/xx039. Set dump window to -3.00 to +2.99 " echo " b005/xx039. Set dump window to -3.00 to +2.99 " -echo " hours for these 5 new GOES-16 tanks. " echo " hours for these 5 new GOES-16 tanks. " -echo " Aug 10 2018 - Updated to run on Dell-p3 (as well as Cray-XC40" echo " Aug 10 2018 - Updated to run on Dell-p3 (as well as Cray-XC40" -echo " and IBM iDataPlex. " echo " and IBM iDataPlex. " -echo " Oct 24 2018 - Add dump of OMPS VSN8 nadir profile (NP) and " echo " Oct 24 2018 - Add dump of OMPS VSN8 nadir profile (NP) and " -echo " total column (TC), Indiasat AMV's in tanks " echo " total column (TC), Indiasat AMV's in tanks " -echo " b005/xx024 b005/xx025 b005/xx026. " echo " b005/xx024 b005/xx025 b005/xx026. " -echo " Mar 15 2019 - Add dumps of BUFR-feed drifting (NC001102) & " echo " Mar 15 2019 - Add dumps of BUFR-feed drifting (NC001102) & " -echo " moored (NC001203) buoys to DUMP group #2 " echo " moored (NC001203) buoys to DUMP group #2 " -echo " Aug 15 2019 - rm'd background threads; now run serially; " echo " Aug 15 2019 - rm'd background threads; now run serially; " -echo " added "./" on "thread_*", needed for Dell/ph3 " echo " added "./" on "thread_*", needed for Dell/ph3 " -echo " Disabled processing that copies sstoi files to " echo " Disabled processing that copies sstoi files to " -echo " COMSP. " echo " COMSP. " -echo " Oct 15 2019 - set DTIM_LATEST_00110[23]=+2.99, as in sfcshp " echo " Oct 15 2019 - set DTIM_LATEST_00110[23]=+2.99, as in sfcshp " -echo " Oct 26 2019 - set DTIM_EARLIEST_005091=-3.00 and " echo " Oct 26 2019 - set DTIM_EARLIEST_005091=-3.00 and " -echo " DTIM_LATEST_005091=+2.99. DTIM_* settings for " echo " DTIM_LATEST_005091=+2.99. DTIM_* settings for " -echo " 005090 are obsolete; added SKIP_005090=YES " echo " 005090 are obsolete; added SKIP_005090=YES " -echo " Jan 03 2020 - turn on gdas.tCCz.saphir.tm00.bufr_d alerts, " echo " Jan 03 2020 - turn on gdas.tCCz.saphir.tm00.bufr_d alerts, " -echo " per DF request " echo " per DF request " -echo " Apr 06 2020 - Update to remove crisdb and escris and replace " echo " Apr 06 2020 - Update to remove crisdb and escris and replace " -echo " with crsfdb and escrsf. On Apr 22, 2020, NSR " echo " with crsfdb and escrsf. On Apr 22, 2020, NSR " -echo " CrIS data from NPP is being replaced by FSR. " echo " CrIS data from NPP is being replaced by FSR. " -echo " The cris dump group is obsolete with the change" echo " The cris dump group is obsolete with the change" -echo " to FSR. " echo " to FSR. " -echo " Aug 20 2020 - Incremented subsets for the adpsfc and sfcshp " echo " Aug 20 2020 - Incremented subsets for the adpsfc and sfcshp " -echo " dump groups to match bufr_dumplist. " echo " dump groups to match bufr_dumplist. " -echo " Removed mbuoyb and dbuoyb from dump group #2. " echo " Removed mbuoyb and dbuoyb from dump group #2. " -echo " Removed obsolete dump groups escrsf and goesfv " echo " Removed obsolete dump groups escrsf and goesfv " -echo " from dump group #7. " echo " from dump group #7. " -echo " - In Dump group #8, disabled processing of legacy" echo " - In Dump group #8, disabled processing of legacy" -echo " EUMETSAT AMV subsets 005064, 005065, and " echo " EUMETSAT AMV subsets 005064, 005065, and " -echo " 005066. Added DTIM settings for new WMO BUFR " echo " 005066. Added DTIM settings for new WMO BUFR " -echo " format EUMETSAT AMV subsets 005067, 005068, and" echo " format EUMETSAT AMV subsets 005067, 005068, and" -echo " 005069. On Oct 6, 2020, EUMETSAT AMV format " echo " 005069. On Oct 6, 2020, EUMETSAT AMV format " -echo " changes to use new WMO BUFR sequence, 3-10-077." echo " changes to use new WMO BUFR sequence, 3-10-077." -echo " Oct 09 2020 - Update to dump gsrcsr, gsrasr, ompslp, ahicsr, " echo " Oct 09 2020 - Update to dump gsrcsr, gsrasr, ompslp, ahicsr, " -echo " sstvcw, sstvpw, leogeo, hdob " echo " sstvcw, sstvpw, leogeo, hdob " -echo " Update to remove obsolete GOES-15 data. " echo " Update to remove obsolete GOES-15 data. " -echo " Update to remove legacy VIIRS AMV data. " echo " Update to remove legacy VIIRS AMV data. " -echo " Feb 22 2021 - Disabled DBN alerts for gpsro dump files. " echo " Feb 22 2021 - Disabled DBN alerts for gpsro dump files. " -echo " These gpsro dump files have the potential to " echo " These gpsro dump files have the potential to " -echo " contain commercial data. The equivalent " echo " contain commercial data. The equivalent " -echo " non-restricted gpsro dump files are alerted " echo " non-restricted gpsro dump files are alerted " -echo " instead. " echo " instead. " -echo " Sep 21 2020 - Incremented subsets for the sfcshp dump groups " echo " Sep 21 2020 - Incremented subsets for the sfcshp dump groups " -echo " to match bufr_dumplist. Removed tideg from " echo " to match bufr_dumplist. Removed tideg from " -echo " sfcshp dump group to make individual dump file." echo " sfcshp dump group to make individual dump file." -echo " - Copy bufr_dumplist to COMOUT. " echo " - Copy bufr_dumplist to COMOUT. " -echo " Dec 16 2021 - modified to work on WCOSS2 " echo " Dec 16 2021 - modified to work on WCOSS2 " -echo " Mar 09 2022 - Enable the dumping of 002017 in vadwnd dump " echo " Mar 09 2022 - Enable the dumping of 002017 in vadwnd dump " -echo " group. " echo " group. " -echo " Aug 10 2022 - subpfl and saldrn added to dump group #1. " echo " Aug 10 2022 - subpfl and saldrn added to dump group #1. " -echo " gmi1cr added to dump group #9 " echo " gmi1cr added to dump group #9 " -echo " snocvr added to dump group #2 " echo " snocvr added to dump group #2 " -echo " b005/xx081 added to satwnd " echo " b005/xx081 added to satwnd " -echo " subpfl aadded to nsstbufr file " echo " subpfl aadded to nsstbufr file " -echo " DBN alerts are also enabled for subpfl,saldrn, " echo " DBN alerts are also enabled for subpfl,saldrn, " -echo " gmi1cr,and snocvr " echo " gmi1cr,and snocvr " - > echo " Oct 17 2022 - Split up groups 1 and 10 into a new group 12 " - > echo " for better optimization. " -############################################################################# ############################################################################# - -# NOTE: NET is changed to gdas in the parent Job script for the gdas RUN # NOTE: NET is changed to gdas in the parent Job script for the gdas RUN -# (was gfs - NET remains gfs for gfs RUN) # (was gfs - NET remains gfs for gfs RUN) -# ----------------------------------------------------------------------- # ----------------------------------------------------------------------- - -set -xau set -xau - -# function to highlight an echoed msg with surrounding hashed separator lines. # function to highlight an echoed msg with surrounding hashed separator lines. - echo_hashed_msg () { echo_hashed_msg () { - set +x set +x - msg=$* msg=$* - echo -e "\n ${msg//?/#}" echo -e "\n ${msg//?/#}" - echo " ${msg}" echo " ${msg}" - echo -e " ${msg//?/#}\n" echo -e " ${msg//?/#}\n" - set -x set -x - } } -# end of function setup # end of function setup -# # -# set some variables if they have not already been set # set some variables if they have not already been set - -set +u set +u - -# JOB_NUMBER = 1 indicates the prepbufr dump job. # JOB_NUMBER = 1 indicates the prepbufr dump job. -# JOB_NUMBER = 2 indicates the non-prepbufr dump job. # JOB_NUMBER = 2 indicates the non-prepbufr dump job. -# JOB_NUMBER not present indicates dump BOTH prepbufr and non-prepbufr data. # JOB_NUMBER not present indicates dump BOTH prepbufr and non-prepbufr data. -# ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- -# Dump group #1 (non-pb, TIME_TRIM defaults to OFF) = # Dump group #1 (non-pb, TIME_TRIM defaults to OFF) = -# avcsam eshrs3 ssmisu saphir atms 1bhrs4 sevcsr tesac mls | # avcsam eshrs3 ssmisu saphir 1bhrs4 sevcsr tesac mls -# esatms crisfs gsrcsr ahicsr sstvcw subpfl saldrn | # esatms gsrcsr ahicsr sstvcw subpfl saldrn -# # -# Dump group #2 (pb, TIME_TRIM defaults to OFF) = # Dump group #2 (pb, TIME_TRIM defaults to OFF) = -# sfcshp tideg atovs* adpsfc ascatt # sfcshp tideg atovs* adpsfc ascatt -# * - for GDAS only # * - for GDAS only -# # -# Dump group #3 (pb, TIME_TRIM defaults to OFF) = # Dump group #3 (pb, TIME_TRIM defaults to OFF) = -# adpupa # adpupa -# # -# Dump group #4 (pb, TIME_TRIM defaults to ON) = # Dump group #4 (pb, TIME_TRIM defaults to ON) = -# aircar aircft proflr vadwnd rassda gpsipw hdob # aircar aircft proflr vadwnd rassda gpsipw hdob -# # -# Dump group #5 (pb, TIME_TRIM defaults to OFF) = # Dump group #5 (pb, TIME_TRIM defaults to OFF) = -# msonet # msonet -# # -# Dump group #6 (non-pb, TIME_TRIM defaults to OFF) = # Dump group #6 (non-pb, TIME_TRIM defaults to OFF) = -# nexrad # nexrad -# # -# Dump group #7 (non-pb, TIME_TRIM defaults to OFF) = # Dump group #7 (non-pb, TIME_TRIM defaults to OFF) = -# avcspm esmhs 1bmhs airsev atmsdb gome omi trkob gpsro # avcspm esmhs 1bmhs airsev atmsdb gome omi trkob gpsro -# crisf4 # crisf4 -# # -# Dump group #8 (pb, TIME_TRIM defaults to ON) = # Dump group #8 (pb, TIME_TRIM defaults to ON) = -# satwnd # satwnd -# # -# Dump group #9 (non-pb, TIME_TRIM defaults to ON) = # Dump group #9 (non-pb, TIME_TRIM defaults to ON) = -# geoimr # geoimr -# # -# Dump group #10 (non-pb, TIME_TRIM defaults to OFF) = # Dump group #10 (non-pb, TIME_TRIM defaults to OFF) = -# esiasi mtiasi esamua crsfdb iasidb sevasr 1bamua bathy osbuv8 | # esiasi mtiasi esamua sevasr 1bamua bathy osbuv8 -# ompst8 ompsn8 gsrasr ompslp sstvpw # ompst8 ompsn8 gsrasr ompslp sstvpw -# # -# Dump group #11 (non-pb, TIME_TRIM defaults to OFF) = # Dump group #11 (non-pb, TIME_TRIM defaults to OFF) = -# amsr2 # amsr2 -# # -# Dump group #12 STATUS FILE | # Dump group #12 crisfs atms (previously group1) - > # crsfdb iasidb (previously group10) - > # - > # Dump group #13 STATUS FILE -# ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- - -#VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV #VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV -# The settings below are based on a future change when the DUMP job will dump # The settings below are based on a future change when the DUMP job will dump -# only types that go into PREPBUFR and the DUMP2 job will dump only types that # only types that go into PREPBUFR and the DUMP2 job will dump only types that -# do not go into PREPBUFR. This will speed up the DUMP + PREP processing. # do not go into PREPBUFR. This will speed up the DUMP + PREP processing. -# Although the logic is in place to now do this (see below), for now we will # Although the logic is in place to now do this (see below), for now we will -# continue to run only a DUMP job which will dump ALL types (no DUMP2 job) - # continue to run only a DUMP job which will dump ALL types (no DUMP2 job) - -# since JOB_NUMBER is not imported to this script, the logic below will dump # since JOB_NUMBER is not imported to this script, the logic below will dump -# all types ... # all types ... -# ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- -#^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -if [ -n "$JOB_NUMBER" ]; then if [ -n "$JOB_NUMBER" ]; then -set -u set -u - if [ $JOB_NUMBER = 2 ]; then if [ $JOB_NUMBER = 2 ]; then - dump_ind=DUMP2 dump_ind=DUMP2 - DUMP_group1=${DUMP_group1:-"YES"} DUMP_group1=${DUMP_group1:-"YES"} - DUMP_group2=${DUMP_group2:-"NO"} DUMP_group2=${DUMP_group2:-"NO"} - DUMP_group3=${DUMP_group3:-"NO"} DUMP_group3=${DUMP_group3:-"NO"} - DUMP_group4=${DUMP_group4:-"NO"} DUMP_group4=${DUMP_group4:-"NO"} - DUMP_group5=${DUMP_group5:-"NO"} DUMP_group5=${DUMP_group5:-"NO"} - DUMP_group6=${DUMP_group6:-"NO"} DUMP_group6=${DUMP_group6:-"NO"} - DUMP_group7=${DUMP_group7:-"YES"} DUMP_group7=${DUMP_group7:-"YES"} - DUMP_group8=${DUMP_group8:-"NO"} DUMP_group8=${DUMP_group8:-"NO"} - DUMP_group9=${DUMP_group9:-"YES"} DUMP_group9=${DUMP_group9:-"YES"} - DUMP_group10=${DUMP_group10:-"YES"} DUMP_group10=${DUMP_group10:-"YES"} - DUMP_group11=${DUMP_group11:-"YES"} DUMP_group11=${DUMP_group11:-"YES"} - > DUMP_group12=${DUMP_group12:-"YES"} - else else - dump_ind=DUMP dump_ind=DUMP - DUMP_group1=${DUMP_group1:-"NO"} DUMP_group1=${DUMP_group1:-"NO"} - DUMP_group2=${DUMP_group2:-"YES"} DUMP_group2=${DUMP_group2:-"YES"} - DUMP_group3=${DUMP_group3:-"YES"} DUMP_group3=${DUMP_group3:-"YES"} - DUMP_group4=${DUMP_group4:-"YES"} DUMP_group4=${DUMP_group4:-"YES"} - DUMP_group5=${DUMP_group5:-"NO"} DUMP_group5=${DUMP_group5:-"NO"} - DUMP_group6=${DUMP_group6:-"NO"} DUMP_group6=${DUMP_group6:-"NO"} - DUMP_group7=${DUMP_group7:-"NO"} DUMP_group7=${DUMP_group7:-"NO"} - DUMP_group8=${DUMP_group8:-"YES"} DUMP_group8=${DUMP_group8:-"YES"} - DUMP_group9=${DUMP_group9:-"NO"} DUMP_group9=${DUMP_group9:-"NO"} - DUMP_group10=${DUMP_group10:-"NO"} DUMP_group10=${DUMP_group10:-"NO"} - DUMP_group11=${DUMP_group11:-"NO"} DUMP_group11=${DUMP_group11:-"NO"} - > DUMP_group12=${DUMP_group12:-"NO"} - fi fi -else else - dump_ind=DUMP dump_ind=DUMP - DUMP_group1=${DUMP_group1:-"YES"} DUMP_group1=${DUMP_group1:-"YES"} - DUMP_group2=${DUMP_group2:-"YES"} DUMP_group2=${DUMP_group2:-"YES"} - DUMP_group3=${DUMP_group3:-"YES"} DUMP_group3=${DUMP_group3:-"YES"} - DUMP_group4=${DUMP_group4:-"YES"} DUMP_group4=${DUMP_group4:-"YES"} - DUMP_group5=${DUMP_group5:-"NO"} DUMP_group5=${DUMP_group5:-"NO"} - DUMP_group6=${DUMP_group6:-"NO"} DUMP_group6=${DUMP_group6:-"NO"} - DUMP_group7=${DUMP_group7:-"YES"} DUMP_group7=${DUMP_group7:-"YES"} - DUMP_group8=${DUMP_group8:-"YES"} DUMP_group8=${DUMP_group8:-"YES"} - DUMP_group9=${DUMP_group9:-"YES"} DUMP_group9=${DUMP_group9:-"YES"} - DUMP_group10=${DUMP_group10:-"YES"} DUMP_group10=${DUMP_group10:-"YES"} - DUMP_group11=${DUMP_group11:-"YES"} DUMP_group11=${DUMP_group11:-"YES"} - > DUMP_group12=${DUMP_group12:-"YES"} -fi fi - -if [ "$NET" = 'gfs' ]; then if [ "$NET" = 'gfs' ]; then - ADPUPA_wait=${ADPUPA_wait:-"YES"} ADPUPA_wait=${ADPUPA_wait:-"YES"} -########ADPUPA_wait=${ADPUPA_wait:-"NO"} # saves time if ADPUPA_wait=NO ########ADPUPA_wait=${ADPUPA_wait:-"NO"} # saves time if ADPUPA_wait=NO -else else - ADPUPA_wait=${ADPUPA_wait:-"NO"} ADPUPA_wait=${ADPUPA_wait:-"NO"} -fi fi - - -# send extra output of DUMP2 for monitoring purposes. # send extra output of DUMP2 for monitoring purposes. -set +u set +u -if [ -n "$JOB_NUMBER" ]; then if [ -n "$JOB_NUMBER" ]; then - [ $JOB_NUMBER = 2 ] && export PS4='$SECONDS + ' [ $JOB_NUMBER = 2 ] && export PS4='$SECONDS + ' -fi fi -set -u set -u - -# Make sure we are in the $DATA directory # Make sure we are in the $DATA directory -cd $DATA cd $DATA - -msg="HAS BEGUN on `hostname`" msg="HAS BEGUN on `hostname`" -$DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" - -cat break > $pgmout cat break > $pgmout - -export dumptime=`cut -c7-16 ncepdate` export dumptime=`cut -c7-16 ncepdate` -export cycp=`echo $dumptime|cut -c9-10` export cycp=`echo $dumptime|cut -c9-10` - -export NET_uc=$(echo $NET | tr [a-z] [A-Z]) export NET_uc=$(echo $NET | tr [a-z] [A-Z]) -export tmmark_uc=$(echo $tmmark | tr [a-z] [A-Z]) export tmmark_uc=$(echo $tmmark | tr [a-z] [A-Z]) - -msg="$NET_uc ANALYSIS TIME IS $PDY$cyc" msg="$NET_uc ANALYSIS TIME IS $PDY$cyc" -$DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" - -set +x set +x -echo echo -echo "CENTER DATA DUMP DATE-TIME FOR $tmmark_uc $NET_uc IS $dumptime" echo "CENTER DATA DUMP DATE-TIME FOR $tmmark_uc $NET_uc IS $dumptime" -echo echo -set -x set -x - -export COMSP=$COMOUT/$RUN.${cycle}. export COMSP=$COMOUT/$RUN.${cycle}. - -if [ "$PROCESS_GRIBFLDS" = 'YES' ]; then if [ "$PROCESS_GRIBFLDS" = 'YES' ]; then - -######################################################## ######################################################## -######################################################## ######################################################## -## The following files are not *required* but will still ## The following files are not *required* but will still -# be processed here for the near term (missing files # be processed here for the near term (missing files -# will not cause job to fail) # will not cause job to fail) -# # -# copy snogrb (0.5 deg) from $TANK_GRIBFLDS # copy snogrb (0.5 deg) from $TANK_GRIBFLDS -# copy snogrb_t574 from $TANK_GRIBFLDS # copy snogrb_t574 from $TANK_GRIBFLDS -# copy engicegrb from $COM_ENGICE # copy engicegrb from $COM_ENGICE -# copy sstgrb from $COM_SSTOI # copy sstgrb from $COM_SSTOI -# generate sstgrb index file # generate sstgrb index file -######################################################## ######################################################## -######################################################## ######################################################## - - # JY - 05/02: remove the dependency of snowdepth files created from isnowgrib job # JY - 05/02: remove the dependency of snowdepth files created from isnowgrib job - #snogrb=$TANK_GRIBFLDS/$PDY/wgrbbul/snowdepth.global.grb #snogrb=$TANK_GRIBFLDS/$PDY/wgrbbul/snowdepth.global.grb - #snoold=$TANK_GRIBFLDS/$PDYm1/wgrbbul/snowdepth.global.grb #snoold=$TANK_GRIBFLDS/$PDYm1/wgrbbul/snowdepth.global.grb - - #if [ -s $snogrb ]; then #if [ -s $snogrb ]; then - # cp $snogrb ${COMSP}snogrb # cp $snogrb ${COMSP}snogrb - # msg="todays 0.5 degree snow grib file located and copied to ${COMSP}snogrb" # msg="todays 0.5 degree snow grib file located and copied to ${COMSP}snogrb" - # $DATA/postmsg "$jlogfile" "$msg" # $DATA/postmsg "$jlogfile" "$msg" - #elif [ -s $snoold ]; then #elif [ -s $snoold ]; then - # cp $snoold ${COMSP}snogrb # cp $snoold ${COMSP}snogrb - # msg="**todays 0.5 degree snow grib file not located - copy 1-day old file" # msg="**todays 0.5 degree snow grib file not located - copy 1-day old file" - # $DATA/postmsg "$jlogfile" "$msg" # $DATA/postmsg "$jlogfile" "$msg" - #else #else - # set +x # set +x - # echo " " # echo " " - # echo " #####################################################" # echo " #####################################################" - # echo " cannot locate 0.5 degree snow grib file" # echo " cannot locate 0.5 degree snow grib file" - # echo " #####################################################" # echo " #####################################################" - # echo " " # echo " " - # set -x # set -x - # msg="***WARNING: CANNOT LOCATE 0.5 DEGREE SNOW GRIB FILE. Not critical." # msg="***WARNING: CANNOT LOCATE 0.5 DEGREE SNOW GRIB FILE. Not critical." - # $DATA/postmsg "$jlogfile" "$msg" # $DATA/postmsg "$jlogfile" "$msg" - #fi #fi - - #snogrb_t574=$TANK_GRIBFLDS/$PDY/wgrbbul/snowdepth.t574.grb #snogrb_t574=$TANK_GRIBFLDS/$PDY/wgrbbul/snowdepth.t574.grb - #snoold_t574=$TANK_GRIBFLDS/$PDYm1/wgrbbul/snowdepth.t574.grb #snoold_t574=$TANK_GRIBFLDS/$PDYm1/wgrbbul/snowdepth.t574.grb - - #if [ -s $snogrb_t574 ]; then #if [ -s $snogrb_t574 ]; then - # cp $snogrb_t574 ${COMSP}snogrb_t574 # cp $snogrb_t574 ${COMSP}snogrb_t574 - # msg="todays T574 snow grib file located and copied to ${COMSP}snogrb_t574" # msg="todays T574 snow grib file located and copied to ${COMSP}snogrb_t574" - # $DATA/postmsg "$jlogfile" "$msg" # $DATA/postmsg "$jlogfile" "$msg" - #elif [ -s $snoold_t574 ]; then #elif [ -s $snoold_t574 ]; then - # cp $snoold_t574 ${COMSP}snogrb_t574 # cp $snoold_t574 ${COMSP}snogrb_t574 - # msg="**todays T574 snow grib file not located - copy 1-day old file" # msg="**todays T574 snow grib file not located - copy 1-day old file" - # $DATA/postmsg "$jlogfile" "$msg" # $DATA/postmsg "$jlogfile" "$msg" - #else #else - # set +x # set +x - # echo " " # echo " " - # echo " ###############################################" # echo " ###############################################" - # echo " cannot locate T574 snow grib file" # echo " cannot locate T574 snow grib file" - # echo " ###############################################" # echo " ###############################################" - # echo " " # echo " " - # set -x # set -x - # msg="***WARNING: CANNOT LOCATE T574 SNOW GRIB FILE. Not critical." # msg="***WARNING: CANNOT LOCATE T574 SNOW GRIB FILE. Not critical." - # $DATA/postmsg "$jlogfile" "$msg" # $DATA/postmsg "$jlogfile" "$msg" - #fi #fi - - engicegrb=${COM_ENGICE}.$PDY/engice.t00z.grb engicegrb=${COM_ENGICE}.$PDY/engice.t00z.grb - engiceold=${COM_ENGICE}.$PDYm1/engice.t00z.grb engiceold=${COM_ENGICE}.$PDYm1/engice.t00z.grb - - if [ -s $engicegrb ]; then if [ -s $engicegrb ]; then - cp $engicegrb ${COMSP}engicegrb cp $engicegrb ${COMSP}engicegrb - msg="todays engice grib file located and copied to ${COMSP}engicegrb" msg="todays engice grib file located and copied to ${COMSP}engicegrb" - $DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" - elif [ -s $engiceold ]; then elif [ -s $engiceold ]; then - cp $engiceold ${COMSP}engicegrb cp $engiceold ${COMSP}engicegrb - msg="**todays engice grib file not located - copy 1-day old file" msg="**todays engice grib file not located - copy 1-day old file" - $DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" - else else - set +x set +x - echo " " echo " " - echo " ############################################" echo " ############################################" - echo " cannot locate engice grib file" echo " cannot locate engice grib file" - echo " ############################################" echo " ############################################" - echo " " echo " " - set -x set -x - msg="***WARNING: CANNOT LOCATE LOW RES ENGICE GRIB FILE. Not critical." msg="***WARNING: CANNOT LOCATE LOW RES ENGICE GRIB FILE. Not critical." - $DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" - fi fi - -# Disabled w/ GFSv15.2 b/c sstoi file no longer needed # Disabled w/ GFSv15.2 b/c sstoi file no longer needed -# sstgrb=${COM_SSTOI}.$PDY/sstoi_grb # sstgrb=${COM_SSTOI}.$PDY/sstoi_grb -# sstold=${COM_SSTOI}.$PDYm1/sstoi_grb # sstold=${COM_SSTOI}.$PDYm1/sstoi_grb - -# if [ -s $sstgrb ]; then # if [ -s $sstgrb ]; then -# cp $sstgrb ${COMSP}sstgrb # cp $sstgrb ${COMSP}sstgrb -# msg="todays lowres sst grib file located and copied to ${COMSP}sstgrb" # msg="todays lowres sst grib file located and copied to ${COMSP}sstgrb" -# $DATA/postmsg "$jlogfile" "$msg" # $DATA/postmsg "$jlogfile" "$msg" -# elif [ -s $sstold ]; then # elif [ -s $sstold ]; then -# cp $sstold ${COMSP}sstgrb # cp $sstold ${COMSP}sstgrb -# msg="**todays lowres sst grib file not located - copy 1-day old file" # msg="**todays lowres sst grib file not located - copy 1-day old file" -# $DATA/postmsg "$jlogfile" "$msg" # $DATA/postmsg "$jlogfile" "$msg" -# else # else -# set +x # set +x -# echo " " # echo " " -# echo " #########################################" # echo " #########################################" -# echo " cannot locate lowres sst grib file" # echo " cannot locate lowres sst grib file" -# echo " #########################################" # echo " #########################################" -# echo " " # echo " " -# set -x # set -x -# msg="***WARNING: CANNOT LOCATE LOW RES SST GRIB FILE. Not critical." # msg="***WARNING: CANNOT LOCATE LOW RES SST GRIB FILE. Not critical." -# $DATA/postmsg "$jlogfile" "$msg" # $DATA/postmsg "$jlogfile" "$msg" -# fi # fi - -# if [ -s ${COMSP}sstgrb ]; then # if [ -s ${COMSP}sstgrb ]; then -# rm errfile # rm errfile -# $GRBINDEX ${COMSP}sstgrb ${COMSP}sstgrb.index 2> errfile # $GRBINDEX ${COMSP}sstgrb ${COMSP}sstgrb.index 2> errfile -# errindx=$? # errindx=$? -# [ "$errindx" -ne '0' ] && cat errfile # [ "$errindx" -ne '0' ] && cat errfile -# rm errfile # rm errfile -# else # else -# echo_hashed_msg "cannot create grib index since sst file does not exist" # echo_hashed_msg "cannot create grib index since sst file does not exist" -# fi # fi - -# The following may no longer be needed, but leave them in place for now. # The following may no longer be needed, but leave them in place for now. -# Print msg in the rare case the grib2 files cannot be created. # Print msg in the rare case the grib2 files cannot be created. - if [ "$NET" = 'gdas' ]; then if [ "$NET" = 'gdas' ]; then - if [ -s ${COMSP}engicegrb ]; then if [ -s ${COMSP}engicegrb ]; then - $CNVGRIB -g12 -p40 ${COMSP}engicegrb ${COMSP}engicegrb.grib2 $CNVGRIB -g12 -p40 ${COMSP}engicegrb ${COMSP}engicegrb.grib2 - else else - echo_hashed_msg "Skip engicegrb.grib2 since grib1 file does not exist" echo_hashed_msg "Skip engicegrb.grib2 since grib1 file does not exist" - fi fi - # Disabled w/ GFSv15.2 b/c sstoi file no longer needed # Disabled w/ GFSv15.2 b/c sstoi file no longer needed - #if [ -s ${COMSP}sstgrb ]; then #if [ -s ${COMSP}sstgrb ]; then - # $CNVGRIB -g12 -p40 ${COMSP}sstgrb ${COMSP}sstgrb.grib2 # $CNVGRIB -g12 -p40 ${COMSP}sstgrb ${COMSP}sstgrb.grib2 - #else #else - # echo_hashed_msg "Skip sstgrb.grib2 since grib1 file does not exist" # echo_hashed_msg "Skip sstgrb.grib2 since grib1 file does not exist" - #fi #fi - #if [ -s ${COMSP}snogrb ]; then #if [ -s ${COMSP}snogrb ]; then - # $CNVGRIB -g12 -p40 ${COMSP}snogrb ${COMSP}snogrb.grib2 # $CNVGRIB -g12 -p40 ${COMSP}snogrb ${COMSP}snogrb.grib2 - #else #else - # echo_hashed_msg "Skip snogrb.grib2 since grib1 file does not exist" # echo_hashed_msg "Skip snogrb.grib2 since grib1 file does not exist" - #fi #fi - fi fi - - -###################################################################### ###################################################################### -###################################################################### ###################################################################### -# For the following, try as far as $ndaysback to find recent file. # # For the following, try as far as $ndaysback to find recent file. # -# Post warning if no file found for $ndaysback_warn or beyond. # # Post warning if no file found for $ndaysback_warn or beyond. # -# The job will continue if no suitable file is available. # # The job will continue if no suitable file is available. # -# ---------------------------------------------------------------- # # ---------------------------------------------------------------- # -# copy NPR.SNWN.SP.S1200.MESH16 from $TANK_GRIBFLDS # # copy NPR.SNWN.SP.S1200.MESH16 from $TANK_GRIBFLDS # -# copy NPR.SNWS.SP.S1200.MESH16 from $TANK_GRIBFLDS # # copy NPR.SNWS.SP.S1200.MESH16 from $TANK_GRIBFLDS # -# copy imssnow96.grb.grib2 from $TANK_GRIBFLDS # # copy imssnow96.grb.grib2 from $TANK_GRIBFLDS # -# copy seaice.t00z.5min.grb from $COM_ICE5MIN # # copy seaice.t00z.5min.grb from $COM_ICE5MIN # -# copy seaice.t00z.5min.grb.grib2 from $COM_ICE5MIN # # copy seaice.t00z.5min.grb.grib2 from $COM_ICE5MIN # -# copy rtgssthr_grb_0.083 from $COM_SSTRTG # # copy rtgssthr_grb_0.083 from $COM_SSTRTG # -# copy rtgssthr_grb_0.083.grib2 from $COM_SSTRTG # # copy rtgssthr_grb_0.083.grib2 from $COM_SSTRTG # -###################################################################### ###################################################################### -###################################################################### ###################################################################### - for gribfile in \ for gribfile in \ - NPR.SNWN.SP.S1200.MESH16 \ NPR.SNWN.SP.S1200.MESH16 \ - NPR.SNWS.SP.S1200.MESH16 \ NPR.SNWS.SP.S1200.MESH16 \ - imssnow96.grb.grib2 \ imssnow96.grb.grib2 \ - seaice.t00z.5min.grb \ seaice.t00z.5min.grb \ - seaice.t00z.5min.grb.grib2 \ seaice.t00z.5min.grb.grib2 \ - rtgssthr_grb_0.083 \ rtgssthr_grb_0.083 \ - rtgssthr_grb_0.083.grib2 rtgssthr_grb_0.083.grib2 - do do -# set the values specific to each file # set the values specific to each file - case $gribfile in case $gribfile in - NPR.SNWN.SP.S1200.MESH16 | NPR.SNWS.SP.S1200.MESH16 ) # AFWA snow NPR.SNWN.SP.S1200.MESH16 | NPR.SNWS.SP.S1200.MESH16 ) # AFWA snow - grib_source='$TANK_GRIBFLDS/$DDATE/wgrbbul'; grib_source='$TANK_GRIBFLDS/$DDATE/wgrbbul'; - target_filename=$gribfile.grb target_filename=$gribfile.grb - ndaysback=1; ndaysback=1; - ndaysback_warn=1;; ndaysback_warn=1;; - imssnow96.grb.grib2 ) # IMS snow imssnow96.grb.grib2 ) # IMS snow - grib_source='$TANK_GRIBFLDS/$DDATE/wgrbbul'; grib_source='$TANK_GRIBFLDS/$DDATE/wgrbbul'; - target_filename=imssnow96.grib2 target_filename=imssnow96.grib2 - ndaysback=1; ndaysback=1; - ndaysback_warn=1;; ndaysback_warn=1;; - seaice.t00z.5min.grb ) seaice.t00z.5min.grb ) - grib_source='${COM_ICE5MIN}.$DDATE'; grib_source='${COM_ICE5MIN}.$DDATE'; - target_filename=seaice.5min.grb target_filename=seaice.5min.grb - ndaysback=7; ndaysback=7; - ndaysback_warn=1;; ndaysback_warn=1;; - seaice.t00z.5min.grb.grib2 ) seaice.t00z.5min.grb.grib2 ) - grib_source='${COM_ICE5MIN}.$DDATE'; grib_source='${COM_ICE5MIN}.$DDATE'; - target_filename=seaice.5min.grib2 target_filename=seaice.5min.grib2 - ndaysback=7; ndaysback=7; - ndaysback_warn=1;; ndaysback_warn=1;; - rtgssthr_grb_0.083 ) rtgssthr_grb_0.083 ) - grib_source='${COM_SSTRTG}.$DDATE'; grib_source='${COM_SSTRTG}.$DDATE'; - target_filename=rtgssthr.grb target_filename=rtgssthr.grb - ndaysback=10; ndaysback=10; - ndaysback_warn=1;; ndaysback_warn=1;; - rtgssthr_grb_0.083.grib2 ) rtgssthr_grb_0.083.grib2 ) - grib_source='${COM_SSTRTG}.$DDATE'; grib_source='${COM_SSTRTG}.$DDATE'; - target_filename=rtgssthr.grib2 target_filename=rtgssthr.grib2 - ndaysback=10; ndaysback=10; - ndaysback_warn=1;; ndaysback_warn=1;; - *) *) - msg="***FATAL ERROR: unexpected grib field file $gribfile"; msg="***FATAL ERROR: unexpected grib field file $gribfile"; - echo_hashed_msg "$msg" echo_hashed_msg "$msg" - $DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" - $DATA/err_exit;; $DATA/err_exit;; - esac esac -# set up string of dates to check # set up string of dates to check - if [ $ndaysback -gt 0 ];then if [ $ndaysback -gt 0 ];then -set +x; echo -e "\n---> path to finddate.sh below is: `which finddate.sh`"; set -x set +x; echo -e "\n---> path to finddate.sh below is: `which finddate.sh`"; set -x - CHECK_DATES="$PDY $(finddate.sh $PDY s-$ndaysback)" CHECK_DATES="$PDY $(finddate.sh $PDY s-$ndaysback)" - else else - CHECK_DATES=$PDY CHECK_DATES=$PDY - fi fi - set +x; set +x; - echo -e "\nWill check as far back as ${CHECK_DATES##* } for $gribfile" echo -e "\nWill check as far back as ${CHECK_DATES##* } for $gribfile" - set -x set -x - ndtry=0 ndtry=0 - found=false found=false -# loop through dates to check for this file type # loop through dates to check for this file type - for DDATE in $CHECK_DATES;do for DDATE in $CHECK_DATES;do - ndtry=`expr $ndtry + 1` ndtry=`expr $ndtry + 1` - eval tryfile=$grib_source/$gribfile eval tryfile=$grib_source/$gribfile - if [ -s $tryfile ];then if [ -s $tryfile ];then - set +x; echo -e "\nPicking up file $tryfile\n"; set -x set +x; echo -e "\nPicking up file $tryfile\n"; set -x - cp $tryfile ${COMSP}$target_filename cp $tryfile ${COMSP}$target_filename - found=true found=true - break break - fi fi - if [ $DDATE -ne ${CHECK_DATES##* } ]; then if [ $DDATE -ne ${CHECK_DATES##* } ]; then - set +x;echo -e "\n$tryfile not available. Try previous day.\n" set +x;echo -e "\n$tryfile not available. Try previous day.\n" - set -x set -x - else else - set +x;echo -e "\n$tryfile not available.\n";set -x set +x;echo -e "\n$tryfile not available.\n";set -x - fi fi - if [ $ndtry -gt $ndaysback_warn ];then if [ $ndtry -gt $ndaysback_warn ];then - msg="***WARNING: INVESTIGATE UNEXPECTED ABSENCE OF $tryfile" msg="***WARNING: INVESTIGATE UNEXPECTED ABSENCE OF $tryfile" - echo_hashed_msg "$msg" echo_hashed_msg "$msg" - $DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" - fi fi - done done - if [ $found != true ]; then if [ $found != true ]; then - msg="***WARNING: NO USEFUL RECENT FILES FOUND FOR $gribfile!!!" msg="***WARNING: NO USEFUL RECENT FILES FOUND FOR $gribfile!!!" - echo_hashed_msg "$msg" echo_hashed_msg "$msg" - $DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" - fi fi - done done - if [ "$SENDECF" = "YES" ]; then if [ "$SENDECF" = "YES" ]; then - ecflow_client --event=release_sfcprep ecflow_client --event=release_sfcprep - fi fi - -# endif loop $PROCESS_GRIBFLDS # endif loop $PROCESS_GRIBFLDS -fi fi - - -echo "=======> Dump group 1 (thread_1) not executed." > $DATA/1.out echo "=======> Dump group 1 (thread_1) not executed." > $DATA/1.out -echo "=======> Dump group 2 (thread_2) not executed." > $DATA/2.out echo "=======> Dump group 2 (thread_2) not executed." > $DATA/2.out -echo "=======> Dump group 3 (thread_3) not executed." > $DATA/3.out echo "=======> Dump group 3 (thread_3) not executed." > $DATA/3.out -echo "=======> Dump group 4 (thread_4) not executed." > $DATA/4.out echo "=======> Dump group 4 (thread_4) not executed." > $DATA/4.out -echo "=======> Dump group 5 (thread_5) not executed." > $DATA/5.out echo "=======> Dump group 5 (thread_5) not executed." > $DATA/5.out -echo "=======> Dump group 6 (thread_6) not executed." > $DATA/6.out echo "=======> Dump group 6 (thread_6) not executed." > $DATA/6.out -echo "=======> Dump group 7 (thread_7) not executed." > $DATA/7.out echo "=======> Dump group 7 (thread_7) not executed." > $DATA/7.out -echo "=======> Dump group 8 (thread_8) not executed." > $DATA/8.out echo "=======> Dump group 8 (thread_8) not executed." > $DATA/8.out -echo "=======> Dump group 9 (thread_9) not executed." > $DATA/9.out echo "=======> Dump group 9 (thread_9) not executed." > $DATA/9.out -echo "=======> Dump group 10 (thread_10) not executed." > $DATA/10.out echo "=======> Dump group 10 (thread_10) not executed." > $DATA/10.out -echo "=======> Dump group 11 (thread_11) not executed." > $DATA/11.out echo "=======> Dump group 11 (thread_11) not executed." > $DATA/11.out - > echo "=======> Dump group 12 (thread_12) not executed." > $DATA/12.out - > - -err1=0 err1=0 -err2=0 err2=0 -err3=0 err3=0 -err4=0 err4=0 -err5=0 err5=0 -err6=0 err6=0 -err7=0 err7=0 -err8=0 err8=0 -err9=0 err9=0 -err10=0 err10=0 -err11=0 err11=0 - > err12=0 -if [ "$PROCESS_DUMP" = 'YES' ]; then if [ "$PROCESS_DUMP" = 'YES' ]; then - -#################################### #################################### -#################################### #################################### -# The data "dump" script for tm00 # The data "dump" script for tm00 -#################################### #################################### -#################################### #################################### - -msg="START THE $tmmark_uc $NET_uc DATA $dump_ind CENTERED ON $dumptime" msg="START THE $tmmark_uc $NET_uc DATA $dump_ind CENTERED ON $dumptime" -$DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" - -set +x set +x -#---------------------------------------------------------------- #---------------------------------------------------------------- -cat<<\EOF>thread_1; chmod +x thread_1 cat<<\EOF>thread_1; chmod +x thread_1 -set -uax set -uax - -cd $DATA cd $DATA - -{ echo { echo -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_1 echo Script thread_1 -echo Executing on node `hostname` echo Executing on node `hostname` -echo Starting time: `date -u` echo Starting time: `date -u` -echo "********************************************************************" echo "********************************************************************" -echo echo -set -x set -x - -export STATUS=NO export STATUS=NO -export DUMP_NUMBER=1 export DUMP_NUMBER=1 - -#========================================================================= #========================================================================= -# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: -# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types -# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) -# # -#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- -# Dump # 1 : AVCSAM: 1 subtype(s) # Dump # 1 : AVCSAM: 1 subtype(s) -# ESHRS3: 1 subtype(s) # ESHRS3: 1 subtype(s) -# SSMISU: 1 subtype(s) # SSMISU: 1 subtype(s) -# SAPHIR: 1 subtype(s) # SAPHIR: 1 subtype(s) -# ATMS: 1 subtype(s) (if present in past 10 days of tanks) < -# 1BHRS4: 1 subtype(s) # 1BHRS4: 1 subtype(s) -# SEVCSR: 1 subtype(s) # SEVCSR: 1 subtype(s) -# TESAC: 1 subtype(s) # TESAC: 1 subtype(s) -# MLS: 1 subtype(s) (if present in past 10 days of tanks) # MLS: 1 subtype(s) (if present in past 10 days of tanks) -# ESATMS: 1 subtype(s) (if present in past 10 days of tanks) # ESATMS: 1 subtype(s) (if present in past 10 days of tanks) -# CRISFS: 1 subtype(s) (if present in past 10 days of tanks) < -# GSRCSR: 1 subtype(s) # GSRCSR: 1 subtype(s) -# AHICSR: 1 subtype(s) # AHICSR: 1 subtype(s) -# SSTVCW: 1 subtype(s) # SSTVCW: 1 subtype(s) -# -------------------- # -------------------- -# TOTAL NUMBER OF SUBTYPES = 14 # TOTAL NUMBER OF SUBTYPES = 14 -# # -#========================================================================= #========================================================================= - -DTIM_latest_avcsam=${DTIM_latest_avcsam:-"+2.99"} DTIM_latest_avcsam=${DTIM_latest_avcsam:-"+2.99"} -DTIM_latest_eshrs3=${DTIM_latest_eshrs3:-"+2.99"} DTIM_latest_eshrs3=${DTIM_latest_eshrs3:-"+2.99"} -DTIM_latest_ssmisu=${DTIM_latest_ssmisu:-"+2.99"} DTIM_latest_ssmisu=${DTIM_latest_ssmisu:-"+2.99"} -#----------------------------------------------- < -DTIM_latest_saphir=${DTIM_latest_saphir:-"+2.99"} DTIM_latest_saphir=${DTIM_latest_saphir:-"+2.99"} -#----------------------------------------------- < -# check for atms tank presence in past 10 days < -atms="" < -err_check_tanks=0 < -sh $USHbufr_dump/check_tanks.sh atms < -err_check_tanks=$? < -if [ $err_check_tanks -eq 0 ];then < - atms=atms < - DTIM_latest_atms=${DTIM_latest_atms:-"+2.99"} < -fi < -#----------------------------------------------- < -DTIM_latest_saldrn=${DTIM_latest_saldrn:-"+2.99"} DTIM_latest_saldrn=${DTIM_latest_saldrn:-"+2.99"} -DTIM_latest_1bhrs4=${DTIM_latest_1bhrs4:-"+2.99"} DTIM_latest_1bhrs4=${DTIM_latest_1bhrs4:-"+2.99"} -DTIM_latest_sevcsr=${DTIM_latest_sevcsr:-"+2.99"} DTIM_latest_sevcsr=${DTIM_latest_sevcsr:-"+2.99"} -DTIM_latest_tesac=${DTIM_latest_tesac:-"+2.99"} DTIM_latest_tesac=${DTIM_latest_tesac:-"+2.99"} -#----------------------------------------------- #----------------------------------------------- -# check for mls tank presence in past 10 days # check for mls tank presence in past 10 days -mls="" mls="" -err_check_tanks=0 err_check_tanks=0 -sh $USHbufr_dump/check_tanks.sh mls sh $USHbufr_dump/check_tanks.sh mls -err_check_tanks=$? err_check_tanks=$? -if [ $err_check_tanks -eq 0 ];then if [ $err_check_tanks -eq 0 ];then - mls=mls mls=mls - DTIM_latest_mls=${DTIM_latest_mls:-"+2.99"} DTIM_latest_mls=${DTIM_latest_mls:-"+2.99"} -fi fi -#----------------------------------------------- #----------------------------------------------- -#----------------------------------------------- #----------------------------------------------- -# check for esatms tank presence in past 10 days # check for esatms tank presence in past 10 days -esatms="" esatms="" -err_check_tanks=0 err_check_tanks=0 -sh $USHbufr_dump/check_tanks.sh esatms sh $USHbufr_dump/check_tanks.sh esatms -err_check_tanks=$? err_check_tanks=$? -if [ $err_check_tanks -eq 0 ];then if [ $err_check_tanks -eq 0 ];then - esatms=esatms esatms=esatms - DTIM_latest_esatms=${DTIM_latest_esatms:-"+2.99"} DTIM_latest_esatms=${DTIM_latest_esatms:-"+2.99"} -fi fi -#----------------------------------------------- #----------------------------------------------- -#----------------------------------------------- < -# check for crisfs tank presence in past 10 days < -crisfs="" < -err_check_tanks=0 < -sh $USHbufr_dump/check_tanks.sh crisfs < -err_check_tanks=$? < -if [ $err_check_tanks -eq 0 ];then < - crisfs=crisfs < - DTIM_latest_crisfs=${DTIM_latest_crisfs:-"+2.99"} < -fi < -#----------------------------------------------- < -DTIM_latest_gsrcsr=${DTIM_latest_gsrcsr:-"+2.99"} DTIM_latest_gsrcsr=${DTIM_latest_gsrcsr:-"+2.99"} -DTIM_latest_ahicsr=${DTIM_latest_ahicsr:-"+2.99"} DTIM_latest_ahicsr=${DTIM_latest_ahicsr:-"+2.99"} -DTIM_latest_sstvcw=${DTIM_latest_sstvcw:-"+2.99"} DTIM_latest_sstvcw=${DTIM_latest_sstvcw:-"+2.99"} - -TIME_TRIM=${TIME_TRIM:-${TIME_TRIM1:-off}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM1:-off}} - -$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 avcsam eshrs3 ssmisu \ $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 avcsam eshrs3 ssmisu \ - saphir $atms 1bhrs4 sevcsr tesac $mls $esatms $crisfs gsrcsr ahicsr sstvcw subpfl saldrn | saphir 1bhrs4 sevcsr tesac $mls $esatms gsrcsr ahicsr sstvcw subpfl saldrn -error1=$? error1=$? -echo "$error1" > $DATA/error1 echo "$error1" > $DATA/error1 - -if [ "$SENDDBN" = "YES" ]; then if [ "$SENDDBN" = "YES" ]; then - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_avcsam $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_avcsam $job \ - ${COMSP}avcsam.tm00.bufr_d ${COMSP}avcsam.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_eshrs3 $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_eshrs3 $job \ - ${COMSP}eshrs3.tm00.bufr_d ${COMSP}eshrs3.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ssmisu $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ssmisu $job \ - ${COMSP}ssmisu.tm00.bufr_d ${COMSP}ssmisu.tm00.bufr_d - if [ "${NET}" = "gdas" ]; then if [ "${NET}" = "gdas" ]; then - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_saphir $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_saphir $job \ - ${COMSP}saphir.tm00.bufr_d ### restricted, only GDAS, turn on 01/13/2020 ${COMSP}saphir.tm00.bufr_d ### restricted, only GDAS, turn on 01/13/2020 - fi fi - if [ "$atms" = atms ];then < - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_atms $job \ < - ${COMSP}atms.tm00.bufr_d < - fi < - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_1bhrs4 $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_1bhrs4 $job \ - ${COMSP}1bhrs4.tm00.bufr_d ${COMSP}1bhrs4.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sevcsr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sevcsr $job \ - ${COMSP}sevcsr.tm00.bufr_d ${COMSP}sevcsr.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_tesac $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_tesac $job \ - ${COMSP}tesac.tm00.bufr_d ${COMSP}tesac.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_saldrn $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_saldrn $job \ - ${COMSP}saldrn.tm00.bufr_d ${COMSP}saldrn.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_subpfl $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_subpfl $job \ - ${COMSP}subpfl.tm00.bufr_d ${COMSP}subpfl.tm00.bufr_d - if [ "$mls" = mls ];then if [ "$mls" = mls ];then - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_mls $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_mls $job \ - ${COMSP}mls.tm00.bufr_d ${COMSP}mls.tm00.bufr_d - fi fi - if [ "$esatms" = esatms ];then if [ "$esatms" = esatms ];then - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_esatms $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_esatms $job \ - ${COMSP}esatms.tm00.bufr_d ${COMSP}esatms.tm00.bufr_d - fi fi -####### ALERTS TURNED OFF UNTIL REQUESTED BY USER ######################### < -# if [ "$crisfs" = crisfs ];then < -# $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_crisfs $job \ < -# ${COMSP}crisfs.tm00.bufr_d < -# fi < -########################################################################### < - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gsrcsr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gsrcsr $job \ - ${COMSP}gsrcsr.tm00.bufr_d ${COMSP}gsrcsr.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ahicsr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ahicsr $job \ - ${COMSP}ahicsr.tm00.bufr_d ${COMSP}ahicsr.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sstvcw $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sstvcw $job \ - ${COMSP}sstvcw.tm00.bufr_d ${COMSP}sstvcw.tm00.bufr_d -fi fi - -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_1 echo Script thread_1 -echo Finished executing on node `hostname` echo Finished executing on node `hostname` -echo Ending time : `date -u` echo Ending time : `date -u` -echo "********************************************************************" echo "********************************************************************" -set -x set -x -} > $DATA/1.out 2>&1 } > $DATA/1.out 2>&1 -EOF EOF -set -x set -x - -set +x set +x -#---------------------------------------------------------------- #---------------------------------------------------------------- -cat<<\EOF>thread_2; chmod +x thread_2 cat<<\EOF>thread_2; chmod +x thread_2 -set -uax set -uax - -cd $DATA cd $DATA - -{ echo { echo -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_2 echo Script thread_2 -echo Executing on node `hostname` echo Executing on node `hostname` -echo Starting time: `date -u` echo Starting time: `date -u` -echo "********************************************************************" echo "********************************************************************" -echo echo -set -x set -x - -export STATUS=NO export STATUS=NO -export DUMP_NUMBER=2 export DUMP_NUMBER=2 - -#========================================================================== #========================================================================== -# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: -# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types -# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) -# (3) GDAS GSI doesn't use ATOVS, but NASA/GSFC is pulling them off our # (3) GDAS GSI doesn't use ATOVS, but NASA/GSFC is pulling them off our -# server, also CDAS may be running special tests using data from GDAS # server, also CDAS may be running special tests using data from GDAS -# cutoff time (ATOVS is not dumped in GFS) # cutoff time (ATOVS is not dumped in GFS) -# # -#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- -# GDAS: # GDAS: -# Dump # 2 : SFCSHP: 11 subtype(s) (added shipsb & shipub in dumplist) # Dump # 2 : SFCSHP: 11 subtype(s) (added shipsb & shipub in dumplist) -# TIDEG: 1 subtype(s) # TIDEG: 1 subtype(s) -# ATOVS: 1 subtype(s) # ATOVS: 1 subtype(s) -# ADPSFC: 7 subtype(s) # ADPSFC: 7 subtype(s) -# ASCATT: 1 subtype(s) # ASCATT: 1 subtype(s) -# xxxxxxxxx WNDSAT: 1 subtype(s) (if present in past 10 days of tanks) # xxxxxxxxx WNDSAT: 1 subtype(s) (if present in past 10 days of tanks) -# ===> Dumping of WNDSAT removed from here until new ingest feed is established # ===> Dumping of WNDSAT removed from here until new ingest feed is established -# (had been dumped with a time window radius of -3.00 to +2.99 hours) # (had been dumped with a time window radius of -3.00 to +2.99 hours) -# -------------------- # -------------------- -# TOTAL NUMBER OF SUBTYPES = 20 - 21 # TOTAL NUMBER OF SUBTYPES = 20 - 21 -# # -#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- -# GFS: # GFS: -# Dump # 2 : SFCSHP: 11 subtype(s) (added shipsb & shipub in dumplist) # Dump # 2 : SFCSHP: 11 subtype(s) (added shipsb & shipub in dumplist) -# TIDEG: 1 subtype(s) # TIDEG: 1 subtype(s) -# ADPSFC: 7 subtype(s) # ADPSFC: 7 subtype(s) -# ASCATT: 1 subtype(s) # ASCATT: 1 subtype(s) -# xxxxxxxxx WNDSAT: 1 subtype(s) (if present in past 10 days of tanks) # xxxxxxxxx WNDSAT: 1 subtype(s) (if present in past 10 days of tanks) -# ===> Dumping of WNDSAT removed from here until new ingest feed is established # ===> Dumping of WNDSAT removed from here until new ingest feed is established -# (had been dumped with a time window radius of -3.00 to +2.99 hours) # (had been dumped with a time window radius of -3.00 to +2.99 hours) -# -------------------- # -------------------- -# TOTAL NUMBER OF SUBTYPES = 19 - 20 # TOTAL NUMBER OF SUBTYPES = 19 - 20 -# # -#========================================================================== #========================================================================== -DTIM_latest_snocvr=${DTIM_latest_snocvr:-"+2.99"} DTIM_latest_snocvr=${DTIM_latest_snocvr:-"+2.99"} -DTIM_latest_sfcshp=${DTIM_latest_sfcshp:-"+2.99"} DTIM_latest_sfcshp=${DTIM_latest_sfcshp:-"+2.99"} -DTIM_latest_tideg=${DTIM_latest_tideg:-"+2.99"} DTIM_latest_tideg=${DTIM_latest_tideg:-"+2.99"} - -atovs="" atovs="" -if [ "$NET" = 'gdas' ]; then if [ "$NET" = 'gdas' ]; then - atovs=atovs atovs=atovs - DTIM_latest_atovs=${DTIM_latest_atovs:-"+2.99"} DTIM_latest_atovs=${DTIM_latest_atovs:-"+2.99"} -fi fi - -DTIM_latest_adpsfc=${DTIM_latest_adpsfc:-"+2.99"} DTIM_latest_adpsfc=${DTIM_latest_adpsfc:-"+2.99"} -DTIM_latest_ascatt=${DTIM_latest_ascatt:-"+2.99"} DTIM_latest_ascatt=${DTIM_latest_ascatt:-"+2.99"} -#----------------------------------------------- #----------------------------------------------- -# check for wndsat tank presence in past 10 days # check for wndsat tank presence in past 10 days -wndsat="" wndsat="" -err_check_tanks=0 err_check_tanks=0 -##########sh $USHbufr_dump/check_tanks.sh wndsat ##########sh $USHbufr_dump/check_tanks.sh wndsat -##########err_check_tanks=$? ##########err_check_tanks=$? -err_check_tanks=99 # comment out 2 lines above & add this line to ensure wndsat err_check_tanks=99 # comment out 2 lines above & add this line to ensure wndsat - # is not ever dumped # is not ever dumped -if [ $err_check_tanks -eq 0 ];then if [ $err_check_tanks -eq 0 ];then - wndsat=wndsat wndsat=wndsat - DTIM_latest_wndsat=${DTIM_latest_wndsat:-"+2.99"} DTIM_latest_wndsat=${DTIM_latest_wndsat:-"+2.99"} -fi fi -#----------------------------------------------- #----------------------------------------------- - -TIME_TRIM=${TIME_TRIM:-${TIME_TRIM2:-off}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM2:-off}} - -$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 sfcshp tideg $atovs adpsfc snocvr ascatt $wndsat $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 sfcshp tideg $atovs adpsfc snocvr ascatt $wndsat -error2=$? error2=$? -echo "$error2" > $DATA/error2 echo "$error2" > $DATA/error2 - -if [ "$SENDDBN" = "YES" ]; then if [ "$SENDDBN" = "YES" ]; then - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sfcshp $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sfcshp $job \ - ${COMSP}sfcshp.tm00.bufr_d ${COMSP}sfcshp.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_tideg $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_tideg $job \ - ${COMSP}tideg.tm00.bufr_d ${COMSP}tideg.tm00.bufr_d - [ -f ${COMSP}atovs.tm00.bufr_d ] && \ [ -f ${COMSP}atovs.tm00.bufr_d ] && \ - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_atovs $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_atovs $job \ - ${COMSP}atovs.tm00.bufr_d ${COMSP}atovs.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_adpsfc $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_adpsfc $job \ - ${COMSP}adpsfc.tm00.bufr_d ${COMSP}adpsfc.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ascatt $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ascatt $job \ - ${COMSP}ascatt.tm00.bufr_d ${COMSP}ascatt.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_snocvr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_snocvr $job \ - ${COMSP}snocvr.tm00.bufr_d ${COMSP}snocvr.tm00.bufr_d - if [ "$NET" = 'gdas' ]; then if [ "$NET" = 'gdas' ]; then - ####### ALERT TURNED ON for GDAS only ######################## ####### ALERT TURNED ON for GDAS only ######################## - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ascatw $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ascatw $job \ - ${COMSP}ascatw.tm00.bufr_d ${COMSP}ascatw.tm00.bufr_d - fi fi - if [ "$wndsat" = wndsat ];then if [ "$wndsat" = wndsat ];then - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_wndsat $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_wndsat $job \ - ${COMSP}wndsat.tm00.bufr_d ${COMSP}wndsat.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_wdsatr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_wdsatr $job \ - ${COMSP}wdsatr.tm00.bufr_d ${COMSP}wdsatr.tm00.bufr_d - fi fi -fi fi - -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_2 echo Script thread_2 -echo Finished executing on node `hostname` echo Finished executing on node `hostname` -echo Ending time : `date -u` echo Ending time : `date -u` -echo "********************************************************************" echo "********************************************************************" -set -x set -x -} > $DATA/2.out 2>&1 } > $DATA/2.out 2>&1 -EOF EOF -set -x set -x - -set +x set +x -#---------------------------------------------------------------- #---------------------------------------------------------------- -cat<<\EOF>thread_3; chmod +x thread_3 cat<<\EOF>thread_3; chmod +x thread_3 -set -uax set -uax - -cd $DATA cd $DATA - -{ echo { echo -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_3 echo Script thread_3 -echo Executing on node `hostname` echo Executing on node `hostname` -echo Starting time: `date -u` echo Starting time: `date -u` -echo "********************************************************************" echo "********************************************************************" -echo echo -set -x set -x - -export STATUS=NO export STATUS=NO -export DUMP_NUMBER=3 export DUMP_NUMBER=3 - -#==================================================================== #==================================================================== -# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: -# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types -# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) -# # -#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- -# Dump #3: ADPUPA: 6 subtype(s) # Dump #3: ADPUPA: 6 subtype(s) -# -------------------- # -------------------- -# TOTAL NUMBER OF SUBTYPES = 6 # TOTAL NUMBER OF SUBTYPES = 6 -# # -#==================================================================== #==================================================================== - -DTIM_latest_adpupa=${DTIM_latest_adpupa:-"+2.99"} DTIM_latest_adpupa=${DTIM_latest_adpupa:-"+2.99"} - -TIME_TRIM=${TIME_TRIM:-${TIME_TRIM3:-off}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM3:-off}} - -$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 adpupa $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 adpupa -error3=$? error3=$? -echo "$error3" > $DATA/error3 echo "$error3" > $DATA/error3 - -if [ "$SENDDBN" = "YES" ]; then if [ "$SENDDBN" = "YES" ]; then - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_adpupa $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_adpupa $job \ - ${COMSP}adpupa.tm00.bufr_d ${COMSP}adpupa.tm00.bufr_d -fi fi - -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_3 echo Script thread_3 -echo Finished executing on node `hostname` echo Finished executing on node `hostname` -echo Ending time : `date -u` echo Ending time : `date -u` -echo "********************************************************************" echo "********************************************************************" -set -x set -x -} > $DATA/3.out 2>&1 } > $DATA/3.out 2>&1 -EOF EOF -set -x set -x - -set +x set +x -#---------------------------------------------------------------- #---------------------------------------------------------------- -cat<<\EOF>thread_4; chmod +x thread_4 cat<<\EOF>thread_4; chmod +x thread_4 -set -uax set -uax - -cd $DATA cd $DATA - -{ echo { echo -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_4 echo Script thread_4 -echo Executing on node `hostname` echo Executing on node `hostname` -echo Starting time: `date -u` echo Starting time: `date -u` -echo "********************************************************************" echo "********************************************************************" -echo echo -set -x set -x - -export STATUS=NO export STATUS=NO -export DUMP_NUMBER=4 export DUMP_NUMBER=4 - -#======================================================================= #======================================================================= -# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: -# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types -# EXCEPT: AIRCFT where it is +/- 3.25 hours # EXCEPT: AIRCFT where it is +/- 3.25 hours -# AIRCAR where it is +/- 3.25 hours # AIRCAR where it is +/- 3.25 hours -# PROFLR where it is -4.00 to +3.99 hours # PROFLR where it is -4.00 to +3.99 hours -# GSPIPW where it is +/- 0.05 hours (+/- 3min) # GSPIPW where it is +/- 0.05 hours (+/- 3min) -# (2) TIME TRIMMING IS DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS DONE IN THIS DUMP (default, unless overridden) -# # -#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- -# Dump # 4 : AIRCAR: 2 subtype(s) # Dump # 4 : AIRCAR: 2 subtype(s) -# AIRCFT: 8 subtype(s) # AIRCFT: 8 subtype(s) -# PROFLR: 4 subtype(s) # PROFLR: 4 subtype(s) -# VADWND: 1 subtype(s) # VADWND: 1 subtype(s) -# RASSDA: 1 subtype(s) # RASSDA: 1 subtype(s) -# GPSIPW: 1 subtype(s) # GPSIPW: 1 subtype(s) -# HDOB : 1 subtype(s) # HDOB : 1 subtype(s) -# -------------------- # -------------------- -# TOTAL NUMBER OF SUBTYPES = 18 # TOTAL NUMBER OF SUBTYPES = 18 -# # -#======================================================================= #======================================================================= - -# Skip NeXRaD VAD WINDS FROM LEVEL 2 DECODER (not ready to be handled in GSI) (002017) # Skip NeXRaD VAD WINDS FROM LEVEL 2 DECODER (not ready to be handled in GSI) (002017) -# 3/9/2022 -- enable the dumping of 002017 in the vadwnd dump group. # 3/9/2022 -- enable the dumping of 002017 in the vadwnd dump group. -#export SKIP_002017=YES #export SKIP_002017=YES - -# Dump AIRCFT and AIRCAR with wide time window to improve PREPOBS_PREPACQC # Dump AIRCFT and AIRCAR with wide time window to improve PREPOBS_PREPACQC -# track-check performance # track-check performance -# (time window will be winnowed down to +/- 3.00 hours in output from # (time window will be winnowed down to +/- 3.00 hours in output from -# PREPOBS_PREPACQC) # PREPOBS_PREPACQC) - -# Dump PROFLR with wide time window to improve PREPOBS_PROFCQC performance # Dump PROFLR with wide time window to improve PREPOBS_PROFCQC performance -# (time window will be winnowed down in output from PREPOBS_PROFCQC, see # (time window will be winnowed down in output from PREPOBS_PROFCQC, see -# parm cards for output time window) # parm cards for output time window) - -# Dump GPSIPW with narrow (+/- 3-min) time window since new Ground Based # Dump GPSIPW with narrow (+/- 3-min) time window since new Ground Based -# GPS-IPW/ZTD (from U.S.-ENI and foreign GNSS providers) is currently limited # GPS-IPW/ZTD (from U.S.-ENI and foreign GNSS providers) is currently limited -# to obs only at cycle-time # to obs only at cycle-time - -DTIM_earliest_aircft=${DTIM_earliest_aircft:-"-3.25"} DTIM_earliest_aircft=${DTIM_earliest_aircft:-"-3.25"} -DTIM_latest_aircft=${DTIM_latest_aircft:-"+3.25"} DTIM_latest_aircft=${DTIM_latest_aircft:-"+3.25"} - -DTIM_earliest_aircar=${DTIM_earliest_aircar:-"-3.25"} DTIM_earliest_aircar=${DTIM_earliest_aircar:-"-3.25"} -DTIM_latest_aircar=${DTIM_latest_aircar:-"+3.25"} DTIM_latest_aircar=${DTIM_latest_aircar:-"+3.25"} - -DTIM_earliest_proflr=${DTIM_earliest_proflr:-"-4.00"} DTIM_earliest_proflr=${DTIM_earliest_proflr:-"-4.00"} -DTIM_latest_proflr=${DTIM_latest_proflr:-"+3.99"} DTIM_latest_proflr=${DTIM_latest_proflr:-"+3.99"} - -DTIM_latest_vadwnd=${DTIM_latest_vadwnd:-"+2.99"} DTIM_latest_vadwnd=${DTIM_latest_vadwnd:-"+2.99"} -DTIM_latest_rassda=${DTIM_latest_rassda:-"+2.99"} DTIM_latest_rassda=${DTIM_latest_rassda:-"+2.99"} - -DTIM_earliest_gpsipw=${DTIM_latest_gpsipw:-"-0.05"} DTIM_earliest_gpsipw=${DTIM_latest_gpsipw:-"-0.05"} -DTIM_latest_gpsipw=${DTIM_latest_gpsipw:-"+0.05"} DTIM_latest_gpsipw=${DTIM_latest_gpsipw:-"+0.05"} - -TIME_TRIM=${TIME_TRIM:-${TIME_TRIM4:-on}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM4:-on}} - -$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 aircar aircft proflr vadwnd \ $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 aircar aircft proflr vadwnd \ - rassda gpsipw hdob rassda gpsipw hdob -error4=$? error4=$? -echo "$error4" > $DATA/error4 echo "$error4" > $DATA/error4 - -if [ "$SENDDBN" = "YES" ]; then if [ "$SENDDBN" = "YES" ]; then - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_aircar $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_aircar $job \ - ${COMSP}aircar.tm00.bufr_d ${COMSP}aircar.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_aircft $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_aircft $job \ - ${COMSP}aircft.tm00.bufr_d ${COMSP}aircft.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_proflr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_proflr $job \ - ${COMSP}proflr.tm00.bufr_d ${COMSP}proflr.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_vadwnd $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_vadwnd $job \ - ${COMSP}vadwnd.tm00.bufr_d ${COMSP}vadwnd.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_rassda $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_rassda $job \ - ${COMSP}rassda.tm00.bufr_d ${COMSP}rassda.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gpsipw $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gpsipw $job \ - ${COMSP}gpsipw.tm00.bufr_d ${COMSP}gpsipw.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_hdob $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_hdob $job \ - ${COMSP}hdob.tm00.bufr_d ${COMSP}hdob.tm00.bufr_d -fi fi - -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_4 echo Script thread_4 -echo Finished executing on node `hostname` echo Finished executing on node `hostname` -echo Ending time : `date -u` echo Ending time : `date -u` -echo "********************************************************************" echo "********************************************************************" -set -x set -x -} > $DATA/4.out 2>&1 } > $DATA/4.out 2>&1 -EOF EOF -set -x set -x - -set +x set +x -#---------------------------------------------------------------- #---------------------------------------------------------------- -cat<<\EOF>thread_5; chmod +x thread_5 cat<<\EOF>thread_5; chmod +x thread_5 -set -uax set -uax - -cd $DATA cd $DATA - -{ echo { echo -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_5 echo Script thread_5 -echo Executing on node `hostname` echo Executing on node `hostname` -echo Starting time: `date -u` echo Starting time: `date -u` -echo "********************************************************************" echo "********************************************************************" -echo echo -set -x set -x - -export STATUS=NO export STATUS=NO -export DUMP_NUMBER=5 export DUMP_NUMBER=5 - -#=================================================================== #=================================================================== -# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: -# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types -# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) -# # -#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- -# Currently not executed in GDAS or GFS: # Currently not executed in GDAS or GFS: -# Dump # 5 : MSONET: 30 subtype(s) # Dump # 5 : MSONET: 30 subtype(s) -# --------------------- # --------------------- -# TOTAL NUMBER OF SUBTYPES = 30 # TOTAL NUMBER OF SUBTYPES = 30 -# # -#=================================================================== #=================================================================== - -DTIM_latest_msonet=${DTIM_latest_msonet:-"+2.99"} DTIM_latest_msonet=${DTIM_latest_msonet:-"+2.99"} - -TIME_TRIM=${TIME_TRIM:-${TIME_TRIM5:-off}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM5:-off}} - -$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 msonet $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 msonet -error5=$? error5=$? -echo "$error5" > $DATA/error5 echo "$error5" > $DATA/error5 - -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_5 echo Script thread_5 -echo Finished executing on node `hostname` echo Finished executing on node `hostname` -echo Ending time : `date -u` echo Ending time : `date -u` -echo "********************************************************************" echo "********************************************************************" -set -x set -x -} > $DATA/5.out 2>&1 } > $DATA/5.out 2>&1 -EOF EOF -set -x set -x - -set +x set +x -#---------------------------------------------------------------- #---------------------------------------------------------------- -cat<<\EOF>thread_6; chmod +x thread_6 cat<<\EOF>thread_6; chmod +x thread_6 -set -uax set -uax - -cd $DATA cd $DATA - -{ echo { echo -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_6 echo Script thread_6 -echo Executing on node `hostname` echo Executing on node `hostname` -echo Starting time: `date -u` echo Starting time: `date -u` -echo "********************************************************************" echo "********************************************************************" -echo echo -set -x set -x - -export STATUS=NO export STATUS=NO -export DUMP_NUMBER=6 export DUMP_NUMBER=6 - -#=================================================================== #=================================================================== -# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: -# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types -# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) -# # -#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- -# Currently not executed in GDAS or GFS: # Currently not executed in GDAS or GFS: -# Dump # 6 : NEXRAD: 8 subtype(s) # Dump # 6 : NEXRAD: 8 subtype(s) -# -------------------- # -------------------- -# TOTAL NUMBER OF SUBTYPES = 8 # TOTAL NUMBER OF SUBTYPES = 8 -# # -#=================================================================== #=================================================================== - -DTIM_latest_nexrad=${DTIM_latest_nexrad:-"+2.99"} DTIM_latest_nexrad=${DTIM_latest_nexrad:-"+2.99"} - -TIME_TRIM=${TIME_TRIM:-${TIME_TRIM6:-off}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM6:-off}} - -# NEXRAD tanks are hourly # NEXRAD tanks are hourly -# Process only those hourly tanks w/i requested dump center cycle time window # Process only those hourly tanks w/i requested dump center cycle time window - -SKIP_006010=YES # radial wind 00Z SKIP_006010=YES # radial wind 00Z -SKIP_006011=YES # radial wind 01Z SKIP_006011=YES # radial wind 01Z -SKIP_006012=YES # radial wind 02Z SKIP_006012=YES # radial wind 02Z -SKIP_006013=YES # radial wind 03Z SKIP_006013=YES # radial wind 03Z -SKIP_006014=YES # radial wind 04Z SKIP_006014=YES # radial wind 04Z -SKIP_006015=YES # radial wind 05Z SKIP_006015=YES # radial wind 05Z -SKIP_006016=YES # radial wind 06Z SKIP_006016=YES # radial wind 06Z -SKIP_006017=YES # radial wind 07Z SKIP_006017=YES # radial wind 07Z -SKIP_006018=YES # radial wind 08Z SKIP_006018=YES # radial wind 08Z -SKIP_006019=YES # radial wind 09Z SKIP_006019=YES # radial wind 09Z -SKIP_006020=YES # radial wind 10Z SKIP_006020=YES # radial wind 10Z -SKIP_006021=YES # radial wind 11Z SKIP_006021=YES # radial wind 11Z -SKIP_006022=YES # radial wind 12Z SKIP_006022=YES # radial wind 12Z -SKIP_006023=YES # radial wind 13Z SKIP_006023=YES # radial wind 13Z -SKIP_006024=YES # radial wind 14Z SKIP_006024=YES # radial wind 14Z -SKIP_006025=YES # radial wind 15Z SKIP_006025=YES # radial wind 15Z -SKIP_006026=YES # radial wind 16Z SKIP_006026=YES # radial wind 16Z -SKIP_006027=YES # radial wind 17Z SKIP_006027=YES # radial wind 17Z -SKIP_006028=YES # radial wind 18Z SKIP_006028=YES # radial wind 18Z -SKIP_006029=YES # radial wind 19Z SKIP_006029=YES # radial wind 19Z -SKIP_006030=YES # radial wind 20Z SKIP_006030=YES # radial wind 20Z -SKIP_006031=YES # radial wind 21Z SKIP_006031=YES # radial wind 21Z -SKIP_006032=YES # radial wind 22Z SKIP_006032=YES # radial wind 22Z -SKIP_006033=YES # radial wind 23Z SKIP_006033=YES # radial wind 23Z - -SKIP_006040=YES # reflectivity 00Z SKIP_006040=YES # reflectivity 00Z -SKIP_006041=YES # reflectivity 01Z SKIP_006041=YES # reflectivity 01Z -SKIP_006042=YES # reflectivity 02Z SKIP_006042=YES # reflectivity 02Z -SKIP_006043=YES # reflectivity 03Z SKIP_006043=YES # reflectivity 03Z -SKIP_006044=YES # reflectivity 04Z SKIP_006044=YES # reflectivity 04Z -SKIP_006045=YES # reflectivity 05Z SKIP_006045=YES # reflectivity 05Z -SKIP_006046=YES # reflectivity 06Z SKIP_006046=YES # reflectivity 06Z -SKIP_006047=YES # reflectivity 07Z SKIP_006047=YES # reflectivity 07Z -SKIP_006048=YES # reflectivity 08Z SKIP_006048=YES # reflectivity 08Z -SKIP_006049=YES # reflectivity 09Z SKIP_006049=YES # reflectivity 09Z -SKIP_006050=YES # reflectivity 10Z SKIP_006050=YES # reflectivity 10Z -SKIP_006051=YES # reflectivity 11Z SKIP_006051=YES # reflectivity 11Z -SKIP_006052=YES # reflectivity 12Z SKIP_006052=YES # reflectivity 12Z -SKIP_006053=YES # reflectivity 13Z SKIP_006053=YES # reflectivity 13Z -SKIP_006054=YES # reflectivity 14Z SKIP_006054=YES # reflectivity 14Z -SKIP_006055=YES # reflectivity 15Z SKIP_006055=YES # reflectivity 15Z -SKIP_006056=YES # reflectivity 16Z SKIP_006056=YES # reflectivity 16Z -SKIP_006057=YES # reflectivity 17Z SKIP_006057=YES # reflectivity 17Z -SKIP_006058=YES # reflectivity 18Z SKIP_006058=YES # reflectivity 18Z -SKIP_006059=YES # reflectivity 19Z SKIP_006059=YES # reflectivity 19Z -SKIP_006060=YES # reflectivity 20Z SKIP_006060=YES # reflectivity 20Z -SKIP_006061=YES # reflectivity 21Z SKIP_006061=YES # reflectivity 21Z -SKIP_006062=YES # reflectivity 22Z SKIP_006062=YES # reflectivity 22Z -SKIP_006063=YES # reflectivity 23Z SKIP_006063=YES # reflectivity 23Z - -if [ $cycp -eq 00 ]; then # (22.5 - 01.5 Z) if [ $cycp -eq 00 ]; then # (22.5 - 01.5 Z) - unset SKIP_006032 # radial wind 22Z unset SKIP_006032 # radial wind 22Z - unset SKIP_006033 # radial wind 23Z unset SKIP_006033 # radial wind 23Z - unset SKIP_006010 # radial wind 00Z unset SKIP_006010 # radial wind 00Z - unset SKIP_006011 # radial wind 01Z unset SKIP_006011 # radial wind 01Z - unset SKIP_006062 # reflectivity 22Z unset SKIP_006062 # reflectivity 22Z - unset SKIP_006063 # reflectivity 23Z unset SKIP_006063 # reflectivity 23Z - unset SKIP_006040 # reflectivity 00Z unset SKIP_006040 # reflectivity 00Z - unset SKIP_006041 # reflectivity 01Z unset SKIP_006041 # reflectivity 01Z -elif [ $cycp -eq 06 ]; then # (04.5 - 07.5 Z) elif [ $cycp -eq 06 ]; then # (04.5 - 07.5 Z) - unset SKIP_006014 # radial wind 04Z unset SKIP_006014 # radial wind 04Z - unset SKIP_006015 # radial wind 05Z unset SKIP_006015 # radial wind 05Z - unset SKIP_006016 # radial wind 06Z unset SKIP_006016 # radial wind 06Z - unset SKIP_006017 # radial wind 07Z unset SKIP_006017 # radial wind 07Z - unset SKIP_006044 # reflectivity 04Z unset SKIP_006044 # reflectivity 04Z - unset SKIP_006045 # reflectivity 05Z unset SKIP_006045 # reflectivity 05Z - unset SKIP_006046 # reflectivity 06Z unset SKIP_006046 # reflectivity 06Z - unset SKIP_006047 # reflectivity 07Z unset SKIP_006047 # reflectivity 07Z -elif [ $cycp -eq 12 ]; then # (10.5 - 13.5 Z) elif [ $cycp -eq 12 ]; then # (10.5 - 13.5 Z) - unset SKIP_006020 # radial wind 10Z unset SKIP_006020 # radial wind 10Z - unset SKIP_006021 # radial wind 11Z unset SKIP_006021 # radial wind 11Z - unset SKIP_006022 # radial wind 12Z unset SKIP_006022 # radial wind 12Z - unset SKIP_006023 # radial wind 13Z unset SKIP_006023 # radial wind 13Z - unset SKIP_006050 # reflectivity 10Z unset SKIP_006050 # reflectivity 10Z - unset SKIP_006051 # reflectivity 11Z unset SKIP_006051 # reflectivity 11Z - unset SKIP_006052 # reflectivity 12Z unset SKIP_006052 # reflectivity 12Z - unset SKIP_006053 # reflectivity 13Z unset SKIP_006053 # reflectivity 13Z -elif [ $cycp -eq 18 ]; then # (16.5 - 19.5 Z) elif [ $cycp -eq 18 ]; then # (16.5 - 19.5 Z) - unset SKIP_006026 # radial wind 16Z unset SKIP_006026 # radial wind 16Z - unset SKIP_006027 # radial wind 17Z unset SKIP_006027 # radial wind 17Z - unset SKIP_006028 # radial wind 18Z unset SKIP_006028 # radial wind 18Z - unset SKIP_006029 # radial wind 19Z unset SKIP_006029 # radial wind 19Z - unset SKIP_006056 # reflectivity 16Z unset SKIP_006056 # reflectivity 16Z - unset SKIP_006057 # reflectivity 17Z unset SKIP_006057 # reflectivity 17Z - unset SKIP_006058 # reflectivity 18Z unset SKIP_006058 # reflectivity 18Z - unset SKIP_006059 # reflectivity 19Z unset SKIP_006059 # reflectivity 19Z -fi fi - -$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 nexrad $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 nexrad -error6=$? error6=$? -echo "$error6" > $DATA/error6 echo "$error6" > $DATA/error6 - -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_6 echo Script thread_6 -echo Finished executing on node `hostname` echo Finished executing on node `hostname` -echo Ending time : `date -u` echo Ending time : `date -u` -echo "********************************************************************" echo "********************************************************************" -set -x set -x -} > $DATA/6.out 2>&1 } > $DATA/6.out 2>&1 -EOF EOF -set -x set -x - -set +x set +x -#------------------------------------------------------------------------------ #------------------------------------------------------------------------------ -cat<<\EOF>thread_7; chmod +x thread_7 cat<<\EOF>thread_7; chmod +x thread_7 -set -uax set -uax - -cd $DATA cd $DATA - -{ echo { echo -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_7 echo Script thread_7 -echo Executing on node `hostname` echo Executing on node `hostname` -echo Starting time: `date -u` echo Starting time: `date -u` -echo "********************************************************************" echo "********************************************************************" -echo echo -set -x set -x - -export STATUS=NO export STATUS=NO -export DUMP_NUMBER=7 export DUMP_NUMBER=7 - -#========================================================================= #========================================================================= -# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: -# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types -# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) -# # -#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- -# Dump # 7 : AVCSPM: 1 subtype(s) # Dump # 7 : AVCSPM: 1 subtype(s) -# ESMHS: 1 subtype(s) # ESMHS: 1 subtype(s) -# 1BMHS: 1 subtype(s) # 1BMHS: 1 subtype(s) -# AIRSEV: 1 subtype(s) # AIRSEV: 1 subtype(s) -# ATMSDB: 1 subtype(s) # ATMSDB: 1 subtype(s) -# GOME: 1 subtype(s) # GOME: 1 subtype(s) -# OMI: 1 subtype(s) # OMI: 1 subtype(s) -# TRKOB: 1 subtype(s) # TRKOB: 1 subtype(s) -# GPSRO: 1 subtype(s) # GPSRO: 1 subtype(s) -# CRISF4: 1 subtype(s) (if present in past 10 days of tanks) # CRISF4: 1 subtype(s) (if present in past 10 days of tanks) -# -------------------- # -------------------- -# TOTAL NUMBER OF SUBTYPES = 10 # TOTAL NUMBER OF SUBTYPES = 10 -# # -#========================================================================= #========================================================================= - -DTIM_latest_avcspm=${DTIM_latest_avcspm:-"+2.99"} DTIM_latest_avcspm=${DTIM_latest_avcspm:-"+2.99"} -DTIM_latest_esmhs=${DTIM_latest_esmhs:-"+2.99"} DTIM_latest_esmhs=${DTIM_latest_esmhs:-"+2.99"} -DTIM_latest_1bmhs=${DTIM_latest_1bmhs:-"+2.99"} DTIM_latest_1bmhs=${DTIM_latest_1bmhs:-"+2.99"} -DTIM_latest_airsev=${DTIM_latest_airsev:-"+2.99"} DTIM_latest_airsev=${DTIM_latest_airsev:-"+2.99"} -DTIM_latest_atmsdb=${DTIM_latest_atmsdb:-"+2.99"} DTIM_latest_atmsdb=${DTIM_latest_atmsdb:-"+2.99"} -DTIM_latest_gome=${DTIM_latest_gome:-"+2.99"} DTIM_latest_gome=${DTIM_latest_gome:-"+2.99"} -DTIM_latest_omi=${DTIM_latest_omi:-"+2.99"} DTIM_latest_omi=${DTIM_latest_omi:-"+2.99"} -DTIM_latest_trkob=${DTIM_latest_trkob:-"+2.99"} DTIM_latest_trkob=${DTIM_latest_trkob:-"+2.99"} -DTIM_latest_gpsro=${DTIM_latest_gpsro:-"+2.99"} DTIM_latest_gpsro=${DTIM_latest_gpsro:-"+2.99"} -#----------------------------------------------- #----------------------------------------------- -# check for crisf4 tank presence in past 10 days # check for crisf4 tank presence in past 10 days -crisf4="" crisf4="" -err_check_tanks=0 err_check_tanks=0 -sh $USHbufr_dump/check_tanks.sh crisf4 sh $USHbufr_dump/check_tanks.sh crisf4 -err_check_tanks=$? err_check_tanks=$? -if [ $err_check_tanks -eq 0 ];then if [ $err_check_tanks -eq 0 ];then - crisf4=crisf4 crisf4=crisf4 - DTIM_latest_crisf4=${DTIM_latest_crisf4:-"+2.99"} DTIM_latest_crisf4=${DTIM_latest_crisf4:-"+2.99"} -fi fi -#----------------------------------------------- #----------------------------------------------- - -TIME_TRIM=${TIME_TRIM:-${TIME_TRIM7:-off}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM7:-off}} - -$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 avcspm esmhs 1bmhs \ $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 avcspm esmhs 1bmhs \ - airsev atmsdb gome omi trkob gpsro $crisf4 airsev atmsdb gome omi trkob gpsro $crisf4 -error7=$? error7=$? -echo "$error7" > $DATA/error7 echo "$error7" > $DATA/error7 - -if [ "$SENDDBN" = "YES" ]; then if [ "$SENDDBN" = "YES" ]; then - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_avcspm $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_avcspm $job \ - ${COMSP}avcspm.tm00.bufr_d ${COMSP}avcspm.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_esmhs $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_esmhs $job \ - ${COMSP}esmhs.tm00.bufr_d ${COMSP}esmhs.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_1bmhs $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_1bmhs $job \ - ${COMSP}1bmhs.tm00.bufr_d ${COMSP}1bmhs.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_airsev $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_airsev $job \ - ${COMSP}airsev.tm00.bufr_d ${COMSP}airsev.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_atmsdb $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_atmsdb $job \ - ${COMSP}atmsdb.tm00.bufr_d ${COMSP}atmsdb.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gome $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gome $job \ - ${COMSP}gome.tm00.bufr_d ${COMSP}gome.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_omi $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_omi $job \ - ${COMSP}omi.tm00.bufr_d ${COMSP}omi.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_trkob $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_trkob $job \ - ${COMSP}trkob.tm00.bufr_d ${COMSP}trkob.tm00.bufr_d -# gpsro dump file has nr version which is alerted from # gpsro dump file has nr version which is alerted from -# exdump_post.sh.ecf # exdump_post.sh.ecf - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gpsro $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gpsro $job \ - ${COMSP}gpsro.tm00.bufr_d ${COMSP}gpsro.tm00.bufr_d - if [ "$crisf4" = crisf4 ];then if [ "$crisf4" = crisf4 ];then - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_crisf4 $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_crisf4 $job \ - ${COMSP}crisf4.tm00.bufr_d ${COMSP}crisf4.tm00.bufr_d - fi fi -fi fi - -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_7 echo Script thread_7 -echo Finished executing on node `hostname` echo Finished executing on node `hostname` -echo Ending time : `date -u` echo Ending time : `date -u` -echo "********************************************************************" echo "********************************************************************" -set -x set -x -} > $DATA/7.out 2>&1 } > $DATA/7.out 2>&1 -EOF EOF -set -x set -x - -set +x set +x -#---------------------------------------------------------------- #---------------------------------------------------------------- -cat<<\EOF>thread_8; chmod +x thread_8 cat<<\EOF>thread_8; chmod +x thread_8 -set -uax set -uax - -cd $DATA cd $DATA - -{ echo { echo -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_8 echo Script thread_8 -echo Executing on node `hostname` echo Executing on node `hostname` -echo Starting time: `date -u` echo Starting time: `date -u` -echo "********************************************************************" echo "********************************************************************" -echo echo -set -x set -x - -export STATUS=NO export STATUS=NO -export DUMP_NUMBER=8 export DUMP_NUMBER=8 - -#======================================================================= #======================================================================= -# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: -# (1) time window radius is +/- 1.5 hrs for all SATWND types # (1) time window radius is +/- 1.5 hrs for all SATWND types -# EXCEPT: SATWND subtypes 005/030, 005/031, 005/032, 005/034, 005/039, # EXCEPT: SATWND subtypes 005/030, 005/031, 005/032, 005/034, 005/039, -# 005/064, 005/065, 005/066, 005/067, 005/068, 005/069, # 005/064, 005/065, 005/066, 005/067, 005/068, 005/069, -# 005/070, 005/071, 005/072, 005/080 and 005/091 # 005/070, 005/071, 005/072, 005/080 and 005/091 -# where it is # where it is -# -3.00 to +2.99 hours. # -3.00 to +2.99 hours. -# (2) TIME TRIMMING IS DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS DONE IN THIS DUMP (default, unless overridden) -# # -#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- -# Dump # 8 : SATWND: 16 subtype(s) (bufr_dumplist.v2.3.0) # Dump # 8 : SATWND: 16 subtype(s) (bufr_dumplist.v2.3.0) -# --------------------- # --------------------- -# TOTAL NUMBER OF SUBTYPES = 25 # TOTAL NUMBER OF SUBTYPES = 25 -# # -#======================================================================= #======================================================================= - -ADD_satwnd="005024 005025 005026 005030 005031 005032 005034 005039 005072" ADD_satwnd="005024 005025 005026 005030 005031 005032 005034 005039 005072" - -# Skip old bufr METEOSAT AMVs; for testing skip in trigger or version file # Skip old bufr METEOSAT AMVs; for testing skip in trigger or version file -#export SKIP_005064=YES #export SKIP_005064=YES -#export SKIP_005065=YES #export SKIP_005065=YES -#export SKIP_005066=YES #export SKIP_005066=YES - -# satwnd types # satwnd types -# ------------ # ------------ -DTIM_earliest_005024=${DTIM_earliest_005024:-"-3.00"} DTIM_earliest_005024=${DTIM_earliest_005024:-"-3.00"} -DTIM_latest_005024=${DTIM_latest_005024:-"+2.99"} DTIM_latest_005024=${DTIM_latest_005024:-"+2.99"} -DTIM_earliest_005025=${DTIM_earliest_005025:-"-3.00"} DTIM_earliest_005025=${DTIM_earliest_005025:-"-3.00"} -DTIM_latest_005025=${DTIM_latest_005025:-"+2.99"} DTIM_latest_005025=${DTIM_latest_005025:-"+2.99"} -DTIM_earliest_005026=${DTIM_earliest_005026:-"-3.00"} DTIM_earliest_005026=${DTIM_earliest_005026:-"-3.00"} -DTIM_latest_005026=${DTIM_latest_005026:-"+2.99"} DTIM_latest_005026=${DTIM_latest_005026:-"+2.99"} -DTIM_earliest_005030=${DTIM_earliest_005030:-"-3.00"} DTIM_earliest_005030=${DTIM_earliest_005030:-"-3.00"} -DTIM_latest_005030=${DTIM_latest_005030:-"+2.99"} DTIM_latest_005030=${DTIM_latest_005030:-"+2.99"} -DTIM_earliest_005031=${DTIM_earliest_005031:-"-3.00"} DTIM_earliest_005031=${DTIM_earliest_005031:-"-3.00"} -DTIM_latest_005031=${DTIM_latest_005031:-"+2.99"} DTIM_latest_005031=${DTIM_latest_005031:-"+2.99"} -DTIM_earliest_005032=${DTIM_earliest_005032:-"-3.00"} DTIM_earliest_005032=${DTIM_earliest_005032:-"-3.00"} -DTIM_latest_005032=${DTIM_latest_005032:-"+2.99"} DTIM_latest_005032=${DTIM_latest_005032:-"+2.99"} -DTIM_earliest_005034=${DTIM_earliest_005034:-"-3.00"} DTIM_earliest_005034=${DTIM_earliest_005034:-"-3.00"} -DTIM_latest_005034=${DTIM_latest_005034:-"+2.99"} DTIM_latest_005034=${DTIM_latest_005034:-"+2.99"} -DTIM_earliest_005039=${DTIM_earliest_005039:-"-3.00"} DTIM_earliest_005039=${DTIM_earliest_005039:-"-3.00"} -DTIM_latest_005039=${DTIM_latest_005039:-"+2.99"} DTIM_latest_005039=${DTIM_latest_005039:-"+2.99"} -DTIM_earliest_005064=${DTIM_earliest_005064:-"-3.00"} DTIM_earliest_005064=${DTIM_earliest_005064:-"-3.00"} -DTIM_latest_005064=${DTIM_latest_005064:-"+2.99"} DTIM_latest_005064=${DTIM_latest_005064:-"+2.99"} -DTIM_earliest_005065=${DTIM_earliest_005065:-"-3.00"} DTIM_earliest_005065=${DTIM_earliest_005065:-"-3.00"} -DTIM_latest_005065=${DTIM_latest_005065:-"+2.99"} DTIM_latest_005065=${DTIM_latest_005065:-"+2.99"} -DTIM_earliest_005066=${DTIM_earliest_005066:-"-3.00"} DTIM_earliest_005066=${DTIM_earliest_005066:-"-3.00"} -DTIM_latest_005066=${DTIM_latest_005066:-"+2.99"} DTIM_latest_005066=${DTIM_latest_005066:-"+2.99"} -DTIM_earliest_005067=${DTIM_earliest_005067:-"-3.00"} DTIM_earliest_005067=${DTIM_earliest_005067:-"-3.00"} -DTIM_latest_005067=${DTIM_latest_005067:-"+2.99"} DTIM_latest_005067=${DTIM_latest_005067:-"+2.99"} -DTIM_earliest_005068=${DTIM_earliest_005068:-"-3.00"} DTIM_earliest_005068=${DTIM_earliest_005068:-"-3.00"} -DTIM_latest_005068=${DTIM_latest_005068:-"+2.99"} DTIM_latest_005068=${DTIM_latest_005068:-"+2.99"} -DTIM_earliest_005069=${DTIM_earliest_005069:-"-3.00"} DTIM_earliest_005069=${DTIM_earliest_005069:-"-3.00"} -DTIM_latest_005069=${DTIM_latest_005069:-"+2.99"} DTIM_latest_005069=${DTIM_latest_005069:-"+2.99"} -DTIM_earliest_005070=${DTIM_earliest_005070:-"-3.00"} DTIM_earliest_005070=${DTIM_earliest_005070:-"-3.00"} -DTIM_latest_005070=${DTIM_latest_005070:-"+2.99"} DTIM_latest_005070=${DTIM_latest_005070:-"+2.99"} -DTIM_earliest_005071=${DTIM_earliest_005071:-"-3.00"} DTIM_earliest_005071=${DTIM_earliest_005071:-"-3.00"} -DTIM_latest_005071=${DTIM_latest_005071:-"+2.99"} DTIM_latest_005071=${DTIM_latest_005071:-"+2.99"} -DTIM_earliest_005072=${DTIM_earliest_005072:-"-3.00"} DTIM_earliest_005072=${DTIM_earliest_005072:-"-3.00"} -DTIM_latest_005072=${DTIM_latest_005072:-"+2.99"} DTIM_latest_005072=${DTIM_latest_005072:-"+2.99"} -DTIM_earliest_005080=${DTIM_earliest_005080:-"-3.00"} DTIM_earliest_005080=${DTIM_earliest_005080:-"-3.00"} -DTIM_latest_005080=${DTIM_latest_005080:-"+2.99"} DTIM_latest_005080=${DTIM_latest_005080:-"+2.99"} -DTIM_earliest_005081=${DTIM_earliest_005081:-"-3.00"} DTIM_earliest_005081=${DTIM_earliest_005081:-"-3.00"} -DTIM_latest_005081=${DTIM_latest_005081:-"+2.99"} DTIM_latest_005081=${DTIM_latest_005081:-"+2.99"} -DTIM_earliest_005091=${DTIM_earliest_005091:-"-3.00"} DTIM_earliest_005091=${DTIM_earliest_005091:-"-3.00"} -DTIM_latest_005091=${DTIM_latest_005091:-"+2.99"} DTIM_latest_005091=${DTIM_latest_005091:-"+2.99"} - - - -TIME_TRIM=${TIME_TRIM:-${TIME_TRIM8:-on}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM8:-on}} - -$ushscript_dump/bufr_dump_obs.sh $dumptime 1.5 1 satwnd $ushscript_dump/bufr_dump_obs.sh $dumptime 1.5 1 satwnd -error8=$? error8=$? -echo "$error8" > $DATA/error8 echo "$error8" > $DATA/error8 - -if [ "$SENDDBN" = "YES" ]; then if [ "$SENDDBN" = "YES" ]; then - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_satwnd $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_satwnd $job \ - ${COMSP}satwnd.tm00.bufr_d ${COMSP}satwnd.tm00.bufr_d -fi fi - -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_8 echo Script thread_8 -echo Finished executing on node `hostname` echo Finished executing on node `hostname` -echo Ending time : `date -u` echo Ending time : `date -u` -echo "********************************************************************" echo "********************************************************************" -set -x set -x -} > $DATA/8.out 2>&1 } > $DATA/8.out 2>&1 -EOF EOF -set -x set -x - -set +x set +x -#---------------------------------------------------------------- #---------------------------------------------------------------- -cat<<\EOF>thread_9; chmod +x thread_9 cat<<\EOF>thread_9; chmod +x thread_9 -set -uax set -uax - -cd $DATA cd $DATA - -{ echo { echo -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_9 echo Script thread_9 -echo Executing on node `hostname` echo Executing on node `hostname` -echo Starting time: `date -u` echo Starting time: `date -u` -echo "********************************************************************" echo "********************************************************************" -echo echo -set -x set -x - -export STATUS=NO export STATUS=NO -export DUMP_NUMBER=9 export DUMP_NUMBER=9 - -#======================================================================= #======================================================================= -# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: -# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types -# EXCEPT: GEOIMR where it is -0.50 to +0.50 hour # EXCEPT: GEOIMR where it is -0.50 to +0.50 hour -# (2) TIME TRIMMING IS DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS DONE IN THIS DUMP (default, unless overridden) -# # -#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- -# Dump # 9 : GEOIMR: 1 subtype(s) # Dump # 9 : GEOIMR: 1 subtype(s) -# -------------------- # -------------------- -# TOTAL NUMBER OF SUBTYPES = 1 # TOTAL NUMBER OF SUBTYPES = 1 -# # -#======================================================================= #======================================================================= -DTIM_earliest_gmi1cr=${DTIM_earliest_gmi1cr:-"-3.00"} DTIM_earliest_gmi1cr=${DTIM_earliest_gmi1cr:-"-3.00"} -DTIM_latest_gmi1cr=${DTIM_latest_gmi1cr:-"+2.99"} DTIM_latest_gmi1cr=${DTIM_latest_gmi1cr:-"+2.99"} - -DTIM_earliest_geoimr=${DTIM_earliest_geoimr:-"-0.50"} DTIM_earliest_geoimr=${DTIM_earliest_geoimr:-"-0.50"} -DTIM_latest_geoimr=${DTIM_latest_geoimr:-"+0.50"} DTIM_latest_geoimr=${DTIM_latest_geoimr:-"+0.50"} - -TIME_TRIM=${TIME_TRIM:-${TIME_TRIM9:-on}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM9:-on}} - -$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 geoimr gmi1cr $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 geoimr gmi1cr - -error9=$? error9=$? -echo "$error9" > $DATA/error9 echo "$error9" > $DATA/error9 - -if [ "$SENDDBN" = "YES" ]; then if [ "$SENDDBN" = "YES" ]; then - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_geoimr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_geoimr $job \ - ${COMSP}geoimr.tm00.bufr_d ${COMSP}geoimr.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gmi1cr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gmi1cr $job \ - ${COMSP}gmi1cr.tm00.bufr_d ${COMSP}gmi1cr.tm00.bufr_d -fi fi - -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_9 echo Script thread_9 -echo Finished executing on node `hostname` echo Finished executing on node `hostname` -echo Ending time : `date -u` echo Ending time : `date -u` -echo "********************************************************************" echo "********************************************************************" -set -x set -x -} > $DATA/9.out 2>&1 } > $DATA/9.out 2>&1 -EOF EOF -set -x set -x - -set +x set +x -#------------------------------------------------------------------------------ #------------------------------------------------------------------------------ -cat<<\EOF>thread_10; chmod +x thread_10 cat<<\EOF>thread_10; chmod +x thread_10 -set -uax set -uax - -cd $DATA cd $DATA - -{ echo { echo -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_10 echo Script thread_10 -echo Executing on node `hostname` echo Executing on node `hostname` -echo Starting time: `date -u` echo Starting time: `date -u` -echo "********************************************************************" echo "********************************************************************" -echo echo -set -x set -x - -export STATUS=NO export STATUS=NO -export DUMP_NUMBER=10 export DUMP_NUMBER=10 - -#========================================================================= #========================================================================= -# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: -# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types -# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) -# # -#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- -# Dump #10 : ESIASI: 1 subtype(s) # Dump #10 : ESIASI: 1 subtype(s) -# MTIASI: 1 subtype(s) # MTIASI: 1 subtype(s) -# ESAMUA: 1 subtype(s) # ESAMUA: 1 subtype(s) -# CRSFDB: 1 subtype(s) < -# IASIDB: 1 subtype(s) < -# SEVASR: 1 subtype(s) # SEVASR: 1 subtype(s) -# 1BAMUA: 1 subtype(s) # 1BAMUA: 1 subtype(s) -# BATHY: 1 subtype(s) # BATHY: 1 subtype(s) -# OSBUV8: 1 subtype(s) # OSBUV8: 1 subtype(s) -# OMPSN8: 1 subtype(s) # OMPSN8: 1 subtype(s) -# OMPST8: 1 subtype(s) # OMPST8: 1 subtype(s) -# GSRASR: 1 subtype(s) # GSRASR: 1 subtype(s) -# OMPSLP: 1 subtype(s) # OMPSLP: 1 subtype(s) -# -------------------- # -------------------- -# TOTAL NUMBER OF SUBTYPES = 13 # TOTAL NUMBER OF SUBTYPES = 13 -# # -#========================================================================= #========================================================================= - -DTIM_latest_esiasi=${DTIM_latest_esiasi:-"+2.99"} DTIM_latest_esiasi=${DTIM_latest_esiasi:-"+2.99"} -DTIM_latest_mtiasi=${DTIM_latest_mtiasi:-"+2.99"} DTIM_latest_mtiasi=${DTIM_latest_mtiasi:-"+2.99"} -DTIM_latest_esamua=${DTIM_latest_esamua:-"+2.99"} DTIM_latest_esamua=${DTIM_latest_esamua:-"+2.99"} -DTIM_latest_crsfdb=${DTIM_latest_crsfdb:-"+2.99"} < -DTIM_latest_iasidb=${DTIM_latest_iasidb:-"+2.99"} < -DTIM_latest_sevasr=${DTIM_latest_sevasr:-"+2.99"} DTIM_latest_sevasr=${DTIM_latest_sevasr:-"+2.99"} -DTIM_latest_1bamua=${DTIM_latest_1bamua:-"+2.99"} DTIM_latest_1bamua=${DTIM_latest_1bamua:-"+2.99"} -DTIM_latest_bathy=${DTIM_latest_bathy:-"+2.99"} DTIM_latest_bathy=${DTIM_latest_bathy:-"+2.99"} -DTIM_latest_osbuv8=${DTIM_latest_osbuv8:-"+2.99"} DTIM_latest_osbuv8=${DTIM_latest_osbuv8:-"+2.99"} -DTIM_latest_ompsn8=${DTIM_latest_ompsn8:-"+2.99"} DTIM_latest_ompsn8=${DTIM_latest_ompsn8:-"+2.99"} -DTIM_latest_ompst8=${DTIM_latest_ompst8:-"+2.99"} DTIM_latest_ompst8=${DTIM_latest_ompst8:-"+2.99"} -DTIM_latest_gsrasr=${DTIM_latest_gsrasr:-"+2.99"} DTIM_latest_gsrasr=${DTIM_latest_gsrasr:-"+2.99"} -DTIM_latest_ompslp=${DTIM_latest_ompslp:-"+2.99"} DTIM_latest_ompslp=${DTIM_latest_ompslp:-"+2.99"} -DTIM_latest_sstvpw=${DTIM_latest_sstvpw:-"+2.99"} DTIM_latest_sstvpw=${DTIM_latest_sstvpw:-"+2.99"} - -TIME_TRIM=${TIME_TRIM:-${TIME_TRIM10:-off}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM10:-off}} - -$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 esiasi mtiasi esamua \ $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 esiasi mtiasi esamua \ - crsfdb iasidb sevasr 1bamua bathy osbuv8 ompsn8 ompst8 gsrasr ompslp \ | sevasr 1bamua bathy osbuv8 ompsn8 ompst8 gsrasr ompslp sstvpw - sstvpw < -error10=$? error10=$? -echo "$error10" > $DATA/error10 echo "$error10" > $DATA/error10 - -if [ "$SENDDBN" = "YES" ]; then if [ "$SENDDBN" = "YES" ]; then - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_esiasi $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_esiasi $job \ - ${COMSP}esiasi.tm00.bufr_d ${COMSP}esiasi.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_mtiasi $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_mtiasi $job \ - ${COMSP}mtiasi.tm00.bufr_d ${COMSP}mtiasi.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_esamua $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_esamua $job \ - ${COMSP}esamua.tm00.bufr_d ${COMSP}esamua.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_crsfdb $job \ < - ${COMSP}crsfdb.tm00.bufr_d < - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_iasidb $job \ < - ${COMSP}iasidb.tm00.bufr_d < - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sevasr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sevasr $job \ - ${COMSP}sevasr.tm00.bufr_d ${COMSP}sevasr.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_1bamua $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_1bamua $job \ - ${COMSP}1bamua.tm00.bufr_d ${COMSP}1bamua.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_bathy $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_bathy $job \ - ${COMSP}bathy.tm00.bufr_d ${COMSP}bathy.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_osbuv8 $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_osbuv8 $job \ - ${COMSP}osbuv8.tm00.bufr_d ${COMSP}osbuv8.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ompsn8 $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ompsn8 $job \ - ${COMSP}ompsn8.tm00.bufr_d ${COMSP}ompsn8.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ompst8 $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ompst8 $job \ - ${COMSP}ompst8.tm00.bufr_d ${COMSP}ompst8.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gsrasr $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gsrasr $job \ - ${COMSP}gsrasr.tm00.bufr_d ${COMSP}gsrasr.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ompslp $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ompslp $job \ - ${COMSP}ompslp.tm00.bufr_d ${COMSP}ompslp.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sstvpw $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sstvpw $job \ - ${COMSP}sstvpw.tm00.bufr_d ${COMSP}sstvpw.tm00.bufr_d -fi fi - -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_10 echo Script thread_10 -echo Finished executing on node `hostname` echo Finished executing on node `hostname` -echo Ending time : `date -u` echo Ending time : `date -u` -echo "********************************************************************" echo "********************************************************************" -set -x set -x -} > $DATA/10.out 2>&1 } > $DATA/10.out 2>&1 -EOF EOF -set -x set -x - -set +x set +x -#------------------------------------------------------------------------------ #------------------------------------------------------------------------------ -cat<<\EOF>thread_11; chmod +x thread_11 cat<<\EOF>thread_11; chmod +x thread_11 -set -uax set -uax - -cd $DATA cd $DATA - -{ echo { echo -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_11 echo Script thread_11 -echo Executing on node `hostname` echo Executing on node `hostname` -echo Starting time: `date -u` echo Starting time: `date -u` -echo "********************************************************************" echo "********************************************************************" -echo echo -set -x set -x - -export STATUS=NO export STATUS=NO -export DUMP_NUMBER=11 export DUMP_NUMBER=11 - -#========================================================================= #========================================================================= -# NOTES ABOUT THIS DUMP GROUP: # NOTES ABOUT THIS DUMP GROUP: -# (1) time window radius is -3.00 to +2.99 hours on all types # (1) time window radius is -3.00 to +2.99 hours on all types -# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) # (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) -# # -#-------------------------------------------------------------------------- #-------------------------------------------------------------------------- -# Dump #11 : AMSR2: 1 subtype(s) # Dump #11 : AMSR2: 1 subtype(s) -# -------------------- # -------------------- -# TOTAL NUMBER OF SUBTYPES = 1 # TOTAL NUMBER OF SUBTYPES = 1 -# # -#========================================================================= #========================================================================= - -DTIM_latest_amsr2=${DTIM_latest_amsr2:-"+2.99"} DTIM_latest_amsr2=${DTIM_latest_amsr2:-"+2.99"} - -TIME_TRIM=${TIME_TRIM:-${TIME_TRIM11:-off}} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM11:-off}} - -$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 amsr2 $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 amsr2 -error11=$? error11=$? -echo "$error11" > $DATA/error11 echo "$error11" > $DATA/error11 - -if [ "$SENDDBN" = "YES" ]; then if [ "$SENDDBN" = "YES" ]; then - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_amsr2 $job \ $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_amsr2 $job \ - ${COMSP}amsr2.tm00.bufr_d ${COMSP}amsr2.tm00.bufr_d -fi fi - -set +x set +x -echo "********************************************************************" echo "********************************************************************" -echo Script thread_11 echo Script thread_11 -echo Finished executing on node `hostname` echo Finished executing on node `hostname` -echo Ending time : `date -u` echo Ending time : `date -u` -echo "********************************************************************" echo "********************************************************************" -set -x set -x -} > $DATA/11.out 2>&1 } > $DATA/11.out 2>&1 -EOF EOF -set -x set -x - - > set +x - > #---------------------------------------------------------------- - > cat<<\EOF>thread_12; chmod +x thread_12 - > set -uax - > - > cd $DATA - > - > { echo - > set +x - > echo "********************************************************************" - > echo Script thread_12 - > echo Executing on node `hostname` - > echo Starting time: `date -u` - > echo "********************************************************************" - > echo - > set -x - > - > export STATUS=NO - > export DUMP_NUMBER=12 - > - > #========================================================================= - > # NOTES ABOUT THIS DUMP GROUP: - > # (1) time window radius is -3.00 to +2.99 hours on all types - > # (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) - > # - > #-------------------------------------------------------------------------- - > # Dump # 12 : ATMS: 1 subtype(s) (if present in past 10 days of tanks) - > # CRISFS: 1 subtype(s) (if present in past 10 days of tanks) - > # CRSFDB: 1 subtype(s) - > # IASIDB: 1 subtype(s) - > # -------------------- - > # TOTAL NUMBER OF SUBTYPES = 4 - > # - > #========================================================================= - > #----------------------------------------------- - > #----------------------------------------------- - > # check for atms tank presence in past 10 days - > atms="" - > err_check_tanks=0 - > sh $USHbufr_dump/check_tanks.sh atms - > err_check_tanks=$? - > if [ $err_check_tanks -eq 0 ];then - > atms=atms - > DTIM_latest_atms=${DTIM_latest_atms:-"+2.99"} - > fi - > #----------------------------------------------- - > #----------------------------------------------- - > # check for crisfs tank presence in past 10 days - > crisfs="" - > err_check_tanks=0 - > sh $USHbufr_dump/check_tanks.sh crisfs - > err_check_tanks=$? - > if [ $err_check_tanks -eq 0 ];then - > crisfs=crisfs - > DTIM_latest_crisfs=${DTIM_latest_crisfs:-"+2.99"} - > fi - > #----------------------------------------------- - > - > DTIM_latest_crsfdb=${DTIM_latest_crsfdb:-"+2.99"} - > DTIM_latest_iasidb=${DTIM_latest_iasidb:-"+2.99"} - > - > TIME_TRIM=${TIME_TRIM:-${TIME_TRIM1:-off}} - > - > $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 $atms $crisfs crsfdb iasidb - > error12=$? - > echo "$error12" > $DATA/error12 - > - > if [ "$SENDDBN" = "YES" ]; then - > if [ "$atms" = atms ];then - > $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_atms $job \ - > ${COMSP}atms.tm00.bufr_d - > fi - > ####### ALERTS TURNED OFF UNTIL REQUESTED BY USER ######################### - > # if [ "$crisfs" = crisfs ];then - > # $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_crisfs $job \ - > # ${COMSP}crisfs.tm00.bufr_d - > # fi - > ########################################################################### - > $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_crsfdb $job \ - > ${COMSP}crsfdb.tm00.bufr_d - > $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_iasidb $job \ - > ${COMSP}iasidb.tm00.bufr_d - > fi - > - > set +x - > echo "********************************************************************" - > echo Script thread_12 - > echo Finished executing on node `hostname` - > echo Ending time : `date -u` - > echo "********************************************************************" - > set -x - > } > $DATA/12.out 2>&1 - > EOF - > set -x - -#---------------------------------------------------------------- #---------------------------------------------------------------- -# Now launch the threads # Now launch the threads - -# determine local system name and type if available # determine local system name and type if available -# ------------------------------------------------- # ------------------------------------------------- - -SITE=${SITE:-""} SITE=${SITE:-""} - -set +u set +u -launcher=${launcher:-"cfp"} # if not "cfp", threads will be run serially. launcher=${launcher:-"cfp"} # if not "cfp", threads will be run serially. - -if [ "$launcher" = cfp ]; then if [ "$launcher" = cfp ]; then - > $DATA/poe.cmdfile > $DATA/poe.cmdfile -# To better take advantage of cfp, execute the longer running commands first. # To better take advantage of cfp, execute the longer running commands first. -# Some reordering was done here based on recent sample runtimes. # Some reordering was done here based on recent sample runtimes. - [ $DUMP_group7 = YES ] && echo ./thread_7 >> $DATA/poe.cmdfile # moved up [ $DUMP_group7 = YES ] && echo ./thread_7 >> $DATA/poe.cmdfile # moved up - [ $DUMP_group1 = YES ] && echo ./thread_1 >> $DATA/poe.cmdfile [ $DUMP_group1 = YES ] && echo ./thread_1 >> $DATA/poe.cmdfile - [ $DUMP_group5 = YES ] && echo ./thread_5 >> $DATA/poe.cmdfile # moved up [ $DUMP_group5 = YES ] && echo ./thread_5 >> $DATA/poe.cmdfile # moved up - [ $DUMP_group6 = YES ] && echo ./thread_6 >> $DATA/poe.cmdfile # moved up [ $DUMP_group6 = YES ] && echo ./thread_6 >> $DATA/poe.cmdfile # moved up - [ $DUMP_group8 = YES ] && echo ./thread_8 >> $DATA/poe.cmdfile # moved up [ $DUMP_group8 = YES ] && echo ./thread_8 >> $DATA/poe.cmdfile # moved up - [ $DUMP_group11 = YES ] && echo ./thread_11 >> $DATA/poe.cmdfile # moved up [ $DUMP_group11 = YES ] && echo ./thread_11 >> $DATA/poe.cmdfile # moved up - [ $DUMP_group10 = YES ] && echo ./thread_10 >> $DATA/poe.cmdfile # moved up [ $DUMP_group10 = YES ] && echo ./thread_10 >> $DATA/poe.cmdfile # moved up - [ $DUMP_group2 = YES ] && echo ./thread_2 >> $DATA/poe.cmdfile [ $DUMP_group2 = YES ] && echo ./thread_2 >> $DATA/poe.cmdfile - [ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && echo ./thread_3 >> $DATA/poe.cmdfile [ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && echo ./thread_3 >> $DATA/poe.cmdfile - [ $DUMP_group4 = YES ] && echo ./thread_4 >> $DATA/poe.cmdfile [ $DUMP_group4 = YES ] && echo ./thread_4 >> $DATA/poe.cmdfile - [ $DUMP_group9 = YES ] && echo ./thread_9 >> $DATA/poe.cmdfile [ $DUMP_group9 = YES ] && echo ./thread_9 >> $DATA/poe.cmdfile - > [ $DUMP_group12 = YES ] && echo ./thread_12 >> $DATA/poe.cmdfile - - - if [ -s $DATA/poe.cmdfile ]; then if [ -s $DATA/poe.cmdfile ]; then - export MP_CSS_INTERRUPT=yes export MP_CSS_INTERRUPT=yes - launcher_DUMP=${launcher_DUMP:-mpiexec} launcher_DUMP=${launcher_DUMP:-mpiexec} - $launcher_DUMP -np 14 --cpu-bind verbose,core cfp $DATA/poe.cmdfile 2>&1 $launcher_DUMP -np 14 --cpu-bind verbose,core cfp $DATA/poe.cmdfile 2>&1 - errpoe=$? errpoe=$? - if [ $errpoe -ne 0 ]; then if [ $errpoe -ne 0 ]; then - $DATA/err_exit "***FATAL: EXIT STATUS $errpoe RUNNING POE COMMAND FILE" $DATA/err_exit "***FATAL: EXIT STATUS $errpoe RUNNING POE COMMAND FILE" - fi fi - else else - echo echo - echo "==> There are no tasks in POE Command File - POE not run" echo "==> There are no tasks in POE Command File - POE not run" - echo echo - fi fi -else else - echo "Running threads serially" echo "Running threads serially" - [ $DUMP_group1 = YES ] && ./thread_1 [ $DUMP_group1 = YES ] && ./thread_1 - [ $DUMP_group2 = YES ] && ./thread_2 [ $DUMP_group2 = YES ] && ./thread_2 - [ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && ./thread_3 [ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && ./thread_3 - [ $DUMP_group4 = YES ] && ./thread_4 [ $DUMP_group4 = YES ] && ./thread_4 - [ $DUMP_group5 = YES ] && ./thread_5 [ $DUMP_group5 = YES ] && ./thread_5 - [ $DUMP_group6 = YES ] && ./thread_6 [ $DUMP_group6 = YES ] && ./thread_6 - [ $DUMP_group7 = YES ] && ./thread_7 [ $DUMP_group7 = YES ] && ./thread_7 - [ $DUMP_group8 = YES ] && ./thread_8 [ $DUMP_group8 = YES ] && ./thread_8 - [ $DUMP_group9 = YES ] && ./thread_9 [ $DUMP_group9 = YES ] && ./thread_9 - [ $DUMP_group10 = YES ] && ./thread_10 [ $DUMP_group10 = YES ] && ./thread_10 - [ $DUMP_group11 = YES ] && ./thread_11 [ $DUMP_group11 = YES ] && ./thread_11 - > [ $DUMP_group12 = YES ] && ./thread_12 -# wait # wait -fi fi - -# if ADPUPA_wait is YES, adpupa is dumped AFTER all other dump threads have # if ADPUPA_wait is YES, adpupa is dumped AFTER all other dump threads have -# run (normally done in real-time GFS runs to dump as late as possible in # run (normally done in real-time GFS runs to dump as late as possible in -# order to maximize data availability in GFS network, particularly DROPs) # order to maximize data availability in GFS network, particularly DROPs) -# -------------------------------------------------------------------------- # -------------------------------------------------------------------------- - -[ $DUMP_group3 = YES -a $ADPUPA_wait = YES ] && ./thread_3 [ $DUMP_group3 = YES -a $ADPUPA_wait = YES ] && ./thread_3 - -cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.o | cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.o - -set +x set +x -echo " " echo " " -echo " " echo " " -set -x set -x - -[ -s $DATA/error1 ] && err1=`cat $DATA/error1` [ -s $DATA/error1 ] && err1=`cat $DATA/error1` -[ -s $DATA/error2 ] && err2=`cat $DATA/error2` [ -s $DATA/error2 ] && err2=`cat $DATA/error2` -[ -s $DATA/error3 ] && err3=`cat $DATA/error3` [ -s $DATA/error3 ] && err3=`cat $DATA/error3` -[ -s $DATA/error4 ] && err4=`cat $DATA/error4` [ -s $DATA/error4 ] && err4=`cat $DATA/error4` -[ -s $DATA/error5 ] && err5=`cat $DATA/error5` [ -s $DATA/error5 ] && err5=`cat $DATA/error5` -[ -s $DATA/error6 ] && err6=`cat $DATA/error6` [ -s $DATA/error6 ] && err6=`cat $DATA/error6` -[ -s $DATA/error7 ] && err7=`cat $DATA/error7` [ -s $DATA/error7 ] && err7=`cat $DATA/error7` -[ -s $DATA/error8 ] && err8=`cat $DATA/error8` [ -s $DATA/error8 ] && err8=`cat $DATA/error8` -[ -s $DATA/error9 ] && err9=`cat $DATA/error9` [ -s $DATA/error9 ] && err9=`cat $DATA/error9` -[ -s $DATA/error10 ] && err10=`cat $DATA/error10` [ -s $DATA/error10 ] && err10=`cat $DATA/error10` -[ -s $DATA/error11 ] && err11=`cat $DATA/error11` [ -s $DATA/error11 ] && err11=`cat $DATA/error11` - > [ -s $DATA/error12 ] && err12=`cat $DATA/error12` - - -#=============================================================================== #=============================================================================== - -export STATUS=YES export STATUS=YES -export DUMP_NUMBER=12 | export DUMP_NUMBER=13 -$ushscript_dump/bufr_dump_obs.sh $dumptime 3.00 1 null $ushscript_dump/bufr_dump_obs.sh $dumptime 3.00 1 null - -# endif loop $PROCESS_DUMP # endif loop $PROCESS_DUMP -fi fi - -echo " " >> $pgmout echo " " >> $pgmout -echo "##################################################################\ echo "##################################################################\ -####################" >> $pgmout ####################" >> $pgmout -echo " " >> $pgmout echo " " >> $pgmout - -#================================================================ #================================================================ -#================================================================ #================================================================ - - -if [ "$PROCESS_DUMP" = 'YES' ]; then if [ "$PROCESS_DUMP" = 'YES' ]; then - - if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o \ if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o \ - "$err4" -gt '5' -o "$err5" -gt '5' -o "$err6" -gt '5' -o \ "$err4" -gt '5' -o "$err5" -gt '5' -o "$err6" -gt '5' -o \ - "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' -o \ "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' -o \ - "$err10" -gt '5' -o "$err11" -gt '5' ]; then | "$err10" -gt '5' -o "$err11" -gt '5' -o "$err12" -gt '5' ]; then - for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 $err11 | for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 $err11 $err12 - do do - if [ "$n" -gt '5' ]; then if [ "$n" -gt '5' ]; then - if [ "$n" -ne '11' -a "$n" -ne '22' ]; then if [ "$n" -ne '11' -a "$n" -ne '22' ]; then - -## fatal error in dumping of BUFR obs. files ## fatal error in dumping of BUFR obs. files - - set +x set +x -echo echo -echo " ###################################################### " echo " ###################################################### " -echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7, $err8, $err9, $err10, $err11 " | $err5, $err6, $err7, $err8, $err9, $err10, $err11, $err12 " -echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " -echo " ###################################################### " echo " ###################################################### " -echo echo - set -x set -x - $DATA/err_exit $DATA/err_exit - exit 9 exit 9 - fi fi - fi fi - done done - -## a status code of 11 or 22 from dumping of BUFR obs. files ## a status code of 11 or 22 from dumping of BUFR obs. files -## is non-fatal but still worth noting ## is non-fatal but still worth noting - - set +x set +x - echo echo - echo " ###################################################### " echo " ###################################################### " - echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7, $err8, $err9, $err10, $err11 " | $err5, $err6, $err7, $err8, $err9, $err10, $err11, $err12 " - echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " - echo " ###################################################### " echo " ###################################################### " - echo echo - set -x set -x - fi fi - -# endif loop $PROCESS_DUMP # endif loop $PROCESS_DUMP -fi fi - -# # -# copy bufr_dumplist to $COMOUT per NCO SPA request # copy bufr_dumplist to $COMOUT per NCO SPA request -# ------------------------------------------------- # ------------------------------------------------- -echo "Copy bufr_dumplist to comout" echo "Copy bufr_dumplist to comout" -LIST_cp=$COMOUT/${RUN}.t${cyc}z.bufr_dumplist.${tmmark} LIST_cp=$COMOUT/${RUN}.t${cyc}z.bufr_dumplist.${tmmark} -cp ${FIXbufr_dump}/bufr_dumplist $LIST_cp cp ${FIXbufr_dump}/bufr_dumplist $LIST_cp -chmod 644 $LIST_cp chmod 644 $LIST_cp - -# GOOD RUN # GOOD RUN -set +x set +x -echo " " echo " " -echo " ****** PROCESSING COMPLETED NORMALLY" echo " ****** PROCESSING COMPLETED NORMALLY" -echo " ****** PROCESSING COMPLETED NORMALLY" echo " ****** PROCESSING COMPLETED NORMALLY" -echo " ****** PROCESSING COMPLETED NORMALLY" echo " ****** PROCESSING COMPLETED NORMALLY" -echo " ****** PROCESSING COMPLETED NORMALLY" echo " ****** PROCESSING COMPLETED NORMALLY" -echo " " echo " " -set -x set -x - - -# save standard output # save standard output -cat break $pgmout break > allout cat break $pgmout break > allout -cat allout cat allout -# rm allout # rm allout - -sleep 10 sleep 10 - -msg='ENDED NORMALLY.' msg='ENDED NORMALLY.' -$DATA/postmsg "$jlogfile" "$msg" $DATA/postmsg "$jlogfile" "$msg" - -################## END OF SCRIPT ####################### ################## END OF SCRIPT ####################### From c08e5e2a64fa299a08dd3e9c074fd2ca034a65f0 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Fri, 28 Jul 2023 02:25:15 +0000 Subject: [PATCH 32/53] Stop sevcsr and saphir --- scripts/exglobal_dump.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/exglobal_dump.sh b/scripts/exglobal_dump.sh index ff22abb..19259cc 100755 --- a/scripts/exglobal_dump.sh +++ b/scripts/exglobal_dump.sh @@ -116,9 +116,9 @@ set +u # JOB_NUMBER not present indicates dump BOTH prepbufr and non-prepbufr data. # ----------------------------------------------------------------------------- # Dump group #1 (non-pb, TIME_TRIM defaults to OFF) = -# avcsam eshrs3 ssmisu saphir 1bhrs4 sevcsr tesac mls +# avcsam eshrs3 ssmisu 1bhrs4 tesac mls # esatms gsrcsr ahicsr sstvcw subpfl saldrn -# +# Stop: sevcsr, saphir in v1.2.0 # Dump group #2 (pb, TIME_TRIM defaults to OFF) = # sfcshp tideg atovs* adpsfc ascatt snocvr # * - for GDAS only @@ -145,8 +145,8 @@ set +u # Dump group #9 (non-pb, TIME_TRIM defaults to ON) = # geoimr gmi1cr satwhr # Dump group #10 (non-pb, TIME_TRIM defaults to OFF) = -# esiasi mtiasi esamua sevasr 1bamua bathy osbuv8 -# ompst8 ompsn8 gsrasr ompslp sstvpw +# esiasi mtiasi esamua sevasr 1bamua bathy +# osbuv8 ompst8 ompsn8 gsrasr ompslp sstvpw # # Dump group #11 (non-pb, TIME_TRIM defaults to OFF) = # amsr2 @@ -592,10 +592,10 @@ export DUMP_NUMBER=1 DTIM_latest_avcsam=${DTIM_latest_avcsam:-"+2.99"} DTIM_latest_eshrs3=${DTIM_latest_eshrs3:-"+2.99"} DTIM_latest_ssmisu=${DTIM_latest_ssmisu:-"+2.99"} -DTIM_latest_saphir=${DTIM_latest_saphir:-"+2.99"} +#DTIM_latest_saphir=${DTIM_latest_saphir:-"+2.99"} DTIM_latest_saldrn=${DTIM_latest_saldrn:-"+2.99"} DTIM_latest_1bhrs4=${DTIM_latest_1bhrs4:-"+2.99"} -DTIM_latest_sevcsr=${DTIM_latest_sevcsr:-"+2.99"} +#DTIM_latest_sevcsr=${DTIM_latest_sevcsr:-"+2.99"} DTIM_latest_tesac=${DTIM_latest_tesac:-"+2.99"} #----------------------------------------------- # check for mls tank presence in past 10 days @@ -626,7 +626,7 @@ DTIM_latest_sstvcw=${DTIM_latest_sstvcw:-"+2.99"} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM1:-off}} $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 avcsam eshrs3 ssmisu \ - saphir 1bhrs4 sevcsr tesac $mls $esatms gsrcsr ahicsr sstvcw subpfl saldrn + 1bhrs4 tesac $mls $esatms gsrcsr ahicsr sstvcw subpfl saldrn error1=$? echo "$error1" > $DATA/error1 @@ -638,13 +638,13 @@ if [ "$SENDDBN" = "YES" ]; then $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ssmisu $job \ ${COMSP}ssmisu.tm00.bufr_d if [ "${NET}" = "gdas" ]; then - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_saphir $job \ - ${COMSP}saphir.tm00.bufr_d ### restricted, only GDAS, turn on 01/13/2020 +# $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_saphir $job \ +# ${COMSP}saphir.tm00.bufr_d ### restricted, only GDAS, turn on 01/13/2020 fi $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_1bhrs4 $job \ ${COMSP}1bhrs4.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sevcsr $job \ - ${COMSP}sevcsr.tm00.bufr_d +# $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sevcsr $job \ +# ${COMSP}sevcsr.tm00.bufr_d $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_tesac $job \ ${COMSP}tesac.tm00.bufr_d $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_saldrn $job \ From 5fb74148503a8568a01121d2d51dbfbe6bf38400 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Fri, 28 Jul 2023 03:56:29 +0000 Subject: [PATCH 33/53] RN --- docs/Release_Notes.txt | 154 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/docs/Release_Notes.txt b/docs/Release_Notes.txt index 342c973..e837b4d 100644 --- a/docs/Release_Notes.txt +++ b/docs/Release_Notes.txt @@ -1,3 +1,157 @@ +obsproc v1.2.0 RELEASED AUG ??,2023 + v1.2.? IMPLEMENTED ??? ??,2023 + +FILE CHANGES + + +D docs/ReleaseNotes.txt +M docs/Release_Notes.txt +M jobs/JOBSPROC_CDAS_PREP1 +M jobs/JOBSPROC_GLOBAL_DUMP_POST +M jobs/JOBSPROC_GLOBAL_PREP +M jobs/JOBSPROC_GLOBAL_PREP_POST +M jobs/JOBSPROC_RAP_PREP +A modulefiles/obsproc_jet.lua +M scripts/excdas_dump.sh +M scripts/exdump_post.sh +M scripts/exglobal_dump.sh +M scripts/exrap_dump.sh +M scripts/exrtma_dump.sh +M sorc/bufr_remorest.fd/remorest.f +M ush/build.sh +D versions/README +M versions/run.ver +M versions/VERSION +(M-modified; A-added; D-deleted) + +DOCS CHANGES + + docs/ReleaseNotes.txt + Delete repetitive file with similar name + + docs/Release_Notes.txt + Reflect this release changes. + +JOBS CHANGES + + jobs/JOBSPROC_CDAS_PREP1 + Increase errPREPDATA_limit in order to allow code to generate prepbufr file if/when one or more key files + (adpsfc, adpupa, uprair) are missing + + jobs/JOBSPROC_GLOBAL_DUMP_POST + Add functionality to unrestrict aircraft data (PROCESS_REMOREST_dm2) + + jobs/JOBSPROC_GLOBAL_PREP + Instruct to add uprair dump files data to GDAS and GFS prepbufr files + + jobs/JOBSPROC_GLOBAL_PREP_POST + Add functionality to unrestrict aircraft data (PROCESS_REMOREST_dm2) + + jobs/JOBSPROC_RAP_PREP + Instruct to add uprair dump files data to RAP's prepbufr file + +MODULEFILES CHANGES + + modulefiles/obsproc_jet.lua + Add support for functionality on Jet + +SCRIPTS CHANGES + + scripts/excdas_dump.sh + Instruct to dump uprair file for CDAS network + + scripts/exdump_post.sh + Add functionality to unrestrict aircraft data (PROCESS_REMOREST_dm2) + + scripts/exglobal_dump.sh + Split groups 1 and 10 and add an additional DUMP_group12 to allow code to run on one more cpu and faster + Dump BUFR formated high-res rawinsonde profiles into *uprair* dump files + + scripts/exrap_dump.sh + Dump BUFR formated high-res rawinsonde profiles into *uprair* dump files + + scripts/exrtma_dump.sh + Relax DTIM_latest_000000 for 00:00UTC run + +SORC CHANGES + + sorc/bufr_remorest.fd/remorest.f + Add functionality to unrestrict aircraft data (PROCESS_REMOREST_dm2) + +USH CHANGES + + ush/build.sh + Add support for functionality on Jet + +VERSIONS CHANGES + + versions/README + Delete old notes + + versions/run.ver + Set bufr_dump_ver=1.2.0 and prepbufr_ver=1.1.0 + + versions/VERSION + Updated version to 1.2.0 + +ECF + + Contents of ECF directory are not EMC's responsibility + However,note that obsproc.v1.2.0 , bufr-dump,v1.2.0 and prepobs.v1.2.0 + will require more memory in the PBS card for: + gdas, gfs, cdas, rap runs (all 4 steps for each network) + See the example logs for : /lfs/h2/emc/stmp/iliana.genkova/CRON/R12/output + +CHANGES TO OUTPUT AND FILE SIZES + + Generate new dumps: + *satwhr* (gdas,gfs), *uprair* (gdas,gfs,cdas,rap) + Update dumps (gdas, gfs): + *prepbufr* (when this release is synced with bufr-dump.v1.2.0 and prepobs.v1.1.0) + + The newly generated dumps (*satwhr*, *uprair*) vary in size from 20Mb to 500Mb depending on network, cycle time, and season. + *uprair* will be largest for 00UTC and 12UTC. + *satwhr* will be largest during hurricane season. + + The updated *prepbur* files are up to ~20% larger in size + For examples, see: /lfs/h2/emc/stmp/iliana.genkova/CRON/R12/com + +CHANGES TO RESOURCES + + Memory usage was increased for all steps in the processing of: + gdas,gfs,cdas, rap + See the example logs for : /lfs/h2/emc/stmp/iliana.genkova/CRON/R12/output + +PRE-IMPLEMENTATION TESTING REQUIREMENTS + + obsproc needs modules bufr-dump and prepobs in order to run + See docs/README.md for build instructions + See versions/build.ver and versions/run.ver for libaries + + Run jobs/JOBSPROC_networkname_DUMP and jobs/JOBSPROC_networkname_DUMP_POST + for the following networkname values: GLOBAL, NAM, RAP, URMA, RTMA, CDAS + + Good to know: + - the RAP uprair dumps might be 0 size for the hours not around 00,06,12,18 UTC + - the CDAS prepbufr files are 6 different flavours, it takes about 26 hours for all to be generated + - only GDAS and GFS networks will generate *satwhr* + - only GDAS and GFS networks would support the unrestrictiom of aircraft data, and only on a DEV machine + - some *bufr_d files might be 0 size until the relevant data stream is recovered (airs, ombusv8) + - please contact obsproc_support@noaa.gov should you have any Qs regarding testing + +DISSEMINATION INFORMATION + + For NOMAD proposed changes, a SCN will be issued/submitted to Data Flow + +SPECIAL INSTRUCTIONS + + 1.Implement release/obsproc.v1.2.0, release/bufr-dump.v1.2.0, and release/prepobs.v1.1.0 at the same time + + 2.Implement release/obsproc.v1.2.0, release/bufr-dump.v1.2.0, and release/prepobs.v1.1.0 after confirming GFS global-workflow is ready for the new versions (first 2 digits) + + + +XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX obsproc v1.1.0 RELEASED AUG 18,2022 v1.1.2 IMPLEMENTED NOV 30,2022 From 739d6c434da3ad3c3f437f8404f2bd68a242f4e9 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Mon, 31 Jul 2023 20:32:39 +0000 Subject: [PATCH 34/53] Add dbn_alert for global satwhr and global and rap uprair dump --- scripts/exglobal_dump.sh | 2 ++ scripts/exrap_dump.sh | 3 +++ 2 files changed, 5 insertions(+) diff --git a/scripts/exglobal_dump.sh b/scripts/exglobal_dump.sh index 19259cc..36fb722 100755 --- a/scripts/exglobal_dump.sh +++ b/scripts/exglobal_dump.sh @@ -1441,6 +1441,8 @@ if [ "$SENDDBN" = "YES" ]; then ${COMSP}geoimr.tm00.bufr_d $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_gmi1cr $job \ ${COMSP}gmi1cr.tm00.bufr_d + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_satwhr $job \ + ${COMSP}satwhr.tm00.bufr_d fi set +x diff --git a/scripts/exrap_dump.sh b/scripts/exrap_dump.sh index 5516f90..c5cd58e 100755 --- a/scripts/exrap_dump.sh +++ b/scripts/exrap_dump.sh @@ -1522,6 +1522,9 @@ if [ $SENDDBN = YES ]; then if [ -s ${COMSP}ssmisu.tm00.bufr_d ]; then $DBNROOT/bin/dbn_alert MODEL ${RUN_uc}_BUFR_ssmisu $job ${COMSP}ssmisu.tm00.bufr_d fi + if [ -s ${COMSP}uprair.tm00.bufr_d ]; then + $DBNROOT/bin/dbn_alert MODEL ${RUN_uc}_BUFR_uprair $job ${COMSP}uprair.tm00.bufr_d + fi fi # From 3ec8c676955c34b3f91cc0a2d58d45541e2005e0 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Mon, 31 Jul 2023 20:35:18 +0000 Subject: [PATCH 35/53] Update RN --- docs/Release_Notes.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/Release_Notes.txt b/docs/Release_Notes.txt index e837b4d..552315b 100644 --- a/docs/Release_Notes.txt +++ b/docs/Release_Notes.txt @@ -66,9 +66,11 @@ SCRIPTS CHANGES scripts/exglobal_dump.sh Split groups 1 and 10 and add an additional DUMP_group12 to allow code to run on one more cpu and faster Dump BUFR formated high-res rawinsonde profiles into *uprair* dump files + Add DBNet dbn_alert for uprair and satwhr dump files going to NOMADS scripts/exrap_dump.sh Dump BUFR formated high-res rawinsonde profiles into *uprair* dump files + Add DBNet dbn_alert for uprair dump files going to NOMADS scripts/exrtma_dump.sh Relax DTIM_latest_000000 for 00:00UTC run From 52d14ae4aa6f396d34a1ee358f137076055325f2 Mon Sep 17 00:00:00 2001 From: Ashley Stanfield <92459593+AshleyStanfield-NOAA@users.noreply.github.com> Date: Tue, 22 Aug 2023 12:52:36 -0400 Subject: [PATCH 36/53] Update Release_Notes.txt Added comments to release notes on the removal of *saphir* and *sevcsr* dumps. --- docs/Release_Notes.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/Release_Notes.txt b/docs/Release_Notes.txt index 552315b..06f73ef 100644 --- a/docs/Release_Notes.txt +++ b/docs/Release_Notes.txt @@ -110,6 +110,8 @@ CHANGES TO OUTPUT AND FILE SIZES *satwhr* (gdas,gfs), *uprair* (gdas,gfs,cdas,rap) Update dumps (gdas, gfs): *prepbufr* (when this release is synced with bufr-dump.v1.2.0 and prepobs.v1.1.0) + Remove dumps (gdas, gfs): + *saphir* , *sevcsr* The newly generated dumps (*satwhr*, *uprair*) vary in size from 20Mb to 500Mb depending on network, cycle time, and season. *uprair* will be largest for 00UTC and 12UTC. @@ -118,6 +120,8 @@ CHANGES TO OUTPUT AND FILE SIZES The updated *prepbur* files are up to ~20% larger in size For examples, see: /lfs/h2/emc/stmp/iliana.genkova/CRON/R12/com + Both *saphir* and *sevcsr* dumps have been removed as they are no longer available. + CHANGES TO RESOURCES Memory usage was increased for all steps in the processing of: From 4d7929d3ca57cc6542662f15478c648661538337 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Wed, 20 Sep 2023 16:25:03 +0000 Subject: [PATCH 37/53] Reactiavte fix for rtma_ru 0000 UTC --- scripts/exrtma_dump.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/exrtma_dump.sh b/scripts/exrtma_dump.sh index bf774d1..a6489eb 100755 --- a/scripts/exrtma_dump.sh +++ b/scripts/exrtma_dump.sh @@ -243,10 +243,9 @@ DTIM_earliest_saldrn=-0.50 DTIM_latest_saldrn=+0.50 # for rtma_ru_0000 read only from previous day's tank -echo "DTIM_latest_000000=-0.01 changed to DTIM_latest_000000=0.5 by Praveen on 03/13/23" +# Temporary bug fix if [ $RUN = "rtma_ru" -a $cycle = "t0000z" ]; then - DTIM_latest_000000=0.5 -# DTIM_latest_000000=-0.01 + DTIM_latest_000000=-0.01 fi $ushscript_dump/bufr_dump_obs.sh $dumptime 0.5 1 sfcshp tideg adpsfc subpfl saldrn From ff169c6a26d3ca55ca928286cafe959da58037d4 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Wed, 11 Oct 2023 16:05:59 +0000 Subject: [PATCH 38/53] Made adpupa and uprair a new dump group in RAP Removed an empty if-then in exglobal_dump.sh for saphir --- scripts/exglobal_dump.sh | 8 +- scripts/exrap_dump.sh | 160 +++++++++++++++++++++++++++++++++------ 2 files changed, 142 insertions(+), 26 deletions(-) diff --git a/scripts/exglobal_dump.sh b/scripts/exglobal_dump.sh index 36fb722..50b0a9b 100755 --- a/scripts/exglobal_dump.sh +++ b/scripts/exglobal_dump.sh @@ -637,10 +637,10 @@ if [ "$SENDDBN" = "YES" ]; then ${COMSP}eshrs3.tm00.bufr_d $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_ssmisu $job \ ${COMSP}ssmisu.tm00.bufr_d - if [ "${NET}" = "gdas" ]; then +# if [ "${NET}" = "gdas" ]; then # $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_saphir $job \ # ${COMSP}saphir.tm00.bufr_d ### restricted, only GDAS, turn on 01/13/2020 - fi +# fi $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_1bhrs4 $job \ ${COMSP}1bhrs4.tm00.bufr_d # $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_sevcsr $job \ @@ -1739,7 +1739,9 @@ if [ "$launcher" = cfp ]; then if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes launcher_DUMP=${launcher_DUMP:-mpiexec} - $launcher_DUMP -np 14 --cpu-bind verbose,core cfp $DATA/poe.cmdfile 2>&1 + #$launcher_DUMP -np 14 --cpu-bind verbose,core cfp $DATA/poe.cmdfile 2>&1 # 1) + #$launcher_DUMP -np 14 --cpu-bind core cfp $DATA/poe.cmdfile 2>&1 # 2) 3) + $launcher_DUMP -np 14 cfp $DATA/poe.cmdfile 2>&1 # 4) errpoe=$? if [ $errpoe -ne 0 ]; then $DATA/err_exit "***FATAL: EXIT STATUS $errpoe RUNNING POE COMMAND FILE" diff --git a/scripts/exrap_dump.sh b/scripts/exrap_dump.sh index c5cd58e..92f0b7e 100755 --- a/scripts/exrap_dump.sh +++ b/scripts/exrap_dump.sh @@ -55,6 +55,9 @@ echo " Aug 10 2022 - Added subpfl,saldn to dump #1,snocvr to dump #3. " echo " added gmi1cr dump group #7 " echo " b005/xx081 added to satwnd " echo " Sep 30 2022 - Enable dumping of UPRAIR data in group #2. " +echo " Oct 11 2023 - Split msonet to msonet and msone1, msone1=255.030 " +echo " concatenate msonet and msone1 right after dump " +echo " - Pull adpupa and uprair into own Dump group " ################################################################################ set -xau @@ -69,17 +72,19 @@ set +u # ------------------------------------------------------------------------ # Dump group #1 (non-pb) = 1bamua 1bmhs esamua esmhs atms mtiasi sevcsr # gpsro esiasi iasidb esatms atmsdb sevasr amsr2 -# Dump group #2 (pb) = vadwnd satwnd adpupa uprair +# Dump group #2 (pb) = vadwnd satwnd # Dump group #3 (pb) = proflr rassda sfcshp adpsfc ascatt tideg snocvr # subpfl saldrn -# Dump group #4 (pb) = msonet gpsipw +# Dump group #4 (pb) = msonet gpsipw # Dump group #5 (pb) = aircft aircar # Dump group #6 (non-pb) = nexrad # Dump group #7 (non-pb) = airsev 1bhrs4 eshrs3 lgycld ssmisu osbuv8 crsfdb # saphir gmi1cr # Dump group #8 (non-pb) = gsrasr gsrcsr # Dump group #9 (non-pb) = lghtng -# Dump group #10 STATUS FILE +# Dump group #10(pb) = msone1 # ONLY tank b255/xx030, the largest +# Dump group #11(pb) = adpupa uprair +# Dump group #11 STATUS FILE # ------------------------------------------------------------------------ #VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV @@ -106,6 +111,8 @@ set -u DUMP_group7=${DUMP_group7:-"YES"} DUMP_group8=${DUMP_group8:-"YES"} DUMP_group9=${DUMP_group9:-"YES"} + DUMP_group10=${DUMP_group10:-"NO"} + DUMP_group11=${DUMP_group11:-"NO"} else dump_ind=DUMP DUMP_group1=${DUMP_group1:-"NO"} @@ -117,6 +124,8 @@ set -u DUMP_group7=${DUMP_group7:-"NO"} DUMP_group8=${DUMP_group8:-"NO"} DUMP_group9=${DUMP_group9:-"NO"} + DUMP_group10=${DUMP_group10:-"YES"} + DUMP_group11=${DUMP_group11:-"YES"} fi else dump_ind=DUMP @@ -129,6 +138,8 @@ else DUMP_group7=${DUMP_group7:-"YES"} DUMP_group8=${DUMP_group8:-"YES"} DUMP_group9=${DUMP_group9:-"YES"} + DUMP_group10=${DUMP_group10:-"YES"} + DUMP_group11=${DUMP_group11:-"YES"} fi # Oct 2019; disable -- not needed for HRRRv4 @@ -226,6 +237,8 @@ echo "=======> Dump group 6 (thread_6) not executed." > $DATA/6.out echo "=======> Dump group 7 (thread_7) not executed." > $DATA/7.out echo "=======> Dump group 8 (thread_8) not executed." > $DATA/8.out echo "=======> Dump group 9 (thread_9) not executed." > $DATA/9.out +echo "=======> Dump group 10 (thread_10) not executed." > $DATA/10.out +echo "=======> Dump group 11 (thread_11) not executed." > $DATA/11.out err1=0 err2=0 @@ -236,6 +249,9 @@ err6=0 err7=0 err8=0 err9=0 +err10=0 +err11=0 + if [ "$PROCESS_DUMP" = 'YES' ]; then ################################## @@ -418,9 +434,9 @@ export STATUS=NO export DUMP_NUMBER=2 #========================================================================== -# Dump # 2 : VADWND, SATWND, ADPUPA, UPRAIR -# (2) (19) (6) (5) -# -- TOTAL NUMBER OF SUBTYPES = 32 +# Dump # 2 : VADWND, SATWND +# (2) (19) +# -- TOTAL NUMBER OF SUBTYPES = 21 #========================================================================== # Skip all Indian satellite winds in SATWND (not in domain) @@ -440,13 +456,6 @@ For testing, skip in ecflow or obsproc_rap.ver file # Add GOES-16/17/18 DMW data to SATWND export ADD_satwnd="005030 005031 005032 005034 005039" -# Time window -1.00 to +1.00 hours for ADPUPA/UPRAIR w/ full & partial cycle runs -# (note: time window increased over +/- 0.5 hr standard to get more data) -DTIM_earliest_adpupa=${DTIM_earliest_adpupa:-"-1.00"} -DTIM_latest_adpupa=${DTIM_latest_adpupa:-"+1.00"} -DTIM_earliest_uprair=${DTIM_earliest_uprair:-"-1.00"} -DTIM_latest_uprair=${DTIM_latest_uprair:-"+1.00"} - # Time window -1.50 to +1.49 hours for EUMETSAT SATWND for full and partial # cycle runs @@ -520,7 +529,7 @@ else fi $ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_2} 1 vadwnd \ - satwnd adpupa uprair + satwnd error2=$? echo "$error2" > $DATA/error2 @@ -1282,6 +1291,101 @@ set -x EOF set -x +### NEW GROUP MSONET IG #10 +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_10; chmod +x thread_10 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_10 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=10 + +#============================================================================ +# Dump # 10 : MSONE1 -- TOTAL NUMBER OF SUBTYPES = 1 +# (1) +#============================================================================ + +def_time_window_10=0.5 # default time window for dump 10 is -0.5 to +0.5 hours + +# Time window -0.50 to +0.50 hours for MSONET for full and partial cycle runs +# (default) + +$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_10} 1 msone1 +error10=$? +echo "$error10" > $DATA/error10 + +set +x +echo "********************************************************************" +echo Script thread_10 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/10.out 2>&1 +EOF +set -x +### NEW GROUP MSONET IG end #10 + +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_11; chmod +x thread_11 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_11 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=11 + +#========================================================================== +# Dump # 11 : ADPUPA, UPRAIR +# (6) (5) +# -- TOTAL NUMBER OF SUBTYPES = 11 +#========================================================================== + +# Time window -1.00 to +1.00 hours for ADPUPA/UPRAIR w/ full & partial cycle runs +# (note: time window increased over +/- 0.5 hr standard to get more data) +DTIM_earliest_adpupa=${DTIM_earliest_adpupa:-"-1.00"} +DTIM_latest_adpupa=${DTIM_latest_adpupa:-"+1.00"} +DTIM_earliest_uprair=${DTIM_earliest_uprair:-"-1.00"} +DTIM_latest_uprair=${DTIM_latest_uprair:-"+1.00"} + +$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_11} 1 adpupa uprair +error11=$? +echo "$error11" > $DATA/error11 + +set +x +echo "********************************************************************" +echo Script thread_11 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/11.out 2>&1 +EOF +set -x + #---------------------------------------------------------------- # Now launch the threads @@ -1307,13 +1411,14 @@ if [ "$launcher" = cfp ]; then [ $DUMP_group5 = YES ] && echo ./thread_5 >> $DATA/poe.cmdfile # moved up [ $DUMP_group2 = YES ] && echo ./thread_2 >> $DATA/poe.cmdfile [ $DUMP_group3 = YES ] && echo ./thread_3 >> $DATA/poe.cmdfile - + [ $DUMP_group10 = YES ] && echo ./thread_10 >> $DATA/poe.cmdfile + [ $DUMP_group11 = YES ] && echo ./thread_11 >> $DATA/poe.cmdfile if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes # ?? launcher_DUMP=${launcher_DUMP:-mpiexec} #$launcher_DUMP -np 3 --cpu-bind verbose,core cfp $DATA/poe.cmdfile - NPROCS=${NPROCS:-1} + NPROCS=${NPROCS:-11} $launcher_DUMP -np $NPROCS --cpu-bind verbose,core cfp $DATA/poe.cmdfile errpoe=$? if [ $errpoe -ne 0 ]; then @@ -1335,9 +1440,11 @@ else [ $DUMP_group7 = YES ] && ./thread_7 [ $DUMP_group8 = YES ] && ./thread_8 [ $DUMP_group9 = YES ] && ./thread_9 + [ $DUMP_group10 = YES ] && ./thread_10 + [ $DUMP_group11 = YES ] && ./thread_11 fi -cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out $DATA/7.out $DATA/8.out $DATA/9.out +cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.out $DATA/11.out set +x echo " " @@ -1353,12 +1460,13 @@ set -x [ -s $DATA/error7 ] && err7=`cat $DATA/error7` [ -s $DATA/error8 ] && err8=`cat $DATA/error8` [ -s $DATA/error9 ] && err9=`cat $DATA/error9` - +[ -s $DATA/error10 ] && err10=`cat $DATA/error10` +[ -s $DATA/error11 ] && err11=`cat $DATA/error11` #=============================================================================== export STATUS=YES -export DUMP_NUMBER=10 +export DUMP_NUMBER=12 $ushscript_dump/bufr_dump_obs.sh $dumptime 3.00 1 null # endif loop $PROCESS_DUMP @@ -1376,8 +1484,9 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o \ "$err4" -gt '5' -o "$err5" -gt '5' -o "$err6" -gt '5' -o \ - "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' ]; then - for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 + "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' -o \ + "$err10" -gt '5' -o "$err11" -gt '5' ]; then + for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 $err11 do if [ "$n" -gt '5' ]; then if [ "$n" -ne '11' -a "$n" -ne '22' ]; then @@ -1388,7 +1497,7 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then echo echo " ###################################################### " echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7 $err8 $err9 " +$err5, $err6, $err7 $err8 $err9 $err10 $err11" echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " echo " ###################################################### " echo @@ -1406,7 +1515,7 @@ echo echo echo " ###################################################### " echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7 $err8 $err9 " +$err5, $err6, $err7 $err8 $err9 $err10 $err11" echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " echo " ###################################################### " echo @@ -1416,6 +1525,10 @@ $err5, $err6, $err7 $err8 $err9 " # endif loop $PROCESS_DUMP fi +# concatenate msonet and msone1, b/c prepobs only wants one file +cat ${COMSP}msone1.tm00.bufr_d >> ${COMSP}msonet.tm00.bufr_d + + grep -q "004.004 in data group aircar for .............-.........\ .... HAS 0 REPORTS" ${COMSP}status.$tmmark.bufr_d err_grep1=$? @@ -1428,6 +1541,7 @@ if [ $err_grep1 -eq 0 -a $err_grep2 -eq 0 ]; then $DATA/postmsg "$jlogfile" "$msg" fi + if [ $SENDDBN = YES ]; then if [ -s ${COMSP}1bamua.tm00.bufr_d ]; then $DBNROOT/bin/dbn_alert MODEL ${RUN_uc}_BUFR_1bamua $job ${COMSP}1bamua.tm00.bufr_d From 9330088db89370b4ed0c782097c486255072cc32 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Wed, 25 Oct 2023 22:50:43 +0000 Subject: [PATCH 39/53] PAUSE --- scripts/excdas_dump.sh | 88 +++++++++++++++++++++++++++++++++++++----- scripts/exnam_dump.sh | 85 +++++++++++++++++++++++++++++++++++----- scripts/exrap_dump.sh | 25 +++++++++++- 3 files changed, 177 insertions(+), 21 deletions(-) diff --git a/scripts/excdas_dump.sh b/scripts/excdas_dump.sh index c2f1c1d..aac1c78 100755 --- a/scripts/excdas_dump.sh +++ b/scripts/excdas_dump.sh @@ -1,4 +1,5 @@ #!/bin/ksh +# ########################################################################### echo "--------------------------------------------------------------------" echo "excdas_dump.sh - CDAS network data dump processing " @@ -41,6 +42,8 @@ echo " group. " echo " Jul 30 2022 - Subpfl, saldrn, snocvr, and gmi1cr added " echo " to dump group #9. " echo " Sep 30 2022 - Enable dumping of UPRAIR data in group #3. " +echo " Oct 17 2023 - Split msonet to msonet (#5) and msone1 (#10) " +echo " concatenate msonet and msone1 right after dump" ########################################################################### set -xau @@ -82,7 +85,10 @@ set +u # Dump group #9 (non-pb, TIME_TRIM = default = ON) = # geoimr subpfl saldrn snocvr gmi1cr # -# Dump group #10 STATUS FILE +# Dump group #10 (pb, TIME_TRIM = OFF) = +# msone1 # ONLY tank b255/xx030, the largest +# +# Dump group #11 STATUS FILE # ----------------------------------------------------------------------------- #VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV @@ -109,6 +115,7 @@ set -u DUMP_group7=${DUMP_group7:-"YES"} DUMP_group8=${DUMP_group8:-"NO"} DUMP_group9=${DUMP_group9:-"YES"} + DUMP_group10=${DUMP_group10:="NO"} else dump_ind=DUMP DUMP_group1=${DUMP_group1:-"NO"} @@ -120,6 +127,7 @@ set -u DUMP_group7=${DUMP_group7:-"NO"} DUMP_group8=${DUMP_group8:-"YES"} DUMP_group9=${DUMP_group9:-"NO"} + DUMP_group10=${DUMP_group10:="YES"} fi else dump_ind=DUMP @@ -132,6 +140,7 @@ else DUMP_group7=${DUMP_group7:-"YES"} DUMP_group8=${DUMP_group8:-"YES"} DUMP_group9=${DUMP_group9:-"YES"} + DUMP_group10=${DUMP_group10:="YES"} fi @@ -294,6 +303,7 @@ echo "=======> Dump group 6 (thread_6) not executed." > $DATA/6.out echo "=======> Dump group 7 (thread_7) not executed." > $DATA/7.out echo "=======> Dump group 8 (thread_8) not executed." > $DATA/8.out echo "=======> Dump group 9 (thread_9) not executed." > $DATA/9.out +echo "=======> Dump group 10 (thread_10) not executed." > $DATA/10.out err1=0 err2=0 @@ -304,6 +314,8 @@ err6=0 err7=0 err8=0 err9=0 +err10=0 + if [ "$PROCESS_DUMP" = 'YES' ]; then #################################### @@ -1057,6 +1069,56 @@ set -x EOF set -x +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_10; chmod +x thread_10 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_10 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=10 + +#=================================================================== +# NOTES ABOUT THIS DUMP GROUP: +# (1) time window radius is -3.00 to +2.99 hours on all types +# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP +# +#-------------------------------------------------------------------------- +# Dump # 10 : MSONET: 1 subtype(s) 255/030 +# ------------------------ +# TOTAL NUMBER OF SUBTYPES = 1 +# +#=================================================================== + +DTIM_latest_msone1=${DTIM_latest_msone1:-"+2.99"} + +TIME_TRIM=off + +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 msone1 +error10=$? +echo "$error10" > $DATA/error10 + +set +x +echo "********************************************************************" +echo Script thread_10 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/10.out 2>&1 +EOF +set -x #---------------------------------------------------------------- # Now launch the threads @@ -1082,12 +1144,13 @@ if [ "$launcher" = cfp ]; then [ $DUMP_group4 = YES ] && echo ./thread_4 >> $DATA/poe.cmdfile [ $DUMP_group7 = YES ] && echo ./thread_7 >> $DATA/poe.cmdfile [ $DUMP_group9 = YES ] && echo ./thread_9 >> $DATA/poe.cmdfile + [ $DUMP_group10 = YES ] && echo ./thread_10 >> $DATA/poe.cmdfile #msone1 if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes # ?? launcher_DUMP=${launcher_DUMP:-mpiexec} - #$launcher_DUMP -np 8 --cpu-bind verbose,core cfp $DATA/poe.cmdfile - NPROCS=${NPROCS:-8} + #$launcher_DUMP -np 10 --cpu-bind verbose,core cfp $DATA/poe.cmdfile + NPROCS=${NPROCS:-10} $launcher_DUMP -np $NPROCS --cpu-bind verbose,core cfp $DATA/poe.cmdfile errpoe=$? if [ $errpoe -ne 0 ]; then @@ -1109,9 +1172,10 @@ else [ $DUMP_group7 = YES ] && ./thread_7 [ $DUMP_group8 = YES ] && ./thread_8 [ $DUMP_group9 = YES ] && ./thread_9 + [ $DUMP_group10 = YES ] && ./thread_10 fi -cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out $DATA/7.out $DATA/8.out $DATA/9.out +cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.out set +x echo " " @@ -1127,11 +1191,11 @@ set -x [ -s $DATA/error7 ] && err7=`cat $DATA/error7` [ -s $DATA/error8 ] && err8=`cat $DATA/error8` [ -s $DATA/error9 ] && err9=`cat $DATA/error9` - +[ -s $DATA/error10 ] && err10=`cat $DATA/error10` #=============================================================================== export STATUS=YES -export DUMP_NUMBER=10 +export DUMP_NUMBER=11 $ushscript_dump/bufr_dump_obs.sh $dumptime 3.00 1 null if [ "$SENDCOM" = 'YES' -a "$COPY_TO_ARKV" = 'YES' ]; then @@ -1173,8 +1237,9 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o \ "$err4" -gt '5' -o "$err5" -gt '5' -o "$err6" -gt '5' -o \ - "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' ]; then - for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 + "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' -o \ + "$err10" -gt '5' ]; then + for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 do if [ "$n" -gt '5' ]; then if [ "$n" -ne '11' -a "$n" -ne '22' ]; then @@ -1185,7 +1250,7 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then echo echo " ###################################################### " echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7, $err8, $err9 " +$err5, $err6, $err7, $err8, $err9 $err10" echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " echo " ###################################################### " echo @@ -1203,7 +1268,7 @@ echo echo echo " ###################################################### " echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7, $err8, $err9 " +$err5, $err6, $err7, $err8, $err9 $err10 " echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " echo " ###################################################### " echo @@ -1213,6 +1278,9 @@ $err5, $err6, $err7, $err8, $err9 " # endif loop $PROCESS_DUMP fi +# concatenate msonet and msone1, b/c prepobs only wants one file +cat ${COMSP}msone1.tm00.bufr_d >> ${COMSP}msonet.tm00.bufr_d + # # copy bufr_dumplist to $COMOUT per NCO SPA request # ------------------------------------------------- diff --git a/scripts/exnam_dump.sh b/scripts/exnam_dump.sh index b172869..8d52c7b 100755 --- a/scripts/exnam_dump.sh +++ b/scripts/exnam_dump.sh @@ -31,6 +31,7 @@ echo " to match bufr_dumplist. Removed tideg from " echo " sfcshp dump group to make unique dump file. " echo " - Copy bufr_dumplist to COMOUT. " echo " Dec 09 2021 - Updated for use on WCOSS2 " +echo " Oct 17 2023 - Split msonet to msonet and msone1 (b255/xx030)" ############################################################################ set -xau @@ -52,7 +53,8 @@ set +u # Dump group #7 (non-pb) = goesfv lgycld --> dump is GLOBAL # Dump group #8 (non-pb) = 1bhrs4 airsev osbuv8 esmhs ssmisu cris lghtng # Dump group #9 (pb) = adpupa -# Dump group #10 STATUS FILE +# Dump group #10 (pb)= msone1 +# Dump group #11 STATUS FILE # ----------------------------------------------------------------------------- #VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV @@ -92,6 +94,7 @@ set -u DUMP_group7=${DUMP_group7:-"YES"} DUMP_group8=${DUMP_group8:-"YES"} DUMP_group9=${DUMP_group9:-"NO"} + DUMP_group10=${DUMP_group10:-"NO"} else dump_ind=DUMP DUMP_group1=${DUMP_group1:-"NO"} @@ -103,6 +106,7 @@ set -u DUMP_group7=${DUMP_group7:-"NO"} DUMP_group8=${DUMP_group8:-"NO"} DUMP_group9=${DUMP_group9:-"YES"} + DUMP_group10=${DUMP_group10:-"YES"} fi else dump_ind=DUMP @@ -115,6 +119,7 @@ else DUMP_group7=${DUMP_group7:-"YES"} DUMP_group8=${DUMP_group8:-"YES"} DUMP_group9=${DUMP_group9:-"YES"} + DUMP_group10=${DUMP_group10:-"YES"} fi if [ $tmmark = tm00 ]; then @@ -244,6 +249,7 @@ echo "=======> Dump group 6 (thread_6) not executed." > $DATA/6.out echo "=======> Dump group 7 (thread_7) not executed." > $DATA/7.out echo "=======> Dump group 8 (thread_8) not executed." > $DATA/8.out echo "=======> Dump group 9 (thread_9) not executed." > $DATA/9.out +echo "=======> Dump group 10 (thread_10) not executed." > $DATA/10.out err1=0 err2=0 @@ -254,6 +260,7 @@ err6=0 err7=0 err8=0 err9=0 +err10=0 if [ "$PROCESS_DUMP" = 'YES' ]; then ####################################################################### @@ -1045,6 +1052,58 @@ set -x EOF set -x +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_10; chmod +x thread_10 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_10 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=10 + +#=========================================================================== +# Dump # 10 : MSONE1 -- TOTAL NUMBER OF SUBTYPES = 1 +# (1) +# for catch-up cycles tm06-01: +# time window radius is (-0.50,+0.50) hours for MSONE1 +# for on-time tm00 cycles: +# time window radius is (-0.75,+1.50) hours for MSONE1 +#=========================================================================== + +if [ "$tmhr" = "00" ]; then + DTIM_earliest_msone1=${DTIM_earliest_msone1:-"-0.75"} + DTIM_latest_msone1=${DTIM_latest_msone1:-"+1.50"} +fi + +$ushscript_dump/bufr_dump_obs.sh $dumptime 0.5 1 msone1 +error10=$? +echo "$error10" > $DATA/error10 + +#if [ "$SENDDBN" = 'YES' ]; then +# $DBNROOT/bin/dbn_alert MODEL NAM_BUFR_msonet $job ${COMSP}msonet.$tmmark.bufr_d +#fi + +set +x +echo "********************************************************************" +echo Script thread_10 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/10.out 2>&1 +EOF +set -x #---------------------------------------------------------------- # Now launch the threads @@ -1070,12 +1129,13 @@ if [ "$launcher" = cfp ]; then [ $DUMP_group6 = YES ] && echo ./thread_6 >> $DATA/poe.cmdfile [ $DUMP_group7 = YES ] && echo ./thread_7 >> $DATA/poe.cmdfile [ $DUMP_group9 = YES -a $ADPUPA_wait != YES ] && echo ./thread_9 >> $DATA/poe.cmdfile + [ $DUMP_group10 = YES ] && echo ./thread_10 >> $DATA/poe.cmdfile if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes # ?? launcher_DUMP=${launcher_DUMP:-mpiexec} - #$launcher_DUMP -np 7 --cpu-bind verbose,core cfp $DATA/poe.cmdfile - NPROCS=${NPROCS:-7} + #$launcher_DUMP -np 10 --cpu-bind verbose,core cfp $DATA/poe.cmdfile + NPROCS=${NPROCS:-10} $launcher_DUMP -np $NPROCS --cpu-bind verbose,core cfp $DATA/poe.cmdfile errpoe=$? if [ $errpoe -ne 0 ]; then @@ -1097,6 +1157,7 @@ else [ $DUMP_group7 = YES ] && ./thread_7 [ $DUMP_group8 = YES ] && ./thread_8 [ $DUMP_group9 = YES -a $ADPUPA_wait != YES ] && ./thread_9 + [ $DUMP_group10 = YES ] && ./thread_10 fi # if ADPUPA_wait is YES, adpupa is dumped AFTER all other dump threads have @@ -1107,7 +1168,7 @@ fi [ $DUMP_group9 = YES -a $ADPUPA_wait = YES ] && ./thread_9 cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out \ - $DATA/7.out $DATA/8.out $DATA/9.out + $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.out set +x echo " " @@ -1123,12 +1184,12 @@ set -x [ -s $DATA/error7 ] && err7=`cat $DATA/error7` [ -s $DATA/error8 ] && err8=`cat $DATA/error8` [ -s $DATA/error9 ] && err9=`cat $DATA/error9` - +[ -s $DATA/error10 ] && err10=`cat $DATA/error10` #=============================================================================== export STATUS=YES -export DUMP_NUMBER=10 +export DUMP_NUMBER=11 $ushscript_dump/bufr_dump_obs.sh $dumptime 3.00 1 null @@ -1146,8 +1207,9 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o \ "$err4" -gt '5' -o "$err5" -gt '5' -o "$err6" -gt '5' -o \ - "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' ]; then - for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 + "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' -o \ + "$err10" -gt '5']; then + for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 do if [ "$n" -gt '5' ]; then if [ "$n" -ne '11' -a "$n" -ne '22' ]; then @@ -1158,7 +1220,7 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then echo echo " ###################################################### " echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7, $err8, $err9 " +$err5, $err6, $err7, $err8, $err9 $err10" echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " echo " ###################################################### " echo @@ -1176,7 +1238,7 @@ echo echo echo " ###################################################### " echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7, $err8, $err9 " +$err5, $err6, $err7, $err8, $err9 $err10" echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " echo " ###################################################### " echo @@ -1186,6 +1248,9 @@ $err5, $err6, $err7, $err8, $err9 " # endif loop $PROCESS_DUMP fi +# concatenate msonet and msone1, b/c prepobs only wants one file +cat ${COMSP}msone1.tm00.bufr_d >> ${COMSP}msonet.tm00.bufr_d + # # copy bufr_dumplist to $COMOUT per NCO SPA request # ------------------------------------------------- diff --git a/scripts/exrap_dump.sh b/scripts/exrap_dump.sh index 92f0b7e..27bb6b0 100755 --- a/scripts/exrap_dump.sh +++ b/scripts/exrap_dump.sh @@ -84,7 +84,7 @@ set +u # Dump group #9 (non-pb) = lghtng # Dump group #10(pb) = msone1 # ONLY tank b255/xx030, the largest # Dump group #11(pb) = adpupa uprair -# Dump group #11 STATUS FILE +# Dump group #12 STATUS FILE # ------------------------------------------------------------------------ #VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV @@ -1363,9 +1363,32 @@ export DUMP_NUMBER=11 # (6) (5) # -- TOTAL NUMBER OF SUBTYPES = 11 #========================================================================== +# +if [ "$RUN" = 'rap_p' ]; then + +# ===> For RUN = rap_p -- partial cycle runs +# ------------------------------------- + + def_time_window_11=1.5 # default time window for dump 11 is -1.5 to +1.5 hours + +else + +# ===> For RUN = rap, rap_e -- full cycle runs (including early at 00/12z) +# ------------------------------------------------------------------- + + if [ $cyc -eq 00 -o $cyc -eq 12 ]; then + def_time_window_11=1.5 # default time window for dump 11 is -1.5 to +1.5 + # hours for 00 or 12z + else + def_time_window_11=2.5 # default time window for dump 11 is -2.5 to +2.5 + # hours for all other cycles + fi + +fi # Time window -1.00 to +1.00 hours for ADPUPA/UPRAIR w/ full & partial cycle runs # (note: time window increased over +/- 0.5 hr standard to get more data) + DTIM_earliest_adpupa=${DTIM_earliest_adpupa:-"-1.00"} DTIM_latest_adpupa=${DTIM_latest_adpupa:-"+1.00"} DTIM_earliest_uprair=${DTIM_earliest_uprair:-"-1.00"} From d68ad0fe168769b235a67c7895b9b37fd1289af0 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Fri, 27 Oct 2023 00:16:17 +0000 Subject: [PATCH 40/53] Correct COMINgdas, COMINGFS, and COMIN1 --- jobs/JOBSPROC_GLOBAL_PREP_POST | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/JOBSPROC_GLOBAL_PREP_POST b/jobs/JOBSPROC_GLOBAL_PREP_POST index 2493b68..626cb6f 100755 --- a/jobs/JOBSPROC_GLOBAL_PREP_POST +++ b/jobs/JOBSPROC_GLOBAL_PREP_POST @@ -174,8 +174,8 @@ fi # The following are used by $PROCESS_TIMETWINS in the GDAS when it is YES # ----------------------------------------------------------------------- -export COMINgdas=${COMINgfs:-$(compath.py ${envir}/${NET}/${gfs_ver})/${RUN}.} -export COMINGFS=${COMINgfs:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${NET}.${PDY}/${cyc}/${COMPONENT})} +export COMINgdas=${COMINgdas:-$(compath.py ${envir}/${NET}/${gfs_ver})/${RUN}.} +export COMINGFS=${COMINGFS:-$(compath.py ${envir}/${obsNET}/${obsproc_ver}/${NET}.${PDY}/${cyc}/${COMPONENT})} #export COMIN1=${COMIN1:-$(compath.py ${envir}/${NET}/${gfs_ver})/${RUN}.} export COMIN1=${COMIN1:-$(compath.py ${envir}/${obsNET}/${obsproc_ver})/${RUN}.} #twinwind file now write in obsproc package From 25a6cecd4a89c128081226fa5cda1d5a47ead201 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Thu, 2 Nov 2023 01:32:33 +0000 Subject: [PATCH 41/53] para run fixes + RN --- docs/Release_Notes.txt | 18 ++- scripts/excdas_dump.sh | 42 +++++-- scripts/exdump_monitor.sh | 136 +++++++++++++++++++-- scripts/exglobal_dump.sh | 7 +- scripts/exrap_dump.sh | 20 +++ scripts/exrtma_dump.sh | 250 ++++++++++++++++++++++++++++---------- scripts/exurma_dump.sh | 239 ++++++++++++++++++++++++++---------- 7 files changed, 562 insertions(+), 150 deletions(-) diff --git a/docs/Release_Notes.txt b/docs/Release_Notes.txt index 06f73ef..82fa011 100644 --- a/docs/Release_Notes.txt +++ b/docs/Release_Notes.txt @@ -13,10 +13,13 @@ M jobs/JOBSPROC_GLOBAL_PREP_POST M jobs/JOBSPROC_RAP_PREP A modulefiles/obsproc_jet.lua M scripts/excdas_dump.sh +M scripts/exdump_monitor.sh M scripts/exdump_post.sh M scripts/exglobal_dump.sh +M scripts/exnam_dump.sh M scripts/exrap_dump.sh M scripts/exrtma_dump.sh +M scripts/exurma_dump.sh M sorc/bufr_remorest.fd/remorest.f M ush/build.sh D versions/README @@ -40,12 +43,14 @@ JOBS CHANGES jobs/JOBSPROC_GLOBAL_DUMP_POST Add functionality to unrestrict aircraft data (PROCESS_REMOREST_dm2) + Separate processing of uprair to its own dump group jobs/JOBSPROC_GLOBAL_PREP Instruct to add uprair dump files data to GDAS and GFS prepbufr files jobs/JOBSPROC_GLOBAL_PREP_POST Add functionality to unrestrict aircraft data (PROCESS_REMOREST_dm2) + Correct COMIN1 and COMINgdas patha jobs/JOBSPROC_RAP_PREP Instruct to add uprair dump files data to RAP's prepbufr file @@ -58,7 +63,8 @@ MODULEFILES CHANGES SCRIPTS CHANGES scripts/excdas_dump.sh - Instruct to dump uprair file for CDAS network + Instruct NOT to dump uprair file for CDAS network (too slow) + Introduce msone1 group and redistribute group scripts/exdump_post.sh Add functionality to unrestrict aircraft data (PROCESS_REMOREST_dm2) @@ -67,13 +73,21 @@ SCRIPTS CHANGES Split groups 1 and 10 and add an additional DUMP_group12 to allow code to run on one more cpu and faster Dump BUFR formated high-res rawinsonde profiles into *uprair* dump files Add DBNet dbn_alert for uprair and satwhr dump files going to NOMADS + Redistribute groups scripts/exrap_dump.sh Dump BUFR formated high-res rawinsonde profiles into *uprair* dump files Add DBNet dbn_alert for uprair dump files going to NOMADS + Introduce msone1 group and redistribute groups + + scripts/exnam_dump.sh + Introduce msone1 group and redistribute groups/add cpus scripts/exrtma_dump.sh - Relax DTIM_latest_000000 for 00:00UTC run + Introduce msone1 group and redistribute groups + + scripts/exurma_dump.sh + Introduce msone1 group and redistribute groups SORC CHANGES diff --git a/scripts/excdas_dump.sh b/scripts/excdas_dump.sh index aac1c78..cd8adbc 100755 --- a/scripts/excdas_dump.sh +++ b/scripts/excdas_dump.sh @@ -65,8 +65,8 @@ set +u # sfcshp tideg atovs adpsfc # # Dump group #3 (pb, TIME_TRIM = OFF) = -# adpupa uprair gpsipw ascatt -# +# adpupa gpsipw ascatt +# # pull out uprair, too slow # Dump group #4 (pb, TIME_TRIM = default = ON) = # aircar aircft proflr vadwnd rassda # @@ -316,6 +316,25 @@ err8=0 err9=0 err10=0 +#restrict processing of unexpected big tanks +#this block appear in all /scripts/ex*_dump.sh proessing msonet and msone1 +TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb +TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb +TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb +if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then + export SKIP_255003=YES + echo "WARNING: TANK b005/xx003 exceeds TANK_MAX_255003 => not dumped" | mail iliana.genkova@noaa.gov +fi +if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then + export SKIP_255004=YES + echo "WARNING: TANK b005/xx004 exceeds TANK_MAX_255004 => not dumped" | mail iliana.genkova@noaa.gov +fi +if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then + export SKIP_255030=YES + echo "WARNING: TANK b005/xx030 exceeds TANK_MAX_255030 => not dumped" | mail iliana.genkova@noaa.gov +fi +#end of block + if [ "$PROCESS_DUMP" = 'YES' ]; then #################################### @@ -510,7 +529,7 @@ export DUMP_NUMBER=3 # #-------------------------------------------------------------------------- # Dump #3: ADPUPA: 6 subtype(s) -# UPRAIR: 5 subtype(s) +# #UPRAIR: 5 subtype(s) - pull out, too slow # GPSIPW: 1 subtype(s) # ASCATT: 1 subtype(s) # --------------------- @@ -527,7 +546,8 @@ DTIM_latest_ascatt=${DTIM_latest_ascatt:-"+2.99"} TIME_TRIM=off -$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 adpupa uprair gpsipw ascatt +# $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 adpupa uprair gpsipw ascatt +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 adpupa gpsipw ascatt error3=$? echo "$error3" > $DATA/error3 @@ -654,12 +674,17 @@ export DUMP_NUMBER=5 # #=================================================================== -DTIM_latest_msonet=${DTIM_latest_msonet:-"+2.99"} +#IG +DTIM_earliest_msonet=${DTIM_latest_msonet:-"-1.99"} +DTIM_latest_msonet=${DTIM_latest_msonet:-"+2.00"} + +#DTIM_latest_msonet=${DTIM_latest_msonet:-"+2.99"} export SKIP_255031=YES # Skip for port to Dell since no new data allowed. export SKIP_255101=YES # Also, b/c CDAS has not tested these providers. -TIME_TRIM=off +TIME_TRIM=on +#TIME_TRIM=off $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 msonet error5=$? @@ -1101,9 +1126,10 @@ export DUMP_NUMBER=10 # #=================================================================== -DTIM_latest_msone1=${DTIM_latest_msone1:-"+2.99"} +DTIM_earliest_msone1=${DTIM_latest_msone1:-"-1.99"} +DTIM_latest_msone1=${DTIM_latest_msone1:-"+2.00"} -TIME_TRIM=off +TIME_TRIM=on #off $ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 msone1 error10=$? diff --git a/scripts/exdump_monitor.sh b/scripts/exdump_monitor.sh index 46123f6..dd71952 100755 --- a/scripts/exdump_monitor.sh +++ b/scripts/exdump_monitor.sh @@ -117,8 +117,29 @@ err5=0 err6=0 err7=0 err8=0 -if [ "$PROCESS_DUMP" = 'YES' ]; then +err9=0 +err10=0 + +#restrict processing of unexpected big tanks +#this block appear in all /scripts/ex*_dump.sh proessing msonet and msone1 +TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb +TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb +TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb +if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then + export SKIP_255003=YES + echo "WARNING: TANK b005/xx003 exceeds TANK_MAX_255003 => not dumped" | mail iliana.genkova@noaa.gov +fi +if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then + export SKIP_255004=YES + echo "WARNING: TANK b005/xx004 exceeds TANK_MAX_255004 => not dumped" | mail iliana.genkova@noaa.gov +fi +if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then + export SKIP_255030=YES + echo "WARNING: TANK b005/xx030 exceeds TANK_MAX_255030 => not dumped" | mail iliana.genkova@noaa.gov +fi +#end of block +if [ "$PROCESS_DUMP" = 'YES' ]; then ########################### ########################### # The data "dump" script @@ -913,6 +934,94 @@ set -x EOF set -x +#-------NEW MESON1 + +set +x +cat<<\EOF>thread_9; chmod +x thread_9 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_9 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=9 + +#=================================================================== +# Dump # 9 : MSONE1 +# (1) +# -- TOTAL NUMBER OF SUBTYPES = 1 +# time window radius is -0.50 to +0.49 hours on MSONET +#=================================================================== +DTIM_latest_msone1=+0.49 +DTIM_earliest_msone1=-1.00 + +$ushscript_dump/bufr_dump_obs.sh $dumptime 0.50 1 msone1 +error9=$? +echo "$error9" > $DATA/error9 + +set +x +echo "********************************************************************" +echo Script thread_9 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/9.out 2>&1 +EOF +set -x + +#-------NEW UPRAIR + +set +x +cat<<\EOF>thread_10; chmod +x thread_10 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_10 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=10 + +#=================================================================== +# Dump # 10: UPRAIR +# (1) +# -- TOTAL NUMBER OF SUBTYPES = 1 +#=================================================================== +DTIM_latest_uprair=+0.49 +DTIM_earliest_uprair=-0.49 + +$ushscript_dump/bufr_dump_obs.sh $dumptime 0.50 1 uprair +error10=$? +echo "$error10" > $DATA/error10 + +set +x +echo "********************************************************************" +echo Script thread_10 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/10.out 2>&1 +EOF +set -x #---------------------------------------------------------------- @@ -930,11 +1039,14 @@ if [ "$launcher" = cfp ]; then echo ./thread_8 >> $DATA/poe.cmdfile # moved up echo ./thread_1 >> $DATA/poe.cmdfile echo ./thread_4 >> $DATA/poe.cmdfile - + echo ./thread_9 >> $DATA/poe.cmdfile + echo ./thread_10 >> $DATA/poe.cmdfile + if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes # ?? launcher_DUMP=${launcher_DUMP:-mpiexec} - $launcher_DUMP -np 7 --cpu-bind verbose,core cfp $DATA/poe.cmdfile + NPROCS=${NPROCS:-10} + $launcher_DUMP -np $NPROCS --cpu-bind verbose,core cfp $DATA/poe.cmdfile errpoe=$? if [ $errpoe -ne 0 ]; then $DATA/err_exit "***FATAL: EXIT STATUS $errpoe RUNNING POE COMMAND FILE" @@ -954,9 +1066,11 @@ else ./thread_6 ./thread_7 ./thread_8 + ./thread_9 + ./thread_10 fi cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out \ - $DATA/7.out $DATA/8.out + $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.out set +x echo " " @@ -971,12 +1085,13 @@ err5=`cat $DATA/error5` err6=`cat $DATA/error6` err7=`cat $DATA/error7` err8=`cat $DATA/error8` - +err9=`cat $DATA/error9` +err10=`cat $DATA/error10` #=============================================================================== export STATUS=YES -export DUMP_NUMBER=9 +export DUMP_NUMBER=11 $ushscript_dump/bufr_dump_obs.sh $dumptime 0.50 1 null @@ -1009,8 +1124,9 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o \ "$err4" -gt '5' -o "$err5" -gt '5' -o "$err6" -gt '5' -o \ - "$err7" -gt '5' -o "$err8" -gt '5' ]; then - for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 + "$err7" -gt '5' -o "$err8" -gt '5' -o \ + "$err9" -gt '5' -o "$err10" -gt '5' ]; then + for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 do if [ "$n" -gt '5' ]; then if [ "$n" -ne '11' -a "$n" -ne '22' ]; then @@ -1021,7 +1137,7 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then echo echo " ###################################################### " echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7 $err8 " +$err5, $err6, $err7 $err8 $err9 $err10" echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " echo " ###################################################### " echo @@ -1039,7 +1155,7 @@ echo echo echo " ###################################################### " echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7 $err8 " +$err5, $err6, $err7 $err8 $err9 $err10 " echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " echo " ###################################################### " echo diff --git a/scripts/exglobal_dump.sh b/scripts/exglobal_dump.sh index 50b0a9b..719146f 100755 --- a/scripts/exglobal_dump.sh +++ b/scripts/exglobal_dump.sh @@ -1738,10 +1738,11 @@ if [ "$launcher" = cfp ]; then if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes - launcher_DUMP=${launcher_DUMP:-mpiexec} - #$launcher_DUMP -np 14 --cpu-bind verbose,core cfp $DATA/poe.cmdfile 2>&1 # 1) + launcher_DUMP=${launcher_DUMP:-mpiexec} + NPROCS=${NPROCS:-14} + #$launcher_DUMP -np 12 --cpu-bind verbose,core cfp $DATA/poe.cmdfile 2>&1 # 1) #$launcher_DUMP -np 14 --cpu-bind core cfp $DATA/poe.cmdfile 2>&1 # 2) 3) - $launcher_DUMP -np 14 cfp $DATA/poe.cmdfile 2>&1 # 4) + $launcher_DUMP -np ${NPROCS} cfp $DATA/poe.cmdfile 2>&1 # 4) errpoe=$? if [ $errpoe -ne 0 ]; then $DATA/err_exit "***FATAL: EXIT STATUS $errpoe RUNNING POE COMMAND FILE" diff --git a/scripts/exrap_dump.sh b/scripts/exrap_dump.sh index 27bb6b0..c2c1bf1 100755 --- a/scripts/exrap_dump.sh +++ b/scripts/exrap_dump.sh @@ -252,6 +252,26 @@ err9=0 err10=0 err11=0 +#restrict processing of unexpected big tanks +#this block appear in all /scripts/ex*_dump.sh proessing msonet and msone1 +TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb +TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb +TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb +if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then + export SKIP_255003=YES + echo "WARNING: TANK b005/xx003 exceeds TANK_MAX_255003 => not dumped" | mail iliana.genkova@noaa.gov +fi +if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then + export SKIP_255004=YES + echo "WARNING: TANK b005/xx004 exceeds TANK_MAX_255004 => not dumped" | mail iliana.genkova@noaa.gov +fi +if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then + export SKIP_255030=YES + echo "WARNING: TANK b005/xx030 exceeds TANK_MAX_255030 => not dumped" | mail iliana.genkova@noaa.gov +fi +#end of block + + if [ "$PROCESS_DUMP" = 'YES' ]; then ################################## diff --git a/scripts/exrtma_dump.sh b/scripts/exrtma_dump.sh index a6489eb..3b2b0be 100755 --- a/scripts/exrtma_dump.sh +++ b/scripts/exrtma_dump.sh @@ -28,6 +28,10 @@ echo " sfcshp dump group to make unique dump file. " echo " - Copy bufr_dumplist to COMOUT. " echo " Dec 15 2021 - set for use on WCOSS2. " echo " Jul 31 2022 - Subpfl,saldrn,snocvr & gmi1cr dump group added " +echo " Oct 12 2023 - Split msonet to msonet and msone1, where " +echo " msone1=255.030; concatenate msonet and msone1 " +echo " right after dump. Seperated satwnd to its own " +echo " dump group. " ##################################################################### set -x @@ -79,6 +83,29 @@ export COMSP=$COMOUT/$RUN.${cycle}. err1=0 err2=0 err3=0 +err4=0 +err5=0 + + +#restrict processing of unexpected big tanks +#this block appear in all /scripts/ex*_dump.sh proessing msonet and msone1 +TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb +TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb +TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb +if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then + export SKIP_255003=YES + echo "WARNING: TANK b005/xx003 exceeds TANK_MAX_255003 => not dumped" | mail iliana.genkova@noaa.gov +fi +if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then + export SKIP_255004=YES + echo "WARNING: TANK b005/xx004 exceeds TANK_MAX_255004 => not dumped" | mail iliana.genkova@noaa.gov +fi +if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then + export SKIP_255030=YES + echo "WARNING: TANK b005/xx030 exceeds TANK_MAX_255030 => not dumped" | mail iliana.genkova@noaa.gov +fi +#end of block + if [ "$PROCESS_DUMP" = 'YES' ]; then ################################### @@ -111,9 +138,9 @@ export STATUS=NO export DUMP_NUMBER=1 #======================================================================== -# Dump # 1 : ASCATT, SATWND*, EFCLAM, SNOCVR, GMI1CR -- -# (1) (14) (1) (1) (1) -# TOTAL NUMB OF SUBTYPES = 18 +# Dump # 1 : ASCATT, EFCLAM, SNOCVR, GMI1CR -- +# (1) (1) (1) (1) +# TOTAL NUMB OF SUBTYPES = 14 # ===> Dumping of WNDSAT removed from here until new ingest feed is established # (had been dumped with a time window radius of -6.00 to 0.00 hours) # @@ -121,57 +148,9 @@ export DUMP_NUMBER=1 # monitors nc005090 and does not support nc005091). # # time window radius is -6.00 to 0.00 hours for ASCATT -# for SATWND subtypes 005/010, 005/011, 005/012 and 005/019 at all -# times: -# time window radius is -1.00 to -0.01 hours -# for SATWND subtypes 005/064, 005/065, 005/066, 005/067, 005/068 -# and 005/069 at all times: -# time window radius is -1.50 to +1.49 hours -# for all other SATWND subtypes: -# time window radius is +/- 2.5 hours -# for EFCLAM: -# time window radius is +/- 0.5 hours +# time window radius is +/- 0.5 hours for EFCLAM #======================================================================= -# Skip all Indian satellite winds in SATWND (not in domain) - -export SKIP_005021=YES -export SKIP_005022=YES -export SKIP_005023=YES - -# Skip legacy EUMETSAT AMV subsets; for testing skip in trigger or version file -#export SKIP_005064=YES -#export SKIP_005065=YES -#export SKIP_005066=YES - -# Skip VIIRS NPP/NOAA-20 IR long wave derived cld motion - -export SKIP_005090=YES # old sequence (discontinued) -export SKIP_005091=YES # new sequence (unsupported in GSI) - -DTIM_earliest_005010=-1.00 -DTIM_latest_005010=-0.01 -DTIM_earliest_005011=-1.00 -DTIM_latest_005011=-0.01 -DTIM_earliest_005012=-1.00 -DTIM_latest_005012=-0.01 -DTIM_earliest_005019=-1.00 -DTIM_latest_005019=-0.01 - -DTIM_earliest_005064=-1.50 -DTIM_latest_005064=+1.49 -DTIM_earliest_005065=-1.50 -DTIM_latest_005065=+1.49 -DTIM_earliest_005066=-1.50 -DTIM_latest_005066=+1.49 - -DTIM_earliest_005067=-1.50 -DTIM_latest_005067=+1.49 -DTIM_earliest_005068=-1.50 -DTIM_latest_005068=+1.49 -DTIM_earliest_005069=-1.50 -DTIM_latest_005069=+1.49 - DTIM_earliest_ascatt=-6.00 DTIM_latest_ascatt=0.00 @@ -186,12 +165,7 @@ DTIM_latest_snocvr=${DTIM_latest_snocvr:-"+2.00"} DTIM_earliest_gmi1cr=-1.50 DTIM_latest_gmi1cr=+1.49 -DTIM_earliest_005081=${DTIM_earliest_005081:-"-1.50"} -DTIM_latest_005081=${DTIM_latest_005081:-"+1.49"} -#DTIM_earliest_005081=-1.50 -#DTIM_latest_005081=+1.49 - -$ushscript_dump/bufr_dump_obs.sh $dumptime 2.5 1 ascatt satwnd efclam snocvr gmi1cr +$ushscript_dump/bufr_dump_obs.sh $dumptime 2.5 1 ascatt efclam snocvr gmi1cr error1=$? echo "$error1" > $DATA/error1 @@ -303,6 +277,141 @@ set -x EOF set -x + +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_4; chmod +x thread_4 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_4 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=4 + +#=========================================================================== +# Dump # 4 : SATWND -- TOTAL NUMBER OF SUBTYPES = 14 +# for SATWND subtypes 005/010, 005/011, 005/012 and 005/019 at all +# times: +# time window radius is -1.00 to -0.01 hours +# for SATWND subtypes 005/064, 005/065, 005/066, 005/067, 005/068 +# and 005/069 at all times: +# time window radius is -1.50 to +1.49 hours +# for all other SATWND subtypes: +# time window radius is +/- 2.5 hours +#=========================================================================== + +# Skip all Indian satellite winds in SATWND (not in domain) +export SKIP_005021=YES +export SKIP_005022=YES +export SKIP_005023=YES + +# Skip legacy EUMETSAT AMV subsets; for testing skip in trigger or version file +#export SKIP_005064=YES +#export SKIP_005065=YES +#export SKIP_005066=YES + +# Skip VIIRS NPP/NOAA-20 IR long wave derived cld motion +export SKIP_005090=YES # old sequence (discontinued) +export SKIP_005091=YES # new sequence (unsupported in GSI) + +DTIM_earliest_005010=-1.00 +DTIM_latest_005010=-0.01 +DTIM_earliest_005011=-1.00 +DTIM_latest_005011=-0.01 +DTIM_earliest_005012=-1.00 +DTIM_latest_005012=-0.01 +DTIM_earliest_005019=-1.00 +DTIM_latest_005019=-0.01 + +DTIM_earliest_005064=-1.50 +DTIM_latest_005064=+1.49 +DTIM_earliest_005065=-1.50 +DTIM_latest_005065=+1.49 +DTIM_earliest_005066=-1.50 +DTIM_latest_005066=+1.49 + +DTIM_earliest_005067=-1.50 +DTIM_latest_005067=+1.49 +DTIM_earliest_005068=-1.50 +DTIM_latest_005068=+1.49 +DTIM_earliest_005069=-1.50 +DTIM_latest_005069=+1.49 + +DTIM_earliest_005081=${DTIM_earliest_005081:-"-1.50"} +DTIM_latest_005081=${DTIM_latest_005081:-"+1.49"} +#DTIM_earliest_005081=-1.50 +#DTIM_latest_005081=+1.49 + +$ushscript_dump/bufr_dump_obs.sh $dumptime 2.5 1 satwnd +error4=$? +echo "$error4" > $DATA/error4 + +set +x +echo "********************************************************************" +echo Script thread_4 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/4.out 2>&1 +EOF +set -x + +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_5; chmod +x thread_5 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_5 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=5 + +#============================================================================ +# Dump # 5 : MSONE1 -- TOTAL NUMBER OF SUBTYPES = 1 +# (1) +#============================================================================ + +def_time_window_5=0.5 # default time window for dump 5 is -0.5 to +0.5 hours + +# Time window -0.50 to +0.50 hours for MSONET for full and partial cycle runs +# (default) + +$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_5} 1 msone1 +error5=$? +echo "$error5" > $DATA/error5 + +set +x +echo "********************************************************************" +echo Script thread_5 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/5.out 2>&1 +EOF +set -x + #---------------------------------------------------------------- # Now launch the threads @@ -320,11 +429,13 @@ if [ "$launcher" = cfp ]; then echo ./thread_3 >> $DATA/poe.cmdfile # moved up echo ./thread_1 >> $DATA/poe.cmdfile echo ./thread_2 >> $DATA/poe.cmdfile + echo ./thread_4 >> $DATA/poe.cmdfile + echo ./thread_5 >> $DATA/poe.cmdfile if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes # ?? launcher_DUMP=${launcher_DUMP:-mpiexec} - $launcher_DUMP -np 3 --cpu-bind verbose,core cfp $DATA/poe.cmdfile + $launcher_DUMP -np 5 --cpu-bind verbose,core cfp $DATA/poe.cmdfile errpoe=$? if [ $errpoe -ne 0 ]; then $DATA/err_exit "***FATAL: EXIT STATUS $errpoe RUNNING POE COMMAND FILE" @@ -339,10 +450,12 @@ else ./thread_1 ./thread_2 ./thread_3 + ./thread_4 + ./thread_5 # wait fi -cat $DATA/1.out $DATA/2.out $DATA/3.out +cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out set +x echo " " @@ -352,12 +465,13 @@ set -x err1=`cat $DATA/error1` err2=`cat $DATA/error2` err3=`cat $DATA/error3` - +err4=`cat $DATA/error4` +err5=`cat $DATA/error5` #================================================================ export STATUS=YES -export DUMP_NUMBER=4 +export DUMP_NUMBER=6 $ushscript_dump/bufr_dump_obs.sh $dumptime 3.00 1 null @@ -370,8 +484,9 @@ fi if [ "$PROCESS_DUMP" = 'YES' ]; then - if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' ] ; then - for n in $err1 $err2 $err3 + if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' \ + -o "$err4" -gt '5' -o "$err5" -gt '5' ] ; then + for n in $err1 $err2 $err3 $err4 $err5 do if [ "$n" -gt '5' ]; then if [ "$n" -ne '11' -a "$n" -ne '22' ]; then @@ -381,7 +496,7 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then set +x echo echo " ###################################################### " -echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3" +echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, $err5" echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " echo " ###################################################### " echo @@ -398,7 +513,7 @@ echo set +x echo echo " ###################################################### " - echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3" + echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, $err5" echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " echo " ###################################################### " echo @@ -412,6 +527,9 @@ if [ "$RUN" == "rtma_ru" ] && [ "${SENDDBN^^}" = YES ] && [ -s ${COMSP}satwnd.tm $DBNROOT/bin/dbn_alert MODEL RTMA_RU_BUFR_satwnd $job ${COMSP}satwnd.tm00.bufr_d fi +# concatenate msonet and msone1, b/c prepobs only wants one file +cat ${COMSP}msone1.tm00.bufr_d >> ${COMSP}msonet.tm00.bufr_d + # # copy bufr_dumplist to $COMOUT per NCO SPA request # ------------------------------------------------- diff --git a/scripts/exurma_dump.sh b/scripts/exurma_dump.sh index 44bdbfc..312a58f 100755 --- a/scripts/exurma_dump.sh +++ b/scripts/exurma_dump.sh @@ -30,6 +30,10 @@ echo " - Copy bufr_dumplist to COMOUT. " echo " Dec 15 2021 - set for use on WCOSS2. " echo " Aug 1 2022 - Added SUBPFL, SALDRN, SNOCVR, " echo " and GMI1CR types. " +echo " Oct 12 2023 - Split msonet to msonet and msone1, " +echo " where msone1=255.030; concatenate " +echo " msonet and msone1 right after dump. " +echo " Seperated satwnd to its own dump group." ##################################################################### set -x @@ -61,6 +65,27 @@ err1=0 err2=0 err3=0 err4=0 +err5=0 +err6=0 +#restrict processing of unexpected big tanks +#this block appear in all /scripts/ex*_dump.sh proessing msonet and msone1 +TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb +TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb +TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb +if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then + export SKIP_255003=YES + echo "WARNING: TANK b005/xx003 exceeds TANK_MAX_255003 => not dumped" | mail iliana.genkova@noaa.gov +fi +if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then + export SKIP_255004=YES + echo "WARNING: TANK b005/xx004 exceeds TANK_MAX_255004 => not dumped" | mail iliana.genkova@noaa.gov +fi +if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then + export SKIP_255030=YES + echo "WARNING: TANK b005/xx030 exceeds TANK_MAX_255030 => not dumped" | mail iliana.genkova@noaa.gov +fi +#end of block + if [ "$PROCESS_DUMP" = 'YES' ]; then ################################### @@ -93,76 +118,27 @@ export STATUS=NO export DUMP_NUMBER=1 #======================================================================== -# Dump # 1 : ASCATT, SATWND, EFCLAM, GMI1CR -# (1) (14) (1) (1) -# -- TOTAL NUMBER OF SUBTYPES = 17 +# Dump # 1 : ASCATT, EFCLAM, GMI1CR +# (1) (1) (1) +# -- TOTAL NUMBER OF SUBTYPES = 3 # ===> Dumping of WNDSAT removed from here until new ingest feed is established # (had been dumped with a time window radius of -6.00 to 0.00 hours) # # time window radius is -6.00 to 0.00 hours for ASCATT -# for SATWND subtypes 005/010, 005/011, 005/012 and 005/019 at all -# times: -# time window radius is -1.00 to -0.01 hours -# for SATWND subtypes 005/064, 005/065, 005/066, 005/067, 005/068, -# and 005/069 at all times: -# time window radius is -1.50 to +1.49 hours -# for all other SATWND subtypes: -# time window radius is +/- 2.5 hours -# for EFCLAM: -# time window radius is +/- 0.5 hours +# time window radius is +/- 0.5 hours for EFCLAM #======================================================================= -# Skip all Indian satellite winds in SATWND (not in domain) -export SKIP_005021=YES -export SKIP_005022=YES -export SKIP_005023=YES - -# Skip legacy Meteosat AMV subsets. Replaced by new BUFR sequence subsets Oct 2020. -# for testing skip in trigger or version file -#export SKIP_005064=YES -#export SKIP_005065=YES -#export SKIP_005066=YES - -DTIM_earliest_005010=-1.00 -DTIM_latest_005010=-0.01 -DTIM_earliest_005011=-1.00 -DTIM_latest_005011=-0.01 -DTIM_earliest_005012=-1.00 -DTIM_latest_005012=-0.01 -DTIM_earliest_005019=-1.00 -DTIM_latest_005019=-0.01 - -DTIM_earliest_005064=-1.50 -DTIM_latest_005064=+1.49 -DTIM_earliest_005065=-1.50 -DTIM_latest_005065=+1.49 -DTIM_earliest_005066=-1.50 -DTIM_latest_005066=+1.49 - -DTIM_earliest_005067=-1.50 -DTIM_latest_005067=+1.49 -DTIM_earliest_005068=-1.50 -DTIM_latest_005068=+1.49 -DTIM_earliest_005069=-1.50 -DTIM_latest_005069=+1.49 - DTIM_earliest_ascatt=-6.00 DTIM_latest_ascatt=0.00 DTIM_earliest_efclam=-0.50 DTIM_latest_efclam=+0.50 -#DTIM_earliest_infav1=-2.50 -#DTIM_latest_infav1=+2.49 -#DTIM_earliest_005081=${DTIM_earliest_005081:-"-3.00"} -#DTIM_latest_005081=${DTIM_latest_005081:-"+2.99"} -DTIM_earliest_005081=-3.00 -DTIM_latest_005081=+1.00 DTIM_earliest_gmi1cr=${DTIM_earliest_gmi1cr:-"-3.00"} DTIM_latest_gmi1cr=${DTIM_latest_gmi1cr:-"+2.99"} -$ushscript_dump/bufr_dump_obs.sh $dumptime 2.5 1 ascatt satwnd efclam gmi1cr +$ushscript_dump/bufr_dump_obs.sh $dumptime 2.5 1 ascatt efclam gmi1cr error1=$? echo "$error1" > $DATA/error1 @@ -311,6 +287,137 @@ set -x EOF set -x +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_5; chmod +x thread_5 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_5 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=5 + +#=========================================================================== +# Dump # 5 : SATWND -- TOTAL NUMBER OF SUBTYPES = 14 +# for SATWND subtypes 005/010, 005/011, 005/012 and 005/019 at all +# times: +# time window radius is -1.00 to -0.01 hours +# for SATWND subtypes 005/064, 005/065, 005/066, 005/067, 005/068, +# and 005/069 at all times: +# time window radius is -1.50 to +1.49 hours +# for all other SATWND subtypes: +# time window radius is +/- 2.5 hours +#=========================================================================== + +# Skip all Indian satellite winds in SATWND (not in domain) +export SKIP_005021=YES +export SKIP_005022=YES +export SKIP_005023=YES + +# Skip legacy Meteosat AMV subsets. Replaced by new BUFR sequence subsets Oct 2020. +# for testing skip in trigger or version file +#export SKIP_005064=YES +#export SKIP_005065=YES +#export SKIP_005066=YES + +DTIM_earliest_005010=-1.00 +DTIM_latest_005010=-0.01 +DTIM_earliest_005011=-1.00 +DTIM_latest_005011=-0.01 +DTIM_earliest_005012=-1.00 +DTIM_latest_005012=-0.01 +DTIM_earliest_005019=-1.00 +DTIM_latest_005019=-0.01 + +DTIM_earliest_005064=-1.50 +DTIM_latest_005064=+1.49 +DTIM_earliest_005065=-1.50 +DTIM_latest_005065=+1.49 +DTIM_earliest_005066=-1.50 +DTIM_latest_005066=+1.49 + +DTIM_earliest_005067=-1.50 +DTIM_latest_005067=+1.49 +DTIM_earliest_005068=-1.50 +DTIM_latest_005068=+1.49 +DTIM_earliest_005069=-1.50 +DTIM_latest_005069=+1.49 + +#DTIM_earliest_005081=${DTIM_earliest_005081:-"-3.00"} +#DTIM_latest_005081=${DTIM_latest_005081:-"+2.99"} +DTIM_earliest_005081=-3.00 +DTIM_latest_005081=+1.00 + +$ushscript_dump/bufr_dump_obs.sh $dumptime 2.5 1 satwnd +error5=$? +echo "$error5" > $DATA/error5 + +set +x +echo "********************************************************************" +echo Script thread_5 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/5.out 2>&1 +EOF +set -x + +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_6; chmod +x thread_6 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_6 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=6 + +#============================================================================ +# Dump # 6 : MSONE1 -- TOTAL NUMBER OF SUBTYPES = 1 +# (1) +#============================================================================ + +def_time_window_6=0.5 # default time window for dump 6 is -0.5 to +0.5 hours + +# Time window -0.50 to +0.50 hours for MSONET for full and partial cycle runs +# (default) + +$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_6} 1 msone1 +error6=$? +echo "$error6" > $DATA/error6 + +set +x +echo "********************************************************************" +echo Script thread_6 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/6.out 2>&1 +EOF +set -x + #---------------------------------------------------------------- # Now launch the threads @@ -329,11 +436,13 @@ if [ "$launcher" = cfp ]; then echo ./thread_1 >> $DATA/poe.cmdfile echo ./thread_2 >> $DATA/poe.cmdfile echo ./thread_4 >> $DATA/poe.cmdfile + echo ./thread_5 >> $DATA/poe.cmdfile + echo ./thread_6 >> $DATA/poe.cmdfile if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes # ?? launcher_DUMP=${launcher_DUMP:-mpiexec} - $launcher_DUMP -np 3 --cpu-bind verbose,core cfp $DATA/poe.cmdfile + $launcher_DUMP -np 6 --cpu-bind verbose,core cfp $DATA/poe.cmdfile errpoe=$? if [ $errpoe -ne 0 ]; then $DATA/err_exit "***FATAL: EXIT STATUS $errpoe RUNNING POE COMMAND FILE" @@ -349,10 +458,12 @@ else ./thread_2 ./thread_3 ./thread_4 + ./thread_5 + ./thread_6 fi -cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out +cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out set +x echo " " @@ -363,12 +474,14 @@ err1=`cat $DATA/error1` err2=`cat $DATA/error2` err3=`cat $DATA/error3` err4=`cat $DATA/error4` +err5=`cat $DATA/error5` +err6=`cat $DATA/error6` #================================================================ export STATUS=YES -export DUMP_NUMBER=5 +export DUMP_NUMBER=7 $ushscript_dump/bufr_dump_obs.sh $dumptime 3.00 1 null @@ -381,8 +494,9 @@ fi if [ "$PROCESS_DUMP" = 'YES' ]; then - if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o "$err4" -gt '5' ]; then - for n in $err1 $err2 $err3 $err4 + if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' \ + -o "$err4" -gt '5' -o "$err5" -gt '5' -o "$err6" -gt '5' ]; then + for n in $err1 $err2 $err3 $err4 $err5 $err6 do if [ "$n" -gt '5' ]; then if [ "$n" -ne '11' -a "$n" -ne '22' ]; then @@ -392,7 +506,7 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then set +x echo echo " ###################################################### " -echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4" +echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, $err5, $err6" echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " echo " ###################################################### " echo @@ -409,7 +523,7 @@ echo set +x echo echo " ###################################################### " - echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4" + echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, $err5, $err6" echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " echo " ###################################################### " echo @@ -419,6 +533,9 @@ echo # endif loop $PROCESS_DUMP fi +# concatenate msonet and msone1, b/c prepobs only wants one file +cat ${COMSP}msone1.tm00.bufr_d >> ${COMSP}msonet.tm00.bufr_d + # # copy bufr_dumplist to $COMOUT per NCO SPA request # ------------------------------------------------- From a2f3052a2e0edcc2e00468842b03e907e9a7ac3f Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Wed, 31 Jan 2024 17:37:31 +0000 Subject: [PATCH 42/53] testing early cron for dump scripts global and rap --- scripts/exglobal_dump.sh | 144 +++++++++++++++++++++++++++++++-------- scripts/exrap_dump.sh | 43 ++++++++---- 2 files changed, 144 insertions(+), 43 deletions(-) diff --git a/scripts/exglobal_dump.sh b/scripts/exglobal_dump.sh index 719146f..7de2b1f 100755 --- a/scripts/exglobal_dump.sh +++ b/scripts/exglobal_dump.sh @@ -124,7 +124,7 @@ set +u # * - for GDAS only # # Dump group #3 (pb, TIME_TRIM defaults to OFF) = -# adpupa uprair +# adpupa # # Dump group #4 (pb, TIME_TRIM defaults to ON) = # aircar aircft proflr vadwnd rassda gpsipw hdob @@ -153,8 +153,10 @@ set +u # # Dump group #12 crisfs atms (previously group1) # crsfdb iasidb (previously group10) +# Dump group #13 (pb, TIME_TRIM defaults to OFF) = +# uprair # -# Dump group #13 STATUS FILE +# Dump group #14 STATUS FILE # ----------------------------------------------------------------------------- #VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV @@ -184,6 +186,7 @@ set -u DUMP_group10=${DUMP_group10:-"YES"} DUMP_group11=${DUMP_group11:-"YES"} DUMP_group12=${DUMP_group12:-"YES"} + DUMP_group13=${DUMP_group12:-"NO"} else dump_ind=DUMP DUMP_group1=${DUMP_group1:-"NO"} @@ -198,6 +201,7 @@ set -u DUMP_group10=${DUMP_group10:-"NO"} DUMP_group11=${DUMP_group11:-"NO"} DUMP_group12=${DUMP_group12:-"NO"} + DUMP_group13=${DUMP_group12:-"YES"} fi else dump_ind=DUMP @@ -213,16 +217,20 @@ else DUMP_group10=${DUMP_group10:-"YES"} DUMP_group11=${DUMP_group11:-"YES"} DUMP_group12=${DUMP_group12:-"YES"} + DUMP_group13=${DUMP_group12:-"YES"} fi +# NAP and NAP_adpupa instroduced so that uprair can run early on his own +NAP=${NAP:-600} #b/c cron is moved to run 10min (600s) early if [ "$NET" = 'gfs' ]; then ADPUPA_wait=${ADPUPA_wait:-"YES"} + NAP_adpupa==${NAP_adpupa:-900} #600s(compensate early cron) + 300s(for adpupa data to come) ########ADPUPA_wait=${ADPUPA_wait:-"NO"} # saves time if ADPUPA_wait=NO else ADPUPA_wait=${ADPUPA_wait:-"NO"} + NAP_adpupa==${NAP_adpupa:-600} #like other dump groups fi - # send extra output of DUMP2 for monitoring purposes. set +u if [ -n "$JOB_NUMBER" ]; then @@ -519,7 +527,7 @@ echo "=======> Dump group 9 (thread_9) not executed." > $DATA/9.out echo "=======> Dump group 10 (thread_10) not executed." > $DATA/10.out echo "=======> Dump group 11 (thread_11) not executed." > $DATA/11.out echo "=======> Dump group 12 (thread_12) not executed." > $DATA/12.out - +echo "=======> Dump group 13 (thread_13) not executed." > $DATA/13.out err1=0 err2=0 @@ -533,6 +541,7 @@ err9=0 err10=0 err11=0 err12=0 +err13=0 if [ "$PROCESS_DUMP" = 'YES' ]; then #################################### @@ -561,6 +570,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=1 @@ -695,6 +705,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=2 @@ -821,6 +832,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP_adpupa} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=3 @@ -831,26 +843,22 @@ export DUMP_NUMBER=3 # #-------------------------------------------------------------------------- # Dump #3: ADPUPA: 6 subtype(s) -# UPRAIR: 5 subtype(s) # -------------------- -# TOTAL NUMBER OF SUBTYPES = 11 +# TOTAL NUMBER OF SUBTYPES = 6 # #==================================================================== DTIM_latest_adpupa=${DTIM_latest_adpupa:-"+2.99"} -DTIM_latest_uprair=${DTIM_latest_uprair:-"+2.99"} TIME_TRIM=${TIME_TRIM:-${TIME_TRIM3:-off}} -$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 adpupa uprair +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 adpupa error3=$? echo "$error3" > $DATA/error3 if [ "$SENDDBN" = "YES" ]; then $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_adpupa $job \ ${COMSP}adpupa.tm00.bufr_d - $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_uprair $job \ - ${COMSP}uprair.tm00.bufr_d fi set +x @@ -881,6 +889,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=4 @@ -990,6 +999,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=5 @@ -1042,6 +1052,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=6 @@ -1185,6 +1196,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=7 @@ -1292,6 +1304,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=8 @@ -1403,6 +1416,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=9 @@ -1473,6 +1487,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=10 @@ -1573,6 +1588,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=11 @@ -1629,6 +1645,7 @@ echo "********************************************************************" echo set -x +sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron export STATUS=NO export DUMP_NUMBER=12 @@ -1707,6 +1724,65 @@ set -x EOF set -x +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_13; chmod +x thread_13 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_13 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +# UPRAIR requires early start, no need to NAP +#sleep ${NAP} # to reverse 10min early start of jglobal_dump in cron +export STATUS=NO +export DUMP_NUMBER=13 + +#==================================================================== +# NOTES ABOUT THIS DUMP GROUP: +# (1) time window radius is -3.00 to +2.99 hours on all types +# (2) TIME TRIMMING IS NOT DONE IN THIS DUMP (default, unless overridden) +# +#-------------------------------------------------------------------------- +# Dump #13: UPRAIR: 5 subtype(s) +# -------------------- +# TOTAL NUMBER OF SUBTYPES = 5 +# +#==================================================================== + +DTIM_latest_uprair=${DTIM_latest_uprair:-"+2.99"} + +TIME_TRIM=${TIME_TRIM:-${TIME_TRIM13:-off}} + +$ushscript_dump/bufr_dump_obs.sh $dumptime 3.0 1 uprair +error13=$? +echo "$error13" > $DATA/error13 + +if [ "$SENDDBN" = "YES" ]; then + $DBNROOT/bin/dbn_alert MODEL ${NET_uc}_BUFR_uprair $job \ + ${COMSP}uprair.tm00.bufr_d +fi + +set +x +echo "********************************************************************" +echo Script thread_13 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/13.out 2>&1 +EOF +set -x + + #---------------------------------------------------------------- # Now launch the threads @@ -1720,8 +1796,14 @@ launcher=${launcher:-"cfp"} # if not "cfp", threads will be run serially. if [ "$launcher" = cfp ]; then > $DATA/poe.cmdfile + echo "Running threads in parallel IG2023" + myPDY=`date +\%Y\%m\%d\%H\%M\%S` + echo "DATE IG2023 start " $myPDY # To better take advantage of cfp, execute the longer running commands first. # Some reordering was done here based on recent sample runtimes. + + #[ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && echo ./thread_3 >> $DATA/poe.cmdfile + [ $DUMP_group13 = YES ] && echo ./thread_13 >> $DATA/poe.cmdfile [ $DUMP_group7 = YES ] && echo ./thread_7 >> $DATA/poe.cmdfile # moved up [ $DUMP_group1 = YES ] && echo ./thread_1 >> $DATA/poe.cmdfile [ $DUMP_group5 = YES ] && echo ./thread_5 >> $DATA/poe.cmdfile # moved up @@ -1730,19 +1812,18 @@ if [ "$launcher" = cfp ]; then [ $DUMP_group11 = YES ] && echo ./thread_11 >> $DATA/poe.cmdfile # moved up [ $DUMP_group10 = YES ] && echo ./thread_10 >> $DATA/poe.cmdfile # moved up [ $DUMP_group2 = YES ] && echo ./thread_2 >> $DATA/poe.cmdfile - [ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && echo ./thread_3 >> $DATA/poe.cmdfile [ $DUMP_group4 = YES ] && echo ./thread_4 >> $DATA/poe.cmdfile [ $DUMP_group9 = YES ] && echo ./thread_9 >> $DATA/poe.cmdfile [ $DUMP_group12 = YES ] && echo ./thread_12 >> $DATA/poe.cmdfile - + [ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && echo ./thread_3 >> $DATA/poe.cmdfile if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes launcher_DUMP=${launcher_DUMP:-mpiexec} - NPROCS=${NPROCS:-14} - #$launcher_DUMP -np 12 --cpu-bind verbose,core cfp $DATA/poe.cmdfile 2>&1 # 1) - #$launcher_DUMP -np 14 --cpu-bind core cfp $DATA/poe.cmdfile 2>&1 # 2) 3) - $launcher_DUMP -np ${NPROCS} cfp $DATA/poe.cmdfile 2>&1 # 4) + NPROCS=${NPROCS:-14} # was 12 + $launcher_DUMP -np ${NPROCS} --cpu-bind verbose,core cfp $DATA/poe.cmdfile 2>&1 + #$launcher_DUMP -np 14 --cpu-bind core cfp $DATA/poe.cmdfile 2>&1 # 1) 3) + #$launcher_DUMP -np ${NPROCS} cfp $DATA/poe.cmdfile 2>&1 # 4) Carolyn Pasti tips errpoe=$? if [ $errpoe -ne 0 ]; then $DATA/err_exit "***FATAL: EXIT STATUS $errpoe RUNNING POE COMMAND FILE" @@ -1754,9 +1835,9 @@ if [ "$launcher" = cfp ]; then fi else echo "Running threads serially" + [ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && ./thread_3 [ $DUMP_group1 = YES ] && ./thread_1 [ $DUMP_group2 = YES ] && ./thread_2 - [ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && ./thread_3 [ $DUMP_group4 = YES ] && ./thread_4 [ $DUMP_group5 = YES ] && ./thread_5 [ $DUMP_group6 = YES ] && ./thread_6 @@ -1766,18 +1847,20 @@ else [ $DUMP_group10 = YES ] && ./thread_10 [ $DUMP_group11 = YES ] && ./thread_11 [ $DUMP_group12 = YES ] && ./thread_12 + [ $DUMP_group13 = YES ] && ./thread_13 # wait fi -# if ADPUPA_wait is YES, adpupa and uprair are dumped AFTER all other dump -# threads have run (normally done in real-time GFS runs to dump as late as -# possible in order to maximize data availability in GFS network, -# particularly DROPs) -# -------------------------------------------------------------------------- - -[ $DUMP_group3 = YES -a $ADPUPA_wait = YES ] && ./thread_3 +# long run times for uprair lead to use of NAP and NAP_adpupa variables (see code above) instead of this code +## if ADPUPA_wait is YES, adpupa and uprair are dumped AFTER all other dump +## threads have run (normally done in real-time GFS runs to dump as late as +## possible in order to maximize data availability in GFS network, +## particularly DROPs) +## -------------------------------------------------------------------------- +## +#[ $DUMP_group3 = YES -a $ADPUPA_wait = YES ] && ./thread_3 -cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.out $DATA/11.out $DATA/12.out +cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.out $DATA/11.out $DATA/12.out $DATA/13.out set +x echo " " @@ -1796,12 +1879,13 @@ set -x [ -s $DATA/error10 ] && err10=`cat $DATA/error10` [ -s $DATA/error11 ] && err11=`cat $DATA/error11` [ -s $DATA/error12 ] && err12=`cat $DATA/error12` +[ -s $DATA/error13 ] && err13=`cat $DATA/error13` #=============================================================================== export STATUS=YES -export DUMP_NUMBER=13 +export DUMP_NUMBER=14 $ushscript_dump/bufr_dump_obs.sh $dumptime 3.00 1 null # endif loop $PROCESS_DUMP @@ -1821,8 +1905,8 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o \ "$err4" -gt '5' -o "$err5" -gt '5' -o "$err6" -gt '5' -o \ "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' -o \ - "$err10" -gt '5' -o "$err11" -gt '5' -o "$err12" -gt '5' ]; then - for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 $err11 $err12 + "$err10" -gt '5' -o "$err11" -gt '5' -o "$err12" -gt '5' -o "$err13" -gt '5']; then + for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 $err11 $err12 $err13 do if [ "$n" -gt '5' ]; then if [ "$n" -ne '11' -a "$n" -ne '22' ]; then @@ -1833,7 +1917,7 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then echo echo " ###################################################### " echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7, $err8, $err9, $err10, $err11, $err12 " +$err5, $err6, $err7, $err8, $err9, $err10, $err11, $err12, $err13 " echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " echo " ###################################################### " echo @@ -1851,7 +1935,7 @@ echo echo echo " ###################################################### " echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7, $err8, $err9, $err10, $err11, $err12 " +$err5, $err6, $err7, $err8, $err9, $err10, $err11, $err12, $err13 " echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " echo " ###################################################### " echo diff --git a/scripts/exrap_dump.sh b/scripts/exrap_dump.sh index c2c1bf1..ac193d8 100755 --- a/scripts/exrap_dump.sh +++ b/scripts/exrap_dump.sh @@ -81,9 +81,9 @@ set +u # Dump group #7 (non-pb) = airsev 1bhrs4 eshrs3 lgycld ssmisu osbuv8 crsfdb # saphir gmi1cr # Dump group #8 (non-pb) = gsrasr gsrcsr -# Dump group #9 (non-pb) = lghtng +# Dump group #9 (non-pb) = lghtng + adpupa # Dump group #10(pb) = msone1 # ONLY tank b255/xx030, the largest -# Dump group #11(pb) = adpupa uprair +# Dump group #11(pb) = adpupa uprair - adpupa # Dump group #12 STATUS FILE # ------------------------------------------------------------------------ @@ -300,6 +300,7 @@ echo "********************************************************************" echo set -x +sleep 120 # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=1 @@ -450,6 +451,7 @@ echo "********************************************************************" echo set -x +sleep 120 # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=2 @@ -581,6 +583,7 @@ echo "********************************************************************" echo set -x +sleep 120 # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=3 @@ -674,6 +677,7 @@ echo "********************************************************************" echo set -x +sleep 120 # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=4 @@ -763,6 +767,7 @@ echo "********************************************************************" echo set -x +sleep 120 # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=5 @@ -850,6 +855,7 @@ echo "********************************************************************" echo set -x +sleep 120 # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=6 @@ -1074,6 +1080,7 @@ echo "********************************************************************" echo set -x +sleep 120 # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=7 @@ -1204,6 +1211,7 @@ echo "********************************************************************" echo set -x +sleep 120 # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=8 @@ -1274,18 +1282,25 @@ echo "********************************************************************" echo set -x +sleep 120 # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=9 #========================================================================== -# Dump # 9 : LGHTNG -# TOTAL NUMBER OF SUBTYPES = 1 +# Dump # 9 : LGHTNG + ADPUPA +# TOTAL NUMBER OF SUBTYPES = 1 + 1 #========================================================================= - + # Time window -1.00 to +0.50 hours for LGHTNG for all cycle runs DTIM_earliest_lghtng=${DTIM_earliest_lghtng:-"-1.00"} DTIM_latest_lghtng=${DTIM_latest_lghtng:-"+0.50"} +# Time window -1.00 to +1.00 hours for ADPUPA/UPRAIR w/ full & partial cycle runs +# (note: time window increased over +/- 0.5 hr standard to get more data) + +DTIM_earliest_adpupa=${DTIM_earliest_adpupa:-"-1.00"} +DTIM_latest_adpupa=${DTIM_latest_adpupa:-"+1.00"} + if [ "$RUN" = 'rap_p' ]; then # ===> For RUN = rap_p -- partial cycle runs # ------------------------------------- @@ -1296,7 +1311,7 @@ else def_time_window_9=3.0 # default time window for dump 9 is -3.0 to +3.0 hours fi -$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_9} 1 lghtng +$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_9} 1 lghtng adpupa error9=$? echo "$error9" > $DATA/error9 @@ -1329,6 +1344,7 @@ echo "********************************************************************" echo set -x +sleep 120 # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=10 @@ -1375,13 +1391,15 @@ echo "********************************************************************" echo set -x +# UPRAIR need to start early +#sleep 120 # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=11 #========================================================================== -# Dump # 11 : ADPUPA, UPRAIR -# (6) (5) -# -- TOTAL NUMBER OF SUBTYPES = 11 +# Dump # 11 : ADPUPA minus UPRAIR +# (6) minus (5) +# -- TOTAL NUMBER OF SUBTYPES = 11-5 #========================================================================== # if [ "$RUN" = 'rap_p' ]; then @@ -1409,12 +1427,10 @@ fi # Time window -1.00 to +1.00 hours for ADPUPA/UPRAIR w/ full & partial cycle runs # (note: time window increased over +/- 0.5 hr standard to get more data) -DTIM_earliest_adpupa=${DTIM_earliest_adpupa:-"-1.00"} -DTIM_latest_adpupa=${DTIM_latest_adpupa:-"+1.00"} DTIM_earliest_uprair=${DTIM_earliest_uprair:-"-1.00"} DTIM_latest_uprair=${DTIM_latest_uprair:-"+1.00"} -$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_11} 1 adpupa uprair +$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_11} 1 uprair error11=$? echo "$error11" > $DATA/error11 @@ -1445,6 +1461,7 @@ if [ "$launcher" = cfp ]; then # To better take advantage of cfp, execute the longer running commands first. # Some reordering was done here based on recent sample runtimes. + [ $DUMP_group11 = YES ] && echo ./thread_11 >> $DATA/poe.cmdfile [ $DUMP_group9 = YES ] && echo ./thread_9 >> $DATA/poe.cmdfile # lghtng 1st [ $DUMP_group7 = YES ] && echo ./thread_7 >> $DATA/poe.cmdfile # moved up [ $DUMP_group1 = YES ] && echo ./thread_1 >> $DATA/poe.cmdfile @@ -1455,7 +1472,7 @@ if [ "$launcher" = cfp ]; then [ $DUMP_group2 = YES ] && echo ./thread_2 >> $DATA/poe.cmdfile [ $DUMP_group3 = YES ] && echo ./thread_3 >> $DATA/poe.cmdfile [ $DUMP_group10 = YES ] && echo ./thread_10 >> $DATA/poe.cmdfile - [ $DUMP_group11 = YES ] && echo ./thread_11 >> $DATA/poe.cmdfile + #[ $DUMP_group11 = YES ] && echo ./thread_11 >> $DATA/poe.cmdfile if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes # ?? From 1df2b7545ed64b34cb62d66c958e30a68f34de03 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Mon, 12 Feb 2024 17:14:02 +0000 Subject: [PATCH 43/53] Increase NSPLIT for prep, instroduce NAM*, regroup --- docs/Release_Notes.txt | 45 +++++++++++++++++++++++++++++++++++---- jobs/JOBSPROC_GLOBAL_PREP | 3 ++- jobs/JOBSPROC_RAP_PREP | 2 +- scripts/exdump_post.sh | 2 +- scripts/exglobal_dump.sh | 9 ++++---- scripts/exrap_dump.sh | 24 +++++++++++---------- 6 files changed, 63 insertions(+), 22 deletions(-) diff --git a/docs/Release_Notes.txt b/docs/Release_Notes.txt index 82fa011..ea16de1 100644 --- a/docs/Release_Notes.txt +++ b/docs/Release_Notes.txt @@ -47,6 +47,7 @@ JOBS CHANGES jobs/JOBSPROC_GLOBAL_PREP Instruct to add uprair dump files data to GDAS and GFS prepbufr files + Increase NSPLIT for faster prep step jobs/JOBSPROC_GLOBAL_PREP_POST Add functionality to unrestrict aircraft data (PROCESS_REMOREST_dm2) @@ -54,6 +55,7 @@ JOBS CHANGES jobs/JOBSPROC_RAP_PREP Instruct to add uprair dump files data to RAP's prepbufr file + Increase NSPLIT for faster prep step MODULEFILES CHANGES @@ -73,21 +75,24 @@ SCRIPTS CHANGES Split groups 1 and 10 and add an additional DUMP_group12 to allow code to run on one more cpu and faster Dump BUFR formated high-res rawinsonde profiles into *uprair* dump files Add DBNet dbn_alert for uprair and satwhr dump files going to NOMADS - Redistribute groups + Redistribute groups, introduce NAP and NAP_adpupa to allow earlier job/cron kickoff scripts/exrap_dump.sh Dump BUFR formated high-res rawinsonde profiles into *uprair* dump files Add DBNet dbn_alert for uprair dump files going to NOMADS Introduce msone1 group and redistribute groups + Redistribute groups, introduce NAP to allow earlier job/cron kickoff scripts/exnam_dump.sh Introduce msone1 group and redistribute groups/add cpus scripts/exrtma_dump.sh Introduce msone1 group and redistribute groups + Introduce max size treshold for mesonet tanks scripts/exurma_dump.sh Introduce msone1 group and redistribute groups + Introduce max size treshold for mesonet tanks SORC CHANGES @@ -138,9 +143,41 @@ CHANGES TO OUTPUT AND FILE SIZES CHANGES TO RESOURCES - Memory usage was increased for all steps in the processing of: - gdas,gfs,cdas, rap - See the example logs for : /lfs/h2/emc/stmp/iliana.genkova/CRON/R12/output + Memory and CPU usage was expanded for many steps in the processing. + EMC tests ran with the following recomended configurations (keep the ncpus values!) + gdas/gfs dump: ncpus=14 mem=500gb + gdas/gfs dump_post: ncpus=14 mem=4gb + gdas/gfs prep: ncpus=14 mem=520gb + gdas/gfs prep_post: ncpus=14 mem=4gb + + rap dump: ncpus=11 mem=50gb + rap dump_post: ncpus=14 mem=6b + rap prep: ncpus=4 mem=100gb + rap prep_post: ncpus=1 mem=3gb + + cdas dump: ncpus=10 mem=200gb + cdas dump_post: ncpus=12 mem=300gb + cdas prep1: ncpus=12 mem=100gb + cdas prep2: ncpus=16 mem=120gb + cdas prep1_post: ncpus=1 mem=6gb + + nam dump: ncpus=7 mem=20gb + nam dump2: ncpus=3 mem=15gb + nam dump_post: ncpus=1 mem=3gb + nam prep: ncpus=4 mem=220gb + nam prep_post: ncpus=1 mem=500Mb + + rtma dump: ncpus=5 mem=40gb + rtma dump_post:ncpus=1 mem=5000MB + rtma prep: ncpus=1 mem=2gb + rtma prep_post:ncpus=1 mem=2gb + + urma dump: ncpus=6:mem=40gb + urma dump_post:ncpus=1:mem=6gb + urma prep: ncpus=1:mem=6gb + urma prep_post:ncpus=1:mem=2gb + + dump monitor: ncpus=15:mem=400GB PRE-IMPLEMENTATION TESTING REQUIREMENTS diff --git a/jobs/JOBSPROC_GLOBAL_PREP b/jobs/JOBSPROC_GLOBAL_PREP index 1ca00a3..f8b6819 100755 --- a/jobs/JOBSPROC_GLOBAL_PREP +++ b/jobs/JOBSPROC_GLOBAL_PREP @@ -233,7 +233,8 @@ export POE=${POE:-YES} # (Applies only when either POE=YES or BACK=YES, and PREPDATA=YES) # Consider the size of the backgrounds fields if using threads. -export NSPLIT=${NSPLIT:-4} +#IG export NSPLIT=${NSPLIT:-4} +export NSPLIT=${NSPLIT:-12} # ${NSPLIT:-8} #16 adds time to MERGE step :( # BUFRLIST indicates the list of BUFR data dump file names to process diff --git a/jobs/JOBSPROC_RAP_PREP b/jobs/JOBSPROC_RAP_PREP index f5cc510..75d010a 100755 --- a/jobs/JOBSPROC_RAP_PREP +++ b/jobs/JOBSPROC_RAP_PREP @@ -221,7 +221,7 @@ export POE=${POE:-NO} # (Applies only when either POE=YES or BACK=YES, and PREPDATA=YES) # Consider the size of the backgrounds fields if using threads. -export NSPLIT=${NSPLIT:-1} +export NSPLIT=${NSPLIT:-4} # BUFRLIST indicates the list of BUFR data dump file names to process diff --git a/scripts/exdump_post.sh b/scripts/exdump_post.sh index 8d473b1..c1e9a45 100755 --- a/scripts/exdump_post.sh +++ b/scripts/exdump_post.sh @@ -764,7 +764,7 @@ EOFthread else which cfp which_cfp_err=$? - mpiexec -np 1 --cpu-bind verbose,core cfp $DATA/mpmd.cmdfile + mpiexec -np 14 --cpu-bind verbose,core cfp $DATA/mpmd.cmdfile fi else echo diff --git a/scripts/exglobal_dump.sh b/scripts/exglobal_dump.sh index 7de2b1f..8105886 100755 --- a/scripts/exglobal_dump.sh +++ b/scripts/exglobal_dump.sh @@ -224,11 +224,11 @@ fi NAP=${NAP:-600} #b/c cron is moved to run 10min (600s) early if [ "$NET" = 'gfs' ]; then ADPUPA_wait=${ADPUPA_wait:-"YES"} - NAP_adpupa==${NAP_adpupa:-900} #600s(compensate early cron) + 300s(for adpupa data to come) + NAP_adpupa=${NAP_adpupa:-800} #600s(compensate early cron) + 300s(for adpupa data to come) ########ADPUPA_wait=${ADPUPA_wait:-"NO"} # saves time if ADPUPA_wait=NO else ADPUPA_wait=${ADPUPA_wait:-"NO"} - NAP_adpupa==${NAP_adpupa:-600} #like other dump groups + NAP_adpupa=${NAP_adpupa:-600} #like other dump groups fi # send extra output of DUMP2 for monitoring purposes. @@ -1803,6 +1803,7 @@ if [ "$launcher" = cfp ]; then # Some reordering was done here based on recent sample runtimes. #[ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && echo ./thread_3 >> $DATA/poe.cmdfile + [ $DUMP_group3 = YES ] && echo ./thread_3 >> $DATA/poe.cmdfile # NAP_adpupa covers for ADPUPA_wait [ $DUMP_group13 = YES ] && echo ./thread_13 >> $DATA/poe.cmdfile [ $DUMP_group7 = YES ] && echo ./thread_7 >> $DATA/poe.cmdfile # moved up [ $DUMP_group1 = YES ] && echo ./thread_1 >> $DATA/poe.cmdfile @@ -1815,7 +1816,6 @@ if [ "$launcher" = cfp ]; then [ $DUMP_group4 = YES ] && echo ./thread_4 >> $DATA/poe.cmdfile [ $DUMP_group9 = YES ] && echo ./thread_9 >> $DATA/poe.cmdfile [ $DUMP_group12 = YES ] && echo ./thread_12 >> $DATA/poe.cmdfile - [ $DUMP_group3 = YES -a $ADPUPA_wait != YES ] && echo ./thread_3 >> $DATA/poe.cmdfile if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes @@ -1823,7 +1823,7 @@ if [ "$launcher" = cfp ]; then NPROCS=${NPROCS:-14} # was 12 $launcher_DUMP -np ${NPROCS} --cpu-bind verbose,core cfp $DATA/poe.cmdfile 2>&1 #$launcher_DUMP -np 14 --cpu-bind core cfp $DATA/poe.cmdfile 2>&1 # 1) 3) - #$launcher_DUMP -np ${NPROCS} cfp $DATA/poe.cmdfile 2>&1 # 4) Carolyn Pasti tips + #$launcher_DUMP -np ${NPROCS} cfp $DATA/poe.cmdfile 2>&1 # 4) Carolyn Pasti suggestions errpoe=$? if [ $errpoe -ne 0 ]; then $DATA/err_exit "***FATAL: EXIT STATUS $errpoe RUNNING POE COMMAND FILE" @@ -1852,6 +1852,7 @@ else fi # long run times for uprair lead to use of NAP and NAP_adpupa variables (see code above) instead of this code +# ## if ADPUPA_wait is YES, adpupa and uprair are dumped AFTER all other dump ## threads have run (normally done in real-time GFS runs to dump as late as ## possible in order to maximize data availability in GFS network, diff --git a/scripts/exrap_dump.sh b/scripts/exrap_dump.sh index ac193d8..e271344 100755 --- a/scripts/exrap_dump.sh +++ b/scripts/exrap_dump.sh @@ -227,6 +227,8 @@ to ${COMSP}${i}" done fi # endif loop $PROCESS_GRIBFLDS +# NAP is introduced so that uprair can run early on his own +NAP=${NAP:-120} #b/c cron is moved to run 2min (120s) early echo "=======> Dump group 1 (thread_1) not executed." > $DATA/1.out echo "=======> Dump group 2 (thread_2) not executed." > $DATA/2.out @@ -300,7 +302,7 @@ echo "********************************************************************" echo set -x -sleep 120 # to reverse 2min early start of jrap_dump in cron +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=1 @@ -451,7 +453,7 @@ echo "********************************************************************" echo set -x -sleep 120 # to reverse 2min early start of jrap_dump in cron +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=2 @@ -583,7 +585,7 @@ echo "********************************************************************" echo set -x -sleep 120 # to reverse 2min early start of jrap_dump in cron +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=3 @@ -677,7 +679,7 @@ echo "********************************************************************" echo set -x -sleep 120 # to reverse 2min early start of jrap_dump in cron +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=4 @@ -767,7 +769,7 @@ echo "********************************************************************" echo set -x -sleep 120 # to reverse 2min early start of jrap_dump in cron +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=5 @@ -855,7 +857,7 @@ echo "********************************************************************" echo set -x -sleep 120 # to reverse 2min early start of jrap_dump in cron +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=6 @@ -1080,7 +1082,7 @@ echo "********************************************************************" echo set -x -sleep 120 # to reverse 2min early start of jrap_dump in cron +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=7 @@ -1211,7 +1213,7 @@ echo "********************************************************************" echo set -x -sleep 120 # to reverse 2min early start of jrap_dump in cron +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=8 @@ -1282,7 +1284,7 @@ echo "********************************************************************" echo set -x -sleep 120 # to reverse 2min early start of jrap_dump in cron +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=9 @@ -1344,7 +1346,7 @@ echo "********************************************************************" echo set -x -sleep 120 # to reverse 2min early start of jrap_dump in cron +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=10 @@ -1392,7 +1394,7 @@ echo set -x # UPRAIR need to start early -#sleep 120 # to reverse 2min early start of jrap_dump in cron +#sleep ${NAP} # to reverse 2min early start of jrap_dump in cron export STATUS=NO export DUMP_NUMBER=11 From b6d57fcc43263e0cc368e216636699e216437b81 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Mon, 12 Feb 2024 17:40:47 +0000 Subject: [PATCH 44/53] Expand RN to explain NAP and NAP_adpupa --- docs/Release_Notes.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/Release_Notes.txt b/docs/Release_Notes.txt index ea16de1..3cb00f0 100644 --- a/docs/Release_Notes.txt +++ b/docs/Release_Notes.txt @@ -143,6 +143,19 @@ CHANGES TO OUTPUT AND FILE SIZES CHANGES TO RESOURCES + 1. + The gfs and gdas processing must start 10 min earlier than current processing. + The NAP(600s) and NAP_adpupa(800s) variables in the obsproc script exglobal_dump.sh compensate for the earlier start, + such that only "uprair" dump stars early. This approach will be revisited in future obsproc updates. + + The rap processing must start 2 min earlier than current processing. + The NAP(120s) variables in the obsproc scripts exrap_dump.sh compensate for the earlier start, + such that only "uprair" dump stars early. This approach will be revisited in future obsproc updates. + + Tests with this approach shows no loss of current opertaional obs data for gdas, gfs and rap. + The new uprair data will be captured and instrodyced with no loss for gdas and rap, and possibly minimal loss for gfs. + + 2. Memory and CPU usage was expanded for many steps in the processing. EMC tests ran with the following recomended configurations (keep the ncpus values!) gdas/gfs dump: ncpus=14 mem=500gb From ae3530f7aefca8ca2f2ce48d90a70e7234b5f71a Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Thu, 14 Mar 2024 20:19:21 +0000 Subject: [PATCH 45/53] Split gsrasr and gsrcsr in exrap_dump Make msonet its own group in exnam_dump Stop msonet in excdas_dump --- scripts/excdas_dump.sh | 17 ++++-- scripts/exnam_dump.sh | 119 +++++++++++++++++++++++++++++++---------- scripts/exrap_dump.sh | 108 ++++++++++++++++++++++++++++++------- 3 files changed, 192 insertions(+), 52 deletions(-) diff --git a/scripts/excdas_dump.sh b/scripts/excdas_dump.sh index cd8adbc..8629163 100755 --- a/scripts/excdas_dump.sh +++ b/scripts/excdas_dump.sh @@ -41,9 +41,11 @@ echo " Mar 08 2022 - Enable the dumping of 002017 in vadwnd dump " echo " group. " echo " Jul 30 2022 - Subpfl, saldrn, snocvr, and gmi1cr added " echo " to dump group #9. " -echo " Sep 30 2022 - Enable dumping of UPRAIR data in group #3. " +echo " Sep 30 2022 - Don't / Enable dumping of UPRAIR data in " +echo " group #3./ b/c it is too slow " echo " Oct 17 2023 - Split msonet to msonet (#5) and msone1 (#10) " echo " concatenate msonet and msone1 right after dump" +echo " Turn off msonet and msone1 - not needed " ########################################################################### set -xau @@ -122,12 +124,14 @@ set -u DUMP_group2=${DUMP_group2:-"YES"} DUMP_group3=${DUMP_group3:-"YES"} DUMP_group4=${DUMP_group4:-"YES"} - DUMP_group5=${DUMP_group5:-"YES"} + #DUMP_group5=${DUMP_group5:-"YES"} + DUMP_group5=${DUMP_group5:-"NO"} DUMP_group6=${DUMP_group6:-"NO"} DUMP_group7=${DUMP_group7:-"NO"} DUMP_group8=${DUMP_group8:-"YES"} DUMP_group9=${DUMP_group9:-"NO"} - DUMP_group10=${DUMP_group10:="YES"} + #DUMP_group10=${DUMP_group10:="YES"} + DUMP_group10=${DUMP_group10:="NO"} fi else dump_ind=DUMP @@ -135,12 +139,14 @@ else DUMP_group2=${DUMP_group2:-"YES"} DUMP_group3=${DUMP_group3:-"YES"} DUMP_group4=${DUMP_group4:-"YES"} - DUMP_group5=${DUMP_group5:-"YES"} + #DUMP_group5=${DUMP_group5:-"YES"} + DUMP_group5=${DUMP_group5:-"NO"} DUMP_group6=${DUMP_group6:-"YES"} DUMP_group7=${DUMP_group7:-"YES"} DUMP_group8=${DUMP_group8:-"YES"} DUMP_group9=${DUMP_group9:-"YES"} - DUMP_group10=${DUMP_group10:="YES"} + #DUMP_group10=${DUMP_group10:="YES"} + DUMP_group10=${DUMP_group10:="NO"} fi @@ -1259,6 +1265,7 @@ echo " " >> $pgmout #================================================================ + if [ "$PROCESS_DUMP" = 'YES' ]; then if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o \ diff --git a/scripts/exnam_dump.sh b/scripts/exnam_dump.sh index 8d52c7b..1760a59 100755 --- a/scripts/exnam_dump.sh +++ b/scripts/exnam_dump.sh @@ -32,6 +32,7 @@ echo " sfcshp dump group to make unique dump file. " echo " - Copy bufr_dumplist to COMOUT. " echo " Dec 09 2021 - Updated for use on WCOSS2 " echo " Oct 17 2023 - Split msonet to msonet and msone1 (b255/xx030)" +echo " Mar 14 2024 - Separate msonet to own dump group " ############################################################################ set -xau @@ -47,14 +48,15 @@ set +u # Dump group #1 (non-pb) = 1bamua 1bmhs gpsro mtiasi esamua eshrs3 sevcsr atms # Dump group #2 (pb) = vadwnd satwnd # Dump group #3 (pb) = proflr rassda sfcshp tideg adpsfc ascatt -# Dump group #4 (pb) = msonet gpsipw +# Dump group #4 (pb) = gpsipw (moved msonet) # Dump group #5 (pb) = aircft aircar goesnd --> dump is GLOBAL # Dump group #6 (non-pb) = nexrad --> dump is GLOBAL # Dump group #7 (non-pb) = goesfv lgycld --> dump is GLOBAL # Dump group #8 (non-pb) = 1bhrs4 airsev osbuv8 esmhs ssmisu cris lghtng # Dump group #9 (pb) = adpupa -# Dump group #10 (pb)= msone1 -# Dump group #11 STATUS FILE +# Dump group #10 (pb)= msonet +# Dump group #11 (pb)= msone1 +# Dump group #12 STATUS FILE # ----------------------------------------------------------------------------- #VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV @@ -95,6 +97,7 @@ set -u DUMP_group8=${DUMP_group8:-"YES"} DUMP_group9=${DUMP_group9:-"NO"} DUMP_group10=${DUMP_group10:-"NO"} + DUMP_group11=${DUMP_group11:-"NO"} else dump_ind=DUMP DUMP_group1=${DUMP_group1:-"NO"} @@ -107,6 +110,7 @@ set -u DUMP_group8=${DUMP_group8:-"NO"} DUMP_group9=${DUMP_group9:-"YES"} DUMP_group10=${DUMP_group10:-"YES"} + DUMP_group11=${DUMP_group11:-"YES"} fi else dump_ind=DUMP @@ -120,6 +124,7 @@ else DUMP_group8=${DUMP_group8:-"YES"} DUMP_group9=${DUMP_group9:-"YES"} DUMP_group10=${DUMP_group10:-"YES"} + DUMP_group11=${DUMP_group11:-"YES"} fi if [ $tmmark = tm00 ]; then @@ -250,6 +255,7 @@ echo "=======> Dump group 7 (thread_7) not executed." > $DATA/7.out echo "=======> Dump group 8 (thread_8) not executed." > $DATA/8.out echo "=======> Dump group 9 (thread_9) not executed." > $DATA/9.out echo "=======> Dump group 10 (thread_10) not executed." > $DATA/10.out +echo "=======> Dump group 11 (thread_11) not executed." > $DATA/11.out err1=0 err2=0 @@ -261,6 +267,7 @@ err7=0 err8=0 err9=0 err10=0 +err11=0 if [ "$PROCESS_DUMP" = 'YES' ]; then ####################################################################### @@ -547,8 +554,8 @@ export STATUS=NO export DUMP_NUMBER=4 #=========================================================================== -# Dump # 4 : MSONET, GPSIPW -- TOTAL NUMBER OF SUBTYPES = 31 -# (30) (1) +# Dump # 4 : [MSONET,] GPSIPW -- TOTAL NUMBER OF SUBTYPES = [31] 1 +# [(30)] (1) - moved msone* to separate groups! # for catch-up cycles tm06-01: # time window radius is (-0.50,+0.50) hours for MSONET # for on-time tm00 cycles: @@ -569,15 +576,7 @@ export DUMP_NUMBER=4 DTIM_earliest_gpsipw=${DTIM_earliest_gpsipw:-"-0.22"} DTIM_latest_gpsipw=${DTIM_latest_gpsipw:-"-0.12"} -if [ "$tmhr" = "00" ]; then - DTIM_earliest_msonet=${DTIM_earliest_msonet:-"-0.75"} - DTIM_latest_msonet=${DTIM_latest_msonet:-"+1.50"} -fi - -export SKIP_255031=YES # Skip for port to Dell since no new data allowed. -export SKIP_255101=YES # Also, b/c NAM is frozen; no new data. - -$ushscript_dump/bufr_dump_obs.sh $dumptime 0.5 1 msonet gpsipw +$ushscript_dump/bufr_dump_obs.sh $dumptime 0.5 1 gpsipw error4=$? echo "$error4" > $DATA/error4 @@ -639,7 +638,6 @@ fi if [ "$SENDDBN" = 'YES' ]; then $DBNROOT/bin/dbn_alert MODEL NAM_BUFR_gpsipw $job ${COMSP}gpsipw.$tmmark.bufr_d - $DBNROOT/bin/dbn_alert MODEL NAM_BUFR_msonet $job ${COMSP}msonet.$tmmark.bufr_d fi set +x @@ -1073,20 +1071,23 @@ export STATUS=NO export DUMP_NUMBER=10 #=========================================================================== -# Dump # 10 : MSONE1 -- TOTAL NUMBER OF SUBTYPES = 1 +# Dump # 10 : MSONET -- TOTAL NUMBER OF SUBTYPES = 1 # (1) # for catch-up cycles tm06-01: -# time window radius is (-0.50,+0.50) hours for MSONE1 +# time window radius is (-0.50,+0.50) hours for MSONET # for on-time tm00 cycles: -# time window radius is (-0.75,+1.50) hours for MSONE1 +# time window radius is (-0.75,+1.50) hours for MSONET #=========================================================================== if [ "$tmhr" = "00" ]; then - DTIM_earliest_msone1=${DTIM_earliest_msone1:-"-0.75"} - DTIM_latest_msone1=${DTIM_latest_msone1:-"+1.50"} + DTIM_earliest_msonet=${DTIM_earliest_msonet:-"-0.75"} + DTIM_latest_msonet=${DTIM_latest_msonet:-"+1.50"} fi -$ushscript_dump/bufr_dump_obs.sh $dumptime 0.5 1 msone1 +export SKIP_255031=YES # Skip for port to Dell since no new data allowed. +export SKIP_255101=YES # Also, b/c NAM is frozen; no new data. + +$ushscript_dump/bufr_dump_obs.sh $dumptime 0.5 1 msonet error10=$? echo "$error10" > $DATA/error10 @@ -1105,6 +1106,64 @@ set -x EOF set -x + +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_11; chmod +x thread_11 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_11 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +export STATUS=NO +export DUMP_NUMBER=11 + +#=========================================================================== +# Dump # 11 : MSONE1 -- TOTAL NUMBER OF SUBTYPES = 1 +# (1) +# for catch-up cycles tm06-01: +# time window radius is (-0.50,+0.50) hours for MSONE1 +# for on-time tm00 cycles: +# time window radius is (-0.75,+1.50) hours for MSONE1 +#=========================================================================== + +if [ "$tmhr" = "00" ]; then + DTIM_earliest_msone1=${DTIM_earliest_msone1:-"-0.75"} + DTIM_latest_msone1=${DTIM_latest_msone1:-"+1.50"} +fi + +export SKIP_255031=YES # Skip for port to Dell since no new data allowed. +export SKIP_255101=YES # Also, b/c NAM is frozen; no new data. + +$ushscript_dump/bufr_dump_obs.sh $dumptime 0.5 1 msone1 +error11=$? +echo "$error11" > $DATA/error11 + +#if [ "$SENDDBN" = 'YES' ]; then +# $DBNROOT/bin/dbn_alert MODEL NAM_BUFR_msonet $job ${COMSP}msone1.$tmmark.bufr_d +#fi + +set +x +echo "********************************************************************" +echo Script thread_11 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/11.out 2>&1 +EOF +set -x + + #---------------------------------------------------------------- # Now launch the threads @@ -1130,12 +1189,13 @@ if [ "$launcher" = cfp ]; then [ $DUMP_group7 = YES ] && echo ./thread_7 >> $DATA/poe.cmdfile [ $DUMP_group9 = YES -a $ADPUPA_wait != YES ] && echo ./thread_9 >> $DATA/poe.cmdfile [ $DUMP_group10 = YES ] && echo ./thread_10 >> $DATA/poe.cmdfile + [ $DUMP_group11 = YES ] && echo ./thread_11 >> $DATA/poe.cmdfile if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes # ?? launcher_DUMP=${launcher_DUMP:-mpiexec} #$launcher_DUMP -np 10 --cpu-bind verbose,core cfp $DATA/poe.cmdfile - NPROCS=${NPROCS:-10} + NPROCS=${NPROCS:-11} $launcher_DUMP -np $NPROCS --cpu-bind verbose,core cfp $DATA/poe.cmdfile errpoe=$? if [ $errpoe -ne 0 ]; then @@ -1158,6 +1218,7 @@ else [ $DUMP_group8 = YES ] && ./thread_8 [ $DUMP_group9 = YES -a $ADPUPA_wait != YES ] && ./thread_9 [ $DUMP_group10 = YES ] && ./thread_10 + [ $DUMP_group11 = YES ] && ./thread_11 fi # if ADPUPA_wait is YES, adpupa is dumped AFTER all other dump threads have @@ -1168,7 +1229,7 @@ fi [ $DUMP_group9 = YES -a $ADPUPA_wait = YES ] && ./thread_9 cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out \ - $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.out + $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.out $DATA/11.out set +x echo " " @@ -1185,11 +1246,11 @@ set -x [ -s $DATA/error8 ] && err8=`cat $DATA/error8` [ -s $DATA/error9 ] && err9=`cat $DATA/error9` [ -s $DATA/error10 ] && err10=`cat $DATA/error10` - +[ -s $DATA/error11 ] && err11=`cat $DATA/error11` #=============================================================================== export STATUS=YES -export DUMP_NUMBER=11 +export DUMP_NUMBER=12 $ushscript_dump/bufr_dump_obs.sh $dumptime 3.00 1 null @@ -1208,8 +1269,8 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o \ "$err4" -gt '5' -o "$err5" -gt '5' -o "$err6" -gt '5' -o \ "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' -o \ - "$err10" -gt '5']; then - for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 + "$err10" -gt '5' -o "$err11" -gt '5']; then + for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 $err11 do if [ "$n" -gt '5' ]; then if [ "$n" -ne '11' -a "$n" -ne '22' ]; then @@ -1220,7 +1281,7 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then echo echo " ###################################################### " echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7, $err8, $err9 $err10" +$err5, $err6, $err7, $err8, $err9, $err10, $err11" echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " echo " ###################################################### " echo @@ -1238,7 +1299,7 @@ echo echo echo " ###################################################### " echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7, $err8, $err9 $err10" +$err5, $err6, $err7, $err8, $err9, $err10, $err11" echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " echo " ###################################################### " echo diff --git a/scripts/exrap_dump.sh b/scripts/exrap_dump.sh index e271344..6060097 100755 --- a/scripts/exrap_dump.sh +++ b/scripts/exrap_dump.sh @@ -58,6 +58,7 @@ echo " Sep 30 2022 - Enable dumping of UPRAIR data in group #2. " echo " Oct 11 2023 - Split msonet to msonet and msone1, msone1=255.030 " echo " concatenate msonet and msone1 right after dump " echo " - Pull adpupa and uprair into own Dump group " +echo " Mar 14 2024 - Split gsrasr and gsrcsr to own dump hroups " ################################################################################ set -xau @@ -80,11 +81,12 @@ set +u # Dump group #6 (non-pb) = nexrad # Dump group #7 (non-pb) = airsev 1bhrs4 eshrs3 lgycld ssmisu osbuv8 crsfdb # saphir gmi1cr -# Dump group #8 (non-pb) = gsrasr gsrcsr +# Dump group #8 (non-pb) = gsrasr [gsrcsr] # Dump group #9 (non-pb) = lghtng + adpupa # Dump group #10(pb) = msone1 # ONLY tank b255/xx030, the largest # Dump group #11(pb) = adpupa uprair - adpupa -# Dump group #12 STATUS FILE +# Dump group #12 (non-pb)= gsrcsr +# Dump group #13 STATUS FILE # ------------------------------------------------------------------------ #VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV @@ -113,6 +115,7 @@ set -u DUMP_group9=${DUMP_group9:-"YES"} DUMP_group10=${DUMP_group10:-"NO"} DUMP_group11=${DUMP_group11:-"NO"} + DUMP_group12=${DUMP_group12:-"YES"} else dump_ind=DUMP DUMP_group1=${DUMP_group1:-"NO"} @@ -126,6 +129,7 @@ set -u DUMP_group9=${DUMP_group9:-"NO"} DUMP_group10=${DUMP_group10:-"YES"} DUMP_group11=${DUMP_group11:-"YES"} + DUMP_group12=${DUMP_group12:-"NO"} fi else dump_ind=DUMP @@ -140,6 +144,7 @@ else DUMP_group9=${DUMP_group9:-"YES"} DUMP_group10=${DUMP_group10:-"YES"} DUMP_group11=${DUMP_group11:-"YES"} + DUMP_group12=${DUMP_group12:-"YES"} fi # Oct 2019; disable -- not needed for HRRRv4 @@ -241,7 +246,7 @@ echo "=======> Dump group 8 (thread_8) not executed." > $DATA/8.out echo "=======> Dump group 9 (thread_9) not executed." > $DATA/9.out echo "=======> Dump group 10 (thread_10) not executed." > $DATA/10.out echo "=======> Dump group 11 (thread_11) not executed." > $DATA/11.out - +echo "=======> Dump group 12 (thread_12) not executed." > $DATA/12.out err1=0 err2=0 err3=0 @@ -253,6 +258,7 @@ err8=0 err9=0 err10=0 err11=0 +err12=0 #restrict processing of unexpected big tanks #this block appear in all /scripts/ex*_dump.sh proessing msonet and msone1 @@ -1218,9 +1224,9 @@ export STATUS=NO export DUMP_NUMBER=8 #=============================================================================== -# Dump # 8 : GSRASR, GSRCSR -# (1) (1) -# TOTAL NUMBER OF SUBTYPES = 2 +# Dump # 8 : GSRASR,[ GSRCSR - moved to own group ] +# (1) [ (1)] +# TOTAL NUMBER OF SUBTYPES = 1 [2] #=============================================================================== if [ "$RUN" = 'rap_p' ]; then @@ -1233,8 +1239,6 @@ if [ "$RUN" = 'rap_p' ]; then # Time window is -1.00 to +0.99 hours for GSRASR, GSRCSR DTIM_earliest_gsrasr=${DTIM_earliest_gsrasr:-"-1.00"} DTIM_latest_gsrasr=${DTIM_latest_gsrasr:-"+0.99"} - DTIM_earliest_gsrcsr=${DTIM_earliest_gsrcsr:-"-1.00"} - DTIM_latest_gsrcsr=${DTIM_latest_gsrcsr:-"+0.99"} else @@ -1246,12 +1250,10 @@ else # Time window is -2.00 to +1.99 hours for GSRASR, GSRCSR DTIM_earliest_gsrasr=${DTIM_earliest_gsrasr:-"-2.00"} DTIM_latest_gsrasr=${DTIM_latest_gsrasr:-"+1.99"} - DTIM_earliest_gsrcsr=${DTIM_earliest_gsrcsr:-"-2.00"} - DTIM_latest_gsrcsr=${DTIM_latest_gsrcsr:-"+1.99"} fi -$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_8} 1 gsrasr gsrcsr +$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_8} 1 gsrasr error8=$? echo "$error8" > $DATA/error8 @@ -1448,6 +1450,73 @@ EOF set -x +set +x +#---------------------------------------------------------------- +cat<<\EOF>thread_12; chmod +x thread_12 +set -uax + +cd $DATA + +{ echo +set +x +echo "********************************************************************" +echo Script thread_12 +echo Executing on node `hostname` +echo Starting time: `date -u` +echo "********************************************************************" +echo +set -x + +sleep ${NAP} # to reverse 2min early start of jrap_dump in cron +export STATUS=NO +export DUMP_NUMBER=12 + +#=============================================================================== +# Dump # 12 : GSRCSR +# (1) +# TOTAL NUMBER OF SUBTYPES = 1 +#=============================================================================== + +if [ "$RUN" = 'rap_p' ]; then + +# ===> For RUN = rap_p -- partial cycle runs +# ------------------------------------- + + def_time_window_12=1.0 # default time window for dump 12 is -1.0 to +1.0 hours + +# Time window is -1.00 to +0.99 hours for GSRASR, GSRCSR + DTIM_earliest_gsrcsr=${DTIM_earliest_gsrcsr:-"-1.00"} + DTIM_latest_gsrcsr=${DTIM_latest_gsrcsr:-"+0.99"} + +else + +# ===> For RUN = rap, rap_e -- full cycle runs (including early at 00/12z) +# ------------------------------------------------------------------- + + def_time_window_12=3.0 # default time window for dump 12 is -3.0 to +3.0 hours + +# Time window is -2.00 to +1.99 hours for GSRASR, GSRCSR + DTIM_earliest_gsrcsr=${DTIM_earliest_gsrcsr:-"-2.00"} + DTIM_latest_gsrcsr=${DTIM_latest_gsrcsr:-"+1.99"} + +fi + +$ushscript_dump/bufr_dump_obs.sh $dumptime ${def_time_window_12} 1 gsrcsr +error12=$? +echo "$error12" > $DATA/error12 + +set +x +echo "********************************************************************" +echo Script thread_12 +echo Finished executing on node `hostname` +echo Ending time : `date -u` +echo "********************************************************************" +set -x +} > $DATA/12.out 2>&1 +EOF +set -x + + #---------------------------------------------------------------- # Now launch the threads @@ -1475,12 +1544,13 @@ if [ "$launcher" = cfp ]; then [ $DUMP_group3 = YES ] && echo ./thread_3 >> $DATA/poe.cmdfile [ $DUMP_group10 = YES ] && echo ./thread_10 >> $DATA/poe.cmdfile #[ $DUMP_group11 = YES ] && echo ./thread_11 >> $DATA/poe.cmdfile + [ $DUMP_group12 = YES ] && echo ./thread_12 >> $DATA/poe.cmdfile if [ -s $DATA/poe.cmdfile ]; then export MP_CSS_INTERRUPT=yes # ?? launcher_DUMP=${launcher_DUMP:-mpiexec} #$launcher_DUMP -np 3 --cpu-bind verbose,core cfp $DATA/poe.cmdfile - NPROCS=${NPROCS:-11} + NPROCS=${NPROCS:-13} $launcher_DUMP -np $NPROCS --cpu-bind verbose,core cfp $DATA/poe.cmdfile errpoe=$? if [ $errpoe -ne 0 ]; then @@ -1504,9 +1574,10 @@ else [ $DUMP_group9 = YES ] && ./thread_9 [ $DUMP_group10 = YES ] && ./thread_10 [ $DUMP_group11 = YES ] && ./thread_11 + [ $DUMP_group12 = YES ] && ./thread_12 fi -cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.out $DATA/11.out +cat $DATA/1.out $DATA/2.out $DATA/3.out $DATA/4.out $DATA/5.out $DATA/6.out $DATA/7.out $DATA/8.out $DATA/9.out $DATA/10.out $DATA/11.out $DATA/12.out set +x echo " " @@ -1524,11 +1595,12 @@ set -x [ -s $DATA/error9 ] && err9=`cat $DATA/error9` [ -s $DATA/error10 ] && err10=`cat $DATA/error10` [ -s $DATA/error11 ] && err11=`cat $DATA/error11` +[ -s $DATA/error12 ] && err12=`cat $DATA/error12` #=============================================================================== export STATUS=YES -export DUMP_NUMBER=12 +export DUMP_NUMBER=13 $ushscript_dump/bufr_dump_obs.sh $dumptime 3.00 1 null # endif loop $PROCESS_DUMP @@ -1547,8 +1619,8 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then if [ "$err1" -gt '5' -o "$err2" -gt '5' -o "$err3" -gt '5' -o \ "$err4" -gt '5' -o "$err5" -gt '5' -o "$err6" -gt '5' -o \ "$err7" -gt '5' -o "$err8" -gt '5' -o "$err9" -gt '5' -o \ - "$err10" -gt '5' -o "$err11" -gt '5' ]; then - for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 $err11 + "$err10" -gt '5' -o "$err11" -gt '5' -o "$err12" -gt '5' ]; then + for n in $err1 $err2 $err3 $err4 $err5 $err6 $err7 $err8 $err9 $err10 $err11 $err12 do if [ "$n" -gt '5' ]; then if [ "$n" -ne '11' -a "$n" -ne '22' ]; then @@ -1559,7 +1631,7 @@ if [ "$PROCESS_DUMP" = 'YES' ]; then echo echo " ###################################################### " echo " --> > 22 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7 $err8 $err9 $err10 $err11" +$err5, $err6, $err7, $err8, $err9, $err10, $err11, $err12" echo " --> @@ F A T A L E R R O R @@ -- ABNORMAL EXIT " echo " ###################################################### " echo @@ -1577,7 +1649,7 @@ echo echo echo " ###################################################### " echo " --> > 5 RETURN CODE FROM DATA DUMP, $err1, $err2, $err3, $err4, \ -$err5, $err6, $err7 $err8 $err9 $err10 $err11" +$err5, $err6, $err7, $err8, $err9, $err10, $err11, $err12" echo " --> NOT ALL DATA DUMP FILES ARE COMPLETE - CONTINUE " echo " ###################################################### " echo From 10d2e5985ae4970c88745c302a1703b7f3491ad1 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Thu, 14 Mar 2024 22:35:53 +0000 Subject: [PATCH 46/53] fixed a big in the dump group #13 YES / NO setting --- scripts/exglobal_dump.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/exglobal_dump.sh b/scripts/exglobal_dump.sh index 8105886..d4b9472 100755 --- a/scripts/exglobal_dump.sh +++ b/scripts/exglobal_dump.sh @@ -186,7 +186,7 @@ set -u DUMP_group10=${DUMP_group10:-"YES"} DUMP_group11=${DUMP_group11:-"YES"} DUMP_group12=${DUMP_group12:-"YES"} - DUMP_group13=${DUMP_group12:-"NO"} + DUMP_group13=${DUMP_group13:-"NO"} else dump_ind=DUMP DUMP_group1=${DUMP_group1:-"NO"} @@ -201,7 +201,7 @@ set -u DUMP_group10=${DUMP_group10:-"NO"} DUMP_group11=${DUMP_group11:-"NO"} DUMP_group12=${DUMP_group12:-"NO"} - DUMP_group13=${DUMP_group12:-"YES"} + DUMP_group13=${DUMP_group13:-"YES"} fi else dump_ind=DUMP @@ -217,7 +217,7 @@ else DUMP_group10=${DUMP_group10:-"YES"} DUMP_group11=${DUMP_group11:-"YES"} DUMP_group12=${DUMP_group12:-"YES"} - DUMP_group13=${DUMP_group12:-"YES"} + DUMP_group13=${DUMP_group13:-"YES"} fi # NAP and NAP_adpupa instroduced so that uprair can run early on his own From 08ea709518131b320543fea46cc8c0733abbaba7 Mon Sep 17 00:00:00 2001 From: iliana Genkova <59100254+ilianagenkova@users.noreply.github.com> Date: Mon, 18 Mar 2024 11:47:51 -0500 Subject: [PATCH 47/53] Update excdas_dump.sh msonet and msone1 are turned off, thus concatenating them is turned off too. --- scripts/excdas_dump.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/excdas_dump.sh b/scripts/excdas_dump.sh index 8629163..4b321e1 100755 --- a/scripts/excdas_dump.sh +++ b/scripts/excdas_dump.sh @@ -1311,8 +1311,8 @@ $err5, $err6, $err7, $err8, $err9 $err10 " # endif loop $PROCESS_DUMP fi -# concatenate msonet and msone1, b/c prepobs only wants one file -cat ${COMSP}msone1.tm00.bufr_d >> ${COMSP}msonet.tm00.bufr_d +## concatenate msonet and msone1, b/c prepobs only wants one file +#cat ${COMSP}msone1.tm00.bufr_d >> ${COMSP}msonet.tm00.bufr_d # # copy bufr_dumplist to $COMOUT per NCO SPA request From 8ffcabd9afd1cfa1557433d5ef60e5600be0d713 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Tue, 19 Mar 2024 20:17:50 +0000 Subject: [PATCH 48/53] replaced mail with mail.py --- scripts/excdas_dump.sh | 11 +++++++---- scripts/exdump_monitor.sh | 11 +++++++---- scripts/exrap_dump.sh | 11 +++++++---- scripts/exrtma_dump.sh | 11 +++++++---- scripts/exurma_dump.sh | 11 +++++++---- 5 files changed, 35 insertions(+), 20 deletions(-) diff --git a/scripts/excdas_dump.sh b/scripts/excdas_dump.sh index 8629163..b384dfc 100755 --- a/scripts/excdas_dump.sh +++ b/scripts/excdas_dump.sh @@ -329,15 +329,18 @@ TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES - echo "WARNING: TANK b005/xx003 exceeds TANK_MAX_255003 => not dumped" | mail iliana.genkova@noaa.gov + msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" + echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES - echo "WARNING: TANK b005/xx004 exceeds TANK_MAX_255004 => not dumped" | mail iliana.genkova@noaa.gov -fi + msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" + echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + fi if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES - echo "WARNING: TANK b005/xx030 exceeds TANK_MAX_255030 => not dumped" | mail iliana.genkova@noaa.gov + msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" + echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi #end of block diff --git a/scripts/exdump_monitor.sh b/scripts/exdump_monitor.sh index dd71952..b2fe03a 100755 --- a/scripts/exdump_monitor.sh +++ b/scripts/exdump_monitor.sh @@ -127,15 +127,18 @@ TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES - echo "WARNING: TANK b005/xx003 exceeds TANK_MAX_255003 => not dumped" | mail iliana.genkova@noaa.gov + msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" + echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES - echo "WARNING: TANK b005/xx004 exceeds TANK_MAX_255004 => not dumped" | mail iliana.genkova@noaa.gov -fi + msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" + echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + fi if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES - echo "WARNING: TANK b005/xx030 exceeds TANK_MAX_255030 => not dumped" | mail iliana.genkova@noaa.gov + msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" + echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi #end of block diff --git a/scripts/exrap_dump.sh b/scripts/exrap_dump.sh index 6060097..814b745 100755 --- a/scripts/exrap_dump.sh +++ b/scripts/exrap_dump.sh @@ -267,15 +267,18 @@ TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES - echo "WARNING: TANK b005/xx003 exceeds TANK_MAX_255003 => not dumped" | mail iliana.genkova@noaa.gov + msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" + echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES - echo "WARNING: TANK b005/xx004 exceeds TANK_MAX_255004 => not dumped" | mail iliana.genkova@noaa.gov -fi + msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" + echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + fi if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES - echo "WARNING: TANK b005/xx030 exceeds TANK_MAX_255030 => not dumped" | mail iliana.genkova@noaa.gov + msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" + echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi #end of block diff --git a/scripts/exrtma_dump.sh b/scripts/exrtma_dump.sh index 3b2b0be..0d7c932 100755 --- a/scripts/exrtma_dump.sh +++ b/scripts/exrtma_dump.sh @@ -94,15 +94,18 @@ TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES - echo "WARNING: TANK b005/xx003 exceeds TANK_MAX_255003 => not dumped" | mail iliana.genkova@noaa.gov + msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" + echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES - echo "WARNING: TANK b005/xx004 exceeds TANK_MAX_255004 => not dumped" | mail iliana.genkova@noaa.gov -fi + msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" + echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + fi if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES - echo "WARNING: TANK b005/xx030 exceeds TANK_MAX_255030 => not dumped" | mail iliana.genkova@noaa.gov + msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" + echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi #end of block diff --git a/scripts/exurma_dump.sh b/scripts/exurma_dump.sh index 312a58f..88d13a5 100755 --- a/scripts/exurma_dump.sh +++ b/scripts/exurma_dump.sh @@ -74,15 +74,18 @@ TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES - echo "WARNING: TANK b005/xx003 exceeds TANK_MAX_255003 => not dumped" | mail iliana.genkova@noaa.gov + msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" + echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES - echo "WARNING: TANK b005/xx004 exceeds TANK_MAX_255004 => not dumped" | mail iliana.genkova@noaa.gov -fi + msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" + echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + fi if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES - echo "WARNING: TANK b005/xx030 exceeds TANK_MAX_255030 => not dumped" | mail iliana.genkova@noaa.gov + msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" + echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi #end of block From 1ae9b45503108e610637da6a6f830651312f1b11 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Wed, 20 Mar 2024 03:21:40 +0000 Subject: [PATCH 49/53] improved logic of TANK_MAX_255* checks --- scripts/excdas_dump.sh | 6 +++--- scripts/exdump_monitor.sh | 6 +++--- scripts/exrap_dump.sh | 6 +++--- scripts/exrtma_dump.sh | 6 +++--- scripts/exurma_dump.sh | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/scripts/excdas_dump.sh b/scripts/excdas_dump.sh index 4387208..33338ce 100755 --- a/scripts/excdas_dump.sh +++ b/scripts/excdas_dump.sh @@ -327,17 +327,17 @@ err10=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov diff --git a/scripts/exdump_monitor.sh b/scripts/exdump_monitor.sh index b2fe03a..cdbc302 100755 --- a/scripts/exdump_monitor.sh +++ b/scripts/exdump_monitor.sh @@ -125,17 +125,17 @@ err10=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov diff --git a/scripts/exrap_dump.sh b/scripts/exrap_dump.sh index 814b745..eddd96d 100755 --- a/scripts/exrap_dump.sh +++ b/scripts/exrap_dump.sh @@ -265,17 +265,17 @@ err12=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov diff --git a/scripts/exrtma_dump.sh b/scripts/exrtma_dump.sh index 0d7c932..38f098c 100755 --- a/scripts/exrtma_dump.sh +++ b/scripts/exrtma_dump.sh @@ -92,17 +92,17 @@ err5=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov diff --git a/scripts/exurma_dump.sh b/scripts/exurma_dump.sh index 88d13a5..e1f389e 100755 --- a/scripts/exurma_dump.sh +++ b/scripts/exurma_dump.sh @@ -72,17 +72,17 @@ err6=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov From e05d4a7b8295d80f449d9c6ec82c90bae35ea1b9 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Thu, 4 Apr 2024 20:04:14 +0000 Subject: [PATCH 50/53] Add PPT instrcutions to RN --- docs/Release_Notes.txt | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/docs/Release_Notes.txt b/docs/Release_Notes.txt index 3cb00f0..fe694a4 100644 --- a/docs/Release_Notes.txt +++ b/docs/Release_Notes.txt @@ -66,7 +66,10 @@ SCRIPTS CHANGES scripts/excdas_dump.sh Instruct NOT to dump uprair file for CDAS network (too slow) - Introduce msone1 group and redistribute group + Introduce msone1 group and redistribute group;add MAX_TANK* tank size check; Turn off msonet and msone1 for the time being. + + scripts/exdump_monitor.sh + Add MAX_TANK* tank size check scripts/exdump_post.sh Add functionality to unrestrict aircraft data (PROCESS_REMOREST_dm2) @@ -80,19 +83,19 @@ SCRIPTS CHANGES scripts/exrap_dump.sh Dump BUFR formated high-res rawinsonde profiles into *uprair* dump files Add DBNet dbn_alert for uprair dump files going to NOMADS - Introduce msone1 group and redistribute groups + Introduce msone1 group and redistribute groups; add MAX_TANK* tank size check Redistribute groups, introduce NAP to allow earlier job/cron kickoff scripts/exnam_dump.sh - Introduce msone1 group and redistribute groups/add cpus + Introduce msone1 group and redistribute groups/add cpus; add MAX_TANK* tank size check scripts/exrtma_dump.sh Introduce msone1 group and redistribute groups - Introduce max size treshold for mesonet tanks + Introduce max size treshold for mesonet tanks;add MAX_TANK* tank size check scripts/exurma_dump.sh Introduce msone1 group and redistribute groups - Introduce max size treshold for mesonet tanks + Introduce max size treshold for mesonet tanks;add MAX_TANK* tank size check SORC CHANGES @@ -201,6 +204,19 @@ PRE-IMPLEMENTATION TESTING REQUIREMENTS Run jobs/JOBSPROC_networkname_DUMP and jobs/JOBSPROC_networkname_DUMP_POST for the following networkname values: GLOBAL, NAM, RAP, URMA, RTMA, CDAS + Parallel Production Testing (PPT) + After testing obsproc v1.2 in the /para space, we recommend doing for at least a month, + it is best to do a single cycle Parallel Production Testing (PPT) (i.e. obsproc + GFS + other models). + Copy the following directoris: + $model.YYYYMMDD, where model: gfs,gdas,nam,rap,rap_e,rap_p,urma,cdas,rtma,rtma_ru,dump + $model.YYYYMM, where model: cdas + $model.YYYY, where model:mods + $model, where:gdas, sdm_rtdm + from the para run: + /para/com/obsproc/v1.2/ + to + /prod/com/obsproc/v1.2/ for the PPT. + Good to know: - the RAP uprair dumps might be 0 size for the hours not around 00,06,12,18 UTC - the CDAS prepbufr files are 6 different flavours, it takes about 26 hours for all to be generated From ab1271c2496dad057168d8f66ddafddd47249b20 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Wed, 10 Apr 2024 02:31:00 +0000 Subject: [PATCH 51/53] Removed a bracket in TANK_MAX_ check --- scripts/excdas_dump.sh | 6 +++--- scripts/exdump_monitor.sh | 6 +++--- scripts/exrap_dump.sh | 6 +++--- scripts/exrtma_dump.sh | 6 +++--- scripts/exurma_dump.sh | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/scripts/excdas_dump.sh b/scripts/excdas_dump.sh index 33338ce..79872e9 100755 --- a/scripts/excdas_dump.sh +++ b/scripts/excdas_dump.sh @@ -327,17 +327,17 @@ err10=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ -s ${TANK}/${PDY}/b255/xx003) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx004) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx030) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov diff --git a/scripts/exdump_monitor.sh b/scripts/exdump_monitor.sh index cdbc302..34940c0 100755 --- a/scripts/exdump_monitor.sh +++ b/scripts/exdump_monitor.sh @@ -125,17 +125,17 @@ err10=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ -s ${TANK}/${PDY}/b255/xx003) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx004) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx030) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov diff --git a/scripts/exrap_dump.sh b/scripts/exrap_dump.sh index eddd96d..7ccaf73 100755 --- a/scripts/exrap_dump.sh +++ b/scripts/exrap_dump.sh @@ -265,17 +265,17 @@ err12=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ -s ${TANK}/${PDY}/b255/xx003) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx004) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx030) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov diff --git a/scripts/exrtma_dump.sh b/scripts/exrtma_dump.sh index 38f098c..5dd85cc 100755 --- a/scripts/exrtma_dump.sh +++ b/scripts/exrtma_dump.sh @@ -92,17 +92,17 @@ err5=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ -s ${TANK}/${PDY}/b255/xx003) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx004) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx030) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov diff --git a/scripts/exurma_dump.sh b/scripts/exurma_dump.sh index e1f389e..e8e23b7 100755 --- a/scripts/exurma_dump.sh +++ b/scripts/exurma_dump.sh @@ -72,17 +72,17 @@ err6=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ -s ${TANK}/${PDY}/b255/xx003) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx004) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx030) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov From 7c27321c6ef8861b3fb41510b1ffd98fbd57cd50 Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Fri, 31 May 2024 22:11:39 +0000 Subject: [PATCH 52/53] Remove an extra closing bracket that was breaking the scripts --- docs/Release_Notes.txt | 13 ++++++++----- scripts/excdas_dump.sh | 6 +++--- scripts/exrap_dump.sh | 6 +++--- scripts/exrtma_dump.sh | 6 +++--- scripts/exurma_dump.sh | 6 +++--- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/docs/Release_Notes.txt b/docs/Release_Notes.txt index 3cb00f0..2936ef4 100644 --- a/docs/Release_Notes.txt +++ b/docs/Release_Notes.txt @@ -66,7 +66,10 @@ SCRIPTS CHANGES scripts/excdas_dump.sh Instruct NOT to dump uprair file for CDAS network (too slow) - Introduce msone1 group and redistribute group + Introduce msone1 group and redistribute group;add MAX_TANK* tank size check; Turn off msonet and msone1 for the time being. + + scripts/exdump_monitor.sh + Add MAX_TANK* tank size check scripts/exdump_post.sh Add functionality to unrestrict aircraft data (PROCESS_REMOREST_dm2) @@ -80,19 +83,19 @@ SCRIPTS CHANGES scripts/exrap_dump.sh Dump BUFR formated high-res rawinsonde profiles into *uprair* dump files Add DBNet dbn_alert for uprair dump files going to NOMADS - Introduce msone1 group and redistribute groups + Introduce msone1 group and redistribute groups; add MAX_TANK* tank size check Redistribute groups, introduce NAP to allow earlier job/cron kickoff scripts/exnam_dump.sh - Introduce msone1 group and redistribute groups/add cpus + Introduce msone1 group and redistribute groups/add cpus; add MAX_TANK* tank size check scripts/exrtma_dump.sh Introduce msone1 group and redistribute groups - Introduce max size treshold for mesonet tanks + Introduce max size treshold for mesonet tanks;add MAX_TANK* tank size check scripts/exurma_dump.sh Introduce msone1 group and redistribute groups - Introduce max size treshold for mesonet tanks + Introduce max size treshold for mesonet tanks;add MAX_TANK* tank size check SORC CHANGES diff --git a/scripts/excdas_dump.sh b/scripts/excdas_dump.sh index 33338ce..79872e9 100755 --- a/scripts/excdas_dump.sh +++ b/scripts/excdas_dump.sh @@ -327,17 +327,17 @@ err10=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ -s ${TANK}/${PDY}/b255/xx003) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx004) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx030) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov diff --git a/scripts/exrap_dump.sh b/scripts/exrap_dump.sh index eddd96d..7ccaf73 100755 --- a/scripts/exrap_dump.sh +++ b/scripts/exrap_dump.sh @@ -265,17 +265,17 @@ err12=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ -s ${TANK}/${PDY}/b255/xx003) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx004) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx030) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov diff --git a/scripts/exrtma_dump.sh b/scripts/exrtma_dump.sh index 38f098c..5dd85cc 100755 --- a/scripts/exrtma_dump.sh +++ b/scripts/exrtma_dump.sh @@ -92,17 +92,17 @@ err5=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ -s ${TANK}/${PDY}/b255/xx003) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx004) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx030) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov diff --git a/scripts/exurma_dump.sh b/scripts/exurma_dump.sh index e1f389e..e8e23b7 100755 --- a/scripts/exurma_dump.sh +++ b/scripts/exurma_dump.sh @@ -72,17 +72,17 @@ err6=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ -s ${TANK}/${PDY}/b255/xx003) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx004) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx030) && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov From f1e6c7143fed38de2449db1dd9ec3c70a9da6a5d Mon Sep 17 00:00:00 2001 From: Iliana Genkova Date: Fri, 31 May 2024 22:20:40 +0000 Subject: [PATCH 53/53] Bring last minute NCO changes:bad bracket,do "grep --text" --- scripts/excdas_dump.sh | 15 +++++++++------ scripts/exdump_monitor.sh | 15 +++++++++------ scripts/exrap_dump.sh | 15 +++++++++------ scripts/exrtma_dump.sh | 15 +++++++++------ scripts/exurma_dump.sh | 15 +++++++++------ ush/bufr_avgdata.sh | 4 ++-- ush/bufr_datacount.sh | 8 ++++---- 7 files changed, 51 insertions(+), 36 deletions(-) diff --git a/scripts/excdas_dump.sh b/scripts/excdas_dump.sh index 79872e9..72173e6 100755 --- a/scripts/excdas_dump.sh +++ b/scripts/excdas_dump.sh @@ -327,20 +327,23 @@ err10=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ -s ${TANK}/${PDY}/b255/xx003 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" - echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx004 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" - echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx030 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" - echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov fi #end of block diff --git a/scripts/exdump_monitor.sh b/scripts/exdump_monitor.sh index 34940c0..d18df26 100755 --- a/scripts/exdump_monitor.sh +++ b/scripts/exdump_monitor.sh @@ -125,20 +125,23 @@ err10=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ -s ${TANK}/${PDY}/b255/xx003 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" - echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx004 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" - echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx030 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" - echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov fi #end of block diff --git a/scripts/exrap_dump.sh b/scripts/exrap_dump.sh index 7ccaf73..9af99cb 100755 --- a/scripts/exrap_dump.sh +++ b/scripts/exrap_dump.sh @@ -265,20 +265,23 @@ err12=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ -s ${TANK}/${PDY}/b255/xx003 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" - echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx004 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" - echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + echo $msg | mail.py -s "$msg" +# echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx030 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" - echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov fi #end of block diff --git a/scripts/exrtma_dump.sh b/scripts/exrtma_dump.sh index 5dd85cc..fdbf1f3 100755 --- a/scripts/exrtma_dump.sh +++ b/scripts/exrtma_dump.sh @@ -92,20 +92,23 @@ err5=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ -s ${TANK}/${PDY}/b255/xx003 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" - echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx004 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" - echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx030 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" - echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov fi #end of block diff --git a/scripts/exurma_dump.sh b/scripts/exurma_dump.sh index e8e23b7..9a5f00a 100755 --- a/scripts/exurma_dump.sh +++ b/scripts/exurma_dump.sh @@ -72,20 +72,23 @@ err6=0 TANK_MAX_255003=${TANK_MAX_255003:-3221225472} #3Gb TANK_MAX_255004=${TANK_MAX_255004:-1610612736} #1.5Gb TANK_MAX_255030=${TANK_MAX_255030:-4187593114} #3.9Gb -if [ -s ${TANK}/${PDY}/b255/xx003 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then +if [ -s ${TANK}/${PDY}/b255/xx003 ]&& [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx003)" -gt "$TANK_MAX_255003" ]; then export SKIP_255003=YES msg="WARNING: TANK b255/xx003 exceeds TANK_MAX_255003 => not dumped" - echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx004 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then +if [ -s ${TANK}/${PDY}/b255/xx004 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx004)" -gt "$TANK_MAX_255004" ]; then export SKIP_255004=YES msg="WARNING: TANK b255/xx004 exceeds TANK_MAX_255004 => not dumped" - echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov fi -if [ -s ${TANK}/${PDY}/b255/xx030 && "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then +if [ -s ${TANK}/${PDY}/b255/xx030 ] && [ "$(stat -c '%s' ${TANK}/${PDY}/b255/xx030)" -gt "$TANK_MAX_255030" ]; then export SKIP_255030=YES msg="WARNING: TANK b255/xx030 exceeds TANK_MAX_255030 => not dumped" - echo $msg | mail.py -s $msg -c iliana.genkova@noaa.gov + echo $msg | mail.py -s "$msg" +#echo $msg | mail.py -s "$msg" -c iliana.genkova@noaa.gov fi #end of block diff --git a/ush/bufr_avgdata.sh b/ush/bufr_avgdata.sh index 339fc12..f827a7e 100755 --- a/ush/bufr_avgdata.sh +++ b/ush/bufr_avgdata.sh @@ -338,7 +338,7 @@ for cycle_avg in $cycles; do > accum.obs.counts.${cycle_avg} for dtg in "${arr30d[@]}"; do set +x - grep -h -e " REPORTS" -e "^EOF @@@" \ + grep --text -h -e " REPORTS" -e "^EOF @@@" \ $OBCNTarch/${net}.${dtg}/*${cycle_avg}.status.*bufr_d | \ grep -e " in data group " -e "^EOF @@@" | grep -e " HAS" -e "^EOF @@@" | \ grep -v -e "Domain" >> delete @@ -354,7 +354,7 @@ for cycle_avg in $cycles; do done done -grep -e "^_" -e "^:" $DUMPLIST | grep -Fe "#>" > dumplist +grep --text -e "^_" -e "^:" $DUMPLIST | grep -Fe "#>" > dumplist # Generate a new average file diff --git a/ush/bufr_datacount.sh b/ush/bufr_datacount.sh index 4902aa0..012f9e2 100755 --- a/ush/bufr_datacount.sh +++ b/ush/bufr_datacount.sh @@ -105,7 +105,7 @@ echo $RETC > $COMOUT/${RUN}.${cycle}.dump_alert_flag.$tmmark # and 12-month ago 30-day avg files # ------------------------------------------------------------------------ -grep -e "^_" -e "^:" $LIST_CNT | grep -Fe "#>" > dumplist +grep --text -e "^_" -e "^:" $LIST_CNT | grep -Fe "#>" > dumplist if [ -s $COMIN/${RUN}.${cycle}.status.tm00.bufr_d ]; then status_bufr_d=$COMIN/${RUN}.${cycle}.status.tm00.bufr_d @@ -114,7 +114,7 @@ elif [ -s $COMOUT/${RUN}.${cycle}.status.tm00.bufr_d ]; then else status_bufr_d=/dev/null fi -grep -Fe " REPORTS" $status_bufr_d | grep -Fe " in data group " | \ +grep --text -Fe " REPORTS" $status_bufr_d | grep -Fe " in data group " | \ grep -Fe " HAS" | grep -v -Fe "Domain" > delete awk -F" HAS" '{print$2}' delete | awk -F" REPORTS" '{print$1}' | \ sed "s/[^0-9]/ /g" > delete.r @@ -126,7 +126,7 @@ rm delete delete.l delete.r > obcount_30davg.${NET}.current if [ -s $AVGDarch_IN/obcount_30davg.${NET}.current ]; then - grep "^# " $AVGDarch_IN/obcount_30davg.${NET}.current | \ + grep --text "^# " $AVGDarch_IN/obcount_30davg.${NET}.current | \ sed "s/# ......./ /g" | sed "s/[^0-9]/ /g" \ > obcount_30davg.${NET}.current fi @@ -146,7 +146,7 @@ for months_ago in 3 6 9 12; do > obcount_30davg.${NET}.${months_ago}months_ago if [ -s $AVGDarch_IN/obcount_30davg.${NET}.${arch_year}${arch_month} ]; then - grep "^# " $AVGDarch_IN/obcount_30davg.${NET}.${arch_year}${arch_month} | \ + grep --text "^# " $AVGDarch_IN/obcount_30davg.${NET}.${arch_year}${arch_month} | \ sed "s/# ......./ /g" | sed "s/[^0-9]/ /g" \ > obcount_30davg.${NET}.${months_ago}months_ago fi