From 3182e1785b674e0f5018d20aeb01b254b7a79fc4 Mon Sep 17 00:00:00 2001 From: "Jeffrey.S.Whitaker" Date: Tue, 4 Feb 2025 15:33:38 -0500 Subject: [PATCH 01/90] introduce new env var USE_BUILD_GSINFO --- parm/config/gfs/config.anal | 7 + parm/config/gfs/config.base | 1 + scripts/exglobal_atmos_analysis.sh | 366 +++++++++++++++++++++++------ scripts/exglobal_diag.sh | 5 + 4 files changed, 303 insertions(+), 76 deletions(-) diff --git a/parm/config/gfs/config.anal b/parm/config/gfs/config.anal index 123bd6decdc..55aeba8e625 100644 --- a/parm/config/gfs/config.anal +++ b/parm/config/gfs/config.anal @@ -148,6 +148,13 @@ if [[ ${RUN_ENVIR} == "emc" ]]; then # fi fi +if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then + # these will be set based on date in scripts/exglobal_atmos_analysis.sh + export SATINFO="generate" + export CONVINFO="generate" + export OZINFO="generate" +fi + # Flag to turn on (.true.) or off (.false.) the infrared cloud and aerosol detection software # for AIRS, CrIS, and IASI. Default is .false. export AIRS_CADS=".false." diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index e28e7ef2268..c5a259c8c05 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -379,6 +379,7 @@ export SMOOTH_ENKF="NO" export l4densvar=".true." export lwrite4danl=".true." export DO_CALC_INCREMENT="NO" +export USE_BUILD_GSINFO="NO" # Early-cycle EnKF parameters export NMEM_ENS_GFS=30 diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index bd11b81c392..b38922d205f 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -106,6 +106,7 @@ B1MSU=${B1MSU:-${COMIN_OBS}/${OPREFIX}1bmsu.tm00.bufr_d${OSUFFIX}} #B1HRS4=${B1HRS4:-${COMIN_OBS}/${OPREFIX}1bhrs4.tm00.bufr_d${OSUFFIX}} # HIRS temporarily disabled due to CRTM versioning issues B1AMUA=${B1AMUA:-${COMIN_OBS}/${OPREFIX}1bamua.tm00.bufr_d${OSUFFIX}} B1AMUB=${B1AMUB:-${COMIN_OBS}/${OPREFIX}1bamub.tm00.bufr_d${OSUFFIX}} +AQUAAMUA=${AQUAAMUA:-${COMIN_OBS}/${OPREFIX}aquaamua.tm00.bufr_d${OSUFFIX}} B1MHS=${B1MHS:-${COMIN_OBS}/${OPREFIX}1bmhs.tm00.bufr_d${OSUFFIX}} ESHRS3=${ESHRS3:-${COMIN_OBS}/${OPREFIX}eshrs3.tm00.bufr_d${OSUFFIX}} ESAMUA=${ESAMUA:-${COMIN_OBS}/${OPREFIX}esamua.tm00.bufr_d${OSUFFIX}} @@ -152,6 +153,11 @@ TCVITL=${TCVITL:-${COMIN_OBS}/${OPREFIX}syndata.tcvitals.tm00} B1AVHAM=${B1AVHAM:-${COMIN_OBS}/${OPREFIX}avcsam.tm00.bufr_d${OSUFFIX}} B1AVHPM=${B1AVHPM:-${COMIN_OBS}/${OPREFIX}avcspm.tm00.bufr_d${OSUFFIX}} HDOB=${HDOB:-${COMIN_OBS}/${OPREFIX}hdob.tm00.bufr_d${OSUFFIX}} +OMIEFFNC=${OMIEFFNC:-${COMIN_OBS/OMIeff-adj.${PDY}_${cyc}z.nc} +OMPSNMEFFNC=${OMPSNMEFFNC:-${COMIN_OBS/OMPSNM.${PDY}_${cyc}z.nc} +OMPSNPNC=${OMPSNPNC:-${COMIN_OBS/OMPSNP.${PDY}_${cyc}z.nc} +OMPSLPNC=${OMPSLPNC:-${COMIN_OBS/OMPS-LPoz-Vis.${PDY}_${cyc}z.nc} +MLS55NC=${MLS55NC:-${COMIN_OBS/MLS-v5.0-oz.${PDY}_${cyc}z.nc} # Guess files GPREFIX=${GPREFIX:-""} @@ -363,14 +369,26 @@ cd ${DATA} || exit 99 # Fixed files ${NLN} ${BERROR} berror_stats ${NLN} ${SATANGL} satbias_angle -${NLN} ${SATINFO} satinfo +if [ $SATINFO == "generate" ]; then + sh ${SCRgfs}/create_satinfo.sh ${PDY}${cyc} > satinfo +else + ${NLN} ${SATINFO} satinfo +fi ${NLN} ${RADCLOUDINFO} cloudy_radiance_info.txt ${NLN} ${ATMSFILTER} atms_beamwidth.txt ${NLN} ${ANAVINFO} anavinfo -${NLN} ${CONVINFO} convinfo +if [ $CONVINFO == "generate" ]; then + sh ${SCRgfs}/create_convinfo.sh ${PDY}${cyc} > convinfo +else + ${NLN} ${CONVINFO} convinfo +fi ${NLN} ${vqcdat} vqctp001.dat ${NLN} ${INSITUINFO} insituinfo -${NLN} ${OZINFO} ozinfo +if [ $OZINFO == "generate" ]; then + sh ${SCRgfs}/build_gsinfo/create_ozinfo.sh ${PDY}${cyc} > ozinfo +else + ${NLN} ${OZINFO} ozinfo +fi ${NLN} ${PCPINFO} pcpinfo ${NLN} ${AEROINFO} aeroinfo ${NLN} ${SCANINFO} scaninfo @@ -456,6 +474,9 @@ ${NLN} ${B1MSU} msubufr ${NLN} ${B1AMUA} amsuabufr ${NLN} ${B1AMUB} amsubbufr ${NLN} ${B1MHS} mhsbufr +${NLN} ${B1HRS2} hirs2bufr +${NLN} ${B1HRS3} hirs3bufr +${NLN} ${B1HRS4} hirs4bufr ${NLN} ${ESAMUA} amsuabufrears ${NLN} ${ESAMUB} amsubbufrears #$NLN $ESMHS mhsbufrears @@ -500,6 +521,17 @@ ${NLN} ${ABIBF} abibufr ${NLN} ${HDOB} hdobbufr ${NLN} ${SSTVIIRS} sstviirs +if [ ${USE_BUILD_GSINFO} == "YES" ]; then + # NASA ozone (netcdf) from NNJA + ${NLN} ${OMIEFFNC} omieffnc + ${NLN} ${OMPSNMEFFNC} ompsnmeffnc + ${NLN} ${OMPSNPNC} ompsnpnc + ${NLN} ${OMPSLPNC} ompslpnc + ${NLN} ${MLS55NC} mls55nc + # NASA airs aqua amsua (bufr) from NNJA + ${NLN} ${AQUAAMUA} aquabufr +fi + [[ ${DONST} = "YES" ]] && ${NLN} ${NSSTBF} nsstbufr ############################################################## @@ -700,79 +732,183 @@ if [ ${DONST} = "YES" ]; then NST="nstinfo=${NSTINFO},fac_dtl=${FAC_DTL},fac_tsl=${FAC_TSL},zsea1=${ZSEA1},zsea2=${ZSEA2},${NST}" fi -############################################################## -# Create global_gsi namelist -cat > gsiparm.anl << EOF -&SETUP - miter=2, - niter(1)=100,niter(2)=100, - niter_no_qc(1)=50,niter_no_qc(2)=0, - write_diag(1)=.true.,write_diag(2)=.false.,write_diag(3)=.true., - qoption=2, - gencode=${IGEN:-0},deltim=${DELTIM}, - factqmin=0.5,factqmax=0.0002, - iguess=-1, - tzr_qc=${TZR_QC}, - oneobtest=.false.,retrieval=.false.,l_foto=.false., - use_pbl=.false.,use_compress=.true.,nsig_ext=45,gpstop=50.,commgpstop=45.,commgpserrinf=1.0, - use_gfs_nemsio=.false.,use_gfs_ncio=.true.,sfcnst_comb=.true., - use_readin_anl_sfcmask=${USE_READIN_ANL_SFCMASK}, - lrun_subdirs=${lrun_subdirs}, - crtm_coeffs_path='./crtm_coeffs/', - newpc4pred=.true.,adp_anglebc=.true.,angord=4,passive_bc=.true.,use_edges=.false., - diag_precon=.true.,step_start=1.e-3,emiss_bc=.true.,nhr_obsbin=${nhr_obsbin:-3}, - cwoption=3,imp_physics=${imp_physics},lupp=${lupp},cnvw_option=${cnvw_option},cao_check=${cao_check}, - netcdf_diag=${netcdf_diag},binary_diag=${binary_diag}, - lobsdiag_forenkf=${lobsdiag_forenkf}, - write_fv3_incr=${write_fv3_increment}, - nhr_anal=${IAUFHRS}, - ta2tb=${ta2tb}, - ${WRITE_INCR_ZERO} - ${WRITE_ZERO_STRAT} - ${WRITE_STRAT_EFOLD} - ${SETUP} -/ -&GRIDOPTS - JCAP_B=${JCAP},JCAP=${JCAP_A},NLAT=${NLAT_A},NLON=${NLON_A},nsig=${LEVS}, - regional=.false.,nlayers(63)=3,nlayers(64)=6, - ${GRIDOPTS} -/ -&BKGERR - vs=0.7, - hzscl=1.7,0.8,0.5, - hswgt=0.45,0.3,0.25, - bw=0.0,norsp=4, - bkgv_flowdep=.true.,bkgv_rewgtfct=1.5, - bkgv_write=.false., - cwcoveqqcov=.false., - ${BKGVERR} -/ -&ANBKGERR - anisotropic=.false., - ${ANBKGERR} -/ -&JCOPTS - ljcdfi=.false.,alphajc=0.0,ljcpdry=.true.,bamp_jcpdry=5.0e7, - ${JCOPTS} -/ -&STRONGOPTS - tlnmc_option=2,nstrong=1,nvmodes_keep=8,period_max=6.,period_width=1.5, - ${STRONGOPTS} -/ -&OBSQC - dfact=0.75,dfact1=3.0,noiqc=.true.,oberrflg=.false.,c_varqc=0.02, - use_poq7=.true.,qc_noirjaco3_pole=.true.,vqc=.false.,nvqc=.true., - aircraft_t_bc=.true.,biaspredt=1.0e5,upd_aircraft=.true.,cleanup_tail=.true., - tcp_width=70.0,tcp_ermax=7.35,airs_cads=${AIRS_CADS},cris_cads=${CRIS_CADS}, - iasi_cads=${IASI_CADS}, - ${OBSQC} -/ -&OBS_INPUT - dmesh(1)=145.0,dmesh(2)=150.0,dmesh(3)=100.0,dmesh(4)=50.0,time_window_max=3.0, - hofx_2m_sfcfile=${hofx_2m_sfcfile}, - ${OBSINPUT} -/ -OBS_INPUT:: +if [ ${USE_BUILD_GSINFO} == "YES" ]; then +OBS_INPUT_TABLE=$(cat << EOF +! dfile dtype dplat dsis dval dthin dsfcalc + prepbufr ps null ps 0.0 0 0 + prepbufr t null t 0.0 0 0 + prepbufr q null q 0.0 0 0 + prepbufr pw null pw 0.0 0 0 + prepbufr_profl t null t 0.0 0 0 + prepbufr_profl q null q 0.0 0 0 + prepbufr_profl uv null uv 0.0 0 0 + satwndbufr uv null uv 0.0 0 0 + prepbufr uv null uv 0.0 0 0 + prepbufr spd null spd 0.0 0 0 + prepbufr dw null dw 0.0 0 0 + radarbufr rw null rw 0.0 0 0 + nsstbufr sst nsst sst 0.0 0 0 + gpsrobufr gps_bnd null gps 0.0 0 0 + ssmirrbufr pcp_ssmi dmsp pcp_ssmi 0.0 -1 0 + tmirrbufr pcp_tmi trmm pcp_tmi 0.0 -1 0 + sbuvbufr sbuv2 nim07 sbuv8_nim07 0.0 0 0 + sbuvbufr sbuv2 n09 sbuv8_n09 0.0 0 0 + sbuvbufr sbuv2 n11 sbuv8_n11 0.0 0 0 + sbuvbufr sbuv2 n14 sbuv8_n14 0.0 0 0 + sbuvbufr sbuv2 n16 sbuv8_n16 0.0 0 0 + sbuvbufr sbuv2 n17 sbuv8_n17 0.0 0 0 + sbuvbufr sbuv2 n18 sbuv8_n18 0.0 0 0 + sbuvbufr sbuv2 n19 sbuv8_n19 0.0 0 0 + hirs2bufr hirs2 tirosn hirs2_tirosn 0.0 1 1 + hirs2bufr hirs2 n06 hirs2_n06 0.0 1 1 + hirs2bufr hirs2 n07 hirs2_n07 0.0 1 1 + hirs2bufr hirs2 n08 hirs2_n08 0.0 1 1 + hirs2bufr hirs2 n09 hirs2_n09 0.0 1 1 + hirs2bufr hirs2 n10 hirs2_n10 0.0 1 1 + hirs2bufr hirs2 n11 hirs2_n11 0.0 1 1 + hirs2bufr hirs2 n12 hirs2_n12 0.0 1 1 + hirs2bufr hirs2 n14 hirs2_n14 0.0 1 1 + hirs3bufr hirs3 n15 hirs3_n15 0.0 1 1 + hirs3bufr hirs3 n16 hirs3_n16 0.0 1 1 + hirs3bufr hirs3 n17 hirs3_n17 0.0 1 1 + gimgrbufr goes_img g11 imgr_g11 0.0 1 0 + gimgrbufr goes_img g12 imgr_g12 0.0 1 0 + airsbufr airs aqua airs_aqua 0.0 1 1 + msubufr msu tirosn msu_tirosn 0.0 1 1 + msubufr msu n06 msu_n06 0.0 1 1 + msubufr msu n07 msu_n07 0.0 1 1 + msubufr msu n08 msu_n08 0.0 1 1 + msubufr msu n09 msu_n09 0.0 1 1 + msubufr msu n10 msu_n10 0.0 1 1 + msubufr msu n11 msu_n11 0.0 1 1 + msubufr msu n12 msu_n12 0.0 1 1 + msubufr msu n14 msu_n14 0.0 1 1 + ssubufr ssu tirosn ssu_tirosn 0.0 1 1 + ssubufr ssu n06 ssu_n06 0.0 1 1 + ssubufr ssu n07 ssu_n07 0.0 1 1 + ssubufr ssu n08 ssu_n08 0.0 1 1 + ssubufr ssu n09 ssu_n09 0.0 1 1 + ssubufr ssu n11 ssu_n11 0.0 1 1 + ssubufr ssu n14 ssu_n14 0.0 1 1 + amsuabufr amsua n15 amsua_n15 0.0 1 1 + amsuabufr amsua n16 amsua_n16 0.0 1 1 + amsuabufr amsua n17 amsua_n17 0.0 1 1 + amsuabufr amsua n18 amsua_n18 0.0 1 1 + aquabufr amsua aqua amsua_aqua 0.0 1 1 + amsubbufr amsub n15 amsub_n15 0.0 1 1 + amsubbufr amsub n16 amsub_n16 0.0 1 1 + amsubbufr amsub n17 amsub_n17 0.0 1 1 + mhsbufr mhs n18 mhs_n18 0.0 1 1 + ssmitbufr ssmi f08 ssmi_f08 0.0 1 0 + ssmitbufr ssmi f10 ssmi_f10 0.0 1 0 + ssmitbufr ssmi f11 ssmi_f11 0.0 1 0 + ssmitbufr ssmi f13 ssmi_f13 0.0 1 0 + ssmitbufr ssmi f14 ssmi_f14 0.0 1 0 + ssmitbufr ssmi f15 ssmi_f15 0.0 1 0 + amsrebufr amsre_low aqua amsre_aqua 0.0 1 0 + amsrebufr amsre_mid aqua amsre_aqua 0.0 1 0 + amsrebufr amsre_hig aqua amsre_aqua 0.0 1 0 + ssmisbufr ssmis f16 ssmis_f16 0.0 1 0 + ssmisbufr ssmis f17 ssmis_f17 0.0 1 0 + ssmisbufr ssmis f18 ssmis_f18 0.0 1 0 + ssmisbufr ssmis f19 ssmis_f19 0.0 1 0 + gsnd1bufr sndr g08 sndr_g08 0.0 1 0 + gsnd1bufr sndr g09 sndr_g09 0.0 1 0 + gsnd1bufr sndr g10 sndr_g10 0.0 1 0 + gsnd1bufr sndr g11 sndr_g11 0.0 1 0 + gsnd1bufr sndr g12 sndr_g12 0.0 1 0 + gsnd1bufr sndrd1 g12 sndrD1_g12 0.0 1 0 + gsnd1bufr sndrd2 g12 sndrD2_g12 0.0 1 0 + gsnd1bufr sndrd3 g12 sndrD3_g12 0.0 1 0 + gsnd1bufr sndrd4 g12 sndrD4_g12 0.0 1 0 + gsnd1bufr sndrd1 g11 sndrD1_g11 0.0 1 0 + gsnd1bufr sndrd2 g11 sndrD2_g11 0.0 1 0 + gsnd1bufr sndrd3 g11 sndrD3_g11 0.0 1 0 + gsnd1bufr sndrd4 g11 sndrD4_g11 0.0 1 0 + gsnd1bufr sndrd1 g13 sndrD1_g13 0.0 1 0 + gsnd1bufr sndrd2 g13 sndrD2_g13 0.0 1 0 + gsnd1bufr sndrd3 g13 sndrD3_g13 0.0 1 0 + gsnd1bufr sndrd4 g13 sndrD4_g13 0.0 1 0 + gsnd1bufr sndrd1 g14 sndrD1_g14 0.0 1 0 + gsnd1bufr sndrd2 g14 sndrD2_g14 0.0 1 0 + gsnd1bufr sndrd3 g14 sndrD3_g14 0.0 1 0 + gsnd1bufr sndrd4 g14 sndrD4_g14 0.0 1 0 + gsnd1bufr sndrd1 g15 sndrD1_g15 0.0 1 0 + gsnd1bufr sndrd2 g15 sndrD2_g15 0.0 1 0 + gsnd1bufr sndrd3 g15 sndrD3_g15 0.0 1 0 + gsnd1bufr sndrd4 g15 sndrD4_g15 0.0 1 0 + hirs4bufr hirs4 n19 hirs4_n19 0.0 1 1 + amsuabufr amsua n19 amsua_n19 0.0 1 1 + mhsbufr mhs n19 mhs_n19 0.0 1 1 + tcvitl tcp null tcp 0.0 0 0 + seviribufr seviri m08 seviri_m08 0.0 1 0 + seviribufr seviri m09 seviri_m09 0.0 1 0 + seviribufr seviri m10 seviri_m10 0.0 1 0 + seviribufr seviri m10 seviri_m11 0.0 1 0 + hirs4bufr hirs4 metop-a hirs4_metop-a 0.0 1 1 + hirs4bufr hirs4 metop-b hirs4_metop-b 0.0 1 0 + amsuabufr amsua metop-a amsua_metop-a 0.0 1 1 + amsuabufr amsua metop-b amsua_metop-b 0.0 1 0 + amsuabufr amsua metop-c amsua_metop-c 0.0 1 0 + mhsbufr mhs metop-a mhs_metop-a 0.0 1 1 + mhsbufr mhs metop-b mhs_metop-b 0.0 1 0 + mhsbufr mhs metop-c mhs_metop-c 0.0 1 0 + iasibufr iasi metop-a iasi_metop-a 0.0 1 1 + iasibufr iasi metop-b iasi_metop-b 0.0 1 0 + iasibufr iasi metop-c iasi_metop-c 0.0 1 0 + gomebufr gome metop-a gome_metop-a 0.0 2 0 + gomebufr gome metop-b gome_metop-b 0.0 2 0 + gomebufr gome metop-c gome_metop-c 0.0 2 0 + atmsbufr atms npp atms_npp 0.0 1 1 + atmsbufr atms n20 atms_n20 0.0 1 1 + atmsbufr atms n20 atms_n21 0.0 1 1 + crisbufr cris npp cris_npp 0.0 1 0 + crisfsbufr cris-fsr npp cris-fsr_npp 0.0 1 0 + crisfsbufr cris-fsr n20 cris-fsr_n20 0.0 1 0 + crisfsbufr cris-fsr n21 cris-fsr_n21 0.0 1 0 + avhambufr avhrr n15 avhrr3_n15 0.0 1 0 + avhambufr avhrr n17 avhrr3_n17 0.0 1 0 + avhambufr avhrr metop-a avhrr3_metop-a 0.0 1 0 + avhambufr avhrr metop-b avhrr3_metop-b 0.0 1 0 + avhambufr avhrr metop-c avhrr3_metop-c 0.0 1 0 + avhpmbufr avhrr n14 avhrr2_n14 0.0 1 0 + avhpmbufr avhrr n16 avhrr3_n16 0.0 1 0 + avhpmbufr avhrr n18 avhrr3_n18 0.0 1 0 + avhpmbufr avhrr n19 avhrr3_n19 0.0 1 0 + oscatbufr uv null uv 0.0 0 0 + amsr2bufr amsr2 gcom-w1 amsr2_gcom-w1 0.0 3 0 + gmibufr gmi gpm gmi_gpm 0.0 3 0 + saphirbufr saphir meghat saphir_meghat 0.0 3 0 + ahibufr ahi himawari8 ahi_himawari8 0.0 1 0 + ahibufr ahi himawari9 ahi_himawari9 0.0 1 0 + abibufr abi g16 abi_g16 0.0 1 0 + abibufr abi g17 abi_g17 0.0 1 0 + abibufr abi g18 abi_g18 0.0 1 0 + rapidscatbufr uv null uv 0.0 0 0 + amsuabufr amsua metop-c amsua_metop-c 0.0 1 1 + mhsbufr mhs metop-c mhs_metop-c 0.0 1 1 + iasibufr iasi metop-c iasi_metop-c 0.0 1 1 +!--ozone bufr dumps +! mlsbufr mls30 aura mls30_aura 0.0 0 0 +! ompsnpbufr ompsnp npp ompsnp_npp 0.0 0 0 + ompsnpbufr ompsnp n20 ompsnp_n20 0.0 0 0 + ompsnpbufr ompsnp n21 ompsnp_n21 0.0 0 0 +! ompslpbufr ompslp npp ompslp_npp 0.0 0 0 +! ompslpbufr ompslp n21 ompslp_n21 0.0 0 0 + ompstcbufr ompstc8 npp ompstc8_npp 0.0 2 0 + ompstcbufr ompstc8 n20 ompstc8_n20 0.0 2 0 + ompstcbufr ompstc8 n21 ompstc8_n21 0.0 2 0 +! omibufr omi aura omi_aura 0.0 2 0 +!--nasa netcdf ozone (can't have both bufr and netcdf versions at the same time) + ompslpnc ompslpnc npp ompslpnc_npp 0.0 0 0 + ompsnpnc ompsnpnc npp ompsnpnc_npp 0.0 0 0 + ompsnmeffnc ompsnmeff npp ompsnmeff_npp 0.0 2 0 + mls55nc mls55 aura mls55_aura 0.0 0 0 + omieffnc omieff aura omieff_aura 0.0 2 0 +EOF +) +else +OBS_INPUT_TABLE=$(cat << EOF ! dfile dtype dplat dsis dval dthin dsfcalc prepbufr ps null ps 0.0 0 0 prepbufr t null t 0.0 0 0 @@ -886,6 +1022,84 @@ OBS_INPUT:: ompsnpbufr ompsnp n21 ompsnp_n21 0.0 0 0 ompstcbufr ompstc8 n21 ompstc8_n21 0.0 2 0 gomebufr gome metop-c gome_metop-c 0.0 2 0 +EOF +) +fi + +############################################################## +# Create global_gsi namelist +cat > gsiparm.anl << EOF +&SETUP + miter=2, + niter(1)=100,niter(2)=100, + niter_no_qc(1)=50,niter_no_qc(2)=0, + write_diag(1)=.true.,write_diag(2)=.false.,write_diag(3)=.true., + qoption=2, + gencode=${IGEN:-0},deltim=${DELTIM}, + factqmin=0.5,factqmax=0.0002, + iguess=-1, + tzr_qc=${TZR_QC}, + oneobtest=.false.,retrieval=.false.,l_foto=.false., + use_pbl=.false.,use_compress=.true.,nsig_ext=45,gpstop=50.,commgpstop=45.,commgpserrinf=1.0, + use_gfs_nemsio=.false.,use_gfs_ncio=.true.,sfcnst_comb=.true., + use_readin_anl_sfcmask=${USE_READIN_ANL_SFCMASK}, + lrun_subdirs=${lrun_subdirs}, + crtm_coeffs_path='./crtm_coeffs/', + newpc4pred=.true.,adp_anglebc=.true.,angord=4,passive_bc=.true.,use_edges=.false., + diag_precon=.true.,step_start=1.e-3,emiss_bc=.true.,nhr_obsbin=${nhr_obsbin:-3}, + cwoption=3,imp_physics=${imp_physics},lupp=${lupp},cnvw_option=${cnvw_option},cao_check=${cao_check}, + netcdf_diag=${netcdf_diag},binary_diag=${binary_diag}, + lobsdiag_forenkf=${lobsdiag_forenkf}, + write_fv3_incr=${write_fv3_increment}, + nhr_anal=${IAUFHRS}, + ta2tb=${ta2tb}, + ${WRITE_INCR_ZERO} + ${WRITE_ZERO_STRAT} + ${WRITE_STRAT_EFOLD} + ${SETUP} +/ +&GRIDOPTS + JCAP_B=${JCAP},JCAP=${JCAP_A},NLAT=${NLAT_A},NLON=${NLON_A},nsig=${LEVS}, + regional=.false.,nlayers(63)=3,nlayers(64)=6, + ${GRIDOPTS} +/ +&BKGERR + vs=0.7, + hzscl=1.7,0.8,0.5, + hswgt=0.45,0.3,0.25, + bw=0.0,norsp=4, + bkgv_flowdep=.true.,bkgv_rewgtfct=1.5, + bkgv_write=.false., + cwcoveqqcov=.false., + ${BKGVERR} +/ +&ANBKGERR + anisotropic=.false., + ${ANBKGERR} +/ +&JCOPTS + ljcdfi=.false.,alphajc=0.0,ljcpdry=.true.,bamp_jcpdry=5.0e7, + ${JCOPTS} +/ +&STRONGOPTS + tlnmc_option=2,nstrong=1,nvmodes_keep=8,period_max=6.,period_width=1.5, + ${STRONGOPTS} +/ +&OBSQC + dfact=0.75,dfact1=3.0,noiqc=.true.,oberrflg=.false.,c_varqc=0.02, + use_poq7=.true.,qc_noirjaco3_pole=.true.,vqc=.false.,nvqc=.true., + aircraft_t_bc=.true.,biaspredt=1.0e5,upd_aircraft=.true.,cleanup_tail=.true., + tcp_width=70.0,tcp_ermax=7.35,airs_cads=${AIRS_CADS},cris_cads=${CRIS_CADS}, + iasi_cads=${IASI_CADS}, + ${OBSQC} +/ +&OBS_INPUT + dmesh(1)=145.0,dmesh(2)=150.0,dmesh(3)=100.0,dmesh(4)=50.0,time_window_max=3.0, + hofx_2m_sfcfile=${hofx_2m_sfcfile}, + ${OBSINPUT} +/ +OBS_INPUT:: +${OBS_INPUT_TABLE} :: &SUPEROB_RADAR ${SUPERRAD} diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh index 46a6e9863c2..0a34add4205 100755 --- a/scripts/exglobal_diag.sh +++ b/scripts/exglobal_diag.sh @@ -98,8 +98,13 @@ if [ $GENDIAG = "YES" ] ; then diagtype[0]="conv conv_gps conv_ps conv_pw conv_q conv_sst conv_t conv_tcp conv_uv conv_spd" diagtype[1]="pcp_ssmi_dmsp pcp_tmi_trmm" + if [ ${USE_BUILD_GSINFO} == "YES" ]; then + diagtype[2]=$(cat ${HOMEgfs}/sorc/build_gsinfo/ozinfo/satellites) + diagtype[3]=$(cat ${HOMEgfs}/sorc/build_gsinfo/satinfo/satellites) + else diagtype[2]="sbuv2_n16 sbuv2_n17 sbuv2_n18 sbuv2_n19 gome_metop-a gome_metop-b omi_aura mls30_aura ompsnp_npp ompstc8_npp ompstc8_n20 ompsnp_n20 ompstc8_n21 ompsnp_n21 ompslp_npp gome_metop-c" diagtype[3]="msu_n14 sndr_g08 sndr_g11 sndr_g12 sndr_g13 sndr_g08_prep sndr_g11_prep sndr_g12_prep sndr_g13_prep sndrd1_g11 sndrd2_g11 sndrd3_g11 sndrd4_g11 sndrd1_g12 sndrd2_g12 sndrd3_g12 sndrd4_g12 sndrd1_g13 sndrd2_g13 sndrd3_g13 sndrd4_g13 sndrd1_g14 sndrd2_g14 sndrd3_g14 sndrd4_g14 sndrd1_g15 sndrd2_g15 sndrd3_g15 sndrd4_g15 amsua_n15 amsua_n16 amsua_n17 amsub_n15 amsub_n16 amsub_n17 hsb_aqua airs_aqua amsua_aqua imgr_g08 imgr_g11 imgr_g12 imgr_g14 imgr_g15 ssmi_f13 ssmi_f15 amsua_n18 amsua_metop-a mhs_n18 mhs_metop-a amsre_low_aqua amsre_mid_aqua amsre_hig_aqua ssmis_f16 ssmis_f17 ssmis_f18 ssmis_f19 ssmis_f20 iasi_metop-a amsua_n19 mhs_n19 seviri_m08 seviri_m09 seviri_m10 seviri_m11 cris_npp cris-fsr_npp cris-fsr_n20 atms_npp atms_n20 amsua_metop-b mhs_metop-b iasi_metop-b avhrr_metop-b avhrr_n18 avhrr_n19 avhrr_metop-a amsr2_gcom-w1 gmi_gpm saphir_meghat ahi_himawari8 abi_g16 abi_g17 amsua_metop-c mhs_metop-c iasi_metop-c avhrr_metop-c viirs-m_npp viirs-m_j1 abi_g18 ahi_himawari9 viirs-m_j2 cris-fsr_n21 atms_n21" + fi diaglist[0]=listcnv diaglist[1]=listpcp From a97e131ff2de43baf09c7644e81106e92d39066e Mon Sep 17 00:00:00 2001 From: "Jeffrey.S.Whitaker" Date: Tue, 4 Feb 2025 16:59:21 -0500 Subject: [PATCH 02/90] add scripts for generating GSI info files when USE_BUILD_GSINFO=YES --- scripts/create_convinfo.sh | 18 ++++++++++++++++++ scripts/create_ozinfo.sh | 27 +++++++++++++++++++++++++++ scripts/create_satinfo.sh | 25 +++++++++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 scripts/create_convinfo.sh create mode 100644 scripts/create_ozinfo.sh create mode 100644 scripts/create_satinfo.sh diff --git a/scripts/create_convinfo.sh b/scripts/create_convinfo.sh new file mode 100644 index 00000000000..79b45a88b21 --- /dev/null +++ b/scripts/create_convinfo.sh @@ -0,0 +1,18 @@ +# create global_ozinfo file for a given date +date=$1 +cd ${FIXgfs}/gsi/build_gsinfo/convinfo +# currently using a single convinfo +cat merged_convinfo.txt +#usedate="" +#for datex in [1-2]*; do +# if [ $date -ge $datex ]; then +# usedate=$datex +# fi +#done +## cat matching date file, or quit if date not found +#if [ $usedate != "" ]; then +# cat $usedate +#else +# echo "date not found for $sat" +# exit 1 +#fi diff --git a/scripts/create_ozinfo.sh b/scripts/create_ozinfo.sh new file mode 100644 index 00000000000..2d36105def0 --- /dev/null +++ b/scripts/create_ozinfo.sh @@ -0,0 +1,27 @@ +# create global_ozinfo file for a given date +date=$1 +# header +echo '! For mls data, pressure and obs errors are pulled from bufr, so not listed here' +echo '! sens/instr/sat lev use pressure gross obs b_oz pg_oz' +echo '! error error variational qc' +# loop over satellites +cd ${FIXgfs}/gsi/build_gsinfo/ozinfo +for sat in $(cat satellites); do + # find matching date + usedate="" + for f in $sat/*; do + if [ $f != "$sat/readme" ]; then # skip readme file + datex=`basename $f` + if [ $date -ge $datex ]; then + usedate=$datex + fi + fi + done + # cat matching date file, or quit if date not found + if [ $usedate != "" ]; then + cat $sat/$usedate + else + echo "date not found for $sat" + exit 1 + fi +done diff --git a/scripts/create_satinfo.sh b/scripts/create_satinfo.sh new file mode 100644 index 00000000000..1729554fba1 --- /dev/null +++ b/scripts/create_satinfo.sh @@ -0,0 +1,25 @@ +# create global_satinfo file for a given date +date=$1 +# header +echo '!sensor/instr/sat chan iuse error error_cld ermax var_b var_pg icld_det icloud iaerosol' +# loop over satellites +cd ${FIXgfs}/gsi/build_gsinfo/satinfo +for sat in $(cat satellites); do + # find matching date + usedate="" + for f in $sat/*; do + if [ $f != "$sat/readme" ]; then # skip readme file + datex=`basename $f` + if [ $date -ge $datex ]; then + usedate=$datex + fi + fi + done + # cat matching date file, or quit if date not found + if [ $usedate != "" ]; then + cat $sat/$usedate + else + echo "date not found for $sat" + exit 1 + fi +done From e4094ad17bdb21fda9a0bf443e96862b30f84d5d Mon Sep 17 00:00:00 2001 From: "Jeffrey.S.Whitaker" Date: Tue, 4 Feb 2025 17:01:50 -0500 Subject: [PATCH 03/90] update --- scripts/exglobal_atmos_analysis.sh | 2 +- scripts/exglobal_diag.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index b38922d205f..c2931919fd5 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -385,7 +385,7 @@ fi ${NLN} ${vqcdat} vqctp001.dat ${NLN} ${INSITUINFO} insituinfo if [ $OZINFO == "generate" ]; then - sh ${SCRgfs}/build_gsinfo/create_ozinfo.sh ${PDY}${cyc} > ozinfo + sh ${SCRgfs}/create_ozinfo.sh ${PDY}${cyc} > ozinfo else ${NLN} ${OZINFO} ozinfo fi diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh index 0a34add4205..f1ab6de56e0 100755 --- a/scripts/exglobal_diag.sh +++ b/scripts/exglobal_diag.sh @@ -99,8 +99,8 @@ if [ $GENDIAG = "YES" ] ; then diagtype[0]="conv conv_gps conv_ps conv_pw conv_q conv_sst conv_t conv_tcp conv_uv conv_spd" diagtype[1]="pcp_ssmi_dmsp pcp_tmi_trmm" if [ ${USE_BUILD_GSINFO} == "YES" ]; then - diagtype[2]=$(cat ${HOMEgfs}/sorc/build_gsinfo/ozinfo/satellites) - diagtype[3]=$(cat ${HOMEgfs}/sorc/build_gsinfo/satinfo/satellites) + diagtype[2]=$(cat ${FIXgfs}/gsi/build_gsinfo/ozinfo/satellites) + diagtype[3]=$(cat ${FIXgfs}/gsi/build_gsinfo/satinfo/satellites) else diagtype[2]="sbuv2_n16 sbuv2_n17 sbuv2_n18 sbuv2_n19 gome_metop-a gome_metop-b omi_aura mls30_aura ompsnp_npp ompstc8_npp ompstc8_n20 ompsnp_n20 ompstc8_n21 ompsnp_n21 ompslp_npp gome_metop-c" diagtype[3]="msu_n14 sndr_g08 sndr_g11 sndr_g12 sndr_g13 sndr_g08_prep sndr_g11_prep sndr_g12_prep sndr_g13_prep sndrd1_g11 sndrd2_g11 sndrd3_g11 sndrd4_g11 sndrd1_g12 sndrd2_g12 sndrd3_g12 sndrd4_g12 sndrd1_g13 sndrd2_g13 sndrd3_g13 sndrd4_g13 sndrd1_g14 sndrd2_g14 sndrd3_g14 sndrd4_g14 sndrd1_g15 sndrd2_g15 sndrd3_g15 sndrd4_g15 amsua_n15 amsua_n16 amsua_n17 amsub_n15 amsub_n16 amsub_n17 hsb_aqua airs_aqua amsua_aqua imgr_g08 imgr_g11 imgr_g12 imgr_g14 imgr_g15 ssmi_f13 ssmi_f15 amsua_n18 amsua_metop-a mhs_n18 mhs_metop-a amsre_low_aqua amsre_mid_aqua amsre_hig_aqua ssmis_f16 ssmis_f17 ssmis_f18 ssmis_f19 ssmis_f20 iasi_metop-a amsua_n19 mhs_n19 seviri_m08 seviri_m09 seviri_m10 seviri_m11 cris_npp cris-fsr_npp cris-fsr_n20 atms_npp atms_n20 amsua_metop-b mhs_metop-b iasi_metop-b avhrr_metop-b avhrr_n18 avhrr_n19 avhrr_metop-a amsr2_gcom-w1 gmi_gpm saphir_meghat ahi_himawari8 abi_g16 abi_g17 amsua_metop-c mhs_metop-c iasi_metop-c avhrr_metop-c viirs-m_npp viirs-m_j1 abi_g18 ahi_himawari9 viirs-m_j2 cris-fsr_n21 atms_n21" From 3fdd95c04c00113671e211eef9ab951c43e21394 Mon Sep 17 00:00:00 2001 From: "Jeffrey.S.Whitaker" Date: Tue, 4 Mar 2025 14:39:18 -0500 Subject: [PATCH 04/90] use OBS_INPUT envar to point to obs_input namelist block table text file --- parm/config/gfs/config.anal | 3 + scripts/exglobal_atmos_analysis.sh | 313 +---------------------------- 2 files changed, 13 insertions(+), 303 deletions(-) diff --git a/parm/config/gfs/config.anal b/parm/config/gfs/config.anal index 85dd1ba56f6..f70ded30040 100644 --- a/parm/config/gfs/config.anal +++ b/parm/config/gfs/config.anal @@ -153,6 +153,9 @@ if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then export SATINFO="generate" export CONVINFO="generate" export OZINFO="generate" + export OBS_INPUT=${FIXgfs}/gsi/build_gsinfo/obs_input_reanl_nasa_ozone.txt +else + export OBS_INPUT=${FIXgfs}/gsi/build_gsinfo/obs_input_ops.txt fi # Flag to turn on (.true.) or off (.false.) the infrared cloud and aerosol detection software diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 4192b817ac6..c7f9bc5cd47 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -309,6 +309,7 @@ AEROINFO=${AEROINFO:-${FIXgfs}/gsi/global_aeroinfo.txt} SCANINFO=${SCANINFO:-${FIXgfs}/gsi/global_scaninfo.txt} HYBENSINFO=${HYBENSINFO:-${FIXgfs}/gsi/global_hybens_info.l${LEVS}.txt} OBERROR=${OBERROR:-${FIXgfs}/gsi/prepobs_errtable.global} +OBS_INPUT=${OBS_INPUT:-${FIXgfs}/gsi/build_gsinfo/obs_input_ops.txt} # GSI namelist SETUP=${SETUP:-""} @@ -521,16 +522,14 @@ ${NLN} ${ABIBF} abibufr ${NLN} ${HDOB} hdobbufr ${NLN} ${SSTVIIRS} sstviirs -if [ ${USE_BUILD_GSINFO} == "YES" ]; then - # NASA ozone (netcdf) from NNJA - ${NLN} ${OMIEFFNC} omieffnc - ${NLN} ${OMPSNMEFFNC} ompsnmeffnc - ${NLN} ${OMPSNPNC} ompsnpnc - ${NLN} ${OMPSLPNC} ompslpnc - ${NLN} ${MLS55NC} mls55nc - # NASA airs aqua amsua (bufr) from NNJA - ${NLN} ${AQUAAMUA} aquabufr -fi +# NASA ozone (netcdf) from NNJA +${NLN} ${OMIEFFNC} omieffnc +${NLN} ${OMPSNMEFFNC} ompsnmeffnc +${NLN} ${OMPSNPNC} ompsnpnc +${NLN} ${OMPSLPNC} ompslpnc +${NLN} ${MLS55NC} mls55nc +# NASA airs aqua amsua (bufr) from NNJA +${NLN} ${AQUAAMUA} aquabufr if [[ "${DONST}" == "YES" ]]; then ${NLN} "${NSSTBF}" nsstbufr @@ -756,299 +755,7 @@ if [ ${DONST} = "YES" ]; then NST="nstinfo=${NSTINFO},fac_dtl=${FAC_DTL},fac_tsl=${FAC_TSL},zsea1=${ZSEA1},zsea2=${ZSEA2},${NST}" fi -if [ ${USE_BUILD_GSINFO} == "YES" ]; then -OBS_INPUT_TABLE=$(cat << EOF -! dfile dtype dplat dsis dval dthin dsfcalc - prepbufr ps null ps 0.0 0 0 - prepbufr t null t 0.0 0 0 - prepbufr q null q 0.0 0 0 - prepbufr pw null pw 0.0 0 0 - prepbufr_profl t null t 0.0 0 0 - prepbufr_profl q null q 0.0 0 0 - prepbufr_profl uv null uv 0.0 0 0 - satwndbufr uv null uv 0.0 0 0 - prepbufr uv null uv 0.0 0 0 - prepbufr spd null spd 0.0 0 0 - prepbufr dw null dw 0.0 0 0 - radarbufr rw null rw 0.0 0 0 - nsstbufr sst nsst sst 0.0 0 0 - gpsrobufr gps_bnd null gps 0.0 0 0 - ssmirrbufr pcp_ssmi dmsp pcp_ssmi 0.0 -1 0 - tmirrbufr pcp_tmi trmm pcp_tmi 0.0 -1 0 - sbuvbufr sbuv2 nim07 sbuv8_nim07 0.0 0 0 - sbuvbufr sbuv2 n09 sbuv8_n09 0.0 0 0 - sbuvbufr sbuv2 n11 sbuv8_n11 0.0 0 0 - sbuvbufr sbuv2 n14 sbuv8_n14 0.0 0 0 - sbuvbufr sbuv2 n16 sbuv8_n16 0.0 0 0 - sbuvbufr sbuv2 n17 sbuv8_n17 0.0 0 0 - sbuvbufr sbuv2 n18 sbuv8_n18 0.0 0 0 - sbuvbufr sbuv2 n19 sbuv8_n19 0.0 0 0 - hirs2bufr hirs2 tirosn hirs2_tirosn 0.0 1 1 - hirs2bufr hirs2 n06 hirs2_n06 0.0 1 1 - hirs2bufr hirs2 n07 hirs2_n07 0.0 1 1 - hirs2bufr hirs2 n08 hirs2_n08 0.0 1 1 - hirs2bufr hirs2 n09 hirs2_n09 0.0 1 1 - hirs2bufr hirs2 n10 hirs2_n10 0.0 1 1 - hirs2bufr hirs2 n11 hirs2_n11 0.0 1 1 - hirs2bufr hirs2 n12 hirs2_n12 0.0 1 1 - hirs2bufr hirs2 n14 hirs2_n14 0.0 1 1 - hirs3bufr hirs3 n15 hirs3_n15 0.0 1 1 - hirs3bufr hirs3 n16 hirs3_n16 0.0 1 1 - hirs3bufr hirs3 n17 hirs3_n17 0.0 1 1 - gimgrbufr goes_img g11 imgr_g11 0.0 1 0 - gimgrbufr goes_img g12 imgr_g12 0.0 1 0 - airsbufr airs aqua airs_aqua 0.0 1 1 - msubufr msu tirosn msu_tirosn 0.0 1 1 - msubufr msu n06 msu_n06 0.0 1 1 - msubufr msu n07 msu_n07 0.0 1 1 - msubufr msu n08 msu_n08 0.0 1 1 - msubufr msu n09 msu_n09 0.0 1 1 - msubufr msu n10 msu_n10 0.0 1 1 - msubufr msu n11 msu_n11 0.0 1 1 - msubufr msu n12 msu_n12 0.0 1 1 - msubufr msu n14 msu_n14 0.0 1 1 - ssubufr ssu tirosn ssu_tirosn 0.0 1 1 - ssubufr ssu n06 ssu_n06 0.0 1 1 - ssubufr ssu n07 ssu_n07 0.0 1 1 - ssubufr ssu n08 ssu_n08 0.0 1 1 - ssubufr ssu n09 ssu_n09 0.0 1 1 - ssubufr ssu n11 ssu_n11 0.0 1 1 - ssubufr ssu n14 ssu_n14 0.0 1 1 - amsuabufr amsua n15 amsua_n15 0.0 1 1 - amsuabufr amsua n16 amsua_n16 0.0 1 1 - amsuabufr amsua n17 amsua_n17 0.0 1 1 - amsuabufr amsua n18 amsua_n18 0.0 1 1 - aquabufr amsua aqua amsua_aqua 0.0 1 1 - amsubbufr amsub n15 amsub_n15 0.0 1 1 - amsubbufr amsub n16 amsub_n16 0.0 1 1 - amsubbufr amsub n17 amsub_n17 0.0 1 1 - mhsbufr mhs n18 mhs_n18 0.0 1 1 - ssmitbufr ssmi f08 ssmi_f08 0.0 1 0 - ssmitbufr ssmi f10 ssmi_f10 0.0 1 0 - ssmitbufr ssmi f11 ssmi_f11 0.0 1 0 - ssmitbufr ssmi f13 ssmi_f13 0.0 1 0 - ssmitbufr ssmi f14 ssmi_f14 0.0 1 0 - ssmitbufr ssmi f15 ssmi_f15 0.0 1 0 - amsrebufr amsre_low aqua amsre_aqua 0.0 1 0 - amsrebufr amsre_mid aqua amsre_aqua 0.0 1 0 - amsrebufr amsre_hig aqua amsre_aqua 0.0 1 0 - ssmisbufr ssmis f16 ssmis_f16 0.0 1 0 - ssmisbufr ssmis f17 ssmis_f17 0.0 1 0 - ssmisbufr ssmis f18 ssmis_f18 0.0 1 0 - ssmisbufr ssmis f19 ssmis_f19 0.0 1 0 - gsnd1bufr sndr g08 sndr_g08 0.0 1 0 - gsnd1bufr sndr g09 sndr_g09 0.0 1 0 - gsnd1bufr sndr g10 sndr_g10 0.0 1 0 - gsnd1bufr sndr g11 sndr_g11 0.0 1 0 - gsnd1bufr sndr g12 sndr_g12 0.0 1 0 - gsnd1bufr sndrd1 g12 sndrD1_g12 0.0 1 0 - gsnd1bufr sndrd2 g12 sndrD2_g12 0.0 1 0 - gsnd1bufr sndrd3 g12 sndrD3_g12 0.0 1 0 - gsnd1bufr sndrd4 g12 sndrD4_g12 0.0 1 0 - gsnd1bufr sndrd1 g11 sndrD1_g11 0.0 1 0 - gsnd1bufr sndrd2 g11 sndrD2_g11 0.0 1 0 - gsnd1bufr sndrd3 g11 sndrD3_g11 0.0 1 0 - gsnd1bufr sndrd4 g11 sndrD4_g11 0.0 1 0 - gsnd1bufr sndrd1 g13 sndrD1_g13 0.0 1 0 - gsnd1bufr sndrd2 g13 sndrD2_g13 0.0 1 0 - gsnd1bufr sndrd3 g13 sndrD3_g13 0.0 1 0 - gsnd1bufr sndrd4 g13 sndrD4_g13 0.0 1 0 - gsnd1bufr sndrd1 g14 sndrD1_g14 0.0 1 0 - gsnd1bufr sndrd2 g14 sndrD2_g14 0.0 1 0 - gsnd1bufr sndrd3 g14 sndrD3_g14 0.0 1 0 - gsnd1bufr sndrd4 g14 sndrD4_g14 0.0 1 0 - gsnd1bufr sndrd1 g15 sndrD1_g15 0.0 1 0 - gsnd1bufr sndrd2 g15 sndrD2_g15 0.0 1 0 - gsnd1bufr sndrd3 g15 sndrD3_g15 0.0 1 0 - gsnd1bufr sndrd4 g15 sndrD4_g15 0.0 1 0 - hirs4bufr hirs4 n19 hirs4_n19 0.0 1 1 - amsuabufr amsua n19 amsua_n19 0.0 1 1 - mhsbufr mhs n19 mhs_n19 0.0 1 1 - tcvitl tcp null tcp 0.0 0 0 - seviribufr seviri m08 seviri_m08 0.0 1 0 - seviribufr seviri m09 seviri_m09 0.0 1 0 - seviribufr seviri m10 seviri_m10 0.0 1 0 - seviribufr seviri m10 seviri_m11 0.0 1 0 - hirs4bufr hirs4 metop-a hirs4_metop-a 0.0 1 1 - hirs4bufr hirs4 metop-b hirs4_metop-b 0.0 1 0 - amsuabufr amsua metop-a amsua_metop-a 0.0 1 1 - amsuabufr amsua metop-b amsua_metop-b 0.0 1 0 - amsuabufr amsua metop-c amsua_metop-c 0.0 1 0 - mhsbufr mhs metop-a mhs_metop-a 0.0 1 1 - mhsbufr mhs metop-b mhs_metop-b 0.0 1 0 - mhsbufr mhs metop-c mhs_metop-c 0.0 1 0 - iasibufr iasi metop-a iasi_metop-a 0.0 1 1 - iasibufr iasi metop-b iasi_metop-b 0.0 1 0 - iasibufr iasi metop-c iasi_metop-c 0.0 1 0 - gomebufr gome metop-a gome_metop-a 0.0 2 0 - gomebufr gome metop-b gome_metop-b 0.0 2 0 - gomebufr gome metop-c gome_metop-c 0.0 2 0 - atmsbufr atms npp atms_npp 0.0 1 1 - atmsbufr atms n20 atms_n20 0.0 1 1 - atmsbufr atms n20 atms_n21 0.0 1 1 - crisbufr cris npp cris_npp 0.0 1 0 - crisfsbufr cris-fsr npp cris-fsr_npp 0.0 1 0 - crisfsbufr cris-fsr n20 cris-fsr_n20 0.0 1 0 - crisfsbufr cris-fsr n21 cris-fsr_n21 0.0 1 0 - avhambufr avhrr n15 avhrr3_n15 0.0 1 0 - avhambufr avhrr n17 avhrr3_n17 0.0 1 0 - avhambufr avhrr metop-a avhrr3_metop-a 0.0 1 0 - avhambufr avhrr metop-b avhrr3_metop-b 0.0 1 0 - avhambufr avhrr metop-c avhrr3_metop-c 0.0 1 0 - avhpmbufr avhrr n14 avhrr2_n14 0.0 1 0 - avhpmbufr avhrr n16 avhrr3_n16 0.0 1 0 - avhpmbufr avhrr n18 avhrr3_n18 0.0 1 0 - avhpmbufr avhrr n19 avhrr3_n19 0.0 1 0 - oscatbufr uv null uv 0.0 0 0 - amsr2bufr amsr2 gcom-w1 amsr2_gcom-w1 0.0 3 0 - gmibufr gmi gpm gmi_gpm 0.0 3 0 - saphirbufr saphir meghat saphir_meghat 0.0 3 0 - ahibufr ahi himawari8 ahi_himawari8 0.0 1 0 - ahibufr ahi himawari9 ahi_himawari9 0.0 1 0 - abibufr abi g16 abi_g16 0.0 1 0 - abibufr abi g17 abi_g17 0.0 1 0 - abibufr abi g18 abi_g18 0.0 1 0 - rapidscatbufr uv null uv 0.0 0 0 - amsuabufr amsua metop-c amsua_metop-c 0.0 1 1 - mhsbufr mhs metop-c mhs_metop-c 0.0 1 1 - iasibufr iasi metop-c iasi_metop-c 0.0 1 1 -!--ozone bufr dumps -! mlsbufr mls30 aura mls30_aura 0.0 0 0 -! ompsnpbufr ompsnp npp ompsnp_npp 0.0 0 0 - ompsnpbufr ompsnp n20 ompsnp_n20 0.0 0 0 - ompsnpbufr ompsnp n21 ompsnp_n21 0.0 0 0 -! ompslpbufr ompslp npp ompslp_npp 0.0 0 0 -! ompslpbufr ompslp n21 ompslp_n21 0.0 0 0 - ompstcbufr ompstc8 npp ompstc8_npp 0.0 2 0 - ompstcbufr ompstc8 n20 ompstc8_n20 0.0 2 0 - ompstcbufr ompstc8 n21 ompstc8_n21 0.0 2 0 -! omibufr omi aura omi_aura 0.0 2 0 -!--nasa netcdf ozone (can't have both bufr and netcdf versions at the same time) - ompslpnc ompslpnc npp ompslpnc_npp 0.0 0 0 - ompsnpnc ompsnpnc npp ompsnpnc_npp 0.0 0 0 - ompsnmeffnc ompsnmeff npp ompsnmeff_npp 0.0 2 0 - mls55nc mls55 aura mls55_aura 0.0 0 0 - omieffnc omieff aura omieff_aura 0.0 2 0 -EOF -) -else -OBS_INPUT_TABLE=$(cat << EOF -! dfile dtype dplat dsis dval dthin dsfcalc - prepbufr ps null ps 0.0 0 0 - prepbufr t null t 0.0 0 0 - prepbufr_profl t null t 0.0 0 0 - hdobbufr t null t 0.0 0 0 - prepbufr q null q 0.0 0 0 - prepbufr_profl q null q 0.0 0 0 - hdobbufr q null q 0.0 0 0 - prepbufr pw null pw 0.0 0 0 - prepbufr uv null uv 0.0 0 0 - prepbufr_profl uv null uv 0.0 0 0 - satwndbufr uv null uv 0.0 0 0 - hdobbufr uv null uv 0.0 0 0 - prepbufr spd null spd 0.0 0 0 - hdobbufr spd null spd 0.0 0 0 - prepbufr dw null dw 0.0 0 0 - radarbufr rw null rw 0.0 0 0 - nsstbufr sst nsst sst 0.0 0 0 - gpsrobufr gps_bnd null gps 0.0 0 0 - ssmirrbufr pcp_ssmi dmsp pcp_ssmi 0.0 -1 0 - tmirrbufr pcp_tmi trmm pcp_tmi 0.0 -1 0 - sbuvbufr sbuv2 n16 sbuv8_n16 0.0 0 0 - sbuvbufr sbuv2 n17 sbuv8_n17 0.0 0 0 - sbuvbufr sbuv2 n18 sbuv8_n18 0.0 0 0 - gimgrbufr goes_img g11 imgr_g11 0.0 1 0 - gimgrbufr goes_img g12 imgr_g12 0.0 1 0 - airsbufr airs aqua airs_aqua 0.0 1 1 - amsuabufr amsua n15 amsua_n15 0.0 1 1 - amsuabufr amsua n18 amsua_n18 0.0 1 1 - amsuabufr amsua metop-a amsua_metop-a 0.0 1 1 - airsbufr amsua aqua amsua_aqua 0.0 1 1 - amsubbufr amsub n17 amsub_n17 0.0 1 1 - mhsbufr mhs n18 mhs_n18 0.0 1 1 - mhsbufr mhs metop-a mhs_metop-a 0.0 1 1 - ssmitbufr ssmi f15 ssmi_f15 0.0 1 0 - amsrebufr amsre_low aqua amsre_aqua 0.0 1 0 - amsrebufr amsre_mid aqua amsre_aqua 0.0 1 0 - amsrebufr amsre_hig aqua amsre_aqua 0.0 1 0 - ssmisbufr ssmis f16 ssmis_f16 0.0 1 0 - ssmisbufr ssmis f17 ssmis_f17 0.0 1 0 - ssmisbufr ssmis f18 ssmis_f18 0.0 1 0 - gsnd1bufr sndrd1 g12 sndrD1_g12 0.0 1 0 - gsnd1bufr sndrd2 g12 sndrD2_g12 0.0 1 0 - gsnd1bufr sndrd3 g12 sndrD3_g12 0.0 1 0 - gsnd1bufr sndrd4 g12 sndrD4_g12 0.0 1 0 - gsnd1bufr sndrd1 g11 sndrD1_g11 0.0 1 0 - gsnd1bufr sndrd2 g11 sndrD2_g11 0.0 1 0 - gsnd1bufr sndrd3 g11 sndrD3_g11 0.0 1 0 - gsnd1bufr sndrd4 g11 sndrD4_g11 0.0 1 0 - gsnd1bufr sndrd1 g13 sndrD1_g13 0.0 1 0 - gsnd1bufr sndrd2 g13 sndrD2_g13 0.0 1 0 - gsnd1bufr sndrd3 g13 sndrD3_g13 0.0 1 0 - gsnd1bufr sndrd4 g13 sndrD4_g13 0.0 1 0 - iasibufr iasi metop-a iasi_metop-a 0.0 1 1 - gomebufr gome metop-a gome_metop-a 0.0 2 0 - omibufr omi aura omi_aura 0.0 2 0 - sbuvbufr sbuv2 n19 sbuv8_n19 0.0 0 0 - amsuabufr amsua n19 amsua_n19 0.0 1 1 - mhsbufr mhs n19 mhs_n19 0.0 1 1 - tcvitl tcp null tcp 0.0 0 0 - seviribufr seviri m08 seviri_m08 0.0 1 0 - seviribufr seviri m09 seviri_m09 0.0 1 0 - seviribufr seviri m10 seviri_m10 0.0 1 0 - seviribufr seviri m11 seviri_m11 0.0 1 0 - amsuabufr amsua metop-b amsua_metop-b 0.0 1 1 - mhsbufr mhs metop-b mhs_metop-b 0.0 1 1 - iasibufr iasi metop-b iasi_metop-b 0.0 1 1 - gomebufr gome metop-b gome_metop-b 0.0 2 0 - atmsbufr atms npp atms_npp 0.0 1 1 - atmsbufr atms n20 atms_n20 0.0 1 1 - atmsbufr atms n21 atms_n21 0.0 1 1 - crisbufr cris npp cris_npp 0.0 1 0 - crisfsbufr cris-fsr npp cris-fsr_npp 0.0 1 0 - crisfsbufr cris-fsr n20 cris-fsr_n20 0.0 1 0 - crisfsbufr cris-fsr n21 cris-fsr_n21 0.0 1 0 - gsnd1bufr sndrd1 g14 sndrD1_g14 0.0 1 0 - gsnd1bufr sndrd2 g14 sndrD2_g14 0.0 1 0 - gsnd1bufr sndrd3 g14 sndrD3_g14 0.0 1 0 - gsnd1bufr sndrd4 g14 sndrD4_g14 0.0 1 0 - gsnd1bufr sndrd1 g15 sndrD1_g15 0.0 1 0 - gsnd1bufr sndrd2 g15 sndrD2_g15 0.0 1 0 - gsnd1bufr sndrd3 g15 sndrD3_g15 0.0 1 0 - gsnd1bufr sndrd4 g15 sndrD4_g15 0.0 1 0 - oscatbufr uv null uv 0.0 0 0 - mlsbufr mls30 aura mls30_aura 0.0 0 0 - avhambufr avhrr metop-a avhrr3_metop-a 0.0 4 0 - avhpmbufr avhrr n18 avhrr3_n18 0.0 4 0 - avhambufr avhrr metop-b avhrr3_metop-b 0.0 4 0 - avhambufr avhrr metop-c avhrr3_metop-c 0.0 4 0 - avhpmbufr avhrr n19 avhrr3_n19 0.0 4 0 - amsr2bufr amsr2 gcom-w1 amsr2_gcom-w1 0.0 3 0 - gmibufr gmi gpm gmi_gpm 0.0 1 0 - saphirbufr saphir meghat saphir_meghat 0.0 3 0 - ahibufr ahi himawari8 ahi_himawari8 0.0 1 0 - abibufr abi g16 abi_g16 0.0 1 0 - abibufr abi g17 abi_g17 0.0 1 0 - abibufr abi g18 abi_g18 0.0 1 0 - rapidscatbufr uv null uv 0.0 0 0 - ompsnpbufr ompsnp npp ompsnp_npp 0.0 0 0 - ompslpbufr ompslp npp ompslp_npp 0.0 0 0 - ompstcbufr ompstc8 npp ompstc8_npp 0.0 2 0 - ompsnpbufr ompsnp n20 ompsnp_n20 0.0 0 0 - ompstcbufr ompstc8 n20 ompstc8_n20 0.0 2 0 - amsuabufr amsua metop-c amsua_metop-c 0.0 1 1 - mhsbufr mhs metop-c mhs_metop-c 0.0 1 1 - iasibufr iasi metop-c iasi_metop-c 0.0 1 1 - sstviirs viirs-m npp viirs-m_npp 0.0 4 0 - sstviirs viirs-m j1 viirs-m_j1 0.0 4 0 - ahibufr ahi himawari9 ahi_himawari9 0.0 1 0 - sstviirs viirs-m j2 viirs-m_j2 0.0 4 0 - ompsnpbufr ompsnp n21 ompsnp_n21 0.0 0 0 - ompstcbufr ompstc8 n21 ompstc8_n21 0.0 2 0 - gomebufr gome metop-c gome_metop-c 0.0 2 0 -EOF -) -fi +OBS_INPUT_TABLE=$(cat ${OBS_INPUT}) ############################################################## # Create global_gsi namelist From a8a3473fe94179d80f86e43dcb4ae6eb2ceaf1d8 Mon Sep 17 00:00:00 2001 From: "Jeffrey.S.Whitaker" Date: Tue, 4 Mar 2025 17:25:52 -0500 Subject: [PATCH 05/90] fix for bad hirs coeff files (workaround until new crtm) --- scripts/exglobal_atmos_analysis.sh | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index c7f9bc5cd47..4898475fd03 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -100,10 +100,10 @@ OSCATBF=${OSCATBF:-${COMIN_OBS}/${OPREFIX}oscatw.tm00.bufr_d${OSUFFIX}} RAPIDSCATBF=${RAPIDSCATBF:-${COMIN_OBS}/${OPREFIX}rapidscatw.tm00.bufr_d${OSUFFIX}} GSNDBF=${GSNDBF:-${COMIN_OBS}/${OPREFIX}goesnd.tm00.bufr_d${OSUFFIX}} GSNDBF1=${GSNDBF1:-${COMIN_OBS}/${OPREFIX}goesfv.tm00.bufr_d${OSUFFIX}} -#B1HRS2=${B1HRS2:-${COMIN_OBS}/${OPREFIX}1bhrs2.tm00.bufr_d${OSUFFIX}} # HIRS temporarily disabled due to CRTM versioning issues +B1HRS2=${B1HRS2:-${COMIN_OBS}/${OPREFIX}1bhrs2.tm00.bufr_d${OSUFFIX}} # HIRS temporarily disabled due to CRTM versioning issues B1MSU=${B1MSU:-${COMIN_OBS}/${OPREFIX}1bmsu.tm00.bufr_d${OSUFFIX}} -#B1HRS3=${B1HRS3:-${COMIN_OBS}/${OPREFIX}1bhrs3.tm00.bufr_d${OSUFFIX}} # HIRS temporarily disabled due to CRTM versioning issues -#B1HRS4=${B1HRS4:-${COMIN_OBS}/${OPREFIX}1bhrs4.tm00.bufr_d${OSUFFIX}} # HIRS temporarily disabled due to CRTM versioning issues +B1HRS3=${B1HRS3:-${COMIN_OBS}/${OPREFIX}1bhrs3.tm00.bufr_d${OSUFFIX}} # HIRS temporarily disabled due to CRTM versioning issues +B1HRS4=${B1HRS4:-${COMIN_OBS}/${OPREFIX}1bhrs4.tm00.bufr_d${OSUFFIX}} # HIRS temporarily disabled due to CRTM versioning issues B1AMUA=${B1AMUA:-${COMIN_OBS}/${OPREFIX}1bamua.tm00.bufr_d${OSUFFIX}} B1AMUB=${B1AMUB:-${COMIN_OBS}/${OPREFIX}1bamub.tm00.bufr_d${OSUFFIX}} AQUAAMUA=${AQUAAMUA:-${COMIN_OBS}/${OPREFIX}aquaamua.tm00.bufr_d${OSUFFIX}} @@ -153,11 +153,11 @@ TCVITL=${TCVITL:-${COMIN_OBS}/${OPREFIX}syndata.tcvitals.tm00} B1AVHAM=${B1AVHAM:-${COMIN_OBS}/${OPREFIX}avcsam.tm00.bufr_d${OSUFFIX}} B1AVHPM=${B1AVHPM:-${COMIN_OBS}/${OPREFIX}avcspm.tm00.bufr_d${OSUFFIX}} HDOB=${HDOB:-${COMIN_OBS}/${OPREFIX}hdob.tm00.bufr_d${OSUFFIX}} -OMIEFFNC=${OMIEFFNC:-${COMIN_OBS/OMIeff-adj.${PDY}_${cyc}z.nc} -OMPSNMEFFNC=${OMPSNMEFFNC:-${COMIN_OBS/OMPSNM.${PDY}_${cyc}z.nc} -OMPSNPNC=${OMPSNPNC:-${COMIN_OBS/OMPSNP.${PDY}_${cyc}z.nc} -OMPSLPNC=${OMPSLPNC:-${COMIN_OBS/OMPS-LPoz-Vis.${PDY}_${cyc}z.nc} -MLS55NC=${MLS55NC:-${COMIN_OBS/MLS-v5.0-oz.${PDY}_${cyc}z.nc} +OMIEFFNC=${OMIEFFNC:-${COMIN_OBS}/OMIeff-adj.${PDY}_${cyc}z.nc}} +OMPSNMEFFNC=${OMPSNMEFFNC:-${COMIN_OBS}/OMPSNM.${PDY}_${cyc}z.nc}} +OMPSNPNC=${OMPSNPNC:-${COMIN_OBS}/OMPSNP.${PDY}_${cyc}z.nc}} +OMPSLPNC=${OMPSLPNC:-${COMIN_OBS}/OMPS-LPoz-Vis.${PDY}_${cyc}z.nc}} +MLS55NC=${MLS55NC:-${COMIN_OBS}/MLS-v5.0-oz.${PDY}_${cyc}z.nc}} # Guess files GPREFIX=${GPREFIX:-""} @@ -310,6 +310,7 @@ SCANINFO=${SCANINFO:-${FIXgfs}/gsi/global_scaninfo.txt} HYBENSINFO=${HYBENSINFO:-${FIXgfs}/gsi/global_hybens_info.l${LEVS}.txt} OBERROR=${OBERROR:-${FIXgfs}/gsi/prepobs_errtable.global} OBS_INPUT=${OBS_INPUT:-${FIXgfs}/gsi/build_gsinfo/obs_input_ops.txt} +HIRS_FIX=${HIRS_FIX:-${CRTM_FIX}} # GSI namelist SETUP=${SETUP:-""} @@ -434,7 +435,12 @@ fi # CRTM Spectral and Transmittance coefficients mkdir -p crtm_coeffs for file in $(awk '{if($1!~"!"){print $1}}' satinfo | sort | uniq); do - ${NLN} ${CRTM_FIX}/${file}.SpcCoeff.bin ./crtm_coeffs/${file}.SpcCoeff.bin + instr=`echo $file | cut -c1-4` + if [ $instr == "hirs" ]; then + ${NLN} ${HIRS_FIX}/${file}.SpcCoeff.bin ./crtm_coeffs/${file}.SpcCoeff.bin + else + ${NLN} ${CRTM_FIX}/${file}.SpcCoeff.bin ./crtm_coeffs/${file}.SpcCoeff.bin + fi ${NLN} ${CRTM_FIX}/${file}.TauCoeff.bin ./crtm_coeffs/${file}.TauCoeff.bin done ${NLN} ${CRTM_FIX}/amsua_metop-a_v2.SpcCoeff.bin ./crtm_coeffs/amsua_metop-a_v2.SpcCoeff.bin From b589b9bea6d031b6f4565e3fc1a47369856b4287 Mon Sep 17 00:00:00 2001 From: "Jeffrey.S.Whitaker" Date: Tue, 4 Mar 2025 17:26:21 -0500 Subject: [PATCH 06/90] add OBS_INPUT and HIRS_FIX --- parm/config/gfs/config.anal | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.anal b/parm/config/gfs/config.anal index f70ded30040..d8cc7e86f1a 100644 --- a/parm/config/gfs/config.anal +++ b/parm/config/gfs/config.anal @@ -153,10 +153,11 @@ if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then export SATINFO="generate" export CONVINFO="generate" export OZINFO="generate" - export OBS_INPUT=${FIXgfs}/gsi/build_gsinfo/obs_input_reanl_nasa_ozone.txt + export OBS_INPUT=${FIXgfs}/gsi/build_gsinfo/obs_input/obs_input_reanl_nasa_ozone.txt else - export OBS_INPUT=${FIXgfs}/gsi/build_gsinfo/obs_input_ops.txt + export OBS_INPUT=${FIXgfs}/gsi/build_gsinfo/obs_input/obs_input_ops.txt fi +export HIRS_FIX=/gpfs/f6/drsa-precip4/proj-shared/Jeffrey.S.Whitaker/hirs_fix # Flag to turn on (.true.) or off (.false.) the infrared cloud and aerosol detection software # for AIRS, CrIS, and IASI. Default is .false. From 49b5ed89ab9ab630624af271ff88592c52006884 Mon Sep 17 00:00:00 2001 From: jswhit2 Date: Tue, 25 Mar 2025 18:08:05 +0000 Subject: [PATCH 07/90] move scripts from scripts to ush, add shebang --- {scripts => ush}/create_convinfo.sh | 1 + {scripts => ush}/create_ozinfo.sh | 1 + {scripts => ush}/create_satinfo.sh | 1 + 3 files changed, 3 insertions(+) rename {scripts => ush}/create_convinfo.sh (95%) rename {scripts => ush}/create_ozinfo.sh (97%) rename {scripts => ush}/create_satinfo.sh (97%) diff --git a/scripts/create_convinfo.sh b/ush/create_convinfo.sh similarity index 95% rename from scripts/create_convinfo.sh rename to ush/create_convinfo.sh index 79b45a88b21..05d70d8e20a 100644 --- a/scripts/create_convinfo.sh +++ b/ush/create_convinfo.sh @@ -1,3 +1,4 @@ +#! /usr/bin/env bash # create global_ozinfo file for a given date date=$1 cd ${FIXgfs}/gsi/build_gsinfo/convinfo diff --git a/scripts/create_ozinfo.sh b/ush/create_ozinfo.sh similarity index 97% rename from scripts/create_ozinfo.sh rename to ush/create_ozinfo.sh index 2d36105def0..bf17d30e9ab 100644 --- a/scripts/create_ozinfo.sh +++ b/ush/create_ozinfo.sh @@ -1,3 +1,4 @@ +#! /usr/bin/env bash # create global_ozinfo file for a given date date=$1 # header diff --git a/scripts/create_satinfo.sh b/ush/create_satinfo.sh similarity index 97% rename from scripts/create_satinfo.sh rename to ush/create_satinfo.sh index 1729554fba1..fcd4fc4f361 100644 --- a/scripts/create_satinfo.sh +++ b/ush/create_satinfo.sh @@ -1,3 +1,4 @@ +#! /usr/bin/env bash # create global_satinfo file for a given date date=$1 # header From dd77bb4140cc1ce30b542da475e323d15a7fa56f Mon Sep 17 00:00:00 2001 From: jswhit2 Date: Tue, 25 Mar 2025 18:28:44 +0000 Subject: [PATCH 08/90] address shell linting errors --- ush/create_satinfo.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ush/create_satinfo.sh b/ush/create_satinfo.sh index fcd4fc4f361..95b4287de56 100644 --- a/ush/create_satinfo.sh +++ b/ush/create_satinfo.sh @@ -4,21 +4,22 @@ date=$1 # header echo '!sensor/instr/sat chan iuse error error_cld ermax var_b var_pg icld_det icloud iaerosol' # loop over satellites -cd ${FIXgfs}/gsi/build_gsinfo/satinfo -for sat in $(cat satellites); do +cd "${FIXgfs}/gsi/build_gsinfo/satinfo" || exit 1 +grep -v '^ *#' satellites | while IFS= read -r sat +do # find matching date usedate="" - for f in $sat/*; do - if [ $f != "$sat/readme" ]; then # skip readme file - datex=`basename $f` - if [ $date -ge $datex ]; then + for f in "${sat}"/*; do + if [[ $f != "${sat}/readme" ]]; then # skip readme file + datex=$(basename $f) + if [[ $date -ge $datex ]]; then usedate=$datex fi fi done # cat matching date file, or quit if date not found - if [ $usedate != "" ]; then - cat $sat/$usedate + if [[ ${usedate} != "" ]]; then + cat "${sat}/$usedate" || exit 1 else echo "date not found for $sat" exit 1 From 0e87789e3d93f4d17a289dbac6e1b86bbf5d253d Mon Sep 17 00:00:00 2001 From: jswhit2 Date: Tue, 25 Mar 2025 18:31:44 +0000 Subject: [PATCH 09/90] address shell linting errors --- ush/create_ozinfo.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ush/create_ozinfo.sh b/ush/create_ozinfo.sh index bf17d30e9ab..df94714c293 100644 --- a/ush/create_ozinfo.sh +++ b/ush/create_ozinfo.sh @@ -6,21 +6,22 @@ echo '! For mls data, pressure and obs errors are pulled from bufr, so not liste echo '! sens/instr/sat lev use pressure gross obs b_oz pg_oz' echo '! error error variational qc' # loop over satellites -cd ${FIXgfs}/gsi/build_gsinfo/ozinfo -for sat in $(cat satellites); do +cd "${FIXgfs}/gsi/build_gsinfo/ozinfo" || exit 1 +grep -v '^ *#' satellites | while IFS= read -r sat +do # find matching date usedate="" - for f in $sat/*; do - if [ $f != "$sat/readme" ]; then # skip readme file - datex=`basename $f` - if [ $date -ge $datex ]; then + for f in "${sat}"/*; do + if [[ $f != "$sat/readme" ]]; then # skip readme file + datex=$(basename $f) + if [[ $date -ge $datex ]]; then usedate=$datex fi fi done # cat matching date file, or quit if date not found - if [ $usedate != "" ]; then - cat $sat/$usedate + if [[ ${usedate} != "" ]]; then + cat "${sat}/$usedate" || exit 1 else echo "date not found for $sat" exit 1 From 3fcc4f12cb6ea060d07fd2832ca5ff77e91c12d4 Mon Sep 17 00:00:00 2001 From: jswhit2 Date: Tue, 25 Mar 2025 18:34:34 +0000 Subject: [PATCH 10/90] address shell linting errors --- ush/create_convinfo.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ush/create_convinfo.sh b/ush/create_convinfo.sh index 05d70d8e20a..6d90d9ba0da 100644 --- a/ush/create_convinfo.sh +++ b/ush/create_convinfo.sh @@ -1,19 +1,19 @@ #! /usr/bin/env bash -# create global_ozinfo file for a given date +# create global_convinfo file for a given date date=$1 -cd ${FIXgfs}/gsi/build_gsinfo/convinfo +cd "${FIXgfs}/gsi/build_gsinfo/convinfo" || exit # currently using a single convinfo cat merged_convinfo.txt #usedate="" #for datex in [1-2]*; do -# if [ $date -ge $datex ]; then +# if [[ ${date} -ge ${datex} ]]; then # usedate=$datex # fi #done ## cat matching date file, or quit if date not found -#if [ $usedate != "" ]; then -# cat $usedate +#if [[ ${usedate} != "" ]]; then +# cat ${usedate} #else -# echo "date not found for $sat" +# echo "date not found" # exit 1 #fi From 1e7b9f8c994fd8e8bcc04215e2aa6dfc147ce295 Mon Sep 17 00:00:00 2001 From: jswhit2 Date: Tue, 25 Mar 2025 18:45:37 +0000 Subject: [PATCH 11/90] fixed crtm2 hirs2 coeff files now included in build_gsinfo --- parm/config/gfs/config.anal | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parm/config/gfs/config.anal b/parm/config/gfs/config.anal index 4b094d7d8b3..ad43570e7b6 100644 --- a/parm/config/gfs/config.anal +++ b/parm/config/gfs/config.anal @@ -157,7 +157,8 @@ if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then else export OBS_INPUT=${FIXgfs}/gsi/build_gsinfo/obs_input/obs_input_ops.txt fi -export HIRS_FIX=/gpfs/f6/drsa-precip4/proj-shared/Jeffrey.S.Whitaker/hirs_fix +# path to "fixed" crtm coefficient files (needed for crtm < 3) +export HIRS_FIX=${FIXgfs}/gsi/build_gsinfo/hirs_fix # Flag to turn on (.true.) or off (.false.) the infrared cloud and aerosol detection software # for AIRS, CrIS, and IASI. Default is .false. From 339dfdb8507128a494b37ea77c7433ed12614232 Mon Sep 17 00:00:00 2001 From: jswhit2 Date: Tue, 25 Mar 2025 19:02:59 +0000 Subject: [PATCH 12/90] address shell linting errors --- scripts/exglobal_atmos_analysis.sh | 20 ++++++++++---------- scripts/exglobal_diag.sh | 2 +- ush/create_convinfo.sh | 2 +- ush/create_ozinfo.sh | 12 ++++++------ ush/create_satinfo.sh | 10 +++++----- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 3e05e9af085..7f95d73ece4 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -151,11 +151,11 @@ TCVITL=${TCVITL:-${COMIN_OBS}/${OPREFIX}syndata.tcvitals.tm00} B1AVHAM=${B1AVHAM:-${COMIN_OBS}/${OPREFIX}avcsam.tm00.bufr_d${OSUFFIX}} B1AVHPM=${B1AVHPM:-${COMIN_OBS}/${OPREFIX}avcspm.tm00.bufr_d${OSUFFIX}} HDOB=${HDOB:-${COMIN_OBS}/${OPREFIX}hdob.tm00.bufr_d${OSUFFIX}} -OMIEFFNC=${OMIEFFNC:-${COMIN_OBS}/OMIeff-adj.${PDY}_${cyc}z.nc}} -OMPSNMEFFNC=${OMPSNMEFFNC:-${COMIN_OBS}/OMPSNM.${PDY}_${cyc}z.nc}} -OMPSNPNC=${OMPSNPNC:-${COMIN_OBS}/OMPSNP.${PDY}_${cyc}z.nc}} -OMPSLPNC=${OMPSLPNC:-${COMIN_OBS}/OMPS-LPoz-Vis.${PDY}_${cyc}z.nc}} -MLS55NC=${MLS55NC:-${COMIN_OBS}/MLS-v5.0-oz.${PDY}_${cyc}z.nc}} +OMIEFFNC=${OMIEFFNC:-${COMIN_OBS}/OMIeff-adj.${PDY}_${cyc}z.nc} +OMPSNMEFFNC=${OMPSNMEFFNC:-${COMIN_OBS}/OMPSNM.${PDY}_${cyc}z.nc} +OMPSNPNC=${OMPSNPNC:-${COMIN_OBS}/OMPSNP.${PDY}_${cyc}z.nc} +OMPSLPNC=${OMPSLPNC:-${COMIN_OBS}/OMPS-LPoz-Vis.${PDY}_${cyc}z.nc} +MLS55NC=${MLS55NC:-${COMIN_OBS}/MLS-v5.0-oz.${PDY}_${cyc}z.nc} # Guess files GPREFIX=${GPREFIX:-""} @@ -369,7 +369,7 @@ cd ${DATA} || exit 99 # Fixed files ${NLN} ${BERROR} berror_stats ${NLN} ${SATANGL} satbias_angle -if [ $SATINFO == "generate" ]; then +if [[ ${SATINFO} == "generate" ]]; then sh ${SCRgfs}/create_satinfo.sh ${PDY}${cyc} > satinfo else ${NLN} ${SATINFO} satinfo @@ -377,14 +377,14 @@ fi ${NLN} ${RADCLOUDINFO} cloudy_radiance_info.txt ${NLN} ${ATMSFILTER} atms_beamwidth.txt ${NLN} ${ANAVINFO} anavinfo -if [ $CONVINFO == "generate" ]; then +if [[ ${CONVINFO} == "generate" ]]; then sh ${SCRgfs}/create_convinfo.sh ${PDY}${cyc} > convinfo else ${NLN} ${CONVINFO} convinfo fi ${NLN} ${vqcdat} vqctp001.dat ${NLN} ${INSITUINFO} insituinfo -if [ $OZINFO == "generate" ]; then +if [[ ${OZINFO} == "generate" ]]; then sh ${SCRgfs}/create_ozinfo.sh ${PDY}${cyc} > ozinfo else ${NLN} ${OZINFO} ozinfo @@ -433,8 +433,8 @@ fi # CRTM Spectral and Transmittance coefficients mkdir -p crtm_coeffs for file in $(awk '{if($1!~"!"){print $1}}' satinfo | sort | uniq); do - instr=`echo $file | cut -c1-4` - if [ $instr == "hirs" ]; then + instr=$(echo $file | cut -c1-4) + if [[ ${instr} == "hirs" ]]; then ${NLN} ${HIRS_FIX}/${file}.SpcCoeff.bin ./crtm_coeffs/${file}.SpcCoeff.bin else ${NLN} ${CRTM_FIX}/${file}.SpcCoeff.bin ./crtm_coeffs/${file}.SpcCoeff.bin diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh index 311ff4a6f93..56efa5798ee 100755 --- a/scripts/exglobal_diag.sh +++ b/scripts/exglobal_diag.sh @@ -106,7 +106,7 @@ if [ $GENDIAG = "YES" ] ; then diagtype[0]="conv conv_gps conv_ps conv_pw conv_q conv_sst conv_t conv_tcp conv_uv conv_spd" diagtype[1]="pcp_ssmi_dmsp pcp_tmi_trmm" - if [ ${USE_BUILD_GSINFO} == "YES" ]; then + if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then diagtype[2]=$(cat ${FIXgfs}/gsi/build_gsinfo/ozinfo/satellites) diagtype[3]=$(cat ${FIXgfs}/gsi/build_gsinfo/satinfo/satellites) else diff --git a/ush/create_convinfo.sh b/ush/create_convinfo.sh index 6d90d9ba0da..ac5c25aa79b 100644 --- a/ush/create_convinfo.sh +++ b/ush/create_convinfo.sh @@ -1,6 +1,6 @@ #! /usr/bin/env bash # create global_convinfo file for a given date -date=$1 +#date=$1 cd "${FIXgfs}/gsi/build_gsinfo/convinfo" || exit # currently using a single convinfo cat merged_convinfo.txt diff --git a/ush/create_ozinfo.sh b/ush/create_ozinfo.sh index df94714c293..e67ae809f0f 100644 --- a/ush/create_ozinfo.sh +++ b/ush/create_ozinfo.sh @@ -12,18 +12,18 @@ do # find matching date usedate="" for f in "${sat}"/*; do - if [[ $f != "$sat/readme" ]]; then # skip readme file - datex=$(basename $f) - if [[ $date -ge $datex ]]; then - usedate=$datex + if [[ ${f} != "${sat}/readme" ]]; then # skip readme file + datex=$(basename "${f}") + if [[ ${date} -ge ${datex} ]]; then + usedate=${datex} fi fi done # cat matching date file, or quit if date not found if [[ ${usedate} != "" ]]; then - cat "${sat}/$usedate" || exit 1 + cat "${sat}/${usedate}" || exit 1 else - echo "date not found for $sat" + echo "date not found for ${sat}" exit 1 fi done diff --git a/ush/create_satinfo.sh b/ush/create_satinfo.sh index 95b4287de56..872cf1357c8 100644 --- a/ush/create_satinfo.sh +++ b/ush/create_satinfo.sh @@ -10,18 +10,18 @@ do # find matching date usedate="" for f in "${sat}"/*; do - if [[ $f != "${sat}/readme" ]]; then # skip readme file - datex=$(basename $f) + if [[ ${f} != "${sat}/readme" ]]; then # skip readme file + datex=$(basename "${f}") if [[ $date -ge $datex ]]; then - usedate=$datex + usedate=${datex} fi fi done # cat matching date file, or quit if date not found if [[ ${usedate} != "" ]]; then - cat "${sat}/$usedate" || exit 1 + cat "${sat}/${usedate}" || exit 1 else - echo "date not found for $sat" + echo "date not found for ${sat}" exit 1 fi done From 25ab1b0954f8d3b9594b2fcf9edffca528673a75 Mon Sep 17 00:00:00 2001 From: jswhit2 Date: Tue, 25 Mar 2025 19:11:56 +0000 Subject: [PATCH 13/90] address more shell linting errors --- scripts/exglobal_atmos_analysis.sh | 4 ++-- ush/create_ozinfo.sh | 2 +- ush/create_satinfo.sh | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 7f95d73ece4..a5fc1f3688f 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -433,7 +433,7 @@ fi # CRTM Spectral and Transmittance coefficients mkdir -p crtm_coeffs for file in $(awk '{if($1!~"!"){print $1}}' satinfo | sort | uniq); do - instr=$(echo $file | cut -c1-4) + instr=$(echo ${file} | cut -c1-4) if [[ ${instr} == "hirs" ]]; then ${NLN} ${HIRS_FIX}/${file}.SpcCoeff.bin ./crtm_coeffs/${file}.SpcCoeff.bin else @@ -759,7 +759,7 @@ if [ ${DONST} = "YES" ]; then NST="nstinfo=${NSTINFO},fac_dtl=${FAC_DTL},fac_tsl=${FAC_TSL},zsea1=${ZSEA1},zsea2=${ZSEA2},${NST}" fi -OBS_INPUT_TABLE=$(cat ${OBS_INPUT}) +OBS_INPUT_TABLE=$(cat "${OBS_INPUT}") ############################################################## # Create global_gsi namelist diff --git a/ush/create_ozinfo.sh b/ush/create_ozinfo.sh index e67ae809f0f..1679c0f05a3 100644 --- a/ush/create_ozinfo.sh +++ b/ush/create_ozinfo.sh @@ -7,7 +7,7 @@ echo '! sens/instr/sat lev use pressure gross obs b_oz pg_oz' echo '! error error variational qc' # loop over satellites cd "${FIXgfs}/gsi/build_gsinfo/ozinfo" || exit 1 -grep -v '^ *#' satellites | while IFS= read -r sat +grep -v '^ *#' satellites | while IFS= read -r sat || true do # find matching date usedate="" diff --git a/ush/create_satinfo.sh b/ush/create_satinfo.sh index 872cf1357c8..2f0116383ff 100644 --- a/ush/create_satinfo.sh +++ b/ush/create_satinfo.sh @@ -5,14 +5,14 @@ date=$1 echo '!sensor/instr/sat chan iuse error error_cld ermax var_b var_pg icld_det icloud iaerosol' # loop over satellites cd "${FIXgfs}/gsi/build_gsinfo/satinfo" || exit 1 -grep -v '^ *#' satellites | while IFS= read -r sat +grep -v '^ *#' satellites | while IFS= read -r sat || true do # find matching date usedate="" for f in "${sat}"/*; do if [[ ${f} != "${sat}/readme" ]]; then # skip readme file datex=$(basename "${f}") - if [[ $date -ge $datex ]]; then + if [[ ${date} -ge ${datex} ]]; then usedate=${datex} fi fi From 25b787126cd8adc6d7a333e8e71753e097eb0df4 Mon Sep 17 00:00:00 2001 From: jswhit2 Date: Tue, 25 Mar 2025 19:13:51 +0000 Subject: [PATCH 14/90] scripts now in USHgfs --- scripts/exglobal_atmos_analysis.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index a5fc1f3688f..e72eba5ab7f 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -370,7 +370,7 @@ cd ${DATA} || exit 99 ${NLN} ${BERROR} berror_stats ${NLN} ${SATANGL} satbias_angle if [[ ${SATINFO} == "generate" ]]; then - sh ${SCRgfs}/create_satinfo.sh ${PDY}${cyc} > satinfo + sh ${USHgfs}/create_satinfo.sh ${PDY}${cyc} > satinfo else ${NLN} ${SATINFO} satinfo fi @@ -378,14 +378,14 @@ ${NLN} ${RADCLOUDINFO} cloudy_radiance_info.txt ${NLN} ${ATMSFILTER} atms_beamwidth.txt ${NLN} ${ANAVINFO} anavinfo if [[ ${CONVINFO} == "generate" ]]; then - sh ${SCRgfs}/create_convinfo.sh ${PDY}${cyc} > convinfo + sh ${USHgfs}/create_convinfo.sh ${PDY}${cyc} > convinfo else ${NLN} ${CONVINFO} convinfo fi ${NLN} ${vqcdat} vqctp001.dat ${NLN} ${INSITUINFO} insituinfo if [[ ${OZINFO} == "generate" ]]; then - sh ${SCRgfs}/create_ozinfo.sh ${PDY}${cyc} > ozinfo + sh ${USHgfs}/create_ozinfo.sh ${PDY}${cyc} > ozinfo else ${NLN} ${OZINFO} ozinfo fi From 11ad3f0fae44a6dae689045cde82a0714ba36b12 Mon Sep 17 00:00:00 2001 From: "Jeffrey.S.Whitaker" Date: Tue, 25 Mar 2025 15:33:05 -0400 Subject: [PATCH 15/90] add check for existence of satellites file --- ush/create_ozinfo.sh | 3 ++- ush/create_satinfo.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ush/create_ozinfo.sh b/ush/create_ozinfo.sh index 1679c0f05a3..5c03c36495a 100644 --- a/ush/create_ozinfo.sh +++ b/ush/create_ozinfo.sh @@ -7,7 +7,8 @@ echo '! sens/instr/sat lev use pressure gross obs b_oz pg_oz' echo '! error error variational qc' # loop over satellites cd "${FIXgfs}/gsi/build_gsinfo/ozinfo" || exit 1 -grep -v '^ *#' satellites | while IFS= read -r sat || true +[ ! -f satellites ] && echo "satellites file not found!" || exit 1 +grep -v '^ *#' satellites | while IFS= read -r sat do # find matching date usedate="" diff --git a/ush/create_satinfo.sh b/ush/create_satinfo.sh index 2f0116383ff..85e586be946 100644 --- a/ush/create_satinfo.sh +++ b/ush/create_satinfo.sh @@ -5,7 +5,8 @@ date=$1 echo '!sensor/instr/sat chan iuse error error_cld ermax var_b var_pg icld_det icloud iaerosol' # loop over satellites cd "${FIXgfs}/gsi/build_gsinfo/satinfo" || exit 1 -grep -v '^ *#' satellites | while IFS= read -r sat || true +[ ! -f satellites ] && echo "satellites file not found!" || exit 1 +grep -v '^ *#' satellites | while IFS= read -r sat do # find matching date usedate="" From 1dd34500cda72a5c723db0b13d4de00a282e3bb1 Mon Sep 17 00:00:00 2001 From: "Jeffrey.S.Whitaker" Date: Tue, 25 Mar 2025 15:35:13 -0400 Subject: [PATCH 16/90] update --- ush/create_ozinfo.sh | 2 +- ush/create_satinfo.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/create_ozinfo.sh b/ush/create_ozinfo.sh index 5c03c36495a..8db1dc4e10d 100644 --- a/ush/create_ozinfo.sh +++ b/ush/create_ozinfo.sh @@ -7,7 +7,7 @@ echo '! sens/instr/sat lev use pressure gross obs b_oz pg_oz' echo '! error error variational qc' # loop over satellites cd "${FIXgfs}/gsi/build_gsinfo/ozinfo" || exit 1 -[ ! -f satellites ] && echo "satellites file not found!" || exit 1 +[[ ! -f satellites ]] && echo "satellites file not found!" || exit 1 grep -v '^ *#' satellites | while IFS= read -r sat do # find matching date diff --git a/ush/create_satinfo.sh b/ush/create_satinfo.sh index 85e586be946..3134d5c07fe 100644 --- a/ush/create_satinfo.sh +++ b/ush/create_satinfo.sh @@ -5,7 +5,7 @@ date=$1 echo '!sensor/instr/sat chan iuse error error_cld ermax var_b var_pg icld_det icloud iaerosol' # loop over satellites cd "${FIXgfs}/gsi/build_gsinfo/satinfo" || exit 1 -[ ! -f satellites ] && echo "satellites file not found!" || exit 1 +[[ ! -f satellites ]] && echo "satellites file not found!" || exit 1 grep -v '^ *#' satellites | while IFS= read -r sat do # find matching date From ac22e7d3bfd17a9ec17ec4c817680d157e5262bd Mon Sep 17 00:00:00 2001 From: "Jeffrey.S.Whitaker" Date: Tue, 25 Mar 2025 20:14:26 -0400 Subject: [PATCH 17/90] update --- ush/create_ozinfo.sh | 1 - ush/create_satinfo.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/ush/create_ozinfo.sh b/ush/create_ozinfo.sh index 8db1dc4e10d..5c4a24c485b 100644 --- a/ush/create_ozinfo.sh +++ b/ush/create_ozinfo.sh @@ -7,7 +7,6 @@ echo '! sens/instr/sat lev use pressure gross obs b_oz pg_oz' echo '! error error variational qc' # loop over satellites cd "${FIXgfs}/gsi/build_gsinfo/ozinfo" || exit 1 -[[ ! -f satellites ]] && echo "satellites file not found!" || exit 1 grep -v '^ *#' satellites | while IFS= read -r sat do # find matching date diff --git a/ush/create_satinfo.sh b/ush/create_satinfo.sh index 3134d5c07fe..2c133187774 100644 --- a/ush/create_satinfo.sh +++ b/ush/create_satinfo.sh @@ -5,7 +5,6 @@ date=$1 echo '!sensor/instr/sat chan iuse error error_cld ermax var_b var_pg icld_det icloud iaerosol' # loop over satellites cd "${FIXgfs}/gsi/build_gsinfo/satinfo" || exit 1 -[[ ! -f satellites ]] && echo "satellites file not found!" || exit 1 grep -v '^ *#' satellites | while IFS= read -r sat do # find matching date From e572cd41b76ed7be800e13c0030ac1b4ec87344b Mon Sep 17 00:00:00 2001 From: Jeffrey Whitaker Date: Fri, 23 May 2025 10:07:04 -0600 Subject: [PATCH 18/90] make executable, use BUILD_GSINFO_DIR, remove tabs --- ush/create_convinfo.sh | 2 +- ush/create_ozinfo.sh | 6 +++--- ush/create_satinfo.sh | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) mode change 100644 => 100755 ush/create_convinfo.sh mode change 100644 => 100755 ush/create_ozinfo.sh mode change 100644 => 100755 ush/create_satinfo.sh diff --git a/ush/create_convinfo.sh b/ush/create_convinfo.sh old mode 100644 new mode 100755 index ac5c25aa79b..efcbc6d56d6 --- a/ush/create_convinfo.sh +++ b/ush/create_convinfo.sh @@ -1,7 +1,7 @@ #! /usr/bin/env bash # create global_convinfo file for a given date #date=$1 -cd "${FIXgfs}/gsi/build_gsinfo/convinfo" || exit +cd "${BUILD_GSINFO_DIR}/convinfo" || exit # currently using a single convinfo cat merged_convinfo.txt #usedate="" diff --git a/ush/create_ozinfo.sh b/ush/create_ozinfo.sh old mode 100644 new mode 100755 index 5c4a24c485b..cc228e81b0d --- a/ush/create_ozinfo.sh +++ b/ush/create_ozinfo.sh @@ -6,7 +6,7 @@ echo '! For mls data, pressure and obs errors are pulled from bufr, so not liste echo '! sens/instr/sat lev use pressure gross obs b_oz pg_oz' echo '! error error variational qc' # loop over satellites -cd "${FIXgfs}/gsi/build_gsinfo/ozinfo" || exit 1 +cd "${BUILD_GSINFO_DIR}/ozinfo" || exit 1 grep -v '^ *#' satellites | while IFS= read -r sat do # find matching date @@ -14,7 +14,7 @@ do for f in "${sat}"/*; do if [[ ${f} != "${sat}/readme" ]]; then # skip readme file datex=$(basename "${f}") - if [[ ${date} -ge ${datex} ]]; then + if [[ ${date} -ge ${datex} ]]; then usedate=${datex} fi fi @@ -24,6 +24,6 @@ do cat "${sat}/${usedate}" || exit 1 else echo "date not found for ${sat}" - exit 1 + exit 1 fi done diff --git a/ush/create_satinfo.sh b/ush/create_satinfo.sh old mode 100644 new mode 100755 index 2c133187774..d22275e6e3f --- a/ush/create_satinfo.sh +++ b/ush/create_satinfo.sh @@ -4,7 +4,7 @@ date=$1 # header echo '!sensor/instr/sat chan iuse error error_cld ermax var_b var_pg icld_det icloud iaerosol' # loop over satellites -cd "${FIXgfs}/gsi/build_gsinfo/satinfo" || exit 1 +cd "${BUILD_GSINFO_DIR}/satinfo" || exit 1 grep -v '^ *#' satellites | while IFS= read -r sat do # find matching date @@ -12,7 +12,7 @@ do for f in "${sat}"/*; do if [[ ${f} != "${sat}/readme" ]]; then # skip readme file datex=$(basename "${f}") - if [[ ${date} -ge ${datex} ]]; then + if [[ ${date} -ge ${datex} ]]; then usedate=${datex} fi fi @@ -22,6 +22,6 @@ do cat "${sat}/${usedate}" || exit 1 else echo "date not found for ${sat}" - exit 1 + exit 1 fi done From 9fbe816cceeddff52b8db18ef42b5afbae02f7ff Mon Sep 17 00:00:00 2001 From: Jeffrey Whitaker Date: Fri, 23 May 2025 10:07:35 -0600 Subject: [PATCH 19/90] use BUILD_GSINFO_DIR --- parm/config/gfs/config.anal | 6 +++--- parm/config/gfs/config.base.j2 | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/parm/config/gfs/config.anal b/parm/config/gfs/config.anal index ad43570e7b6..6824206f070 100644 --- a/parm/config/gfs/config.anal +++ b/parm/config/gfs/config.anal @@ -153,12 +153,12 @@ if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then export SATINFO="generate" export CONVINFO="generate" export OZINFO="generate" - export OBS_INPUT=${FIXgfs}/gsi/build_gsinfo/obs_input/obs_input_reanl_nasa_ozone.txt + export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_reanl_nasa_ozone.txt" else - export OBS_INPUT=${FIXgfs}/gsi/build_gsinfo/obs_input/obs_input_ops.txt + export OBS_INPUT="${BUILD_GSINFO_DIR}obs_input/obs_input_ops.txt" fi # path to "fixed" crtm coefficient files (needed for crtm < 3) -export HIRS_FIX=${FIXgfs}/gsi/build_gsinfo/hirs_fix +export HIRS_FIX="${BUILD_GSINFO_DIR}/hirs_fix" # Flag to turn on (.true.) or off (.false.) the infrared cloud and aerosol detection software # for AIRS, CrIS, and IASI. Default is .false. diff --git a/parm/config/gfs/config.base.j2 b/parm/config/gfs/config.base.j2 index bcf4a01cf09..2ba229bb9fa 100644 --- a/parm/config/gfs/config.base.j2 +++ b/parm/config/gfs/config.base.j2 @@ -385,6 +385,7 @@ export l4densvar=".true." export lwrite4danl=".true." export DO_CALC_INCREMENT="NO" export USE_BUILD_GSINFO="NO" +export BUILD_GSINFO_DIR="${FIXgfs}/gsi/fix/build_gsinfo" # Early-cycle EnKF parameters export NMEM_ENS_GFS="{{ NMEM_ENS_GFS }}" From 1e53c6809bb39c497f9106f7074211294092d954 Mon Sep 17 00:00:00 2001 From: Jeffrey Whitaker Date: Fri, 23 May 2025 10:09:24 -0600 Subject: [PATCH 20/90] create_*info.sh now executable. --- scripts/exglobal_atmos_analysis.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index c2e2ed3c090..3f9eaff7225 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -305,7 +305,7 @@ AEROINFO=${AEROINFO:-${FIXgfs}/gsi/global_aeroinfo.txt} SCANINFO=${SCANINFO:-${FIXgfs}/gsi/global_scaninfo.txt} HYBENSINFO=${HYBENSINFO:-${FIXgfs}/gsi/global_hybens_info.l${LEVS}.txt} OBERROR=${OBERROR:-${FIXgfs}/gsi/prepobs_errtable.global} -OBS_INPUT=${OBS_INPUT:-${FIXgfs}/gsi/build_gsinfo/obs_input_ops.txt} +OBS_INPUT=${OBS_INPUT:-${BUILD_GSINFO_DIR}/obs_input/obs_input_ops.txt} HIRS_FIX=${HIRS_FIX:-${CRTM_FIX}} # GSI namelist @@ -368,7 +368,7 @@ cd ${DATA} || exit 99 ${NLN} ${BERROR} berror_stats ${NLN} ${SATANGL} satbias_angle if [[ ${SATINFO} == "generate" ]]; then - sh ${USHgfs}/create_satinfo.sh ${PDY}${cyc} > satinfo + ${USHgfs}/create_satinfo.sh ${PDY}${cyc} > satinfo else ${NLN} ${SATINFO} satinfo fi @@ -376,14 +376,14 @@ ${NLN} ${RADCLOUDINFO} cloudy_radiance_info.txt ${NLN} ${ATMSFILTER} atms_beamwidth.txt ${NLN} ${ANAVINFO} anavinfo if [[ ${CONVINFO} == "generate" ]]; then - sh ${USHgfs}/create_convinfo.sh ${PDY}${cyc} > convinfo + ${USHgfs}/create_convinfo.sh ${PDY}${cyc} > convinfo else ${NLN} ${CONVINFO} convinfo fi ${NLN} ${vqcdat} vqctp001.dat ${NLN} ${INSITUINFO} insituinfo if [[ ${OZINFO} == "generate" ]]; then - sh ${USHgfs}/create_ozinfo.sh ${PDY}${cyc} > ozinfo + ${USHgfs}/create_ozinfo.sh ${PDY}${cyc} > ozinfo else ${NLN} ${OZINFO} ozinfo fi From e2529349cbd601e31bb84918ca423debbc7d75bb Mon Sep 17 00:00:00 2001 From: Jeffrey Whitaker Date: Fri, 23 May 2025 10:09:44 -0600 Subject: [PATCH 21/90] use OBS_INPUT_DIR --- scripts/exglobal_diag.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh index 64d8790389e..d0aafca9300 100755 --- a/scripts/exglobal_diag.sh +++ b/scripts/exglobal_diag.sh @@ -105,8 +105,8 @@ if [ $GENDIAG = "YES" ] ; then diagtype[0]="conv conv_gps conv_ps conv_pw conv_q conv_sst conv_t conv_tcp conv_uv conv_spd" diagtype[1]="pcp_ssmi_dmsp pcp_tmi_trmm" if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then - diagtype[2]=$(cat ${FIXgfs}/gsi/build_gsinfo/ozinfo/satellites) - diagtype[3]=$(cat ${FIXgfs}/gsi/build_gsinfo/satinfo/satellites) + diagtype[2]=$(cat ${BUILD_GSINFO_DIR}/ozinfo/satellites) + diagtype[3]=$(cat ${BUILD_GSINFO_DIR}/satinfo/satellites) else diagtype[2]="sbuv2_n16 sbuv2_n17 sbuv2_n18 sbuv2_n19 gome_metop-a gome_metop-b omi_aura mls30_aura ompsnp_npp ompstc8_npp ompstc8_n20 ompsnp_n20 ompstc8_n21 ompsnp_n21 ompslp_npp gome_metop-c" diagtype[3]="msu_n14 sndr_g08 sndr_g11 sndr_g12 sndr_g13 sndr_g08_prep sndr_g11_prep sndr_g12_prep sndr_g13_prep sndrd1_g11 sndrd2_g11 sndrd3_g11 sndrd4_g11 sndrd1_g12 sndrd2_g12 sndrd3_g12 sndrd4_g12 sndrd1_g13 sndrd2_g13 sndrd3_g13 sndrd4_g13 sndrd1_g14 sndrd2_g14 sndrd3_g14 sndrd4_g14 sndrd1_g15 sndrd2_g15 sndrd3_g15 sndrd4_g15 amsua_n15 amsua_n16 amsua_n17 amsub_n15 amsub_n16 amsub_n17 hsb_aqua airs_aqua amsua_aqua imgr_g08 imgr_g11 imgr_g12 imgr_g14 imgr_g15 ssmi_f13 ssmi_f15 amsua_n18 amsua_metop-a mhs_n18 mhs_metop-a amsre_low_aqua amsre_mid_aqua amsre_hig_aqua ssmis_f16 ssmis_f17 ssmis_f18 ssmis_f19 ssmis_f20 iasi_metop-a amsua_n19 mhs_n19 seviri_m08 seviri_m09 seviri_m10 seviri_m11 cris_npp cris-fsr_npp cris-fsr_n20 atms_npp atms_n20 amsua_metop-b mhs_metop-b iasi_metop-b avhrr_metop-b avhrr_n18 avhrr_n19 avhrr_metop-a amsr2_gcom-w1 gmi_gpm saphir_meghat ahi_himawari8 abi_g16 abi_g17 amsua_metop-c mhs_metop-c iasi_metop-c avhrr_metop-c viirs-m_npp viirs-m_j1 abi_g18 ahi_himawari9 viirs-m_j2 cris-fsr_n21 atms_n21" From 2814d4ff2803add94b144250421e38e094a22856 Mon Sep 17 00:00:00 2001 From: Jeffrey Whitaker Date: Fri, 23 May 2025 10:17:40 -0600 Subject: [PATCH 22/90] fix typo --- parm/config/gfs/config.anal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.anal b/parm/config/gfs/config.anal index 6824206f070..902b107712f 100644 --- a/parm/config/gfs/config.anal +++ b/parm/config/gfs/config.anal @@ -155,7 +155,7 @@ if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then export OZINFO="generate" export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_reanl_nasa_ozone.txt" else - export OBS_INPUT="${BUILD_GSINFO_DIR}obs_input/obs_input_ops.txt" + export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_ops.txt" fi # path to "fixed" crtm coefficient files (needed for crtm < 3) export HIRS_FIX="${BUILD_GSINFO_DIR}/hirs_fix" From b27e1d9b71812e0b66db0dc1e0d5311f0748a25b Mon Sep 17 00:00:00 2001 From: Jeffrey Whitaker Date: Fri, 23 May 2025 10:21:30 -0600 Subject: [PATCH 23/90] add ignore_2mQM --- scripts/exglobal_atmos_analysis.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 3f9eaff7225..164d076d7d2 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -84,6 +84,7 @@ SENDDBN=${SENDDBN:-"NO"} export gesenvir=${gesenvir:-${envir}} export hofx_2m_sfcfile=${hofx_2m_sfcfile:-".false."} +export ignore_2mQM=${ignore_2mQM:-".false."} # Observations OPREFIX=${OPREFIX:-""} @@ -828,7 +829,7 @@ cat > gsiparm.anl << EOF / &OBS_INPUT dmesh(1)=145.0,dmesh(2)=150.0,dmesh(3)=100.0,dmesh(4)=50.0,time_window_max=3.0, - hofx_2m_sfcfile=${hofx_2m_sfcfile}, + hofx_2m_sfcfile=${hofx_2m_sfcfile},ignore_2mQM=${ignore_2mQM}, ${OBSINPUT} / OBS_INPUT:: From 7e256990f06720677388ac35364f6382c741dc7f Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 2 Jun 2025 09:43:34 -0400 Subject: [PATCH 24/90] Update scripts/exglobal_diag.sh --- scripts/exglobal_diag.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh index 26366f992fc..c543bac9c7e 100755 --- a/scripts/exglobal_diag.sh +++ b/scripts/exglobal_diag.sh @@ -84,8 +84,8 @@ if [[ "${GENDIAG}" == "YES" ]] ; then diagtype[0]="conv conv_gps conv_ps conv_pw conv_q conv_sst conv_t conv_tcp conv_uv conv_spd" diagtype[1]="pcp_ssmi_dmsp pcp_tmi_trmm" if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then - diagtype[2]=$(cat ${BUILD_GSINFO_DIR}/ozinfo/satellites) - diagtype[3]=$(cat ${BUILD_GSINFO_DIR}/satinfo/satellites) + diagtype[2]=$(cat ${BUILD_GSINFO_DIR}/ozinfo/satellites) + diagtype[3]=$(cat ${BUILD_GSINFO_DIR}/satinfo/satellites) else diagtype[2]="sbuv2_n16 sbuv2_n17 sbuv2_n18 sbuv2_n19 gome_metop-a gome_metop-b omi_aura mls30_aura ompsnp_npp ompstc8_npp ompstc8_n20 ompsnp_n20 ompstc8_n21 ompsnp_n21 ompslp_npp gome_metop-c" diagtype[3]="msu_n14 sndr_g08 sndr_g11 sndr_g12 sndr_g13 sndr_g08_prep sndr_g11_prep sndr_g12_prep sndr_g13_prep sndrd1_g11 sndrd2_g11 sndrd3_g11 sndrd4_g11 sndrd1_g12 sndrd2_g12 sndrd3_g12 sndrd4_g12 sndrd1_g13 sndrd2_g13 sndrd3_g13 sndrd4_g13 sndrd1_g14 sndrd2_g14 sndrd3_g14 sndrd4_g14 sndrd1_g15 sndrd2_g15 sndrd3_g15 sndrd4_g15 amsua_n15 amsua_n16 amsua_n17 amsub_n15 amsub_n16 amsub_n17 hsb_aqua airs_aqua amsua_aqua imgr_g08 imgr_g11 imgr_g12 imgr_g14 imgr_g15 ssmi_f13 ssmi_f15 amsua_n18 amsua_metop-a mhs_n18 mhs_metop-a amsre_low_aqua amsre_mid_aqua amsre_hig_aqua ssmis_f16 ssmis_f17 ssmis_f18 ssmis_f19 ssmis_f20 iasi_metop-a amsua_n19 mhs_n19 seviri_m08 seviri_m09 seviri_m10 seviri_m11 cris_npp cris-fsr_npp cris-fsr_n20 atms_npp atms_n20 amsua_metop-b mhs_metop-b iasi_metop-b avhrr_metop-b avhrr_n18 avhrr_n19 avhrr_metop-a amsr2_gcom-w1 gmi_gpm saphir_meghat ahi_himawari8 abi_g16 abi_g17 amsua_metop-c mhs_metop-c iasi_metop-c avhrr_metop-c viirs-m_npp viirs-m_j1 abi_g18 ahi_himawari9 viirs-m_j2 cris-fsr_n21 atms_n21" From 1700e8665f199c4fa33e0910fecfc95c60899664 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 2 Jun 2025 09:43:51 -0400 Subject: [PATCH 25/90] Update ush/create_satinfo.sh --- ush/create_satinfo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/create_satinfo.sh b/ush/create_satinfo.sh index d22275e6e3f..e68f45841fe 100755 --- a/ush/create_satinfo.sh +++ b/ush/create_satinfo.sh @@ -1,6 +1,6 @@ #! /usr/bin/env bash # create global_satinfo file for a given date -date=$1 +date_in=$1 # header echo '!sensor/instr/sat chan iuse error error_cld ermax var_b var_pg icld_det icloud iaerosol' # loop over satellites From 124ea8d6fe7caf2dfb1605af11c19528f45d5d65 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 2 Jun 2025 09:45:05 -0400 Subject: [PATCH 26/90] Update ush/create_ozinfo.sh --- ush/create_ozinfo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/create_ozinfo.sh b/ush/create_ozinfo.sh index cc228e81b0d..bd922122830 100755 --- a/ush/create_ozinfo.sh +++ b/ush/create_ozinfo.sh @@ -1,6 +1,6 @@ #! /usr/bin/env bash # create global_ozinfo file for a given date -date=$1 +date_in=$1 # header echo '! For mls data, pressure and obs errors are pulled from bufr, so not listed here' echo '! sens/instr/sat lev use pressure gross obs b_oz pg_oz' From 90ea7185ccb0cfa50c4407629223002b73c52032 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 2 Jun 2025 09:45:18 -0400 Subject: [PATCH 27/90] Update ush/create_ozinfo.sh --- ush/create_ozinfo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/create_ozinfo.sh b/ush/create_ozinfo.sh index bd922122830..38f357abac0 100755 --- a/ush/create_ozinfo.sh +++ b/ush/create_ozinfo.sh @@ -14,7 +14,7 @@ do for f in "${sat}"/*; do if [[ ${f} != "${sat}/readme" ]]; then # skip readme file datex=$(basename "${f}") - if [[ ${date} -ge ${datex} ]]; then + if [[ ${date_in} -ge ${datex} ]]; then usedate=${datex} fi fi From 3232b113a9e4138ea9c11469ef74bd1200aeae70 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 2 Jun 2025 09:45:27 -0400 Subject: [PATCH 28/90] Update ush/create_satinfo.sh --- ush/create_satinfo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/create_satinfo.sh b/ush/create_satinfo.sh index e68f45841fe..e4076e6a8b1 100755 --- a/ush/create_satinfo.sh +++ b/ush/create_satinfo.sh @@ -12,7 +12,7 @@ do for f in "${sat}"/*; do if [[ ${f} != "${sat}/readme" ]]; then # skip readme file datex=$(basename "${f}") - if [[ ${date} -ge ${datex} ]]; then + if [[ ${date_in} -ge ${datex} ]]; then usedate=${datex} fi fi From 88df316aced535e88cab3f3ef5e1a8c90a1c4b05 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 2 Jun 2025 11:11:26 -0400 Subject: [PATCH 29/90] Add double quotes to variables in exglobal_atmos_analysis.sh --- scripts/exglobal_atmos_analysis.sh | 34 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 9269d4344d0..81caa2cb65f 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -57,7 +57,7 @@ ta2tb=${ta2tb:-".true."} netcdf_diag=${netcdf_diag:-".true."} binary_diag=${binary_diag:-".false."} lobsdiag_forenkf=${lobsdiag_forenkf:-".false."} - + # IAU DOIAU=${DOIAU:-"NO"} export IAUFHRS=${IAUFHRS:-"6,"} @@ -82,7 +82,7 @@ RUN=${RUN:-""} SENDECF=${SENDECF:-"NO"} SENDDBN=${SENDDBN:-"NO"} export gesenvir=${gesenvir:-${envir}} - + export hofx_2m_sfcfile=${hofx_2m_sfcfile:-".false."} export ignore_2mQM=${ignore_2mQM:-".false."} @@ -725,22 +725,22 @@ EOFunzip else fname=$(echo ${diag_file} | cut -d'.' -f1) date=$(echo ${diag_file} | cut -d'.' -f2) - ${UNCOMPRESS} ${diag_file} - fnameges=$(echo ${fname}|sed 's/_ges//g') - ${NMV} ${fname}.${date}${DIAG_SUFFIX} ${fnameges} + ${UNCOMPRESS} "${diag_file}" + fnameges=$(echo "${fname}"|sed 's/_ges//g') + ${NMV} "${fname}.${date}${DIAG_SUFFIX}" "${fnameges}" fi done if [[ "${USE_CFP}" == "YES" ]] ; then - chmod 755 ${DATA}/mp_unzip.sh - ncmd=$(cat ${DATA}/mp_unzip.sh | wc -l) + chmod 755 "${DATA}/mp_unzip.sh" + ncmd=$(cat "${DATA}/mp_unzip.sh" | wc -l) if [[ ${ncmd} -gt 0 ]]; then if [[ ${ncmd} -lt ${max_tasks_per_node} ]]; then ncmd_max=${ncmd} else ncmd_max=${max_tasks_per_node} fi - APRUNCFP_UNZIP=$(eval echo ${APRUNCFP}) + APRUNCFP_UNZIP=$(eval echo "${APRUNCFP}") ${APRUNCFP_UNZIP} "${DATA}/mp_unzip.sh" export err=$? if [[ ${err} -ne 0 ]]; then @@ -888,8 +888,8 @@ export OMP_NUM_THREADS=${NTHREADS_GSI} export pgm=${GSIEXEC} . prep_step -cpreq ${GSIEXEC} ${DATA} -${APRUN_GSI} ${DATA}/$(basename ${GSIEXEC}) 1>&1 2>&2 +cpreq "${GSIEXEC}" "${DATA}" +${APRUN_GSI} "${DATA}/$(basename "${GSIEXEC}")" 1>&1 2>&2 export err=$? if [[ ${err} -ne 0 ]]; then err_exit "Failed to run the GSI analysis!" @@ -911,13 +911,13 @@ fi ############################################################## # For eupd if [[ -s satbias_out.int ]]; then - cpfs satbias_out.int ${ABIASe} + cpfs satbias_out.int "${ABIASe}" else - cpfs satbias_in ${ABIASe} + cpfs satbias_in "${ABIASe}" fi # Cat runtime output files. -cat fort.2* > ${GSISTAT} +cat fort.2* > "${GSISTAT}" # If requested, create obsinput tarball from obs_input.* files if [[ ${RUN_SELECT} == "YES" ]]; then @@ -925,11 +925,11 @@ if [[ ${RUN_SELECT} == "YES" ]]; then if [[ -s obsinput.tar ]]; then rm -f obsinput.tar fi - ${NLN} ${SELECT_OBS} obsinput.tar + ${NLN} "${SELECT_OBS}" obsinput.tar ${CHGRP_CMD} obs_input.* tar -cvf obsinput.tar obs_input.* - chmod 750 ${SELECT_OBS} - ${CHGRP_CMD} ${SELECT_OBS} + chmod 750 "${SELECT_OBS}" + ${CHGRP_CMD} "${SELECT_OBS}" rm -f obsinput.tar echo $(date) END tar obs_input >&2 fi @@ -938,7 +938,7 @@ fi # Send alerts if [[ ${SENDDBN} == "YES" ]]; then if [[ ${RUN} == "gfs" ]]; then - ${DBNROOT}/bin/dbn_alert MODEL GFS_abias ${job} ${ABIAS} + "${DBNROOT}/bin/dbn_alert" MODEL GFS_abias "${job}" "${ABIAS}" fi fi From 72aac5c53e26d40558b07f9e6926d39586eeb516 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 2 Jun 2025 11:15:08 -0400 Subject: [PATCH 30/90] Add double quotes to variables in exglobal_diag.sh --- scripts/exglobal_diag.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh index c543bac9c7e..45837a6801e 100755 --- a/scripts/exglobal_diag.sh +++ b/scripts/exglobal_diag.sh @@ -201,7 +201,7 @@ EOFdiag # If requested, compress diagnostic files if [[ "${DIAG_COMPRESS}" == "YES" && "${USE_CFP}" == "NO" ]]; then echo $(date) START "${COMPRESS}" diagnostic files >&2 - for file in $(ls diag_*${CDATE}${DIAG_SUFFIX}); do + for file in $(ls "diag_*${CDATE}${DIAG_SUFFIX}"); do ${COMPRESS} "${file}" done echo "$(date) END ${COMPRESS} diagnostic files" >&2 @@ -246,7 +246,7 @@ EOFdiag TAROPTS="-cvf" fi if [[ ${numfile[n]} -gt 0 ]]; then - tar ${TAROPTS} "${diagfile[n]}" $(cat ${diaglist[n]}) + tar ${TAROPTS} "${diagfile[n]}" $(cat "${diaglist[n]}") export err=$? if [[ ${err} -ne 0 ]]; then err_exit "Unable to create ${diagfile[n]}!" From 96cbc66a5022d34cac81c9a512642f8b3c6b20cf Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 2 Jun 2025 11:21:04 -0400 Subject: [PATCH 31/90] Convert unneeded cat to < operator --- scripts/exglobal_atmos_analysis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 81caa2cb65f..70d2c38bab6 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -733,7 +733,7 @@ EOFunzip if [[ "${USE_CFP}" == "YES" ]] ; then chmod 755 "${DATA}/mp_unzip.sh" - ncmd=$(cat "${DATA}/mp_unzip.sh" | wc -l) + ncmd=$(wc -l < "${DATA}/mp_unzip.sh") if [[ ${ncmd} -gt 0 ]]; then if [[ ${ncmd} -lt ${max_tasks_per_node} ]]; then ncmd_max=${ncmd} From a7b1b2216bf4b4921e542e22d7c1fa20970249e5 Mon Sep 17 00:00:00 2001 From: Jeffrey Whitaker Date: Mon, 15 Sep 2025 18:54:34 -0600 Subject: [PATCH 32/90] update scripts as suggested in PR, have create_convinfo turn on 2m obs if desired --- ush/create_convinfo.sh | 46 +++++++++++++++++++++++++++--------------- ush/create_ozinfo.sh | 4 ++-- ush/create_satinfo.sh | 4 ++-- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/ush/create_convinfo.sh b/ush/create_convinfo.sh index efcbc6d56d6..03e913a6f7f 100755 --- a/ush/create_convinfo.sh +++ b/ush/create_convinfo.sh @@ -1,19 +1,33 @@ #! /usr/bin/env bash # create global_convinfo file for a given date -#date=$1 +date=$1 +# 2m ob use file (DO_GSISOILDA) +if [ -z ${2} ]; then + use2mobs="NO" # NO if not present +else + use2mobs=$2 +fi cd "${BUILD_GSINFO_DIR}/convinfo" || exit -# currently using a single convinfo -cat merged_convinfo.txt -#usedate="" -#for datex in [1-2]*; do -# if [[ ${date} -ge ${datex} ]]; then -# usedate=$datex -# fi -#done -## cat matching date file, or quit if date not found -#if [[ ${usedate} != "" ]]; then -# cat ${usedate} -#else -# echo "date not found" -# exit 1 -#fi +usedate="" +for datex in [1-2]*; do + if [[ ${date} -ge ${datex} ]]; then + usedate=$datex + fi +done +# cat matching date file, or quit if date not found +if [[ ${usedate} != "" ]]; then + if [[ ${use2mobs} == "YES" ]]; then + # turn on 2m t,q obs over land + sed -e "s/t 181 0 -1/t 181 0 1/g" \ + -e "s/t 183 0 -1/t 183 0 1/g" \ + -e "s/t 187 0 -1/t 187 0 1/g" \ + -e "s/q 181 0 -1/q 181 0 1/g" \ + -e "s/q 183 0 -1/q 183 0 1/g" \ + -e "s/q 187 0 -1/q 187 0 1/g" ${usedate} + else + cat ${usedate} + fi +else + echo "date not found" + exit 1 +fi diff --git a/ush/create_ozinfo.sh b/ush/create_ozinfo.sh index 38f357abac0..829fb1f0087 100755 --- a/ush/create_ozinfo.sh +++ b/ush/create_ozinfo.sh @@ -7,7 +7,7 @@ echo '! sens/instr/sat lev use pressure gross obs b_oz pg_oz' echo '! error error variational qc' # loop over satellites cd "${BUILD_GSINFO_DIR}/ozinfo" || exit 1 -grep -v '^ *#' satellites | while IFS= read -r sat +while IFS= read -r sat do # find matching date usedate="" @@ -26,4 +26,4 @@ do echo "date not found for ${sat}" exit 1 fi -done +done < <(grep -v '^ *#' satellites) diff --git a/ush/create_satinfo.sh b/ush/create_satinfo.sh index e4076e6a8b1..15bcda37628 100755 --- a/ush/create_satinfo.sh +++ b/ush/create_satinfo.sh @@ -5,7 +5,7 @@ date_in=$1 echo '!sensor/instr/sat chan iuse error error_cld ermax var_b var_pg icld_det icloud iaerosol' # loop over satellites cd "${BUILD_GSINFO_DIR}/satinfo" || exit 1 -grep -v '^ *#' satellites | while IFS= read -r sat +while IFS= read -r sat do # find matching date usedate="" @@ -24,4 +24,4 @@ do echo "date not found for ${sat}" exit 1 fi -done +done < <(grep -v '^ *#' satellites) From a34a23c9a7c83de68ac16643fb9b8ad792bc01d1 Mon Sep 17 00:00:00 2001 From: Jeffrey Whitaker Date: Mon, 15 Sep 2025 19:21:47 -0600 Subject: [PATCH 33/90] use ncep ozone by default --- dev/parm/config/gfs/config.anal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/parm/config/gfs/config.anal b/dev/parm/config/gfs/config.anal index 2e5ae5c2bf3..917b0ee0126 100644 --- a/dev/parm/config/gfs/config.anal +++ b/dev/parm/config/gfs/config.anal @@ -155,7 +155,7 @@ if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then export SATINFO="generate" export CONVINFO="generate" export OZINFO="generate" - export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_reanl_nasa_ozone.txt" + export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_reanl_ncep_ozone.txt" else export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_ops.txt" fi From a3db7fa58f5eb670918617aac80e917937f23046 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Tue, 16 Sep 2025 17:38:44 +0000 Subject: [PATCH 34/90] Address shellcheck issues. --- ush/create_convinfo.sh | 8 ++++---- ush/create_ozinfo.sh | 2 +- ush/create_satinfo.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ush/create_convinfo.sh b/ush/create_convinfo.sh index 03e913a6f7f..f929cc1f6c5 100755 --- a/ush/create_convinfo.sh +++ b/ush/create_convinfo.sh @@ -2,7 +2,7 @@ # create global_convinfo file for a given date date=$1 # 2m ob use file (DO_GSISOILDA) -if [ -z ${2} ]; then +if [[ $# -ge 2 && -z "${2}" ]]; then use2mobs="NO" # NO if not present else use2mobs=$2 @@ -11,7 +11,7 @@ cd "${BUILD_GSINFO_DIR}/convinfo" || exit usedate="" for datex in [1-2]*; do if [[ ${date} -ge ${datex} ]]; then - usedate=$datex + usedate=${datex} fi done # cat matching date file, or quit if date not found @@ -23,9 +23,9 @@ if [[ ${usedate} != "" ]]; then -e "s/t 187 0 -1/t 187 0 1/g" \ -e "s/q 181 0 -1/q 181 0 1/g" \ -e "s/q 183 0 -1/q 183 0 1/g" \ - -e "s/q 187 0 -1/q 187 0 1/g" ${usedate} + -e "s/q 187 0 -1/q 187 0 1/g" "${usedate}" else - cat ${usedate} + cat "${usedate}" fi else echo "date not found" diff --git a/ush/create_ozinfo.sh b/ush/create_ozinfo.sh index 829fb1f0087..336ea44dbeb 100755 --- a/ush/create_ozinfo.sh +++ b/ush/create_ozinfo.sh @@ -26,4 +26,4 @@ do echo "date not found for ${sat}" exit 1 fi -done < <(grep -v '^ *#' satellites) +done < <(grep -v '^ *#' satellites || true) diff --git a/ush/create_satinfo.sh b/ush/create_satinfo.sh index 15bcda37628..21c92ffe019 100755 --- a/ush/create_satinfo.sh +++ b/ush/create_satinfo.sh @@ -24,4 +24,4 @@ do echo "date not found for ${sat}" exit 1 fi -done < <(grep -v '^ *#' satellites) +done < <(grep -v '^ *#' satellites || true) From 518e89110b749e5f0b5ea5bb229d33c97442ace6 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Tue, 16 Sep 2025 18:00:49 +0000 Subject: [PATCH 35/90] More shellcheck issues. --- scripts/exglobal_atmos_analysis.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 3cdb4421138..99f7cd9a97f 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -723,8 +723,8 @@ EOFunzip nm=$((nm+1)) fi else - fname=$(echo ${diag_file} | cut -d'.' -f1) - date=$(echo ${diag_file} | cut -d'.' -f2) + fname=$(echo "${diag_file}" | cut -d'.' -f1) + date=$(echo "${diag_file}" | cut -d'.' -f2) ${UNCOMPRESS} "${diag_file}" fnameges=$(echo "${fname}"|sed 's/_ges//g') ${NMV} "${fname}.${date}${DIAG_SUFFIX}" "${fnameges}" From 57ca8ed08a4e9761b35d74be6060581c5cc99a35 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 16 Sep 2025 13:53:00 -0500 Subject: [PATCH 36/90] Address shellcheck issues throughout the touched scripts --- scripts/exglobal_atmos_analysis.sh | 258 ++++++++++++++--------------- scripts/exglobal_diag.sh | 8 +- 2 files changed, 133 insertions(+), 133 deletions(-) diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 99f7cd9a97f..4166f19d47a 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -115,7 +115,7 @@ ESIASI=${ESIASI:-${COMIN_OBS}/${OPREFIX}esiasi.tm00.bufr_d${OSUFFIX}} IASIDB=${IASIDB:-${COMIN_OBS}/${OPREFIX}iasidb.tm00.bufr_d${OSUFFIX}} AMSREBF=${AMSREBF:-${COMIN_OBS}/${OPREFIX}amsre.tm00.bufr_d${OSUFFIX}} AMSR2BF=${AMSR2BF:-${COMIN_OBS}/${OPREFIX}amsr2.tm00.bufr_d${OSUFFIX}} -GMI1CRBF=${GMI1CRBF:-${COMIN_OBS}/${OPREFIX}gmi1cr.tm00.bufr_d${OSUFFIX}} +GMI1CRBF=${GMI1CRBF:-${COMIN_OBS}/${OPREFIX}gmi1cr.tm00.bufr_d${OSUFFIX}} SAPHIRBF=${SAPHIRBF:-${COMIN_OBS}/${OPREFIX}saphir.tm00.bufr_d${OSUFFIX}} SEVIRIBF=${SEVIRIBF:-${COMIN_OBS}/${OPREFIX}sevcsr.tm00.bufr_d${OSUFFIX}} AHIBF=${AHIBF:-${COMIN_OBS}/${OPREFIX}ahicsr.tm00.bufr_d${OSUFFIX}} @@ -367,7 +367,7 @@ fi ${NLN} ${BERROR} berror_stats ${NLN} ${SATANGL} satbias_angle if [[ ${SATINFO} == "generate" ]]; then - ${USHgfs}/create_satinfo.sh ${PDY}${cyc} > satinfo + ${USHgfs}/create_satinfo.sh "${PDY}${cyc}" > satinfo else ${NLN} ${SATINFO} satinfo fi @@ -375,27 +375,27 @@ ${NLN} ${RADCLOUDINFO} cloudy_radiance_info.txt ${NLN} ${ATMSFILTER} atms_beamwidth.txt ${NLN} ${ANAVINFO} anavinfo if [[ ${CONVINFO} == "generate" ]]; then - ${USHgfs}/create_convinfo.sh ${PDY}${cyc} > convinfo + ${USHgfs}/create_convinfo.sh "${PDY}${cyc}" > convinfo else - ${NLN} ${CONVINFO} convinfo + ${NLN} "${CONVINFO}" convinfo fi -${NLN} ${vqcdat} vqctp001.dat -${NLN} ${INSITUINFO} insituinfo +${NLN} "${vqcdat}" vqctp001.dat +${NLN} "${INSITUINFO}" insituinfo if [[ ${OZINFO} == "generate" ]]; then ${USHgfs}/create_ozinfo.sh ${PDY}${cyc} > ozinfo else - ${NLN} ${OZINFO} ozinfo + ${NLN} "${OZINFO}" ozinfo fi -${NLN} ${PCPINFO} pcpinfo -${NLN} ${AEROINFO} aeroinfo -${NLN} ${SCANINFO} scaninfo -${NLN} ${HYBENSINFO} hybens_info -${NLN} ${OBERROR} errtable -${NLN} ${BLACKLST} blacklist +${NLN} "${PCPINFO}" pcpinfo +${NLN} "${AEROINFO}" aeroinfo +${NLN} "${SCANINFO}" scaninfo +${NLN} "${HYBENSINFO}" hybens_info +${NLN} "${OBERROR}" errtable +${NLN} "${BLACKLST}" blacklist -${NLN} ${FIXgfs}/gsi/AIRS_CLDDET.NL AIRS_CLDDET.NL -${NLN} ${FIXgfs}/gsi/CRIS_CLDDET.NL CRIS_CLDDET.NL -${NLN} ${FIXgfs}/gsi/IASI_CLDDET.NL IASI_CLDDET.NL +${NLN} "${FIXgfs}/gsi/AIRS_CLDDET.NL" AIRS_CLDDET.NL +${NLN} "${FIXgfs}/gsi/CRIS_CLDDET.NL" CRIS_CLDDET.NL +${NLN} "${FIXgfs}/gsi/IASI_CLDDET.NL" IASI_CLDDET.NL #If using correlated error, link to the covariance files if [[ ${USE_CORRELATED_OBERRS} == "YES" ]]; then @@ -433,24 +433,24 @@ mkdir -p crtm_coeffs for file in $(awk '{if($1!~"!"){print $1}}' satinfo | sort | uniq); do instr=$(echo ${file} | cut -c1-4) if [[ ${instr} == "hirs" ]]; then - ${NLN} ${HIRS_FIX}/${file}.SpcCoeff.bin ./crtm_coeffs/${file}.SpcCoeff.bin + ${NLN} "${HIRS_FIX}/${file}.SpcCoeff.bin" "./crtm_coeffs/${file}.SpcCoeff.bin" else - ${NLN} ${CRTM_FIX}/${file}.SpcCoeff.bin ./crtm_coeffs/${file}.SpcCoeff.bin + ${NLN} "${CRTM_FIX}/${file}.SpcCoeff.bin" "./crtm_coeffs/${file}.SpcCoeff.bin" fi - ${NLN} ${CRTM_FIX}/${file}.TauCoeff.bin ./crtm_coeffs/${file}.TauCoeff.bin + ${NLN} "${CRTM_FIX}/${file}.TauCoeff.bin" "./crtm_coeffs/${file}.TauCoeff.bin" done -${NLN} ${CRTM_FIX}/amsua_metop-a_v2.SpcCoeff.bin ./crtm_coeffs/amsua_metop-a_v2.SpcCoeff.bin - -${NLN} ${CRTM_FIX}/Nalli.IRwater.EmisCoeff.bin ./crtm_coeffs/Nalli.IRwater.EmisCoeff.bin -${NLN} ${CRTM_FIX}/NPOESS.IRice.EmisCoeff.bin ./crtm_coeffs/NPOESS.IRice.EmisCoeff.bin -${NLN} ${CRTM_FIX}/NPOESS.IRland.EmisCoeff.bin ./crtm_coeffs/NPOESS.IRland.EmisCoeff.bin -${NLN} ${CRTM_FIX}/NPOESS.IRsnow.EmisCoeff.bin ./crtm_coeffs/NPOESS.IRsnow.EmisCoeff.bin -${NLN} ${CRTM_FIX}/NPOESS.VISice.EmisCoeff.bin ./crtm_coeffs/NPOESS.VISice.EmisCoeff.bin -${NLN} ${CRTM_FIX}/NPOESS.VISland.EmisCoeff.bin ./crtm_coeffs/NPOESS.VISland.EmisCoeff.bin -${NLN} ${CRTM_FIX}/NPOESS.VISsnow.EmisCoeff.bin ./crtm_coeffs/NPOESS.VISsnow.EmisCoeff.bin -${NLN} ${CRTM_FIX}/NPOESS.VISwater.EmisCoeff.bin ./crtm_coeffs/NPOESS.VISwater.EmisCoeff.bin -${NLN} ${CRTM_FIX}/FASTEM6.MWwater.EmisCoeff.bin ./crtm_coeffs/FASTEM6.MWwater.EmisCoeff.bin -${NLN} ${CRTM_FIX}/AerosolCoeff.bin ./crtm_coeffs/AerosolCoeff.bin +${NLN} "${CRTM_FIX}/amsua_metop-a_v2.SpcCoeff.bin" "./crtm_coeffs/amsua_metop-a_v2.SpcCoeff.bin" + +${NLN} "${CRTM_FIX}/Nalli.IRwater.EmisCoeff.bin" "./crtm_coeffs/Nalli.IRwater.EmisCoeff.bin" +${NLN} "${CRTM_FIX}/NPOESS.IRice.EmisCoeff.bin" "./crtm_coeffs/NPOESS.IRice.EmisCoeff.bin" +${NLN} "${CRTM_FIX}/NPOESS.IRland.EmisCoeff.bin" "./crtm_coeffs/NPOESS.IRland.EmisCoeff.bin" +${NLN} "${CRTM_FIX}/NPOESS.IRsnow.EmisCoeff.bin" "./crtm_coeffs/NPOESS.IRsnow.EmisCoeff.bin" +${NLN} "${CRTM_FIX}/NPOESS.VISice.EmisCoeff.bin" "./crtm_coeffs/NPOESS.VISice.EmisCoeff.bin" +${NLN} "${CRTM_FIX}/NPOESS.VISland.EmisCoeff.bin" "./crtm_coeffs/NPOESS.VISland.EmisCoeff.bin" +${NLN} "${CRTM_FIX}/NPOESS.VISsnow.EmisCoeff.bin" "./crtm_coeffs/NPOESS.VISsnow.EmisCoeff.bin" +${NLN} "${CRTM_FIX}/NPOESS.VISwater.EmisCoeff.bin" "./crtm_coeffs/NPOESS.VISwater.EmisCoeff.bin" +${NLN} "${CRTM_FIX}/FASTEM6.MWwater.EmisCoeff.bin" "./crtm_coeffs/FASTEM6.MWwater.EmisCoeff.bin" +${NLN} "${CRTM_FIX}/AerosolCoeff.bin" "./crtm_coeffs/AerosolCoeff.bin" if (( imp_physics == 8 )); then echo "using CRTM Thompson cloud optical table" ${NLN} "${CRTM_FIX}/CloudCoeff.Thompson08.-109z-1.bin" ./crtm_coeffs/CloudCoeff.bin @@ -466,74 +466,74 @@ fi ############################################################## # Observational data -${NLN} ${PREPQC} prepbufr -${NLN} ${PREPQCPF} prepbufr_profl -${NLN} ${SATWND} satwndbufr -${NLN} ${OSCATBF} oscatbufr -${NLN} ${RAPIDSCATBF} rapidscatbufr -${NLN} ${GSNDBF} gsndrbufr -${NLN} ${GSNDBF1} gsnd1bufr -${NLN} ${B1MSU} msubufr -${NLN} ${B1AMUA} amsuabufr -${NLN} ${B1AMUB} amsubbufr -${NLN} ${B1MHS} mhsbufr -${NLN} ${B1HRS2} hirs2bufr -${NLN} ${B1HRS3} hirs3bufr -${NLN} ${B1HRS4} hirs4bufr -${NLN} ${ESAMUA} amsuabufrears -${NLN} ${ESAMUB} amsubbufrears -#$NLN $ESMHS mhsbufrears -${NLN} ${AMUADB} amsuabufr_db -${NLN} ${AMUBDB} amsubbufr_db -#$NLN $MHSDB mhsbufr_db -${NLN} ${SBUVBF} sbuvbufr -${NLN} ${OMPSNPBF} ompsnpbufr -${NLN} ${OMPSLPBF} ompslpbufr -${NLN} ${OMPSTCBF} ompstcbufr -${NLN} ${GOMEBF} gomebufr -${NLN} ${OMIBF} omibufr -${NLN} ${MLSBF} mlsbufr -${NLN} ${SMIPCP} ssmirrbufr -${NLN} ${TMIPCP} tmirrbufr -${NLN} ${AIRSBF} airsbufr -${NLN} ${IASIBF} iasibufr -${NLN} ${ESIASI} iasibufrears -${NLN} ${IASIDB} iasibufr_db -${NLN} ${AMSREBF} amsrebufr -${NLN} ${AMSR2BF} amsr2bufr -${NLN} ${GMI1CRBF} gmibufr -${NLN} ${SAPHIRBF} saphirbufr -${NLN} ${SEVIRIBF} seviribufr -${NLN} ${CRISBF} crisbufr -${NLN} ${ESCRIS} crisbufrears -${NLN} ${CRISDB} crisbufr_db -${NLN} ${CRISFSBF} crisfsbufr -${NLN} ${ESCRISFS} crisfsbufrears -${NLN} ${CRISFSDB} crisfsbufr_db -${NLN} ${ATMSBF} atmsbufr -${NLN} ${ESATMS} atmsbufrears -${NLN} ${ATMSDB} atmsbufr_db -${NLN} ${SSMITBF} ssmitbufr -${NLN} ${SSMISBF} ssmisbufr -${NLN} ${GPSROBF} gpsrobufr -${NLN} ${TCVITL} tcvitl -${NLN} ${B1AVHAM} avhambufr -${NLN} ${B1AVHPM} avhpmbufr -${NLN} ${AHIBF} ahibufr -${NLN} ${ABIBF} abibufr -${NLN} ${HDOB} hdobbufr -${NLN} ${SSTVIIRS} sstviirs -${NLN} ${SAILDRONE} sdbufr -${NLN} ${GSBBF} wbbufr +${NLN} "${PREPQC}" prepbufr +${NLN} "${PREPQCPF}" prepbufr_profl +${NLN} "${SATWND}" satwndbufr +${NLN} "${OSCATBF}" oscatbufr +${NLN} "${RAPIDSCATBF}" rapidscatbufr +${NLN} "${GSNDBF}" gsndrbufr +${NLN} "${GSNDBF1}" gsnd1bufr +${NLN} "${B1MSU}" msubufr +${NLN} "${B1AMUA}" amsuabufr +${NLN} "${B1AMUB}" amsubbufr +${NLN} "${B1MHS}" mhsbufr +${NLN} "${B1HRS2}" hirs2bufr +${NLN} "${B1HRS3}" hirs3bufr +${NLN} "${B1HRS4}" hirs4bufr +${NLN} "${ESAMUA}" amsuabufrears +${NLN} "${ESAMUB}" amsubbufrears +#${NLN} "${ESMHS}" mhsbufrears +${NLN} "${AMUADB}" amsuabufr_db +${NLN} "${AMUBDB}" amsubbufr_db +#${NLN} "${MHSDB}" mhsbufr_db +${NLN} "${SBUVBF}" sbuvbufr +${NLN} "${OMPSNPBF}" ompsnpbufr +${NLN} "${OMPSLPBF}" ompslpbufr +${NLN} "${OMPSTCBF}" ompstcbufr +${NLN} "${GOMEBF}" gomebufr +${NLN} "${OMIBF}" omibufr +${NLN} "${MLSBF}" mlsbufr +${NLN} "${SMIPCP}" ssmirrbufr +${NLN} "${TMIPCP}" tmirrbufr +${NLN} "${AIRSBF}" airsbufr +${NLN} "${IASIBF}" iasibufr +${NLN} "${ESIASI}" iasibufrears +${NLN} "${IASIDB}" iasibufr_db +${NLN} "${AMSREBF}" amsrebufr +${NLN} "${AMSR2BF}" amsr2bufr +${NLN} "${GMI1CRBF}" gmibufr +${NLN} "${SAPHIRBF}" saphirbufr +${NLN} "${SEVIRIBF}" seviribufr +${NLN} "${CRISBF}" crisbufr +${NLN} "${ESCRIS}" crisbufrears +${NLN} "${CRISDB}" crisbufr_db +${NLN} "${CRISFSBF}" crisfsbufr +${NLN} "${ESCRISFS}" crisfsbufrears +${NLN} "${CRISFSDB}" crisfsbufr_db +${NLN} "${ATMSBF}" atmsbufr +${NLN} "${ESATMS}" atmsbufrears +${NLN} "${ATMSDB}" atmsbufr_db +${NLN} "${SSMITBF}" ssmitbufr +${NLN} "${SSMISBF}" ssmisbufr +${NLN} "${GPSROBF}" gpsrobufr +${NLN} "${TCVITL}" tcvitl +${NLN} "${B1AVHAM}" avhambufr +${NLN} "${B1AVHPM}" avhpmbufr +${NLN} "${AHIBF}" ahibufr +${NLN} "${ABIBF}" abibufr +${NLN} "${HDOB}" hdobbufr +${NLN} "${SSTVIIRS}" sstviirs +${NLN} "${SAILDRONE}" sdbufr +${NLN} "${GSBBF}" wbbufr # NASA ozone (netcdf) from NNJA -${NLN} ${OMIEFFNC} omieffnc -${NLN} ${OMPSNMEFFNC} ompsnmeffnc -${NLN} ${OMPSNPNC} ompsnpnc -${NLN} ${OMPSLPNC} ompslpnc -${NLN} ${MLS55NC} mls55nc +${NLN} "${OMIEFFNC}" omieffnc +${NLN} "${OMPSNMEFFNC}" ompsnmeffnc +${NLN} "${OMPSNPNC}" ompsnpnc +${NLN} "${OMPSLPNC}" ompslpnc +${NLN} "${MLS55NC}" mls55nc # NASA airs aqua amsua (bufr) from NNJA -${NLN} ${AQUAAMUA} aquabufr +${NLN} "${AQUAAMUA}" aquabufr if [[ "${DONST}" == "YES" ]]; then ${NLN} "${NSSTBF}" nsstbufr @@ -541,20 +541,20 @@ fi ############################################################## # Required bias guess files -${NLN} ${GBIAS} satbias_in -${NLN} ${GBIASPC} satbias_pc -${NLN} ${GBIASAIR} aircftbias_in -${NLN} ${GRADSTAT} radstat.gdas +${NLN} "${GBIAS}" satbias_in +${NLN} "${GBIASPC}" satbias_pc +${NLN} "${GBIASAIR}" aircftbias_in +${NLN} "${GRADSTAT}" radstat.gdas ############################################################## # Required model guess files -${NLN} ${ATMG03} sigf03 -${NLN} ${ATMGES} sigf06 -${NLN} ${ATMG09} sigf09 +${NLN} "${ATMG03}" sigf03 +${NLN} "${ATMGES}" sigf06 +${NLN} "${ATMG09}" sigf09 -${NLN} ${SFCG03} sfcf03 -${NLN} ${SFCGES} sfcf06 -${NLN} ${SFCG09} sfcf09 +${NLN} "${SFCG03}" sfcf03 +${NLN} "${SFCGES}" sfcf06 +${NLN} "${SFCG09}" sfcf09 if [[ -f "${ATMG04}" ]]; then ${NLN} "${ATMG04}" sigf04 @@ -604,9 +604,9 @@ if [[ "${DOHYBVAR}" == "YES" ]]; then COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL for fhr in ${fhrs}; do - ${NLN} ${COMIN_ATMOS_HISTORY}/${GPREFIX_ENS}atmf0${fhr}${ENKF_SUFFIX}.nc ./ensemble_data/sigf${fhr}_ens_${memchar} + ${NLN} "${COMIN_ATMOS_HISTORY}/${GPREFIX_ENS}atmf0${fhr}${ENKF_SUFFIX}.nc" "./ensemble_data/sigf${fhr}_ens_${memchar}" if [[ ${cnvw_option} == ".true." ]]; then - ${NLN} ${COMIN_ATMOS_HISTORY}/${GPREFIX_ENS}sfcf0${fhr}.nc ./ensemble_data/sfcf${fhr}_ens_${memchar} + ${NLN} "${COMIN_ATMOS_HISTORY}/${GPREFIX_ENS}sfcf0${fhr}.nc" "./ensemble_data/sfcf${fhr}_ens_${memchar}" fi done done @@ -620,7 +620,7 @@ if [[ ${JCAP} -ne ${JCAP_A} ]]; then if [[ ${DOHYBVAR} == "YES" && ${JCAP_A} == "${JCAP_ENKF}" ]]; then if [[ -e ${SFCGES_ENSMEAN} ]]; then USE_READIN_ANL_SFCMASK=.true. - ${NLN} ${SFCGES_ENSMEAN} sfcf06_anlgrid + ${NLN} "${SFCGES_ENSMEAN}" sfcf06_anlgrid else echo "Warning: Inconsistent sfc mask between analysis and ensemble grids, GSI will interpolate" fi @@ -635,13 +635,13 @@ fi if [[ ${GENDIAG} == "YES" ]] ; then if [[ ${lrun_subdirs} == ".true." ]] ; then if [[ -d ${DIAG_DIR} ]]; then - rm -rf ${DIAG_DIR} + rm -rf "${DIAG_DIR}" fi ntasks_m1="$((ntasks-1))" for pe in $(seq 0 ${ntasks_m1}); do pedir="dir."$(printf %04i ${pe}) - mkdir -p ${DIAG_DIR}/${pedir} - ${NLN} ${DIAG_DIR}/${pedir} ${pedir} + mkdir -p "${DIAG_DIR}/${pedir}" + ${NLN} "${DIAG_DIR}/${pedir}" "${pedir}" done else err_exit "lrun_subdirs must be true. lrun_subdirs=${lrun_subdirs}" @@ -653,21 +653,21 @@ fi ${NLN} ${ATMANL} siganl ${NLN} ${ATMINC} siginc.nc if [[ ${DOHYBVAR} == "YES" && ${l4densvar} == ".true." && ${lwrite4danl} == ".true." ]]; then - ${NLN} ${ATMA03} siga03 - ${NLN} ${ATMI03} sigi03.nc - ${NLN} ${ATMA04} siga04 - ${NLN} ${ATMI04} sigi04.nc - ${NLN} ${ATMA05} siga05 - ${NLN} ${ATMI05} sigi05.nc - ${NLN} ${ATMA07} siga07 - ${NLN} ${ATMI07} sigi07.nc - ${NLN} ${ATMA08} siga08 - ${NLN} ${ATMI08} sigi08.nc - ${NLN} ${ATMA09} siga09 - ${NLN} ${ATMI09} sigi09.nc -fi -${NLN} ${ABIAS} satbias_out -${NLN} ${ABIASPC} satbias_pc.out + ${NLN} "${ATMA03}" siga03 + ${NLN} "${ATMI03}" sigi03.nc + ${NLN} "${ATMA04}" siga04 + ${NLN} "${ATMI04}" sigi04.nc + ${NLN} "${ATMA05}" siga05 + ${NLN} "${ATMI05}" sigi05.nc + ${NLN} "${ATMA07}" siga07 + ${NLN} "${ATMI07}" sigi07.nc + ${NLN} "${ATMA08}" siga08 + ${NLN} "${ATMI08}" sigi08.nc + ${NLN} "${ATMA09}" siga09 + ${NLN} "${ATMI09}" sigi09.nc +fi +${NLN} ${ABIAS}" satbias_out +${NLN} ${ABIASPC}" satbias_pc.out ${NLN} ${ABIASAIR} aircftbias_out if [[ ${DONST} == "YES" ]]; then @@ -686,7 +686,7 @@ if [[ ${USE_SELECT} == "YES" ]]; then else for filetop in $(ls ${SELECT_OBS}/obs_input.*); do fileloc=$(basename ${filetop}) - ${NLN} ${filetop} ${fileloc} + ${NLN} "${filetop}" "${fileloc}" done fi fi @@ -718,7 +718,7 @@ EOFunzip for type in ${listdiag}; do diag_file=$(echo ${type} | cut -d',' -f1) if [[ ${USE_CFP} == "YES" ]] ; then - echo "${nm} ${DATA}/unzip.sh ${diag_file} ${DIAG_SUFFIX}" | tee -a ${DATA}/mp_unzip.sh + echo "${nm} ${DATA}/unzip.sh ${diag_file} ${DIAG_SUFFIX}" | tee -a "${DATA}/mp_unzip.sh" if [[ ${CFP_MP:-"NO"} == "YES" ]]; then nm=$((nm+1)) fi diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh index 9edc73706b6..c471f9c63ee 100755 --- a/scripts/exglobal_diag.sh +++ b/scripts/exglobal_diag.sh @@ -181,14 +181,14 @@ EOFdiag ${CATEXEC} -o "diag_${type}_${string}.${PDY}${cyc}${DIAG_SUFFIX}" "${prefix}${type}_${loop}"* fi fi - echo "diag_${type}_${string}.${PDY}${cyc}*" >> ${diaglist[n]} + echo "diag_${type}_${string}.${PDY}${cyc}*" >> "${diaglist[n]}" numfile[n]=$(expr ${numfile[n]} + 1) elif [[ ${count} -eq 1 ]]; then - cat ${prefix}${type}_${loop}* > "diag_${type}_${string}.${PDY}${cyc}${DIAG_SUFFIX}" + cat "${prefix}${type}_${loop}"* > "diag_${type}_${string}.${PDY}${cyc}${DIAG_SUFFIX}" if [[ "${DIAG_COMPRESS}" == "YES" ]]; then ${COMPRESS} "diag_${type}_${string}.${PDY}${cyc}${DIAG_SUFFIX}" fi - echo diag_${type}_${string}.${PDY}${cyc}* >> ${diaglist[n]} + echo "diag_${type}_${string}.${PDY}${cyc}"* >> "${diaglist[n]}" numfile[n]=$(expr "${numfile[n]}" + 1) fi done @@ -203,7 +203,7 @@ EOFdiag if [[ "${DIAG_COMPRESS}" == "YES" && "${USE_CFP}" == "NO" ]]; then echo $(date) START "${COMPRESS}" diagnostic files >&2 # shellcheck disable=SC2086 - for file in $(ls diag_*${PDY}${cyc}${DIAG_SUFFIX}); do + for file in "diag_"*"${PDY}${cyc}${DIAG_SUFFIX}"; do ${COMPRESS} "${file}" done echo "$(date) END ${COMPRESS} diagnostic files" >&2 From 23fd05575719aa096de02fec89635c4ff60fd15a Mon Sep 17 00:00:00 2001 From: Jeffrey Whitaker Date: Tue, 16 Sep 2025 14:14:05 -0600 Subject: [PATCH 37/90] update --- dev/parm/config/gfs/config.anal | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dev/parm/config/gfs/config.anal b/dev/parm/config/gfs/config.anal index 917b0ee0126..b94e1b9525d 100644 --- a/dev/parm/config/gfs/config.anal +++ b/dev/parm/config/gfs/config.anal @@ -155,7 +155,12 @@ if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then export SATINFO="generate" export CONVINFO="generate" export OZINFO="generate" - export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_reanl_ncep_ozone.txt" + if [[ "${PDY}${cyc}" -lt "2022052712" ]]; then + # use NASA ozone files for reanalysis before 2022052712 + export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_reanl_nasa_ozone.txt" + else # after 2022052712 use NCEP ozone bufr with hoices from gfsv17_historical + export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_reanl_ncep_ozone.txt" + fi else export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_ops.txt" fi From adc0c69892dd917cff54e152f47c231cf313aeb1 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 17 Sep 2025 09:52:52 -0500 Subject: [PATCH 38/90] Make a generic script to handle *stat files --- dev/parm/config/gfs/config.anal | 2 + scripts/exglobal_atmos_analysis.sh | 6 +- ush/create_gsi_info.sh | 124 +++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 3 deletions(-) create mode 100644 ush/create_gsi_info.sh diff --git a/dev/parm/config/gfs/config.anal b/dev/parm/config/gfs/config.anal index 917b0ee0126..42c52bb705f 100644 --- a/dev/parm/config/gfs/config.anal +++ b/dev/parm/config/gfs/config.anal @@ -156,6 +156,8 @@ if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then export CONVINFO="generate" export OZINFO="generate" export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_reanl_ncep_ozone.txt" + # Use 2m observations when constructing conventional obs info file + export BUILD_2M_OBS="NO" else export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_ops.txt" fi diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 4166f19d47a..46ceac76faa 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -367,7 +367,7 @@ fi ${NLN} ${BERROR} berror_stats ${NLN} ${SATANGL} satbias_angle if [[ ${SATINFO} == "generate" ]]; then - ${USHgfs}/create_satinfo.sh "${PDY}${cyc}" > satinfo + ${USHgfs}/create_gsi_info.sh sat "${PDY}${cyc}" else ${NLN} ${SATINFO} satinfo fi @@ -375,14 +375,14 @@ ${NLN} ${RADCLOUDINFO} cloudy_radiance_info.txt ${NLN} ${ATMSFILTER} atms_beamwidth.txt ${NLN} ${ANAVINFO} anavinfo if [[ ${CONVINFO} == "generate" ]]; then - ${USHgfs}/create_convinfo.sh "${PDY}${cyc}" > convinfo + ${USHgfs}/create_gsi_info.sh conv "${PDY}${cyc}" "${USE_2M_OBS}" else ${NLN} "${CONVINFO}" convinfo fi ${NLN} "${vqcdat}" vqctp001.dat ${NLN} "${INSITUINFO}" insituinfo if [[ ${OZINFO} == "generate" ]]; then - ${USHgfs}/create_ozinfo.sh ${PDY}${cyc} > ozinfo + ${USHgfs}/create_gsi_info.sh oz "${PDY}${cyc}" else ${NLN} "${OZINFO}" ozinfo fi diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh new file mode 100644 index 00000000000..35e01007dcd --- /dev/null +++ b/ush/create_gsi_info.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# Generic script to create convinfo, ozinfo, or satinfo for a given date. +# Usage: +# ./create_info.sh [use2mobs] +# : conv, oz, or sat +# : date string to match +# [use2mobs]: (optional, only for conv) YES or NO + +set -e + +type_in=${1:-} +date_in=${2:-} +use2mobs=${3:-NO} + +if [[ -z "${type_in}" || -z "${date_in}" ]]; then + echo "Usage: ${0} [use2mobs]" + echo " : conv, oz, or sat" + echo " : date string to match" + echo " [use2mobs]: (optional, only for conv, whether to use 2m observations) YES or NO" + exit 1 +fi + +# Function to get the most recent data available for the target obs. +# If an empty string is returned, this represents an error. +get_usedate() { + usedate="" + # Loop over files matching date pattern. + for datex in [1-2]*; do + # Skip for loop if there are no matches. + if [[ ! -e "${datex}" ]]; then + continue + fi + + if [[ ${date_in} -ge ${datex} ]]; then + usedate=${datex} + fi + done + + echo "${usedate}" +} + +# Filename to write the info to +info_file="${PWD}/${type_in}info" +if [[ -f "${info_file}" ]]; then + rm -f "${info_file}" +fi + +# Get the starting directory +starting_dir=${PWD} + +case "${type_in}" in + conv) + cd "${BUILD_GSINFO_DIR}/convinfo" || exit 1 + + usedate=$(get_usedate) + if [[ ${usedate} != "" ]]; then + if [[ ${use2mobs} == "YES" ]]; then + # Turn on 2m t,q obs over land + sed -e "s/t 181 0 -1/t 181 0 1/g" \ + -e "s/t 183 0 -1/t 183 0 1/g" \ + -e "s/t 187 0 -1/t 187 0 1/g" \ + -e "s/q 181 0 -1/q 181 0 1/g" \ + -e "s/q 183 0 -1/q 183 0 1/g" \ + -e "s/q 187 0 -1/q 187 0 1/g" "${usedate}" >> "${info_file}" + else + cat "${usedate}" >> "${info_file}" + fi + else + echo "ERROR: No valid conventional info was found!" + exit 1 + fi + ;; + oz) + cd "${BUILD_GSINFO_DIR}/ozinfo" || exit 1 + usedate=$(get_usedate) + if [[ ${usedate} != "" ]]; then + cat "${usedate}" >> "${info_file}" + else + echo "ERROR: No valid ozone info was found!" + exit 1 + fi + ;; + sat) + cd "${BUILD_GSINFO_DIR}/satinfo" || exit 1 + # Read in the satellite list, ignoring comment lines and the readme + satellite_list=$(grep -v '^ *#\|^readme' satellites) + if [[ -z "${satellite_list}" ]]; then + echo "ERROR: No satellites found in the satellite file list!" + exit 1 + fi + + # Header line + echo '!sensor/instr/sat chan iuse error error_cld ermax var_b var_pg icld_det icloud iaerosol' >> "${info_file}" + while IFS= read -r sat + do + usedate="" + # Check that the satellite directory exists + if [[ ! -d "${sat}" ]]; then + echo "ERROR: Directory ${sat} does not exist!" + exit 1 + fi + + cd "${sat}" || exit 1 + + usedate=$(get_usedate) + + cd "${BUILD_GSINFO_DIR}/satinfo" || exit 1 + + if [[ ${usedate} != "" ]]; then + cat "${sat}/${usedate}" >> "${info_file}" + else + echo "ERROR: No valid satellite info was found for satellite target '${sat}'!" + exit 1 + fi + done <<< "${satellite_list}" + ;; + *) + echo "ERROR: Unknown info file type: '${type_in}'. Must be one of: conv, oz, sat" + exit 2 + ;; +esac + +# Return to starting directory +cd "${starting_dir}" || exit 1 From 18933ca308da03973c5c33b0d82725abf6ba9ba9 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 17 Sep 2025 09:56:26 -0500 Subject: [PATCH 39/90] Make create_gsi_info.sh executable --- ush/create_gsi_info.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ush/create_gsi_info.sh diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh old mode 100644 new mode 100755 From a5d769a5c2d2274a381d8bd3009e6717b091724d Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 17 Sep 2025 09:57:14 -0500 Subject: [PATCH 40/90] Remove now-unused create_*info.sh files --- ush/create_convinfo.sh | 33 --------------------------------- ush/create_ozinfo.sh | 29 ----------------------------- ush/create_satinfo.sh | 27 --------------------------- 3 files changed, 89 deletions(-) delete mode 100755 ush/create_convinfo.sh delete mode 100755 ush/create_ozinfo.sh delete mode 100755 ush/create_satinfo.sh diff --git a/ush/create_convinfo.sh b/ush/create_convinfo.sh deleted file mode 100755 index f929cc1f6c5..00000000000 --- a/ush/create_convinfo.sh +++ /dev/null @@ -1,33 +0,0 @@ -#! /usr/bin/env bash -# create global_convinfo file for a given date -date=$1 -# 2m ob use file (DO_GSISOILDA) -if [[ $# -ge 2 && -z "${2}" ]]; then - use2mobs="NO" # NO if not present -else - use2mobs=$2 -fi -cd "${BUILD_GSINFO_DIR}/convinfo" || exit -usedate="" -for datex in [1-2]*; do - if [[ ${date} -ge ${datex} ]]; then - usedate=${datex} - fi -done -# cat matching date file, or quit if date not found -if [[ ${usedate} != "" ]]; then - if [[ ${use2mobs} == "YES" ]]; then - # turn on 2m t,q obs over land - sed -e "s/t 181 0 -1/t 181 0 1/g" \ - -e "s/t 183 0 -1/t 183 0 1/g" \ - -e "s/t 187 0 -1/t 187 0 1/g" \ - -e "s/q 181 0 -1/q 181 0 1/g" \ - -e "s/q 183 0 -1/q 183 0 1/g" \ - -e "s/q 187 0 -1/q 187 0 1/g" "${usedate}" - else - cat "${usedate}" - fi -else - echo "date not found" - exit 1 -fi diff --git a/ush/create_ozinfo.sh b/ush/create_ozinfo.sh deleted file mode 100755 index 336ea44dbeb..00000000000 --- a/ush/create_ozinfo.sh +++ /dev/null @@ -1,29 +0,0 @@ -#! /usr/bin/env bash -# create global_ozinfo file for a given date -date_in=$1 -# header -echo '! For mls data, pressure and obs errors are pulled from bufr, so not listed here' -echo '! sens/instr/sat lev use pressure gross obs b_oz pg_oz' -echo '! error error variational qc' -# loop over satellites -cd "${BUILD_GSINFO_DIR}/ozinfo" || exit 1 -while IFS= read -r sat -do - # find matching date - usedate="" - for f in "${sat}"/*; do - if [[ ${f} != "${sat}/readme" ]]; then # skip readme file - datex=$(basename "${f}") - if [[ ${date_in} -ge ${datex} ]]; then - usedate=${datex} - fi - fi - done - # cat matching date file, or quit if date not found - if [[ ${usedate} != "" ]]; then - cat "${sat}/${usedate}" || exit 1 - else - echo "date not found for ${sat}" - exit 1 - fi -done < <(grep -v '^ *#' satellites || true) diff --git a/ush/create_satinfo.sh b/ush/create_satinfo.sh deleted file mode 100755 index 21c92ffe019..00000000000 --- a/ush/create_satinfo.sh +++ /dev/null @@ -1,27 +0,0 @@ -#! /usr/bin/env bash -# create global_satinfo file for a given date -date_in=$1 -# header -echo '!sensor/instr/sat chan iuse error error_cld ermax var_b var_pg icld_det icloud iaerosol' -# loop over satellites -cd "${BUILD_GSINFO_DIR}/satinfo" || exit 1 -while IFS= read -r sat -do - # find matching date - usedate="" - for f in "${sat}"/*; do - if [[ ${f} != "${sat}/readme" ]]; then # skip readme file - datex=$(basename "${f}") - if [[ ${date_in} -ge ${datex} ]]; then - usedate=${datex} - fi - fi - done - # cat matching date file, or quit if date not found - if [[ ${usedate} != "" ]]; then - cat "${sat}/${usedate}" || exit 1 - else - echo "date not found for ${sat}" - exit 1 - fi -done < <(grep -v '^ *#' satellites || true) From 5b33ba046b84155acdb9ca9da43f0144117cc049 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Wed, 17 Sep 2025 11:39:02 -0400 Subject: [PATCH 41/90] Update dev/parm/config/gfs/config.anal --- dev/parm/config/gfs/config.anal | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/parm/config/gfs/config.anal b/dev/parm/config/gfs/config.anal index a830e608551..546ffb6688d 100644 --- a/dev/parm/config/gfs/config.anal +++ b/dev/parm/config/gfs/config.anal @@ -155,7 +155,6 @@ if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then export SATINFO="generate" export CONVINFO="generate" export OZINFO="generate" - export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_reanl_ncep_ozone.txt" if [[ "${PDY}${cyc}" -lt "2022052712" ]]; then # use NASA ozone files for reanalysis before 2022052712 export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_reanl_nasa_ozone.txt" From 277b3a74a0d6d2d5c3ff6ad223ccb3632aacb568 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Wed, 17 Sep 2025 11:51:20 -0400 Subject: [PATCH 42/90] Update ush/create_gsi_info.sh --- ush/create_gsi_info.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index 35e01007dcd..a86f4af1f15 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -7,7 +7,8 @@ # [use2mobs]: (optional, only for conv) YES or NO set -e - +# Inherit set -e inside of the get_usedate function below +shopt -s inherit_errexit type_in=${1:-} date_in=${2:-} use2mobs=${3:-NO} From 09d41be581a1231a2aa4729c6c6a20f81beca402 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Wed, 17 Sep 2025 12:00:23 -0400 Subject: [PATCH 43/90] Update scripts/exglobal_atmos_analysis.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/exglobal_atmos_analysis.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 46ceac76faa..51631409339 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -666,8 +666,8 @@ if [[ ${DOHYBVAR} == "YES" && ${l4densvar} == ".true." && ${lwrite4danl} == ".tr ${NLN} "${ATMA09}" siga09 ${NLN} "${ATMI09}" sigi09.nc fi -${NLN} ${ABIAS}" satbias_out -${NLN} ${ABIASPC}" satbias_pc.out +${NLN} "${ABIAS}" satbias_out +${NLN} "${ABIASPC}" satbias_pc.out ${NLN} ${ABIASAIR} aircftbias_out if [[ ${DONST} == "YES" ]]; then From 3ef0a377576391172bd391f12298859b577f3e5a Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Wed, 17 Sep 2025 12:01:42 -0400 Subject: [PATCH 44/90] Update ush/create_gsi_info.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- ush/create_gsi_info.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index a86f4af1f15..bafaab78820 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -26,7 +26,7 @@ fi get_usedate() { usedate="" # Loop over files matching date pattern. - for datex in [1-2]*; do + for datex in [12][0-9][0-9][0-9]*; do # Skip for loop if there are no matches. if [[ ! -e "${datex}" ]]; then continue From b69786a6338c3d089ad2fcf5ab30140a2fe7b400 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 17 Sep 2025 11:25:34 -0500 Subject: [PATCH 45/90] Convert grep to egrep for easier-to-understand searching --- ush/create_gsi_info.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index bafaab78820..d948323fd97 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -84,7 +84,7 @@ case "${type_in}" in sat) cd "${BUILD_GSINFO_DIR}/satinfo" || exit 1 # Read in the satellite list, ignoring comment lines and the readme - satellite_list=$(grep -v '^ *#\|^readme' satellites) + satellite_list=$(egrep -v '^ *#|readme' satellites) if [[ -z "${satellite_list}" ]]; then echo "ERROR: No satellites found in the satellite file list!" exit 1 From 1bd07d9bd2e3c763e832b32b0b95b48fd904b110 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 17 Sep 2025 11:28:11 -0500 Subject: [PATCH 46/90] Use grep -E instead --- ush/create_gsi_info.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index d948323fd97..56ec99dda1b 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -84,7 +84,7 @@ case "${type_in}" in sat) cd "${BUILD_GSINFO_DIR}/satinfo" || exit 1 # Read in the satellite list, ignoring comment lines and the readme - satellite_list=$(egrep -v '^ *#|readme' satellites) + satellite_list=$(grep -Ev '^ *#|readme' satellites) if [[ -z "${satellite_list}" ]]; then echo "ERROR: No satellites found in the satellite file list!" exit 1 From df0ac5d768f12e55771a012bd8c6d4b453a98c41 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 17 Sep 2025 11:40:12 -0500 Subject: [PATCH 47/90] Add ozone header; fix regex --- ush/create_gsi_info.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index 56ec99dda1b..2f96158075f 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -6,7 +6,7 @@ # : date string to match # [use2mobs]: (optional, only for conv) YES or NO -set -e +set -eu # Inherit set -e inside of the get_usedate function below shopt -s inherit_errexit type_in=${1:-} @@ -26,7 +26,7 @@ fi get_usedate() { usedate="" # Loop over files matching date pattern. - for datex in [12][0-9][0-9][0-9]*; do + for datex in [1-2][0-9][0-9][0-9]*; do # Skip for loop if there are no matches. if [[ ! -e "${datex}" ]]; then continue @@ -75,6 +75,10 @@ case "${type_in}" in cd "${BUILD_GSINFO_DIR}/ozinfo" || exit 1 usedate=$(get_usedate) if [[ ${usedate} != "" ]]; then + # Header lines + echo '! For mls data, pressure and obs errors are pulled from bufr, so not listed here' >> "${info_file}" + echo '! sens/instr/sat lev use pressure gross obs b_oz pg_oz' >> "${info_file}" + echo '! error error variational qc' >> "${info_file}" cat "${usedate}" >> "${info_file}" else echo "ERROR: No valid ozone info was found!" From 9085f4540dc4eb9a532a02afbde41dccfec3619b Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 17 Sep 2025 11:43:42 -0500 Subject: [PATCH 48/90] Group the header echos --- ush/create_gsi_info.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index 2f96158075f..7abcbadbf4b 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -76,9 +76,11 @@ case "${type_in}" in usedate=$(get_usedate) if [[ ${usedate} != "" ]]; then # Header lines - echo '! For mls data, pressure and obs errors are pulled from bufr, so not listed here' >> "${info_file}" - echo '! sens/instr/sat lev use pressure gross obs b_oz pg_oz' >> "${info_file}" - echo '! error error variational qc' >> "${info_file}" + { + echo '! For mls data, pressure and obs errors are pulled from bufr, so not listed here' + echo '! sens/instr/sat lev use pressure gross obs b_oz pg_oz' + echo '! error error variational qc' + } >> "${info_file}" cat "${usedate}" >> "${info_file}" else echo "ERROR: No valid ozone info was found!" From e6260c5023f7d0f03d465ab92c7dac94caa547ea Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Wed, 17 Sep 2025 12:44:28 -0400 Subject: [PATCH 49/90] Update scripts/exglobal_atmos_analysis.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/exglobal_atmos_analysis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 51631409339..647c5ae817d 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -375,7 +375,7 @@ ${NLN} ${RADCLOUDINFO} cloudy_radiance_info.txt ${NLN} ${ATMSFILTER} atms_beamwidth.txt ${NLN} ${ANAVINFO} anavinfo if [[ ${CONVINFO} == "generate" ]]; then - ${USHgfs}/create_gsi_info.sh conv "${PDY}${cyc}" "${USE_2M_OBS}" + ${USHgfs}/create_gsi_info.sh conv "${PDY}${cyc}" "${BUILD_2M_OBS}" else ${NLN} "${CONVINFO}" convinfo fi From fdccb58c2d0e154075013078e038976a75d5daf2 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 17 Sep 2025 11:54:38 -0500 Subject: [PATCH 50/90] Be more explicit about where we're working --- ush/create_gsi_info.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index 7abcbadbf4b..3388973002f 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -40,15 +40,15 @@ get_usedate() { echo "${usedate}" } +# Get the starting directory +starting_dir="${DATA:-${PWD}}" + # Filename to write the info to -info_file="${PWD}/${type_in}info" +info_file="${starting_dir}/${type_in}info" if [[ -f "${info_file}" ]]; then rm -f "${info_file}" fi -# Get the starting directory -starting_dir=${PWD} - case "${type_in}" in conv) cd "${BUILD_GSINFO_DIR}/convinfo" || exit 1 From d8949abcf7f2567e6ff677c62ee302f5cd50fb64 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Wed, 17 Sep 2025 12:57:31 -0400 Subject: [PATCH 51/90] Update ush/create_gsi_info.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- ush/create_gsi_info.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index 3388973002f..5fd8b403226 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Generic script to create convinfo, ozinfo, or satinfo for a given date. # Usage: -# ./create_info.sh [use2mobs] +# ./create_gsi_info.sh [use2mobs] # : conv, oz, or sat # : date string to match # [use2mobs]: (optional, only for conv) YES or NO From 31db8bbf0903fe6697ef233ad78b63a860d522a0 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 17 Sep 2025 14:50:43 -0500 Subject: [PATCH 52/90] Set 183 obs to monitor --- ush/create_gsi_info.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index 5fd8b403226..983e329d918 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -58,7 +58,7 @@ case "${type_in}" in if [[ ${use2mobs} == "YES" ]]; then # Turn on 2m t,q obs over land sed -e "s/t 181 0 -1/t 181 0 1/g" \ - -e "s/t 183 0 -1/t 183 0 1/g" \ + -e "s/t 183 -1 -1/t 183 0 1/g" \ -e "s/t 187 0 -1/t 187 0 1/g" \ -e "s/q 181 0 -1/q 181 0 1/g" \ -e "s/q 183 0 -1/q 183 0 1/g" \ From 183eb729a6dd74161d48acad79cb5855f5343eb0 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 17 Sep 2025 14:54:28 -0500 Subject: [PATCH 53/90] Tie BUILD_2M_OBS to DO_GSISOILDA --- dev/parm/config/gfs/config.anal | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dev/parm/config/gfs/config.anal b/dev/parm/config/gfs/config.anal index 546ffb6688d..70864f5c84a 100644 --- a/dev/parm/config/gfs/config.anal +++ b/dev/parm/config/gfs/config.anal @@ -161,8 +161,13 @@ if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then else # after 2022052712 use NCEP ozone bufr with hoices from gfsv17_historical export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_reanl_ncep_ozone.txt" fi + # Use 2m observations when constructing conventional obs info file - export BUILD_2M_OBS="NO" + if [[ ${DO_GSISOILDA} == "YES" ]]; then + export BUILD_2M_OBS="YES" + else + export BUILD_2M_OBS="NO" + fi else export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_ops.txt" fi From 5f4233ea8c2af785e0430dbaff35c402ee231542 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 17 Sep 2025 14:55:59 -0500 Subject: [PATCH 54/90] Also monitor 183 q --- ush/create_gsi_info.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index 983e329d918..4f661fe16d7 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -58,7 +58,7 @@ case "${type_in}" in if [[ ${use2mobs} == "YES" ]]; then # Turn on 2m t,q obs over land sed -e "s/t 181 0 -1/t 181 0 1/g" \ - -e "s/t 183 -1 -1/t 183 0 1/g" \ + -e "s/t 183 -1 -1/t 183 -1 1/g" \ -e "s/t 187 0 -1/t 187 0 1/g" \ -e "s/q 181 0 -1/q 181 0 1/g" \ -e "s/q 183 0 -1/q 183 0 1/g" \ From 7cd535e6fe292926b77edc4be29eaeb0cd37cd6d Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 17 Sep 2025 15:18:34 -0500 Subject: [PATCH 55/90] Apply patch from Jeff --- ush/create_gsi_info.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index 4f661fe16d7..fccce61e419 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -58,10 +58,8 @@ case "${type_in}" in if [[ ${use2mobs} == "YES" ]]; then # Turn on 2m t,q obs over land sed -e "s/t 181 0 -1/t 181 0 1/g" \ - -e "s/t 183 -1 -1/t 183 -1 1/g" \ -e "s/t 187 0 -1/t 187 0 1/g" \ -e "s/q 181 0 -1/q 181 0 1/g" \ - -e "s/q 183 0 -1/q 183 0 1/g" \ -e "s/q 187 0 -1/q 187 0 1/g" "${usedate}" >> "${info_file}" else cat "${usedate}" >> "${info_file}" From 6f2d23a50fb26b163678520b8127b81cd14ece33 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Wed, 17 Sep 2025 14:43:42 -0600 Subject: [PATCH 56/90] remove outdated comments --- scripts/exglobal_atmos_analysis.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 647c5ae817d..57d84304196 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -93,10 +93,10 @@ OSCATBF=${OSCATBF:-${COMIN_OBS}/${OPREFIX}oscatw.tm00.bufr_d${OSUFFIX}} RAPIDSCATBF=${RAPIDSCATBF:-${COMIN_OBS}/${OPREFIX}rapidscatw.tm00.bufr_d${OSUFFIX}} GSNDBF=${GSNDBF:-${COMIN_OBS}/${OPREFIX}goesnd.tm00.bufr_d${OSUFFIX}} GSNDBF1=${GSNDBF1:-${COMIN_OBS}/${OPREFIX}goesfv.tm00.bufr_d${OSUFFIX}} -B1HRS2=${B1HRS2:-${COMIN_OBS}/${OPREFIX}1bhrs2.tm00.bufr_d${OSUFFIX}} # HIRS temporarily disabled due to CRTM versioning issues +B1HRS2=${B1HRS2:-${COMIN_OBS}/${OPREFIX}1bhrs2.tm00.bufr_d${OSUFFIX}} B1MSU=${B1MSU:-${COMIN_OBS}/${OPREFIX}1bmsu.tm00.bufr_d${OSUFFIX}} -B1HRS3=${B1HRS3:-${COMIN_OBS}/${OPREFIX}1bhrs3.tm00.bufr_d${OSUFFIX}} # HIRS temporarily disabled due to CRTM versioning issues -B1HRS4=${B1HRS4:-${COMIN_OBS}/${OPREFIX}1bhrs4.tm00.bufr_d${OSUFFIX}} # HIRS temporarily disabled due to CRTM versioning issues +B1HRS3=${B1HRS3:-${COMIN_OBS}/${OPREFIX}1bhrs3.tm00.bufr_d${OSUFFIX}} +B1HRS4=${B1HRS4:-${COMIN_OBS}/${OPREFIX}1bhrs4.tm00.bufr_d${OSUFFIX}} B1AMUA=${B1AMUA:-${COMIN_OBS}/${OPREFIX}1bamua.tm00.bufr_d${OSUFFIX}} B1AMUB=${B1AMUB:-${COMIN_OBS}/${OPREFIX}1bamub.tm00.bufr_d${OSUFFIX}} AQUAAMUA=${AQUAAMUA:-${COMIN_OBS}/${OPREFIX}aquaamua.tm00.bufr_d${OSUFFIX}} From dca61d614d5d2cac84c8976e649412bc339c1e19 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 18 Sep 2025 11:18:46 -0400 Subject: [PATCH 57/90] Remove 'set' from create script (already in preamble) --- ush/create_gsi_info.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index fccce61e419..071c2d74168 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -6,9 +6,6 @@ # : date string to match # [use2mobs]: (optional, only for conv) YES or NO -set -eu -# Inherit set -e inside of the get_usedate function below -shopt -s inherit_errexit type_in=${1:-} date_in=${2:-} use2mobs=${3:-NO} From fae6cd36218d190afe1c1c7eb3ea7d7650e33565 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 18 Sep 2025 14:39:34 -0400 Subject: [PATCH 58/90] Add the directory to the create call --- scripts/exglobal_atmos_analysis.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 57d84304196..290c1c8e584 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -93,10 +93,10 @@ OSCATBF=${OSCATBF:-${COMIN_OBS}/${OPREFIX}oscatw.tm00.bufr_d${OSUFFIX}} RAPIDSCATBF=${RAPIDSCATBF:-${COMIN_OBS}/${OPREFIX}rapidscatw.tm00.bufr_d${OSUFFIX}} GSNDBF=${GSNDBF:-${COMIN_OBS}/${OPREFIX}goesnd.tm00.bufr_d${OSUFFIX}} GSNDBF1=${GSNDBF1:-${COMIN_OBS}/${OPREFIX}goesfv.tm00.bufr_d${OSUFFIX}} -B1HRS2=${B1HRS2:-${COMIN_OBS}/${OPREFIX}1bhrs2.tm00.bufr_d${OSUFFIX}} +B1HRS2=${B1HRS2:-${COMIN_OBS}/${OPREFIX}1bhrs2.tm00.bufr_d${OSUFFIX}} B1MSU=${B1MSU:-${COMIN_OBS}/${OPREFIX}1bmsu.tm00.bufr_d${OSUFFIX}} -B1HRS3=${B1HRS3:-${COMIN_OBS}/${OPREFIX}1bhrs3.tm00.bufr_d${OSUFFIX}} -B1HRS4=${B1HRS4:-${COMIN_OBS}/${OPREFIX}1bhrs4.tm00.bufr_d${OSUFFIX}} +B1HRS3=${B1HRS3:-${COMIN_OBS}/${OPREFIX}1bhrs3.tm00.bufr_d${OSUFFIX}} +B1HRS4=${B1HRS4:-${COMIN_OBS}/${OPREFIX}1bhrs4.tm00.bufr_d${OSUFFIX}} B1AMUA=${B1AMUA:-${COMIN_OBS}/${OPREFIX}1bamua.tm00.bufr_d${OSUFFIX}} B1AMUB=${B1AMUB:-${COMIN_OBS}/${OPREFIX}1bamub.tm00.bufr_d${OSUFFIX}} AQUAAMUA=${AQUAAMUA:-${COMIN_OBS}/${OPREFIX}aquaamua.tm00.bufr_d${OSUFFIX}} @@ -367,7 +367,7 @@ fi ${NLN} ${BERROR} berror_stats ${NLN} ${SATANGL} satbias_angle if [[ ${SATINFO} == "generate" ]]; then - ${USHgfs}/create_gsi_info.sh sat "${PDY}${cyc}" + ${USHgfs}/create_gsi_info.sh sat "${PDY}${cyc}" "${DATA}" else ${NLN} ${SATINFO} satinfo fi @@ -375,14 +375,14 @@ ${NLN} ${RADCLOUDINFO} cloudy_radiance_info.txt ${NLN} ${ATMSFILTER} atms_beamwidth.txt ${NLN} ${ANAVINFO} anavinfo if [[ ${CONVINFO} == "generate" ]]; then - ${USHgfs}/create_gsi_info.sh conv "${PDY}${cyc}" "${BUILD_2M_OBS}" + ${USHgfs}/create_gsi_info.sh conv "${PDY}${cyc}" "${DATA}" "${BUILD_2M_OBS}" else ${NLN} "${CONVINFO}" convinfo fi ${NLN} "${vqcdat}" vqctp001.dat ${NLN} "${INSITUINFO}" insituinfo if [[ ${OZINFO} == "generate" ]]; then - ${USHgfs}/create_gsi_info.sh oz "${PDY}${cyc}" + ${USHgfs}/create_gsi_info.sh oz "${PDY}${cyc}" "${DATA}" else ${NLN} "${OZINFO}" ozinfo fi From 4f2a8dfde90417b8247333b0b845b61757dd9d5d Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 18 Sep 2025 15:21:43 -0400 Subject: [PATCH 59/90] Fix ozone info file creation; write satellite function --- ush/create_gsi_info.sh | 116 +++++++++++++++++++++++------------------ 1 file changed, 65 insertions(+), 51 deletions(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index 071c2d74168..be8e36ed508 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -1,25 +1,30 @@ #!/usr/bin/env bash + # Generic script to create convinfo, ozinfo, or satinfo for a given date. # Usage: # ./create_gsi_info.sh [use2mobs] # : conv, oz, or sat # : date string to match +# : directory where the info file will be created # [use2mobs]: (optional, only for conv) YES or NO type_in=${1:-} date_in=${2:-} -use2mobs=${3:-NO} +write_dir=${3:-} +use2mobs=${4:-NO} -if [[ -z "${type_in}" || -z "${date_in}" ]]; then +if [[ -z "${type_in}" || -z "${date_in}" || -z "${write_dir}" ]]; then echo "Usage: ${0} [use2mobs]" echo " : conv, oz, or sat" echo " : date string to match" + echo " : directory where the info file will be created" echo " [use2mobs]: (optional, only for conv, whether to use 2m observations) YES or NO" exit 1 fi # Function to get the most recent data available for the target obs. # If an empty string is returned, this represents an error. +# Assumes the variable date_in is set. get_usedate() { usedate="" # Loop over files matching date pattern. @@ -38,18 +43,61 @@ get_usedate() { } # Get the starting directory -starting_dir="${DATA:-${PWD}}" +starting_dir="${PWD}" + +# Get the build directory +build_dir="${BUILD_GSINFO_DIR}/${type_in}info" +cd "${build_dir}" || exit 1 + +# Get the list of satellites available +if [[ "${type_in}" != "conv" ]]; then + if [[ ! -f satellites ]]; then + echo "FATAL ERROR: Satellite list file 'satellites' not found in ${build_dir}!" + exit 1 + fi + + satellite_list=$(grep -Ev '^ *#|readme' satellites) + + if [[ -z "${satellite_list}" ]]; then + echo "FATAL ERROR: No satellites found in the satellite file list!" + exit 1 + fi +fi # Filename to write the info to -info_file="${starting_dir}/${type_in}info" +info_file="${write_dir}/${type_in}info" if [[ -f "${info_file}" ]]; then rm -f "${info_file}" fi +# Function to cycle through the list of satellites (oz or sat) and build the info file. +build_info_file() { + while IFS= read -r sat + do + usedate="" + # Check that the satellite directory exists + if [[ ! -d "${sat}" ]]; then + echo "FATAL ERROR: Directory ${sat} does not exist!" + exit 1 + fi + + cd "${sat}" || exit 1 + + usedate=$(get_usedate) + + cd "${info_file}" || exit 1 + + if [[ ${usedate} != "" ]]; then + cat "${sat}/${usedate}" >> "${info_file}" + else + echo "FATAL ERROR: No valid satellite info was found for satellite target '${sat}'!" + exit 1 + fi + done <<< "${satellite_list}" +} + case "${type_in}" in conv) - cd "${BUILD_GSINFO_DIR}/convinfo" || exit 1 - usedate=$(get_usedate) if [[ ${usedate} != "" ]]; then if [[ ${use2mobs} == "YES" ]]; then @@ -62,62 +110,28 @@ case "${type_in}" in cat "${usedate}" >> "${info_file}" fi else - echo "ERROR: No valid conventional info was found!" + echo "FATAL ERROR: No valid conventional info was found!" exit 1 fi ;; oz) - cd "${BUILD_GSINFO_DIR}/ozinfo" || exit 1 - usedate=$(get_usedate) - if [[ ${usedate} != "" ]]; then - # Header lines - { - echo '! For mls data, pressure and obs errors are pulled from bufr, so not listed here' - echo '! sens/instr/sat lev use pressure gross obs b_oz pg_oz' - echo '! error error variational qc' - } >> "${info_file}" - cat "${usedate}" >> "${info_file}" - else - echo "ERROR: No valid ozone info was found!" - exit 1 - fi + # Header lines + { + echo '! For mls data, pressure and obs errors are pulled from bufr, so not listed here' + echo '! sens/instr/sat lev use pressure gross obs b_oz pg_oz' + echo '! error error variational qc' + } >> "${info_file}" + build_info_file ;; + sat) - cd "${BUILD_GSINFO_DIR}/satinfo" || exit 1 - # Read in the satellite list, ignoring comment lines and the readme - satellite_list=$(grep -Ev '^ *#|readme' satellites) - if [[ -z "${satellite_list}" ]]; then - echo "ERROR: No satellites found in the satellite file list!" - exit 1 - fi # Header line echo '!sensor/instr/sat chan iuse error error_cld ermax var_b var_pg icld_det icloud iaerosol' >> "${info_file}" - while IFS= read -r sat - do - usedate="" - # Check that the satellite directory exists - if [[ ! -d "${sat}" ]]; then - echo "ERROR: Directory ${sat} does not exist!" - exit 1 - fi - - cd "${sat}" || exit 1 - - usedate=$(get_usedate) - - cd "${BUILD_GSINFO_DIR}/satinfo" || exit 1 - - if [[ ${usedate} != "" ]]; then - cat "${sat}/${usedate}" >> "${info_file}" - else - echo "ERROR: No valid satellite info was found for satellite target '${sat}'!" - exit 1 - fi - done <<< "${satellite_list}" + build_info_file ;; *) - echo "ERROR: Unknown info file type: '${type_in}'. Must be one of: conv, oz, sat" + echo "FATAL ERROR: Unknown info file type: '${type_in}'. Must be one of: conv, oz, sat" exit 2 ;; esac From 3ba3d2c494fbd755e5047d00999fc4a77ea33ca9 Mon Sep 17 00:00:00 2001 From: Jeffrey Whitaker Date: Fri, 19 Sep 2025 09:16:17 -0600 Subject: [PATCH 60/90] update --- dev/parm/config/gfs/config.anal | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/dev/parm/config/gfs/config.anal b/dev/parm/config/gfs/config.anal index 70864f5c84a..ab6a7fa6424 100644 --- a/dev/parm/config/gfs/config.anal +++ b/dev/parm/config/gfs/config.anal @@ -155,12 +155,8 @@ if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then export SATINFO="generate" export CONVINFO="generate" export OZINFO="generate" - if [[ "${PDY}${cyc}" -lt "2022052712" ]]; then - # use NASA ozone files for reanalysis before 2022052712 - export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_reanl_nasa_ozone.txt" - else # after 2022052712 use NCEP ozone bufr with hoices from gfsv17_historical - export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_reanl_ncep_ozone.txt" - fi + # this can be over-ridden, for example to use NASA's netcdf ozone files use obs_input_reanl_nasa_ozone.txt. + export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_reanl_ncep_ozone.txt" # Use 2m observations when constructing conventional obs info file if [[ ${DO_GSISOILDA} == "YES" ]]; then From 54e1a596ddc75f9cbbc6e12cbfdc060a0552398a Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 19 Sep 2025 13:36:09 -0400 Subject: [PATCH 61/90] Add override option for BUILD_2M_OBS --- dev/parm/config/gfs/config.anal | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/dev/parm/config/gfs/config.anal b/dev/parm/config/gfs/config.anal index 70864f5c84a..6db5460df0d 100644 --- a/dev/parm/config/gfs/config.anal +++ b/dev/parm/config/gfs/config.anal @@ -163,11 +163,8 @@ if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then fi # Use 2m observations when constructing conventional obs info file - if [[ ${DO_GSISOILDA} == "YES" ]]; then - export BUILD_2M_OBS="YES" - else - export BUILD_2M_OBS="NO" - fi + export BUILD_2M_OBS="${BUILD_2M_OBS:-${DO_GSISOILDA}" + else export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_ops.txt" fi From 422a1ba1450494ac1db1788a56e9b278841ca532 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 19 Sep 2025 15:52:47 -0400 Subject: [PATCH 62/90] Update build_gsinfo --- sorc/gsi_enkf.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index b806a99515d..d45287a1341 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit b806a99515d02de3fbfa80bb6336a454eabf8d8b +Subproject commit d45287a1341e0f4caad74b2107a5407791e0975b From d0fe259b513bcae4b308ace81f6aa5cf0fb72788 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 19 Sep 2025 15:55:55 -0400 Subject: [PATCH 63/90] Fix build directory variable name --- ush/create_gsi_info.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index be8e36ed508..d820a38a623 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -85,7 +85,7 @@ build_info_file() { usedate=$(get_usedate) - cd "${info_file}" || exit 1 + cd "${build_dir}" || exit 1 if [[ ${usedate} != "" ]]; then cat "${sat}/${usedate}" >> "${info_file}" From 8167654653d5f97a8bb625840208e87badae4f66 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 19 Sep 2025 15:56:21 -0400 Subject: [PATCH 64/90] Add parm links for gsinfo files --- sorc/link_workflow.sh | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index f57ea472dde..913b3e3749a 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -98,7 +98,7 @@ done # Link fix directories if [[ -n "${FIX_DIR}" ]]; then - if [[ ! -d "${HOMEgfs}/fix" ]]; then mkdir "${HOMEgfs}/fix" || exit 1; fi + mkdir -p "${HOMEgfs}/fix" || exit 1 fi cd "${HOMEgfs}/fix" || exit 1 for dir in aer \ @@ -214,9 +214,7 @@ fi #------------------------------ if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then cd "${HOMEgfs}/fix" || exit 1 - if [[ ! -d gdas ]]; then - mkdir -p gdas - fi + mkdir -p gdas cd gdas || exit 1 for gdas_sub in fv3jedi gsibec obs soca aero snow; do if [[ -d "${gdas_sub}" ]]; then @@ -232,9 +230,7 @@ fi #------------------------------ if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then cd "${HOMEgfs}/parm" || exit 1 - if [[ ! -d gdas ]]; then - mkdir -p gdas - fi + mkdir -p gdas cd gdas || exit 1 declare -a gdasapp_comps=("aero" "atm" "io" "ioda" "snow" "marine" "jcb-gdas" "jcb-algorithms" "anlstat" "analcalc") for comp in "${gdasapp_comps[@]}"; do @@ -294,11 +290,32 @@ if [[ -d "${HOMEgfs}/sorc/gsi_monitor.fd" ]]; then # ${LINK_OR_COPY} "${HOMEgfs}/sorc/gsi_monitor.fd/src/Radiance_Monitor/nwprod/gdas_radmon/parm/gdas_radmon.parm" . fi +#------------------------------ +#--Add GSI conv, sat, and oz info parm files +#------------------------------ +if [[ -d "${HOMEgfs}/sorc/gsi_enkf.fd/fix/build_gsinfo" ]]; then + + cd "${HOMEgfs}/parm" || exit 1 + + mkdir -p gsinfo + + cd gsinfo || exit 1 + + for dir in convinfo satinfo ozinfo; do + if [[ -d "${dir}" ]]; then + rm -rf "${dir}" + fi + ${LINK_OR_COPY} "${HOMEgfs}/sorc/gsi_enkf.fd/fix/build_gsinfo/${dir}" "${dir}" + done +fi + + #------------------------------ #--link executables #------------------------------ -if [[ ! -d "${HOMEgfs}/exec" ]]; then mkdir "${HOMEgfs}/exec" || exit 1; fi +mkdir -p "${HOMEgfs}/exec" || exit 1 + cd "${HOMEgfs}/exec" || exit 1 for utilexe in fbwndgfs.x gaussian_sfcanl.x gfs_bufr.x supvit.x syndat_getjtbul.x \ @@ -398,7 +415,7 @@ fi # GDASApp libraries if [[ -d "${HOMEgfs}/sorc/gdas.cd/install" ]]; then - if [[ ! -d "${HOMEgfs}/lib" ]]; then mkdir "${HOMEgfs}/lib" || exit 1; fi + mkdir -p "${HOMEgfs}/lib" || exit 1 cd "${HOMEgfs}/lib" || exit 1 cp -af "${HOMEgfs}/sorc/gdas.cd/install/lib/." ./ fi From ee61b30f0d6bed31001dbe6d88af636550c7b654 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 19 Sep 2025 15:57:03 -0400 Subject: [PATCH 65/90] Point to parm rather than fix for build_gsinfo --- dev/parm/config/gfs/config.base.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/parm/config/gfs/config.base.j2 b/dev/parm/config/gfs/config.base.j2 index 6026a992549..e404b4d8eab 100644 --- a/dev/parm/config/gfs/config.base.j2 +++ b/dev/parm/config/gfs/config.base.j2 @@ -179,7 +179,7 @@ aero_fcst_runs="gdas" # When to run aerosol forecast: gdas, gfs, or both aero_anl_runs="gdas gfs" # When to run aerosol analysis: gdas, gfs, or both export DO_AERO_FCST="NO" export DO_AERO_ANL="NO" -export DOBNDPNT_WAVE="YES" # Option to create point outputs for downstream boundary points +export DOBNDPNT_WAVE="YES" # Option to create point outputs for downstream boundary points export FRAC_GRID=".true." export DO_NEST="NO" # Whether to run a global-nested domain if [[ "${DO_NEST:-NO}" == "YES" ]] ; then @@ -380,7 +380,7 @@ export l4densvar=".true." export lwrite4danl=".true." export DO_CALC_INCREMENT="NO" export USE_BUILD_GSINFO="NO" -export BUILD_GSINFO_DIR="${FIXgfs}/gsi/fix/build_gsinfo" +export BUILD_GSINFO_DIR="${PARMgfs}/gsinfo" # Early-cycle EnKF parameters export NMEM_ENS_GFS="{{ NMEM_ENS_GFS }}" From 4fc6f73b384aeaefc1cdee5e953940cebffa116d Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 19 Sep 2025 16:12:34 -0400 Subject: [PATCH 66/90] Add gsinfo to ignore list --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e0e68a8f2ec..97f48065f2f 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ parm/gdas/soca parm/gdas/stat parm/gdas/jcb-gdas parm/gdas/jcb-algorithms +parm/gsinfo parm/monitor parm/post/nam_micro_lookup.dat parm/post/optics_luts_DUST.dat From 11d65665defb01edf13453a94b172e13b74c77ed Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 19 Sep 2025 16:13:16 -0400 Subject: [PATCH 67/90] Add missing } --- dev/parm/config/gfs/config.anal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/parm/config/gfs/config.anal b/dev/parm/config/gfs/config.anal index 6db5460df0d..01d03beed08 100644 --- a/dev/parm/config/gfs/config.anal +++ b/dev/parm/config/gfs/config.anal @@ -163,7 +163,7 @@ if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then fi # Use 2m observations when constructing conventional obs info file - export BUILD_2M_OBS="${BUILD_2M_OBS:-${DO_GSISOILDA}" + export BUILD_2M_OBS="${BUILD_2M_OBS:-${DO_GSISOILDA}}" else export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_ops.txt" From 9ca7754bcc6042d33297606bc846b61451e78111 Mon Sep 17 00:00:00 2001 From: Jeffrey Whitaker Date: Tue, 23 Sep 2025 10:39:39 -0600 Subject: [PATCH 68/90] implement reviewer suggestions --- ush/create_gsi_info.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index fccce61e419..69ed078b9d5 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -65,7 +65,7 @@ case "${type_in}" in cat "${usedate}" >> "${info_file}" fi else - echo "ERROR: No valid conventional info was found!" + echo "FATAL ERROR: No valid conventional info was found!" exit 1 fi ;; @@ -81,7 +81,7 @@ case "${type_in}" in } >> "${info_file}" cat "${usedate}" >> "${info_file}" else - echo "ERROR: No valid ozone info was found!" + echo "FATAL ERROR: No valid ozone info was found!" exit 1 fi ;; @@ -90,7 +90,7 @@ case "${type_in}" in # Read in the satellite list, ignoring comment lines and the readme satellite_list=$(grep -Ev '^ *#|readme' satellites) if [[ -z "${satellite_list}" ]]; then - echo "ERROR: No satellites found in the satellite file list!" + echo "FATAL ERROR: No satellites found in the satellite file list!" exit 1 fi @@ -101,7 +101,7 @@ case "${type_in}" in usedate="" # Check that the satellite directory exists if [[ ! -d "${sat}" ]]; then - echo "ERROR: Directory ${sat} does not exist!" + echo "FATAL ERROR: Directory ${sat} does not exist!" exit 1 fi @@ -114,13 +114,13 @@ case "${type_in}" in if [[ ${usedate} != "" ]]; then cat "${sat}/${usedate}" >> "${info_file}" else - echo "ERROR: No valid satellite info was found for satellite target '${sat}'!" + echo "FATAL ERROR: No valid satellite info was found for satellite target '${sat}'!" exit 1 fi done <<< "${satellite_list}" ;; *) - echo "ERROR: Unknown info file type: '${type_in}'. Must be one of: conv, oz, sat" + echo "FATAL ERROR: Unknown info file type: '${type_in}'. Must be one of: conv, oz, sat" exit 2 ;; esac From a3c92a91f4ea75cf5affa2aebdff0bd03a2ed996 Mon Sep 17 00:00:00 2001 From: Jeffrey Whitaker Date: Tue, 23 Sep 2025 10:44:37 -0600 Subject: [PATCH 69/90] set BUILD_2M_OBS --- dev/parm/config/gfs/config.anal | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/dev/parm/config/gfs/config.anal b/dev/parm/config/gfs/config.anal index ab6a7fa6424..cf75ded5cdc 100644 --- a/dev/parm/config/gfs/config.anal +++ b/dev/parm/config/gfs/config.anal @@ -159,11 +159,8 @@ if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_reanl_ncep_ozone.txt" # Use 2m observations when constructing conventional obs info file - if [[ ${DO_GSISOILDA} == "YES" ]]; then - export BUILD_2M_OBS="YES" - else - export BUILD_2M_OBS="NO" - fi + # can use 2m obs without enabling soil DA by setting BUILD_2M_OBS=YES in config.base + export BUILD_2M_OBS="${BUILD_2M_OBS:-DO_GSISOILDA}" else export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_ops.txt" fi From 3affd14ff7eb84d4f0bdb8345fd7b84af7606917 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Tue, 23 Sep 2025 10:48:33 -0600 Subject: [PATCH 70/90] Update ush/create_gsi_info.sh Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ush/create_gsi_info.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index 69ed078b9d5..3c0fe5697ea 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -1,22 +1,27 @@ #!/usr/bin/env bash # Generic script to create convinfo, ozinfo, or satinfo for a given date. # Usage: -# ./create_gsi_info.sh [use2mobs] +# ./create_gsi_info.sh [use2mobs] # : conv, oz, or sat # : date string to match +# : directory to write the info file into # [use2mobs]: (optional, only for conv) YES or NO set -eu # Inherit set -e inside of the get_usedate function below shopt -s inherit_errexit + +# Input arguments type_in=${1:-} date_in=${2:-} -use2mobs=${3:-NO} +dir_in=${3:-} +use2mobs=${4:-NO} -if [[ -z "${type_in}" || -z "${date_in}" ]]; then - echo "Usage: ${0} [use2mobs]" +if [[ -z "${type_in}" || -z "${date_in}" || -z "${dir_in}" ]]; then + echo "Usage: ${0} [use2mobs]" echo " : conv, oz, or sat" echo " : date string to match" + echo " : where to write the new GSI info files into" echo " [use2mobs]: (optional, only for conv, whether to use 2m observations) YES or NO" exit 1 fi From 495729e659c90cc4c330092af1cfee4a918cec37 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Tue, 23 Sep 2025 10:48:56 -0600 Subject: [PATCH 71/90] Update ush/create_gsi_info.sh Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ush/create_gsi_info.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index 3c0fe5697ea..6fab69bcaa2 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -46,7 +46,7 @@ get_usedate() { } # Get the starting directory -starting_dir="${DATA:-${PWD}}" +starting_dir="${PWD}" # Filename to write the info to info_file="${starting_dir}/${type_in}info" From c20e0452dd87fc8a63e8f01aea27343d4e2157f6 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Tue, 23 Sep 2025 10:49:04 -0600 Subject: [PATCH 72/90] Update ush/create_gsi_info.sh Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ush/create_gsi_info.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index 6fab69bcaa2..53697386368 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -49,7 +49,7 @@ get_usedate() { starting_dir="${PWD}" # Filename to write the info to -info_file="${starting_dir}/${type_in}info" +info_file="${dir_in}/${type_in}info" if [[ -f "${info_file}" ]]; then rm -f "${info_file}" fi From 48598f45881cceb5bd07fc15af09f0dbc01d0d48 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 24 Sep 2025 11:38:23 -0400 Subject: [PATCH 73/90] Link in the obs_input directory as well --- sorc/link_workflow.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 913b3e3749a..7afb3118465 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -290,9 +290,9 @@ if [[ -d "${HOMEgfs}/sorc/gsi_monitor.fd" ]]; then # ${LINK_OR_COPY} "${HOMEgfs}/sorc/gsi_monitor.fd/src/Radiance_Monitor/nwprod/gdas_radmon/parm/gdas_radmon.parm" . fi -#------------------------------ +#------------------------------------------- #--Add GSI conv, sat, and oz info parm files -#------------------------------ +#------------------------------------------- if [[ -d "${HOMEgfs}/sorc/gsi_enkf.fd/fix/build_gsinfo" ]]; then cd "${HOMEgfs}/parm" || exit 1 @@ -301,7 +301,7 @@ if [[ -d "${HOMEgfs}/sorc/gsi_enkf.fd/fix/build_gsinfo" ]]; then cd gsinfo || exit 1 - for dir in convinfo satinfo ozinfo; do + for dir in convinfo satinfo ozinfo obs_input; do if [[ -d "${dir}" ]]; then rm -rf "${dir}" fi From 6eca2556ced9434a14bb42f47a540170aed4035a Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 24 Sep 2025 13:00:04 -0400 Subject: [PATCH 74/90] Add hirs_fix to parm from build_gsinfo --- sorc/link_workflow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 7afb3118465..9d5599609f8 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -301,7 +301,7 @@ if [[ -d "${HOMEgfs}/sorc/gsi_enkf.fd/fix/build_gsinfo" ]]; then cd gsinfo || exit 1 - for dir in convinfo satinfo ozinfo obs_input; do + for dir in convinfo satinfo ozinfo obs_input hirs_fix; do if [[ -d "${dir}" ]]; then rm -rf "${dir}" fi From d9f6a0b6b58d26c7791b40419e468cce7f700c80 Mon Sep 17 00:00:00 2001 From: DavidHuber-NOAA Date: Wed, 24 Sep 2025 17:33:48 +0000 Subject: [PATCH 75/90] Update build_gsinfo --- sorc/gsi_enkf.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index d45287a1341..b806a99515d 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit d45287a1341e0f4caad74b2107a5407791e0975b +Subproject commit b806a99515d02de3fbfa80bb6336a454eabf8d8b From a115e59b7b2fcaa8c7f17a15c1bad55ee3d66ee4 Mon Sep 17 00:00:00 2001 From: DavidHuber-NOAA Date: Wed, 24 Sep 2025 17:46:16 +0000 Subject: [PATCH 76/90] Update build_gsinfo --- sorc/gsi_enkf.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index b806a99515d..1f7480c70f6 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit b806a99515d02de3fbfa80bb6336a454eabf8d8b +Subproject commit 1f7480c70f678706aa67fd8526f499cbe073496b From 23aa8b0042c0adce982be04a226940f5b684bd78 Mon Sep 17 00:00:00 2001 From: DavidHuber-NOAA Date: Wed, 24 Sep 2025 19:35:47 +0000 Subject: [PATCH 77/90] Fix merge oops --- dev/parm/config/gfs/config.anal | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/parm/config/gfs/config.anal b/dev/parm/config/gfs/config.anal index 7d4a5287511..a448d9cc9e6 100644 --- a/dev/parm/config/gfs/config.anal +++ b/dev/parm/config/gfs/config.anal @@ -158,6 +158,7 @@ if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then # this can be over-ridden, for example to use NASA's netcdf ozone files use obs_input_reanl_nasa_ozone.txt. export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_reanl_ncep_ozone.txt" + # Use 2m observations when constructing conventional obs info file # Can use 2m obs without enabling soil DA by setting BUILD_2M_OBS=YES in config.base export BUILD_2M_OBS="${BUILD_2M_OBS:-${DO_GSISOILDA}}" else From bc562a05ed4f98f47de205a09ab9ca9f07e2b94f Mon Sep 17 00:00:00 2001 From: DavidHuber-NOAA Date: Wed, 24 Sep 2025 20:00:45 +0000 Subject: [PATCH 78/90] Correct the usage prints --- ush/create_gsi_info.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/create_gsi_info.sh b/ush/create_gsi_info.sh index 42068448a8c..eda4e718329 100755 --- a/ush/create_gsi_info.sh +++ b/ush/create_gsi_info.sh @@ -15,7 +15,7 @@ write_dir=${3:-} use2mobs=${4:-NO} if [[ -z "${type_in}" || -z "${date_in}" || -z "${write_dir}" ]]; then - echo "Usage: ${0} [use2mobs]" + echo "Usage: ${0} [use2mobs]" echo " : conv, oz, or sat" echo " : date string to match" echo " : where to write the new GSI info files into" From 4aa903c33c8361080a1e3c3ebdddfc58b33805ab Mon Sep 17 00:00:00 2001 From: Jeffrey Whitaker Date: Mon, 29 Sep 2025 10:32:49 -0600 Subject: [PATCH 79/90] change BUILD_2M_OBS to USE_2M_OBS --- dev/parm/config/gfs/config.anal | 4 ++-- scripts/exglobal_atmos_analysis.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/parm/config/gfs/config.anal b/dev/parm/config/gfs/config.anal index cf75ded5cdc..7075b53d260 100644 --- a/dev/parm/config/gfs/config.anal +++ b/dev/parm/config/gfs/config.anal @@ -159,8 +159,8 @@ if [[ ${USE_BUILD_GSINFO} == "YES" ]]; then export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_reanl_ncep_ozone.txt" # Use 2m observations when constructing conventional obs info file - # can use 2m obs without enabling soil DA by setting BUILD_2M_OBS=YES in config.base - export BUILD_2M_OBS="${BUILD_2M_OBS:-DO_GSISOILDA}" + # can use 2m obs without enabling soil DA by setting USE_2M_OBS=YES in config.base + export USE_2M_OBS="${USE_2M_OBS:-DO_GSISOILDA}" else export OBS_INPUT="${BUILD_GSINFO_DIR}/obs_input/obs_input_ops.txt" fi diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 57d84304196..10dfa4022d7 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -375,7 +375,7 @@ ${NLN} ${RADCLOUDINFO} cloudy_radiance_info.txt ${NLN} ${ATMSFILTER} atms_beamwidth.txt ${NLN} ${ANAVINFO} anavinfo if [[ ${CONVINFO} == "generate" ]]; then - ${USHgfs}/create_gsi_info.sh conv "${PDY}${cyc}" "${BUILD_2M_OBS}" + ${USHgfs}/create_gsi_info.sh conv "${PDY}${cyc}" "${USE_2M_OBS}" else ${NLN} "${CONVINFO}" convinfo fi From 4438e6b089b41abe4d35b574b9348a529e7d7fe2 Mon Sep 17 00:00:00 2001 From: Jeffrey Whitaker Date: Wed, 1 Oct 2025 14:50:44 -0600 Subject: [PATCH 80/90] update --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 51046f0519f..474415313df 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -150,6 +150,7 @@ ush/bash_utils.sh ush/calcanl_gfs.py @CoryMartin-NOAA ush/calcinc_gfs.py @CoryMartin-NOAA ush/detect_machine.sh +ush/create_gs_info.sh @jswhit2 @DavidHuber-NOAA ush/extractvars_tools.sh @EricSinsky-NOAA ush/file_utils.sh ush/forecast_det.sh @aerorahul From d8a0471188de9b21959f73e608f7af6648c48e29 Mon Sep 17 00:00:00 2001 From: Jeffrey Whitaker Date: Wed, 1 Oct 2025 15:01:52 -0600 Subject: [PATCH 81/90] move * within quotes --- scripts/exglobal_diag.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh index c471f9c63ee..6ad306bd7d0 100755 --- a/scripts/exglobal_diag.sh +++ b/scripts/exglobal_diag.sh @@ -188,7 +188,7 @@ EOFdiag if [[ "${DIAG_COMPRESS}" == "YES" ]]; then ${COMPRESS} "diag_${type}_${string}.${PDY}${cyc}${DIAG_SUFFIX}" fi - echo "diag_${type}_${string}.${PDY}${cyc}"* >> "${diaglist[n]}" + echo "diag_${type}_${string}.${PDY}${cyc}*" >> "${diaglist[n]}" numfile[n]=$(expr "${numfile[n]}" + 1) fi done From c3518bb392a4ca73c24c156fa54d821dc35decfd Mon Sep 17 00:00:00 2001 From: Jeffrey Whitaker Date: Wed, 1 Oct 2025 20:38:35 -0600 Subject: [PATCH 82/90] fix quoting that was causing analdiag to fail with tcp obs --- scripts/exglobal_diag.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh index 6ad306bd7d0..f7d019c0bd2 100755 --- a/scripts/exglobal_diag.sh +++ b/scripts/exglobal_diag.sh @@ -184,7 +184,7 @@ EOFdiag echo "diag_${type}_${string}.${PDY}${cyc}*" >> "${diaglist[n]}" numfile[n]=$(expr ${numfile[n]} + 1) elif [[ ${count} -eq 1 ]]; then - cat "${prefix}${type}_${loop}"* > "diag_${type}_${string}.${PDY}${cyc}${DIAG_SUFFIX}" + cat ${prefix}${type}_${loop}* > "diag_${type}_${string}.${PDY}${cyc}${DIAG_SUFFIX}" if [[ "${DIAG_COMPRESS}" == "YES" ]]; then ${COMPRESS} "diag_${type}_${string}.${PDY}${cyc}${DIAG_SUFFIX}" fi From 11e44c7f91e454f3b9e413790c40613c7e724e74 Mon Sep 17 00:00:00 2001 From: DavidHuber-NOAA Date: Thu, 2 Oct 2025 13:07:54 +0000 Subject: [PATCH 83/90] Update build_gsinfo --- sorc/gsi_enkf.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index 1f7480c70f6..1460f419c70 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit 1f7480c70f678706aa67fd8526f499cbe073496b +Subproject commit 1460f419c700dadb706c3c85f2cbe71cc280aab6 From 59f0cdb16df1cbbac035fba61867cdda0aacb1de Mon Sep 17 00:00:00 2001 From: DavidHuber-NOAA Date: Tue, 14 Oct 2025 17:56:24 +0000 Subject: [PATCH 84/90] Update gsi hash --- sorc/gsi_enkf.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index 1460f419c70..9532be46b29 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit 1460f419c700dadb706c3c85f2cbe71cc280aab6 +Subproject commit 9532be46b29895ec75cf17630514ddb8fa0c46c2 From 39f4fd042f8963fcd0cdf186b0f70ca5b20599ea Mon Sep 17 00:00:00 2001 From: DavidHuber-NOAA Date: Tue, 14 Oct 2025 17:57:18 +0000 Subject: [PATCH 85/90] Add me to gitmodules --- .gitmodules | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index c80d24c03aa..c3d54a0c293 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,9 +13,10 @@ [submodule "sorc/verif-global.fd"] path = sorc/verif-global.fd url = https://github.com/NOAA-EMC/EMC_verif-global.git +# TODO: Point back to NOAA-EMC [submodule "sorc/gsi_enkf.fd"] path = sorc/gsi_enkf.fd - url = https://github.com/NOAA-EMC/GSI.git + url = https://github.com/DavidHuber-NOAA/GSI.git [submodule "sorc/gdas.cd"] path = sorc/gdas.cd url = https://github.com/NOAA-EMC/GDASApp.git From 3ae6d29c951fc6b222301b8e715cd4a675ce7b33 Mon Sep 17 00:00:00 2001 From: jswhit2 Date: Tue, 4 Nov 2025 10:11:43 -0700 Subject: [PATCH 86/90] update .gitmodules, GSI hash, set USE_BUILD_GSINFO=YES --- .gitmodules | 2 +- dev/parm/config/gfs/config.base.j2 | 2 +- sorc/gsi_enkf.fd | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 3e4515ceb1e..bce7b7bc07e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -17,7 +17,7 @@ # TODO: Point back to NOAA-EMC [submodule "sorc/gsi_enkf.fd"] path = sorc/gsi_enkf.fd - url = https://github.com/DavidHuber-NOAA/GSI.git + url = https://github.com/NOAA-EMC/GSI.git [submodule "sorc/gdas.cd"] path = sorc/gdas.cd url = https://github.com/NOAA-EMC/GDASApp.git diff --git a/dev/parm/config/gfs/config.base.j2 b/dev/parm/config/gfs/config.base.j2 index 85eafbd5533..e6bfa5ce68f 100644 --- a/dev/parm/config/gfs/config.base.j2 +++ b/dev/parm/config/gfs/config.base.j2 @@ -385,7 +385,7 @@ export SMOOTH_ENKF="NO" export l4densvar=".true." export lwrite4danl=".true." export DO_CALC_INCREMENT="NO" -export USE_BUILD_GSINFO="NO" +export USE_BUILD_GSINFO="YES" export BUILD_GSINFO_DIR="${PARMgfs}/gsinfo" # Early-cycle EnKF parameters diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index 9532be46b29..840ad1d01af 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit 9532be46b29895ec75cf17630514ddb8fa0c46c2 +Subproject commit 840ad1d01af82b5d0fe6a6369399f9ee509ab7db From ab40906d41584d3b1028f89e37accf57d2014abd Mon Sep 17 00:00:00 2001 From: DavidHuber-NOAA Date: Wed, 5 Nov 2025 13:47:50 +0000 Subject: [PATCH 87/90] Update gsi fix version --- versions/fix.ver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions/fix.ver b/versions/fix.ver index a9430bb76c8..fa2b4311fad 100644 --- a/versions/fix.ver +++ b/versions/fix.ver @@ -15,7 +15,7 @@ export gdas_obs_ver=20240213 export gdas_aero_ver=20240806 export gdas_snow_ver=20241210 export glwu_ver=20220805 -export gsi_ver=20250827 +export gsi_ver=20251105 export lut_ver=20220805 export mom6_ver=20240416 export orog_ver=20240917 From 184a07924d5a2becf6640efb03c2e94c34f6f9fe Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Wed, 5 Nov 2025 08:58:18 -0500 Subject: [PATCH 88/90] Remove completed TODO from .gitmodules Co-authored-by: Cory Martin --- .gitmodules | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index bce7b7bc07e..e608c11ae23 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,7 +14,6 @@ [submodule "sorc/verif-global.fd"] path = sorc/verif-global.fd url = https://github.com/NOAA-EMC/EMC_verif-global.git -# TODO: Point back to NOAA-EMC [submodule "sorc/gsi_enkf.fd"] path = sorc/gsi_enkf.fd url = https://github.com/NOAA-EMC/GSI.git From 6d6c6f28d4d652a941b9b2093ba54d1090a39f44 Mon Sep 17 00:00:00 2001 From: DavidHuber-NOAA Date: Thu, 6 Nov 2025 14:40:18 +0000 Subject: [PATCH 89/90] Update GSI hash to update gsi fix directories --- sorc/gsi_enkf.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index 840ad1d01af..7da30e79df0 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit 840ad1d01af82b5d0fe6a6369399f9ee509ab7db +Subproject commit 7da30e79df0c986712c92b30de8136d7caa00adc From 375ce5204cd533a4d2066b2e4adeb02bca3e6f27 Mon Sep 17 00:00:00 2001 From: DavidHuber-NOAA Date: Thu, 6 Nov 2025 14:58:42 +0000 Subject: [PATCH 90/90] Add document entry for USE_BUILD_GSINFO --- docs/source/configure.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/source/configure.rst b/docs/source/configure.rst index 231b3071814..ee15179dd8f 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -70,3 +70,8 @@ The global-workflow configs contain switches that change how the system runs. Ma | WRITE_DOPOST | Run inline post | .true. | NO | If .true. produces master post output in forecast | | | | | | job | +------------------+----------------------------------+---------------+-------------+---------------------------------------------------+ +| USE_BUILD_GSINFO | Build the GSI info files | YES | NO | If YES, the GSI analysis jobs will build the | +| | | | | satinfo, cnvinfo, and ozinfo files dynamically. | +| | | | | If NO, static versions located in the GSI FIX | +| | | | | directory will be used. | ++------------------+----------------------------------+---------------+-------------+---------------------------------------------------+