From 875fb3ccefc5a6c586b62a5c63ff7dad524ca4be Mon Sep 17 00:00:00 2001 From: jianjunj Date: Thu, 16 Jan 2025 12:55:38 -0500 Subject: [PATCH 01/12] Add precipitable clouds in microwave all-sky assimilations (#63) Add rain, snow, and graupel clouds for CRTM inputs. Add a prior filter to zero clouds over non-water surfaces. Add an option of clouds in the Hydrometeo check. Add gross error check threshold for precipition impacted channels. Add cold-air-outbreak filters. Read cloud fraction from GeoVals. Render list of not-bias corrected channels from JCB tables. Add configurations for atms_n21. --- .../atmosphere/atms_n21.yaml | 43 ++ observations/atmosphere/amsua_n19.yaml.j2 | 142 +++- observations/atmosphere/atms_n20.yaml.j2 | 157 +++- observations/atmosphere/atms_n21.yaml.j2 | 686 ++++++++++++++++++ observations/atmosphere/atms_npp.yaml.j2 | 141 +++- 5 files changed, 1155 insertions(+), 14 deletions(-) create mode 100644 observation_chronicle/atmosphere/atms_n21.yaml create mode 100644 observations/atmosphere/atms_n21.yaml.j2 diff --git a/observation_chronicle/atmosphere/atms_n21.yaml b/observation_chronicle/atmosphere/atms_n21.yaml new file mode 100644 index 0000000..3311e65 --- /dev/null +++ b/observation_chronicle/atmosphere/atms_n21.yaml @@ -0,0 +1,43 @@ +# Instrument metadata +# ------------------- +commissioned: 2023-05-01T12:00:00 + +observer_type: satellite # Type of chronicle to use + +# Instrument initial configuration +# -------------------------------- +channel_variables: + simulated: min + active: min + biascorrtd: min + error: max + error_cld: max + ermax: max + var_b: min + var_pg: min + icld_det: min + icloud: min + iaerosol: min +channel_values: + 1: [ 1, 1, 1, 4.50, 20.00, 4.50, 10.00, 0.00, -2, 1, -1 ] + 2: [ 1, 1, 1, 4.50, 25.00, 4.50, 10.00, 0.00, -2, 1, -1 ] + 3: [ 1, 1, 1, 4.50, 12.00, 3.00, 10.00, 0.00, -2, 1, -1 ] + 4: [ 1, 1, 1, 2.50, 7.00, 3.00, 10.00, 0.00, -2, 1, -1 ] + 5: [ 1, 1, 1, 0.55, 3.50, 1.00, 10.00, 0.00, -2, 1, -1 ] + 6: [ 1, 1, 1, 0.30, 3.00, 1.00, 10.00, 0.00, -2, 1, -1 ] + 7: [ 1, 1, 1, 0.30, 0.80, 1.00, 10.00, 0.00, -2, 1, -1 ] + 8: [ 1, 1, 1, 0.40, 0.40, 1.00, 10.00, 0.00, -2, 1, -1 ] + 9: [ 1, 1, 1, 0.40, 0.40, 1.00, 10.00, 0.00, -2, 1, -1 ] + 10: [ 1, 1, 1, 0.40, 0.40, 1.00, 10.00, 0.00, -2, 1, -1 ] + 11: [ 1, 1, 1, 0.45, 0.45, 1.00, 10.00, 0.00, -2, 1, -1 ] + 12: [ 1, 1, 1, 0.45, 0.45, 1.00, 10.00, 0.00, -2, 1, -1 ] + 13: [ 1, 1, 1, 0.55, 0.55, 1.00, 10.00, 0.00, -2, 1, -1 ] + 14: [ 1, 1, 1, 0.80, 0.80, 2.00, 10.00, 0.00, -2, 1, -1 ] + 15: [ 1, 1, 0, 4.00, 4.00, 4.50, 10.00, 0.00, -2, 1, -1 ] + 16: [ 1, 1, 1, 4.00, 19.00, 4.50, 10.00, 0.00, -2, 1, -1 ] + 17: [ 1, 1, 1, 4.00, 30.00, 2.00, 10.00, 0.00, -2, 1, -1 ] + 18: [ 1, 1, 1, 3.50, 25.00, 2.00, 10.00, 0.00, -2, 1, -1 ] + 19: [ 1, 1, 1, 3.00, 16.50, 2.00, 10.00, 0.00, -2, 1, -1 ] + 20: [ 1, 1, 1, 3.00, 12.00, 2.00, 10.00, 0.00, -2, 1, -1 ] + 21: [ 1, 1, 1, 3.00, 9.00, 2.00, 10.00, 0.00, -2, 1, -1 ] + 22: [ 1, 1, 1, 3.00, 6.50, 2.00, 10.00, 0.00, -2, 1, -1 ] diff --git a/observations/atmosphere/amsua_n19.yaml.j2 b/observations/atmosphere/amsua_n19.yaml.j2 index dba4162..2cc30e3 100644 --- a/observations/atmosphere/amsua_n19.yaml.j2 +++ b/observations/atmosphere/amsua_n19.yaml.j2 @@ -21,9 +21,10 @@ # -------------------- obs operator: name: CRTM - Absorbers: [H2O, O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 + Absorbers: [H2O, O3, CO2] + Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] +# Cloud_Fraction: 1.0 + Cloud_Seeding: true obs options: Sensor_ID: &{{observation_from_jcb}}_sensor_id amsua_n19 EndianType: little_endian @@ -34,6 +35,8 @@ obs bias: input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" + variables without bc: [brightnessTemperature] + channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -64,7 +67,25 @@ # Observation Filters (QC) # ------------------------ - obs filters: + obs prior filters: + # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 + - filter: Variable Assignment + assignments: + - name: MetaData/zeroCloudInCRTM + type: int + function: + name: ObsFunction/Conditional + options: +# firstmatchingcase: false + defaultvalue: 0 # Will not zero clouds by default + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + value: 1 # Will zero clouds by default + + obs post filters: - filter: BlackList filter variables: - name: brightnessTemperature @@ -124,6 +145,7 @@ channels: *{{observation_from_jcb}}_simulated_channels options: channels: *{{observation_from_jcb}}_simulated_channels + Clouds: *{{observation_from_jcb}}_clouds obserr_clearsky: *{{observation_from_jcb}}_err0 clwret_function: name: ObsFunction/CLWRetMW @@ -301,6 +323,8 @@ x1: *{{observation_from_jcb}}_x1 err0: *{{observation_from_jcb}}_err0 err1: *{{observation_from_jcb}}_err1 + threshold: 3 + threshold_precip: 2.5 obserr_bound_max: [{{ get_satellite_variable(observation_from_jcb, "ermax") }}] action: name: reject @@ -334,8 +358,116 @@ minvalue: 1.0e-12 action: name: reject +# Column liquid cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer +# Column ice cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnIceCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer +# Combined ColumnLiquidCloud + ColumnIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnIceCloud + coefs: [1, 1] +# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + value: 1.0e-7 + where: + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + maxvalue: 1.0e-8 - +# ratio of liquid /(Liquid+Ice) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnRatioLiquidCloud + type: float + function: + name: ObsFunction/ElementMultiply + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnLiquidIceCloud + exponents: [1,-1] +# TotalColumnVaporGuess + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/TotalColumnVaporGuess + type: float + function: + name: ObsFunction/TotalColumnVaporGuess +# potentialTemperature at surface + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureSurface + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: surface_pressure +# potentialTemperature near 700 hPa + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: 70000.0 +# stability = Difference between PotentialTemperatur values + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureDiff + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + - name: DerivedMetaData/PotentialTemperatureSurface + coefs: [1, -1] +## cao_check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + where: + - variable: + name: ObsFunction/TotalColumnVaporGuess + maxvalue: 8.0 + - variable: + name: DerivedMetaData/ColumnRatioLiquidCloud + maxvalue: 0.5 + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + minvalue: 1.0e-6 + - variable: + name: DerivedMetaData/PotentialTemperatureDiff + maxvalue: 12.0 + action: + name: reject # GeoVaLs for Driving Observation Operators (testing mode) # -------------------------------------------------------- geovals: diff --git a/observations/atmosphere/atms_n20.yaml.j2 b/observations/atmosphere/atms_n20.yaml.j2 index c16a22c..82d2f50 100644 --- a/observations/atmosphere/atms_n20.yaml.j2 +++ b/observations/atmosphere/atms_n20.yaml.j2 @@ -22,8 +22,8 @@ obs operator: name: CRTM Absorbers: [H2O, O3, CO2] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 + Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] +# Cloud_Fraction: 1.0 Cloud_Seeding: true obs options: Sensor_ID: &{{observation_from_jcb}}_sensor_id atms_n20 @@ -37,6 +37,8 @@ obs bias: input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" + variables without bc: [brightnessTemperature] + channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -65,6 +67,25 @@ ratio for small dataset: 2.0 output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiascovout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovout_suffix}}" + # ------------------------ + obs prior filters: + # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 + - filter: Variable Assignment + assignments: + - name: MetaData/zeroCloudInCRTM + type: int + function: + name: ObsFunction/Conditional + options: +# firstmatchingcase: false + defaultvalue: 0 # Will not zero clouds by default + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + value: 1 # Will zero clouds by default + # Observation Pre Filters (QC) # ---------------------------- obs pre filters: @@ -114,6 +135,7 @@ clwret_ch238: 1 clwret_ch314: 2 clwret_types: [ObsValue] + # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -125,6 +147,7 @@ clwret_ch238: 1 clwret_ch314: 2 clwret_types: [HofX] + # Calculate symmetric retrieved CLW - filter: Variable Assignment assignments: @@ -166,6 +189,7 @@ scatret_ch314: 2 scatret_ch890: 16 scatret_types: [ObsValue] + # Calculate CLW obs/bkg match index - filter: Variable Assignment assignments: @@ -184,6 +208,7 @@ clwret_clearsky: [0.030, 0.030, 0.030, 0.020, 0.030, 0.080, 0.150, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.020, 0.030, 0.030, 0.030, 0.030, 0.050, 0.100] + # Calculate symmetric observation error - filter: Variable Assignment assignments: @@ -209,6 +234,7 @@ err1: [20.000, 25.000, 12.000, 7.000, 3.500, 3.000, 0.800, 0.400, 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 19.000, 30.000, 25.000, 16.500, 12.000, 9.000, 6.500] + # Calculate Innovation@DerivedMetaData - filter: Variable Assignment assignments: @@ -225,6 +251,7 @@ - name: brightnessTemperature@HofX channels: *{{observation_from_jcb}}_simulated_channels coefs: [1, -1] + # Step 0-C: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: @@ -235,6 +262,7 @@ error function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 1: Remove Observations from the Edge of the Scan - filter: Domain Check filter variables: @@ -322,6 +350,7 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + Clouds: *{{observation_from_jcb}}_clouds - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -348,6 +377,7 @@ options: sensor: *{{observation_from_jcb}}_sensor_id channels: *{{observation_from_jcb}}_simulated_channels + - filter: Perform Action filter variables: - name: brightnessTemperature @@ -357,6 +387,7 @@ inflation variable: name: ObsErrorFactorTopo@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: @@ -368,6 +399,7 @@ channels: *{{observation_from_jcb}}_simulated_channels options: channels: *{{observation_from_jcb}}_simulated_channels + - filter: Perform Action filter variables: - name: brightnessTemperature @@ -377,6 +409,7 @@ inflation variable: name: ObsErrorFactorTransmitTop@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: @@ -391,6 +424,7 @@ channels: *{{observation_from_jcb}}_simulated_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + - filter: Perform Action filter variables: - name: brightnessTemperature @@ -400,6 +434,7 @@ inflation variable: name: ObsErrorFactorSurfJacobian@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 9: Situation Dependent Check - filter: Variable Assignment assignments: @@ -427,6 +462,7 @@ obserr_clearsky: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, 3.000, 3.000] + - filter: Perform Action filter variables: - name: brightnessTemperature @@ -436,8 +472,9 @@ inflation variable: name: ObsErrorFactorSituDepend@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 10: Gross check - # Remove data if abs(Obs-HofX) > absolute threhold + # Remove data if abs(Obs-HofX) > absolute threshold - filter: Variable Assignment assignments: - name: ObsErrorFactorLat@DerivedMetaData @@ -446,6 +483,7 @@ name: ObsErrorFactorLatRad@ObsFunction options: latitude_parameters: [25.0, 0.25, 0.04, 3.0] + - filter: Variable Assignment assignments: - name: ObsErrorBound@DerivedMetaData @@ -470,9 +508,11 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - threhold: 3 + threshold: 3 + threshold_precip: 2.5 obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] + - filter: Background Check filter variables: - name: brightnessTemperature @@ -526,7 +566,116 @@ ignore: rejected observations - name: reject +# Column liquid cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer +# Column ice cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnIceCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer +# Combined ColumnLiquidCloud + ColumnIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnIceCloud + coefs: [1, 1] +# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + value: 1.0e-7 + where: + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + maxvalue: 1.0e-8 +# ratio of liquid /(Liquid+Ice) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnRatioLiquidCloud + type: float + function: + name: ObsFunction/ElementMultiply + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnLiquidIceCloud + exponents: [1,-1] +# TotalColumnVaporGuess + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/TotalColumnVaporGuess + type: float + function: + name: ObsFunction/TotalColumnVaporGuess +# potentialTemperature at surface + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureSurface + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: surface_pressure +# potentialTemperature near 700 hPa + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: 70000.0 +# stability = Difference between PotentialTemperatur values + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureDiff + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + - name: DerivedMetaData/PotentialTemperatureSurface + coefs: [1, -1] +## cao_check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + where: + - variable: + name: ObsFunction/TotalColumnVaporGuess + maxvalue: 8.0 + - variable: + name: DerivedMetaData/ColumnRatioLiquidCloud + maxvalue: 0.5 + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + minvalue: 1.0e-6 + - variable: + name: DerivedMetaData/PotentialTemperatureDiff + maxvalue: 12.0 + action: + name: reject # GeoVaLs for Driving Observation Operators (testing mode) # -------------------------------------------------------- geovals: diff --git a/observations/atmosphere/atms_n21.yaml.j2 b/observations/atmosphere/atms_n21.yaml.j2 new file mode 100644 index 0000000..560be3b --- /dev/null +++ b/observations/atmosphere/atms_n21.yaml.j2 @@ -0,0 +1,686 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: ATMS N21 + obsdatain: + engine: + type: H5File + obsfile: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}" + obsdataout: + engine: + type: H5File + obsfile: "{{atmosphere_obsdataout_path}}/{{atmosphere_obsdataout_prefix}}{{observation_from_jcb}}{{atmosphere_obsdataout_suffix}}" + io pool: + max pool size: 1 + simulated variables: [brightnessTemperature] + channels: &{{observation_from_jcb}}_simulated_channels {{ get_satellite_variable(observation_from_jcb, "simulated") }} + + # Observation Operator + # -------------------- + obs operator: + name: CRTM + Absorbers: [H2O, O3, CO2] + Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] +# Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: &{{observation_from_jcb}}_sensor_id atms_n21 + EndianType: little_endian + CoefficientPath: "{{crtm_coefficient_path}}" + linear obs operator: + Absorbers: [H2O, O3] + + # Observation Bias Correction (VarBC) + # ----------------------------------- + obs bias: + input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" + output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" + variables without bc: [brightnessTemperature] + channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: &{{observation_from_jcb}}_tlapse "{{atmosphere_obsbiasin_path}}/{{atmosphere_obstlapsein_prefix}}{{observation_from_jcb}}{{atmosphere_obstlapsein_suffix}}" + - name: lapseRate + tlapse: *{{observation_from_jcb}}_tlapse + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiascovin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovin_suffix}}" + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiascovout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovout_suffix}}" + + # ------------------------ + obs prior filters: + # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 + - filter: Variable Assignment + assignments: + - name: MetaData/zeroCloudInCRTM + type: int + function: + name: ObsFunction/Conditional + options: +# firstmatchingcase: false + defaultvalue: 0 # Will not zero clouds by default + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + value: 1 # Will zero clouds by default + + # Observation Pre Filters (QC) + # ---------------------------- + obs pre filters: + # Step 0-A: Create Diagnostic Flags + - filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false + + # Observation Post Filters (QC) + # ----------------------------- + obs post filters: + # Step 0-B: Calculate derived variables + # Calculate CLW retrieved from observation + - filter: Variable Assignment + assignments: + - name: CLWRetFromObs@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + + # Calculate CLW retrieved from observation + - filter: Variable Assignment + assignments: + - name: CLWRetFromBkg@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + + # Calculate symmetric retrieved CLW + - filter: Variable Assignment + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + value: 1000.0 + + - filter: Variable Assignment + where: + - variable: + name: CLWRetFromObs@DerivedMetaData + minvalue: 0. + maxvalue: 999. + - variable: + name: CLWRetFromBkg@DerivedMetaData + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + function: + name: Arithmetic@ObsFunction + options: + variables: + - name: CLWRetFromObs@DerivedMetaData + - name: CLWRetFromBkg@DerivedMetaData + total coefficient: 0.5 + + # Calculate scattering index from observation + - filter: Variable Assignment + assignments: + - name: SIRetFromObs@DerivedMetaData + type: float + function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 16 + scatret_types: [ObsValue] + + # Calculate CLW obs/bkg match index + - filter: Variable Assignment + assignments: + - name: CLWMatchIndex@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: CLWMatchIndexMW@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwret_clearsky: [0.030, 0.030, 0.030, 0.020, 0.030, 0.080, 0.150, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.020, 0.030, 0.030, + 0.030, 0.030, 0.050, 0.100] + + # Calculate symmetric observation error + - filter: Variable Assignment + assignments: + - name: InitialObsError@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsErrorModelRamp@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + xvar: + name: CLWRetSymmetric@DerivedMetaData + x0: [0.030, 0.030, 0.030, 0.020, 0.030, 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [0.350, 0.380, 0.400, 0.450, 0.500, 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, 3.000, 0.800, 0.400, 0.400, + 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 19.000, 30.000, 25.000, 16.500, + 12.000, 9.000, 6.500] + + # Calculate Innovation@DerivedMetaData + - filter: Variable Assignment + assignments: + - name: Innovation@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *{{observation_from_jcb}}_simulated_channels + options: + variables: + - name: brightnessTemperature@ObsValue + channels: *{{observation_from_jcb}}_simulated_channels + - name: brightnessTemperature@HofX + channels: *{{observation_from_jcb}}_simulated_channels + coefs: [1, -1] + + # Step 0-C: Assign Initial All-Sky Observation Error + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: assign error + error function: + name: InitialObsError@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + + # Step 1: Remove Observations from the Edge of the Scan + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-22 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 7-90 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + + # Step 2: Data Thinning + - filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic + # round_horizontal_bin_count_to_nearest: true + # partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + + # Step 3A: CLW Retrieval Check (observation_based) + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetFromObs@DerivedMetaData + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + + # Step 3B: CLW Retrieval Check (background_based) + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetFromBkg@DerivedMetaData + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + + # Step 4: Window Channel Sanity Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16, 17-22 + test variables: + - name: Innovation@DerivedMetaData + channels: 1, 2, 5-7, 16 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject + + # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: HydrometeorCheckATMS@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + obserr_clearsky: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, + 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, + 3.000, 3.000, 3.000, 3.000] + clwret_function: + name: CLWRetFromObs@DerivedMetaData + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + Clouds: *{{observation_from_jcb}}_clouds + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + test variables: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *{{observation_from_jcb}}_simulated_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheck + ignore: rejected observations + - name: reject + + # Step 6: Observation Error Inflation based on Topography Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorTopo@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + sensor: *{{observation_from_jcb}}_sensor_id + channels: *{{observation_from_jcb}}_simulated_channels + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + + # Step 7: Obs Error Inflation based on TOA Transmittancec Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + + # Step 8: Observation Error Inflation based on Surface Jacobian Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + sensor: *{{observation_from_jcb}}_sensor_id + channels: *{{observation_from_jcb}}_simulated_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + + # Step 9: Situation Dependent Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsErrorFactorSituDependMW@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + sensor: *{{observation_from_jcb}}_sensor_id + channels: *{{observation_from_jcb}}_simulated_channels + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + scatobs_function: + name: SIRetFromObs@DerivedMetaData + clwmatchidx_function: + name: CLWMatchIndex@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + obserr_clearsky: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, + 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, + 3.000, 3.000, 3.000, 3.000] + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + + # Step 10: Gross check + # Remove data if abs(Obs-HofX) > absolute threshold + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorLat@DerivedMetaData + type: float + function: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + + - filter: Variable Assignment + assignments: + - name: ObsErrorBound@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsErrorBoundMW@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + sensor: *{{observation_from_jcb}}_sensor_id + channels: *{{observation_from_jcb}}_simulated_channels + obserr_bound_latitude: + name: ObsErrorFactorLat@DerivedMetaData + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + obserr_bound_topo: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + threshold: 3 + threshold_precip: 2.5 + obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] + + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + function absolute threshold: + - name: ObsErrorBound@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + actions: + - name: set + flag: GrossCheck + ignore: rejected observations + - name: reject + + # Step 11: Inter-Channel Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + test variables: + - name: InterChannelConsistencyCheck@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + use passive_bc: true + sensor: *{{observation_from_jcb}}_sensor_id + use_flag: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, + 1] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + + # Step 12: Useflag Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + use_flag: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, + 1] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject + +# Column liquid cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer +# Column ice cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnIceCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer +# Combined ColumnLiquidCloud + ColumnIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnIceCloud + coefs: [1, 1] +# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + value: 1.0e-7 + where: + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + maxvalue: 1.0e-8 + +# ratio of liquid /(Liquid+Ice) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnRatioLiquidCloud + type: float + function: + name: ObsFunction/ElementMultiply + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnLiquidIceCloud + exponents: [1,-1] +# TotalColumnVaporGuess + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/TotalColumnVaporGuess + type: float + function: + name: ObsFunction/TotalColumnVaporGuess +# potentialTemperature at surface + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureSurface + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: surface_pressure +# potentialTemperature near 700 hPa + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: 70000.0 +# stability = Difference between PotentialTemperatur values + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureDiff + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + - name: DerivedMetaData/PotentialTemperatureSurface + coefs: [1, -1] +## cao_check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + where: + - variable: + name: ObsFunction/TotalColumnVaporGuess + maxvalue: 8.0 + - variable: + name: DerivedMetaData/ColumnRatioLiquidCloud + maxvalue: 0.5 + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + minvalue: 1.0e-6 + - variable: + name: DerivedMetaData/PotentialTemperatureDiff + maxvalue: 12.0 + action: + name: reject + # GeoVaLs for Driving Observation Operators (testing mode) + # -------------------------------------------------------- + geovals: + filename: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}_geoval{{atmosphere_obsdatain_suffix}}" + + # Passed benchmark for UFO testing + # -------------------------------- + passedBenchmark: 0 diff --git a/observations/atmosphere/atms_npp.yaml.j2 b/observations/atmosphere/atms_npp.yaml.j2 index d18985c..309da34 100644 --- a/observations/atmosphere/atms_npp.yaml.j2 +++ b/observations/atmosphere/atms_npp.yaml.j2 @@ -22,8 +22,8 @@ obs operator: name: CRTM Absorbers: [H2O, O3, CO2] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 + Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] +# Cloud_Fraction: 1.0 Cloud_Seeding: true obs options: Sensor_ID: &{{observation_from_jcb}}_sensor_id atms_npp @@ -37,6 +37,8 @@ obs bias: input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" + variables without bc: [brightnessTemperature] + channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -65,6 +67,25 @@ ratio for small dataset: 2.0 output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiascovout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovout_suffix}}" + # ------------------------ + obs prior filters: + # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 + - filter: Variable Assignment + assignments: + - name: MetaData/zeroCloudInCRTM + type: int + function: + name: ObsFunction/Conditional + options: +# firstmatchingcase: false + defaultvalue: 0 # Will not zero clouds by default + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + value: 1 # Will zero clouds by default + # Observation Pre Filters (QC) # ---------------------------- obs pre filters: @@ -329,7 +350,7 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - + Clouds: *{{observation_from_jcb}}_clouds - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -453,7 +474,7 @@ channels: *{{observation_from_jcb}}_simulated_channels # Step 10: Gross check - # Remove data if abs(Obs-HofX) > absolute threhold + # Remove data if abs(Obs-HofX) > absolute threshold - filter: Variable Assignment assignments: - name: ObsErrorFactorLat@DerivedMetaData @@ -487,7 +508,8 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - threhold: 3 + threshold: 3 + threshold_precip: 2.5 obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] @@ -544,7 +566,116 @@ ignore: rejected observations - name: reject +# Column liquid cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer +# Column ice cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnIceCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer +# Combined ColumnLiquidCloud + ColumnIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnIceCloud + coefs: [1, 1] +# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + value: 1.0e-7 + where: + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + maxvalue: 1.0e-8 +# ratio of liquid /(Liquid+Ice) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnRatioLiquidCloud + type: float + function: + name: ObsFunction/ElementMultiply + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnLiquidIceCloud + exponents: [1,-1] +# TotalColumnVaporGuess + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/TotalColumnVaporGuess + type: float + function: + name: ObsFunction/TotalColumnVaporGuess +# potentialTemperature at surface + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureSurface + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: surface_pressure +# potentialTemperature near 700 hPa + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: 70000.0 +# stability = Difference between PotentialTemperatur values + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureDiff + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + - name: DerivedMetaData/PotentialTemperatureSurface + coefs: [1, -1] +## cao_check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + where: + - variable: + name: ObsFunction/TotalColumnVaporGuess + maxvalue: 8.0 + - variable: + name: DerivedMetaData/ColumnRatioLiquidCloud + maxvalue: 0.5 + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + minvalue: 1.0e-6 + - variable: + name: DerivedMetaData/PotentialTemperatureDiff + maxvalue: 12.0 + action: + name: reject # GeoVaLs for Driving Observation Operators (testing mode) # -------------------------------------------------------- geovals: From 0150d282785e3b48d62d7751bc906439baf88125 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> Date: Fri, 17 Jan 2025 21:41:42 -0500 Subject: [PATCH 02/12] Revert " Add precipitable clouds in microwave all-sky assimilations" (#68) --- .../atmosphere/atms_n21.yaml | 43 -- observations/atmosphere/amsua_n19.yaml.j2 | 142 +--- observations/atmosphere/atms_n20.yaml.j2 | 157 +--- observations/atmosphere/atms_n21.yaml.j2 | 686 ------------------ observations/atmosphere/atms_npp.yaml.j2 | 141 +--- 5 files changed, 14 insertions(+), 1155 deletions(-) delete mode 100644 observation_chronicle/atmosphere/atms_n21.yaml delete mode 100644 observations/atmosphere/atms_n21.yaml.j2 diff --git a/observation_chronicle/atmosphere/atms_n21.yaml b/observation_chronicle/atmosphere/atms_n21.yaml deleted file mode 100644 index 3311e65..0000000 --- a/observation_chronicle/atmosphere/atms_n21.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# Instrument metadata -# ------------------- -commissioned: 2023-05-01T12:00:00 - -observer_type: satellite # Type of chronicle to use - -# Instrument initial configuration -# -------------------------------- -channel_variables: - simulated: min - active: min - biascorrtd: min - error: max - error_cld: max - ermax: max - var_b: min - var_pg: min - icld_det: min - icloud: min - iaerosol: min -channel_values: - 1: [ 1, 1, 1, 4.50, 20.00, 4.50, 10.00, 0.00, -2, 1, -1 ] - 2: [ 1, 1, 1, 4.50, 25.00, 4.50, 10.00, 0.00, -2, 1, -1 ] - 3: [ 1, 1, 1, 4.50, 12.00, 3.00, 10.00, 0.00, -2, 1, -1 ] - 4: [ 1, 1, 1, 2.50, 7.00, 3.00, 10.00, 0.00, -2, 1, -1 ] - 5: [ 1, 1, 1, 0.55, 3.50, 1.00, 10.00, 0.00, -2, 1, -1 ] - 6: [ 1, 1, 1, 0.30, 3.00, 1.00, 10.00, 0.00, -2, 1, -1 ] - 7: [ 1, 1, 1, 0.30, 0.80, 1.00, 10.00, 0.00, -2, 1, -1 ] - 8: [ 1, 1, 1, 0.40, 0.40, 1.00, 10.00, 0.00, -2, 1, -1 ] - 9: [ 1, 1, 1, 0.40, 0.40, 1.00, 10.00, 0.00, -2, 1, -1 ] - 10: [ 1, 1, 1, 0.40, 0.40, 1.00, 10.00, 0.00, -2, 1, -1 ] - 11: [ 1, 1, 1, 0.45, 0.45, 1.00, 10.00, 0.00, -2, 1, -1 ] - 12: [ 1, 1, 1, 0.45, 0.45, 1.00, 10.00, 0.00, -2, 1, -1 ] - 13: [ 1, 1, 1, 0.55, 0.55, 1.00, 10.00, 0.00, -2, 1, -1 ] - 14: [ 1, 1, 1, 0.80, 0.80, 2.00, 10.00, 0.00, -2, 1, -1 ] - 15: [ 1, 1, 0, 4.00, 4.00, 4.50, 10.00, 0.00, -2, 1, -1 ] - 16: [ 1, 1, 1, 4.00, 19.00, 4.50, 10.00, 0.00, -2, 1, -1 ] - 17: [ 1, 1, 1, 4.00, 30.00, 2.00, 10.00, 0.00, -2, 1, -1 ] - 18: [ 1, 1, 1, 3.50, 25.00, 2.00, 10.00, 0.00, -2, 1, -1 ] - 19: [ 1, 1, 1, 3.00, 16.50, 2.00, 10.00, 0.00, -2, 1, -1 ] - 20: [ 1, 1, 1, 3.00, 12.00, 2.00, 10.00, 0.00, -2, 1, -1 ] - 21: [ 1, 1, 1, 3.00, 9.00, 2.00, 10.00, 0.00, -2, 1, -1 ] - 22: [ 1, 1, 1, 3.00, 6.50, 2.00, 10.00, 0.00, -2, 1, -1 ] diff --git a/observations/atmosphere/amsua_n19.yaml.j2 b/observations/atmosphere/amsua_n19.yaml.j2 index 2cc30e3..dba4162 100644 --- a/observations/atmosphere/amsua_n19.yaml.j2 +++ b/observations/atmosphere/amsua_n19.yaml.j2 @@ -21,10 +21,9 @@ # -------------------- obs operator: name: CRTM - Absorbers: [H2O, O3, CO2] - Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] -# Cloud_Fraction: 1.0 - Cloud_Seeding: true + Absorbers: [H2O, O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 obs options: Sensor_ID: &{{observation_from_jcb}}_sensor_id amsua_n19 EndianType: little_endian @@ -35,8 +34,6 @@ obs bias: input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" - variables without bc: [brightnessTemperature] - channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -67,25 +64,7 @@ # Observation Filters (QC) # ------------------------ - obs prior filters: - # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 - - filter: Variable Assignment - assignments: - - name: MetaData/zeroCloudInCRTM - type: int - function: - name: ObsFunction/Conditional - options: -# firstmatchingcase: false - defaultvalue: 0 # Will not zero clouds by default - cases: - - where: - - variable: - name: GeoVaLs/water_area_fraction - maxvalue: 0.99 - value: 1 # Will zero clouds by default - - obs post filters: + obs filters: - filter: BlackList filter variables: - name: brightnessTemperature @@ -145,7 +124,6 @@ channels: *{{observation_from_jcb}}_simulated_channels options: channels: *{{observation_from_jcb}}_simulated_channels - Clouds: *{{observation_from_jcb}}_clouds obserr_clearsky: *{{observation_from_jcb}}_err0 clwret_function: name: ObsFunction/CLWRetMW @@ -323,8 +301,6 @@ x1: *{{observation_from_jcb}}_x1 err0: *{{observation_from_jcb}}_err0 err1: *{{observation_from_jcb}}_err1 - threshold: 3 - threshold_precip: 2.5 obserr_bound_max: [{{ get_satellite_variable(observation_from_jcb, "ermax") }}] action: name: reject @@ -358,116 +334,8 @@ minvalue: 1.0e-12 action: name: reject -# Column liquid cloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidCloud - type: float - function: - name: TotalColumnCloudGuess@ObsFunction - options: - cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer -# Column ice cloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnIceCloud - type: float - function: - name: TotalColumnCloudGuess@ObsFunction - options: - cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer -# Combined ColumnLiquidCloud + ColumnIceCloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidIceCloud - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: DerivedMetaData/ColumnLiquidCloud - - name: DerivedMetaData/ColumnIceCloud - coefs: [1, 1] -# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidIceCloud - type: float - value: 1.0e-7 - where: - - variable: - name: DerivedMetaData/ColumnLiquidIceCloud - maxvalue: 1.0e-8 -# ratio of liquid /(Liquid+Ice) - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnRatioLiquidCloud - type: float - function: - name: ObsFunction/ElementMultiply - options: - variables: - - name: DerivedMetaData/ColumnLiquidCloud - - name: DerivedMetaData/ColumnLiquidIceCloud - exponents: [1,-1] -# TotalColumnVaporGuess - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/TotalColumnVaporGuess - type: float - function: - name: ObsFunction/TotalColumnVaporGuess -# potentialTemperature at surface - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureSurface - type: float - function: - name: PotentialTemperatureGuess2D@ObsFunction - options: - approximate pressure: surface_pressure -# potentialTemperature near 700 hPa - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureAt700hPa - type: float - function: - name: PotentialTemperatureGuess2D@ObsFunction - options: - approximate pressure: 70000.0 -# stability = Difference between PotentialTemperatur values - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureDiff - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: DerivedMetaData/PotentialTemperatureAt700hPa - - name: DerivedMetaData/PotentialTemperatureSurface - coefs: [1, -1] -## cao_check - - filter: BlackList - filter variables: - - name: brightnessTemperature - channels: 1-6, 15 - where: - - variable: - name: ObsFunction/TotalColumnVaporGuess - maxvalue: 8.0 - - variable: - name: DerivedMetaData/ColumnRatioLiquidCloud - maxvalue: 0.5 - - variable: - name: DerivedMetaData/ColumnLiquidIceCloud - minvalue: 1.0e-6 - - variable: - name: DerivedMetaData/PotentialTemperatureDiff - maxvalue: 12.0 - action: - name: reject + # GeoVaLs for Driving Observation Operators (testing mode) # -------------------------------------------------------- geovals: diff --git a/observations/atmosphere/atms_n20.yaml.j2 b/observations/atmosphere/atms_n20.yaml.j2 index 82d2f50..c16a22c 100644 --- a/observations/atmosphere/atms_n20.yaml.j2 +++ b/observations/atmosphere/atms_n20.yaml.j2 @@ -22,8 +22,8 @@ obs operator: name: CRTM Absorbers: [H2O, O3, CO2] - Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] -# Cloud_Fraction: 1.0 + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 Cloud_Seeding: true obs options: Sensor_ID: &{{observation_from_jcb}}_sensor_id atms_n20 @@ -37,8 +37,6 @@ obs bias: input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" - variables without bc: [brightnessTemperature] - channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -67,25 +65,6 @@ ratio for small dataset: 2.0 output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiascovout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovout_suffix}}" - # ------------------------ - obs prior filters: - # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 - - filter: Variable Assignment - assignments: - - name: MetaData/zeroCloudInCRTM - type: int - function: - name: ObsFunction/Conditional - options: -# firstmatchingcase: false - defaultvalue: 0 # Will not zero clouds by default - cases: - - where: - - variable: - name: GeoVaLs/water_area_fraction - maxvalue: 0.99 - value: 1 # Will zero clouds by default - # Observation Pre Filters (QC) # ---------------------------- obs pre filters: @@ -135,7 +114,6 @@ clwret_ch238: 1 clwret_ch314: 2 clwret_types: [ObsValue] - # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -147,7 +125,6 @@ clwret_ch238: 1 clwret_ch314: 2 clwret_types: [HofX] - # Calculate symmetric retrieved CLW - filter: Variable Assignment assignments: @@ -189,7 +166,6 @@ scatret_ch314: 2 scatret_ch890: 16 scatret_types: [ObsValue] - # Calculate CLW obs/bkg match index - filter: Variable Assignment assignments: @@ -208,7 +184,6 @@ clwret_clearsky: [0.030, 0.030, 0.030, 0.020, 0.030, 0.080, 0.150, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.020, 0.030, 0.030, 0.030, 0.030, 0.050, 0.100] - # Calculate symmetric observation error - filter: Variable Assignment assignments: @@ -234,7 +209,6 @@ err1: [20.000, 25.000, 12.000, 7.000, 3.500, 3.000, 0.800, 0.400, 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 19.000, 30.000, 25.000, 16.500, 12.000, 9.000, 6.500] - # Calculate Innovation@DerivedMetaData - filter: Variable Assignment assignments: @@ -251,7 +225,6 @@ - name: brightnessTemperature@HofX channels: *{{observation_from_jcb}}_simulated_channels coefs: [1, -1] - # Step 0-C: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: @@ -262,7 +235,6 @@ error function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - # Step 1: Remove Observations from the Edge of the Scan - filter: Domain Check filter variables: @@ -350,7 +322,6 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - Clouds: *{{observation_from_jcb}}_clouds - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -377,7 +348,6 @@ options: sensor: *{{observation_from_jcb}}_sensor_id channels: *{{observation_from_jcb}}_simulated_channels - - filter: Perform Action filter variables: - name: brightnessTemperature @@ -387,7 +357,6 @@ inflation variable: name: ObsErrorFactorTopo@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - # Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: @@ -399,7 +368,6 @@ channels: *{{observation_from_jcb}}_simulated_channels options: channels: *{{observation_from_jcb}}_simulated_channels - - filter: Perform Action filter variables: - name: brightnessTemperature @@ -409,7 +377,6 @@ inflation variable: name: ObsErrorFactorTransmitTop@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - # Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: @@ -424,7 +391,6 @@ channels: *{{observation_from_jcb}}_simulated_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] - - filter: Perform Action filter variables: - name: brightnessTemperature @@ -434,7 +400,6 @@ inflation variable: name: ObsErrorFactorSurfJacobian@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - # Step 9: Situation Dependent Check - filter: Variable Assignment assignments: @@ -462,7 +427,6 @@ obserr_clearsky: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, 3.000, 3.000] - - filter: Perform Action filter variables: - name: brightnessTemperature @@ -472,9 +436,8 @@ inflation variable: name: ObsErrorFactorSituDepend@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - # Step 10: Gross check - # Remove data if abs(Obs-HofX) > absolute threshold + # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: - name: ObsErrorFactorLat@DerivedMetaData @@ -483,7 +446,6 @@ name: ObsErrorFactorLatRad@ObsFunction options: latitude_parameters: [25.0, 0.25, 0.04, 3.0] - - filter: Variable Assignment assignments: - name: ObsErrorBound@DerivedMetaData @@ -508,11 +470,9 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - threshold: 3 - threshold_precip: 2.5 + threhold: 3 obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] - - filter: Background Check filter variables: - name: brightnessTemperature @@ -566,116 +526,7 @@ ignore: rejected observations - name: reject -# Column liquid cloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidCloud - type: float - function: - name: TotalColumnCloudGuess@ObsFunction - options: - cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer -# Column ice cloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnIceCloud - type: float - function: - name: TotalColumnCloudGuess@ObsFunction - options: - cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer -# Combined ColumnLiquidCloud + ColumnIceCloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidIceCloud - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: DerivedMetaData/ColumnLiquidCloud - - name: DerivedMetaData/ColumnIceCloud - coefs: [1, 1] -# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidIceCloud - type: float - value: 1.0e-7 - where: - - variable: - name: DerivedMetaData/ColumnLiquidIceCloud - maxvalue: 1.0e-8 -# ratio of liquid /(Liquid+Ice) - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnRatioLiquidCloud - type: float - function: - name: ObsFunction/ElementMultiply - options: - variables: - - name: DerivedMetaData/ColumnLiquidCloud - - name: DerivedMetaData/ColumnLiquidIceCloud - exponents: [1,-1] -# TotalColumnVaporGuess - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/TotalColumnVaporGuess - type: float - function: - name: ObsFunction/TotalColumnVaporGuess -# potentialTemperature at surface - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureSurface - type: float - function: - name: PotentialTemperatureGuess2D@ObsFunction - options: - approximate pressure: surface_pressure -# potentialTemperature near 700 hPa - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureAt700hPa - type: float - function: - name: PotentialTemperatureGuess2D@ObsFunction - options: - approximate pressure: 70000.0 -# stability = Difference between PotentialTemperatur values - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureDiff - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: DerivedMetaData/PotentialTemperatureAt700hPa - - name: DerivedMetaData/PotentialTemperatureSurface - coefs: [1, -1] -## cao_check - - filter: BlackList - filter variables: - - name: brightnessTemperature - channels: 1-7, 16-22 - where: - - variable: - name: ObsFunction/TotalColumnVaporGuess - maxvalue: 8.0 - - variable: - name: DerivedMetaData/ColumnRatioLiquidCloud - maxvalue: 0.5 - - variable: - name: DerivedMetaData/ColumnLiquidIceCloud - minvalue: 1.0e-6 - - variable: - name: DerivedMetaData/PotentialTemperatureDiff - maxvalue: 12.0 - action: - name: reject # GeoVaLs for Driving Observation Operators (testing mode) # -------------------------------------------------------- geovals: diff --git a/observations/atmosphere/atms_n21.yaml.j2 b/observations/atmosphere/atms_n21.yaml.j2 deleted file mode 100644 index 560be3b..0000000 --- a/observations/atmosphere/atms_n21.yaml.j2 +++ /dev/null @@ -1,686 +0,0 @@ -- - - # Observation Space (I/O) - # ----------------------- - obs space: - name: ATMS N21 - obsdatain: - engine: - type: H5File - obsfile: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}" - obsdataout: - engine: - type: H5File - obsfile: "{{atmosphere_obsdataout_path}}/{{atmosphere_obsdataout_prefix}}{{observation_from_jcb}}{{atmosphere_obsdataout_suffix}}" - io pool: - max pool size: 1 - simulated variables: [brightnessTemperature] - channels: &{{observation_from_jcb}}_simulated_channels {{ get_satellite_variable(observation_from_jcb, "simulated") }} - - # Observation Operator - # -------------------- - obs operator: - name: CRTM - Absorbers: [H2O, O3, CO2] - Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] -# Cloud_Fraction: 1.0 - Cloud_Seeding: true - obs options: - Sensor_ID: &{{observation_from_jcb}}_sensor_id atms_n21 - EndianType: little_endian - CoefficientPath: "{{crtm_coefficient_path}}" - linear obs operator: - Absorbers: [H2O, O3] - - # Observation Bias Correction (VarBC) - # ----------------------------------- - obs bias: - input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" - output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" - variables without bc: [brightnessTemperature] - channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} - variational bc: - predictors: - - name: constant - - name: lapseRate - order: 2 - tlapse: &{{observation_from_jcb}}_tlapse "{{atmosphere_obsbiasin_path}}/{{atmosphere_obstlapsein_prefix}}{{observation_from_jcb}}{{atmosphere_obstlapsein_suffix}}" - - name: lapseRate - tlapse: *{{observation_from_jcb}}_tlapse - - name: emissivityJacobian - - name: sensorScanAngle - order: 4 - - name: sensorScanAngle - order: 3 - - name: sensorScanAngle - order: 2 - - name: sensorScanAngle - covariance: - minimal required obs number: 20 - variance range: [1.0e-6, 10.0] - step size: 1.0e-4 - largest analysis variance: 10000.0 - prior: - input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiascovin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovin_suffix}}" - inflation: - ratio: 1.1 - ratio for small dataset: 2.0 - output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiascovout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovout_suffix}}" - - # ------------------------ - obs prior filters: - # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 - - filter: Variable Assignment - assignments: - - name: MetaData/zeroCloudInCRTM - type: int - function: - name: ObsFunction/Conditional - options: -# firstmatchingcase: false - defaultvalue: 0 # Will not zero clouds by default - cases: - - where: - - variable: - name: GeoVaLs/water_area_fraction - maxvalue: 0.99 - value: 1 # Will zero clouds by default - - # Observation Pre Filters (QC) - # ---------------------------- - obs pre filters: - # Step 0-A: Create Diagnostic Flags - - filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - flags: - - name: ScanEdgeRemoval - initial value: false - force reinitialization: false - - name: Thinning - initial value: false - force reinitialization: false - - name: CLWRetrievalCheck - initial value: false - force reinitialization: false - - name: WindowChannelExtremeResidual - initial value: false - force reinitialization: false - - name: HydrometeorCheck - initial value: false - force reinitialization: false - - name: GrossCheck - initial value: false - force reinitialization: false - - name: InterChannelConsistency - initial value: false - force reinitialization: false - - name: UseflagCheck - initial value: false - force reinitialization: false - - # Observation Post Filters (QC) - # ----------------------------- - obs post filters: - # Step 0-B: Calculate derived variables - # Calculate CLW retrieved from observation - - filter: Variable Assignment - assignments: - - name: CLWRetFromObs@DerivedMetaData - type: float - function: - name: CLWRetMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] - - # Calculate CLW retrieved from observation - - filter: Variable Assignment - assignments: - - name: CLWRetFromBkg@DerivedMetaData - type: float - function: - name: CLWRetMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] - - # Calculate symmetric retrieved CLW - - filter: Variable Assignment - assignments: - - name: CLWRetSymmetric@DerivedMetaData - type: float - value: 1000.0 - - - filter: Variable Assignment - where: - - variable: - name: CLWRetFromObs@DerivedMetaData - minvalue: 0. - maxvalue: 999. - - variable: - name: CLWRetFromBkg@DerivedMetaData - minvalue: 0. - maxvalue: 999. - where operator: and - assignments: - - name: CLWRetSymmetric@DerivedMetaData - type: float - function: - name: Arithmetic@ObsFunction - options: - variables: - - name: CLWRetFromObs@DerivedMetaData - - name: CLWRetFromBkg@DerivedMetaData - total coefficient: 0.5 - - # Calculate scattering index from observation - - filter: Variable Assignment - assignments: - - name: SIRetFromObs@DerivedMetaData - type: float - function: - name: SCATRetMW@ObsFunction - options: - scatret_ch238: 1 - scatret_ch314: 2 - scatret_ch890: 16 - scatret_types: [ObsValue] - - # Calculate CLW obs/bkg match index - - filter: Variable Assignment - assignments: - - name: CLWMatchIndex@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - type: float - function: - name: CLWMatchIndexMW@ObsFunction - channels: *{{observation_from_jcb}}_simulated_channels - options: - channels: *{{observation_from_jcb}}_simulated_channels - clwobs_function: - name: CLWRetFromObs@DerivedMetaData - clwbkg_function: - name: CLWRetFromBkg@DerivedMetaData - clwret_clearsky: [0.030, 0.030, 0.030, 0.020, 0.030, 0.080, 0.150, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.020, 0.030, 0.030, - 0.030, 0.030, 0.050, 0.100] - - # Calculate symmetric observation error - - filter: Variable Assignment - assignments: - - name: InitialObsError@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - type: float - function: - name: ObsErrorModelRamp@ObsFunction - channels: *{{observation_from_jcb}}_simulated_channels - options: - channels: *{{observation_from_jcb}}_simulated_channels - xvar: - name: CLWRetSymmetric@DerivedMetaData - x0: [0.030, 0.030, 0.030, 0.020, 0.030, 0.080, 0.150, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, 0.020, 0.030, 0.030, 0.030, 0.030, - 0.050, 0.100] - x1: [0.350, 0.380, 0.400, 0.450, 0.500, 1.000, 1.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, 0.350, 0.500, 0.500, 0.500, 0.500, - 0.500, 0.500] - err0: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - err1: [20.000, 25.000, 12.000, 7.000, 3.500, 3.000, 0.800, 0.400, 0.400, - 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 19.000, 30.000, 25.000, 16.500, - 12.000, 9.000, 6.500] - - # Calculate Innovation@DerivedMetaData - - filter: Variable Assignment - assignments: - - name: Innovation@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - type: float - function: - name: ObsFunction/Arithmetic - channels: *{{observation_from_jcb}}_simulated_channels - options: - variables: - - name: brightnessTemperature@ObsValue - channels: *{{observation_from_jcb}}_simulated_channels - - name: brightnessTemperature@HofX - channels: *{{observation_from_jcb}}_simulated_channels - coefs: [1, -1] - - # Step 0-C: Assign Initial All-Sky Observation Error - - filter: Perform Action - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - action: - name: assign error - error function: - name: InitialObsError@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - - # Step 1: Remove Observations from the Edge of the Scan - - filter: Domain Check - filter variables: - - name: brightnessTemperature - channels: 1-22 - where: - - variable: - name: MetaData/sensorScanPosition - is_in: 7-90 - actions: - - name: set - flag: ScanEdgeRemoval - - name: reject - - # Step 2: Data Thinning - - filter: Gaussian Thinning - horizontal_mesh: 145 - use_reduced_horizontal_grid: true - distance_norm: geodesic - # round_horizontal_bin_count_to_nearest: true - # partition_longitude_bins_using_mesh: true - actions: - - name: set - flag: Thinning - - name: reject - - # Step 3A: CLW Retrieval Check (observation_based) - - filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: 1-7, 16-22 - test variables: - - name: CLWRetFromObs@DerivedMetaData - maxvalue: 999.0 - actions: - - name: set - flag: CLWRetrievalCheck - - name: reject - - # Step 3B: CLW Retrieval Check (background_based) - - filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: 1-7, 16-22 - test variables: - - name: CLWRetFromBkg@DerivedMetaData - maxvalue: 999.0 - actions: - - name: set - flag: CLWRetrievalCheck - - name: reject - - # Step 4: Window Channel Sanity Check - - filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: 1-7, 16, 17-22 - test variables: - - name: Innovation@DerivedMetaData - channels: 1, 2, 5-7, 16 - maxvalue: 200.0 - minvalue: -200.0 - flag all filter variables if any test variable is out of bounds: true - actions: - - name: set - flag: WindowChannelExtremeResidual - - name: reject - - # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/HydrometeorCheckATMS - channels: *{{observation_from_jcb}}_simulated_channels - type: float - function: - name: HydrometeorCheckATMS@ObsFunction - channels: *{{observation_from_jcb}}_simulated_channels - options: - channels: *{{observation_from_jcb}}_simulated_channels - obserr_clearsky: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, - 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, - 3.000, 3.000, 3.000, 3.000] - clwret_function: - name: CLWRetFromObs@DerivedMetaData - obserr_function: - name: InitialObsError@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - Clouds: *{{observation_from_jcb}}_clouds - - filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - test variables: - - name: DerivedMetaData/HydrometeorCheckATMS - channels: *{{observation_from_jcb}}_simulated_channels - maxvalue: 0.0 - actions: - - name: set - flag: HydrometeorCheck - ignore: rejected observations - - name: reject - - # Step 6: Observation Error Inflation based on Topography Check - - filter: Variable Assignment - assignments: - - name: ObsErrorFactorTopo@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - type: float - function: - name: ObsErrorFactorTopoRad@ObsFunction - channels: *{{observation_from_jcb}}_simulated_channels - options: - sensor: *{{observation_from_jcb}}_sensor_id - channels: *{{observation_from_jcb}}_simulated_channels - - - filter: Perform Action - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - action: - name: inflate error - inflation variable: - name: ObsErrorFactorTopo@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - - # Step 7: Obs Error Inflation based on TOA Transmittancec Check - - filter: Variable Assignment - assignments: - - name: ObsErrorFactorTransmitTop@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - type: float - function: - name: ObsErrorFactorTransmitTopRad@ObsFunction - channels: *{{observation_from_jcb}}_simulated_channels - options: - channels: *{{observation_from_jcb}}_simulated_channels - - - filter: Perform Action - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - action: - name: inflate error - inflation variable: - name: ObsErrorFactorTransmitTop@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - - # Step 8: Observation Error Inflation based on Surface Jacobian Check - - filter: Variable Assignment - assignments: - - name: ObsErrorFactorSurfJacobian@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - type: float - function: - name: ObsErrorFactorSurfJacobianRad@ObsFunction - channels: *{{observation_from_jcb}}_simulated_channels - options: - sensor: *{{observation_from_jcb}}_sensor_id - channels: *{{observation_from_jcb}}_simulated_channels - obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] - obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] - - - filter: Perform Action - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - action: - name: inflate error - inflation variable: - name: ObsErrorFactorSurfJacobian@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - - # Step 9: Situation Dependent Check - - filter: Variable Assignment - assignments: - - name: ObsErrorFactorSituDepend@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - type: float - function: - name: ObsErrorFactorSituDependMW@ObsFunction - channels: *{{observation_from_jcb}}_simulated_channels - options: - sensor: *{{observation_from_jcb}}_sensor_id - channels: *{{observation_from_jcb}}_simulated_channels - clwbkg_function: - name: CLWRetFromBkg@DerivedMetaData - clwobs_function: - name: CLWRetFromObs@DerivedMetaData - scatobs_function: - name: SIRetFromObs@DerivedMetaData - clwmatchidx_function: - name: CLWMatchIndex@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - obserr_function: - name: InitialObsError@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - obserr_clearsky: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, - 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, - 3.000, 3.000, 3.000, 3.000] - - - filter: Perform Action - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - action: - name: inflate error - inflation variable: - name: ObsErrorFactorSituDepend@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - - # Step 10: Gross check - # Remove data if abs(Obs-HofX) > absolute threshold - - filter: Variable Assignment - assignments: - - name: ObsErrorFactorLat@DerivedMetaData - type: float - function: - name: ObsErrorFactorLatRad@ObsFunction - options: - latitude_parameters: [25.0, 0.25, 0.04, 3.0] - - - filter: Variable Assignment - assignments: - - name: ObsErrorBound@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - type: float - function: - name: ObsErrorBoundMW@ObsFunction - channels: *{{observation_from_jcb}}_simulated_channels - options: - sensor: *{{observation_from_jcb}}_sensor_id - channels: *{{observation_from_jcb}}_simulated_channels - obserr_bound_latitude: - name: ObsErrorFactorLat@DerivedMetaData - obserr_bound_transmittop: - name: ObsErrorFactorTransmitTop@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - options: - channels: *{{observation_from_jcb}}_simulated_channels - obserr_bound_topo: - name: ObsErrorFactorTopo@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - obserr_function: - name: InitialObsError@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - threshold: 3 - threshold_precip: 2.5 - obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] - - - filter: Background Check - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - function absolute threshold: - - name: ObsErrorBound@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - actions: - - name: set - flag: GrossCheck - ignore: rejected observations - - name: reject - - # Step 11: Inter-Channel Check - - filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - test variables: - - name: InterChannelConsistencyCheck@ObsFunction - channels: *{{observation_from_jcb}}_simulated_channels - options: - channels: *{{observation_from_jcb}}_simulated_channels - use passive_bc: true - sensor: *{{observation_from_jcb}}_sensor_id - use_flag: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, - 1] - maxvalue: 1.0e-12 - actions: - - name: set - flag: InterChannelConsistency - ignore: rejected observations - - name: reject - - # Step 12: Useflag Check - - filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - test variables: - - name: ObsFunction/ChannelUseflagCheckRad - channels: *{{observation_from_jcb}}_simulated_channels - options: - channels: *{{observation_from_jcb}}_simulated_channels - use_flag: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, - 1] - minvalue: 1.0e-12 - actions: - - name: set - flag: UseflagCheck - ignore: rejected observations - - name: reject - -# Column liquid cloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidCloud - type: float - function: - name: TotalColumnCloudGuess@ObsFunction - options: - cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer -# Column ice cloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnIceCloud - type: float - function: - name: TotalColumnCloudGuess@ObsFunction - options: - cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer -# Combined ColumnLiquidCloud + ColumnIceCloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidIceCloud - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: DerivedMetaData/ColumnLiquidCloud - - name: DerivedMetaData/ColumnIceCloud - coefs: [1, 1] -# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidIceCloud - type: float - value: 1.0e-7 - where: - - variable: - name: DerivedMetaData/ColumnLiquidIceCloud - maxvalue: 1.0e-8 - -# ratio of liquid /(Liquid+Ice) - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnRatioLiquidCloud - type: float - function: - name: ObsFunction/ElementMultiply - options: - variables: - - name: DerivedMetaData/ColumnLiquidCloud - - name: DerivedMetaData/ColumnLiquidIceCloud - exponents: [1,-1] -# TotalColumnVaporGuess - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/TotalColumnVaporGuess - type: float - function: - name: ObsFunction/TotalColumnVaporGuess -# potentialTemperature at surface - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureSurface - type: float - function: - name: PotentialTemperatureGuess2D@ObsFunction - options: - approximate pressure: surface_pressure -# potentialTemperature near 700 hPa - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureAt700hPa - type: float - function: - name: PotentialTemperatureGuess2D@ObsFunction - options: - approximate pressure: 70000.0 -# stability = Difference between PotentialTemperatur values - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureDiff - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: DerivedMetaData/PotentialTemperatureAt700hPa - - name: DerivedMetaData/PotentialTemperatureSurface - coefs: [1, -1] -## cao_check - - filter: BlackList - filter variables: - - name: brightnessTemperature - channels: 1-7, 16-22 - where: - - variable: - name: ObsFunction/TotalColumnVaporGuess - maxvalue: 8.0 - - variable: - name: DerivedMetaData/ColumnRatioLiquidCloud - maxvalue: 0.5 - - variable: - name: DerivedMetaData/ColumnLiquidIceCloud - minvalue: 1.0e-6 - - variable: - name: DerivedMetaData/PotentialTemperatureDiff - maxvalue: 12.0 - action: - name: reject - # GeoVaLs for Driving Observation Operators (testing mode) - # -------------------------------------------------------- - geovals: - filename: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}_geoval{{atmosphere_obsdatain_suffix}}" - - # Passed benchmark for UFO testing - # -------------------------------- - passedBenchmark: 0 diff --git a/observations/atmosphere/atms_npp.yaml.j2 b/observations/atmosphere/atms_npp.yaml.j2 index 309da34..d18985c 100644 --- a/observations/atmosphere/atms_npp.yaml.j2 +++ b/observations/atmosphere/atms_npp.yaml.j2 @@ -22,8 +22,8 @@ obs operator: name: CRTM Absorbers: [H2O, O3, CO2] - Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] -# Cloud_Fraction: 1.0 + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 Cloud_Seeding: true obs options: Sensor_ID: &{{observation_from_jcb}}_sensor_id atms_npp @@ -37,8 +37,6 @@ obs bias: input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" - variables without bc: [brightnessTemperature] - channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -67,25 +65,6 @@ ratio for small dataset: 2.0 output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiascovout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovout_suffix}}" - # ------------------------ - obs prior filters: - # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 - - filter: Variable Assignment - assignments: - - name: MetaData/zeroCloudInCRTM - type: int - function: - name: ObsFunction/Conditional - options: -# firstmatchingcase: false - defaultvalue: 0 # Will not zero clouds by default - cases: - - where: - - variable: - name: GeoVaLs/water_area_fraction - maxvalue: 0.99 - value: 1 # Will zero clouds by default - # Observation Pre Filters (QC) # ---------------------------- obs pre filters: @@ -350,7 +329,7 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - Clouds: *{{observation_from_jcb}}_clouds + - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -474,7 +453,7 @@ channels: *{{observation_from_jcb}}_simulated_channels # Step 10: Gross check - # Remove data if abs(Obs-HofX) > absolute threshold + # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: - name: ObsErrorFactorLat@DerivedMetaData @@ -508,8 +487,7 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - threshold: 3 - threshold_precip: 2.5 + threhold: 3 obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] @@ -566,116 +544,7 @@ ignore: rejected observations - name: reject -# Column liquid cloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidCloud - type: float - function: - name: TotalColumnCloudGuess@ObsFunction - options: - cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer -# Column ice cloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnIceCloud - type: float - function: - name: TotalColumnCloudGuess@ObsFunction - options: - cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer -# Combined ColumnLiquidCloud + ColumnIceCloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidIceCloud - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: DerivedMetaData/ColumnLiquidCloud - - name: DerivedMetaData/ColumnIceCloud - coefs: [1, 1] -# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidIceCloud - type: float - value: 1.0e-7 - where: - - variable: - name: DerivedMetaData/ColumnLiquidIceCloud - maxvalue: 1.0e-8 -# ratio of liquid /(Liquid+Ice) - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnRatioLiquidCloud - type: float - function: - name: ObsFunction/ElementMultiply - options: - variables: - - name: DerivedMetaData/ColumnLiquidCloud - - name: DerivedMetaData/ColumnLiquidIceCloud - exponents: [1,-1] -# TotalColumnVaporGuess - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/TotalColumnVaporGuess - type: float - function: - name: ObsFunction/TotalColumnVaporGuess -# potentialTemperature at surface - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureSurface - type: float - function: - name: PotentialTemperatureGuess2D@ObsFunction - options: - approximate pressure: surface_pressure -# potentialTemperature near 700 hPa - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureAt700hPa - type: float - function: - name: PotentialTemperatureGuess2D@ObsFunction - options: - approximate pressure: 70000.0 -# stability = Difference between PotentialTemperatur values - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureDiff - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: DerivedMetaData/PotentialTemperatureAt700hPa - - name: DerivedMetaData/PotentialTemperatureSurface - coefs: [1, -1] -## cao_check - - filter: BlackList - filter variables: - - name: brightnessTemperature - channels: 1-7, 16-22 - where: - - variable: - name: ObsFunction/TotalColumnVaporGuess - maxvalue: 8.0 - - variable: - name: DerivedMetaData/ColumnRatioLiquidCloud - maxvalue: 0.5 - - variable: - name: DerivedMetaData/ColumnLiquidIceCloud - minvalue: 1.0e-6 - - variable: - name: DerivedMetaData/PotentialTemperatureDiff - maxvalue: 12.0 - action: - name: reject # GeoVaLs for Driving Observation Operators (testing mode) # -------------------------------------------------------- geovals: From 9e069cc585e8e6b09643c350cd0cc9c7d6909119 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 22 Jan 2025 11:18:23 -0700 Subject: [PATCH 03/12] Add option for outputting post-processed soca ice increment (#64) Adds yaml section to output postprocessed ("real" aka what get added to CICE restart) sea ice increment. Part of https://github.com/NOAA-EMC/GDASApp/issues/1402 Co-authored-by: Anna Shlyaeva Co-authored-by: Dan Holdaway <27729500+danholdaway@users.noreply.github.com> --- algorithm/marine/soca_2cice_global.yaml.j2 | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/algorithm/marine/soca_2cice_global.yaml.j2 b/algorithm/marine/soca_2cice_global.yaml.j2 index 01bff96..4ccf033 100644 --- a/algorithm/marine/soca_2cice_global.yaml.j2 +++ b/algorithm/marine/soca_2cice_global.yaml.j2 @@ -29,10 +29,20 @@ variable change: sno_lev: 1 cice output: restart: {{ ice_rst }} + increment output: + datadir: Data + exp: soca2cice + type: incr + date: '{{ fcst_begin }}' + soca increment: + read_from_file: 1 + basename: ./ + ocn_filename: {{ ocn_inc }} + ice_filename: {{ ice_inc }} + date: '{{ fcst_begin }}' output variables: - sea_water_potential_temperature - sea_water_salinity - - sea_water_cell_thickness - sea_ice_area_fraction - sea_ice_thickness - sea_ice_snow_thickness @@ -47,7 +57,6 @@ states: state variables: - sea_water_potential_temperature - sea_water_salinity - - sea_water_cell_thickness - sea_ice_area_fraction - sea_ice_thickness - sea_ice_snow_thickness From 04873d78ce1f462f97b27bc6fcb0652d20be2798 Mon Sep 17 00:00:00 2001 From: Xuanli Li <101414760+XuanliLi-NOAA@users.noreply.github.com> Date: Thu, 23 Jan 2025 16:27:34 -0500 Subject: [PATCH 04/12] Add GNSSRO yaml files (#67) Adding gnssro_**mission**.yaml.j2 data. The GNSS RO data is reorganized so each satellite mission has its own yaml file. --------- Co-authored-by: Dan Holdaway <27729500+danholdaway@users.noreply.github.com> --- .../atmosphere/gnssro_cosmic2.yaml.j2 | 94 +++++++++++++++ observations/atmosphere/gnssro_geoopt.yaml.j2 | 96 ++++++++++++++++ observations/atmosphere/gnssro_grace.yaml.j2 | 94 +++++++++++++++ observations/atmosphere/gnssro_k5.yaml.j2 | 94 +++++++++++++++ observations/atmosphere/gnssro_metop.yaml.j2 | 107 ++++++++++++++++++ observations/atmosphere/gnssro_paz.yaml.j2 | 94 +++++++++++++++ observations/atmosphere/gnssro_piq.yaml.j2 | 96 ++++++++++++++++ observations/atmosphere/gnssro_s6.yaml.j2 | 94 +++++++++++++++ observations/atmosphere/gnssro_spire.yaml.j2 | 96 ++++++++++++++++ observations/atmosphere/gnssro_tdm.yaml.j2 | 94 +++++++++++++++ observations/atmosphere/gnssro_tsx.yaml.j2 | 94 +++++++++++++++ 11 files changed, 1053 insertions(+) create mode 100644 observations/atmosphere/gnssro_cosmic2.yaml.j2 create mode 100644 observations/atmosphere/gnssro_geoopt.yaml.j2 create mode 100644 observations/atmosphere/gnssro_grace.yaml.j2 create mode 100644 observations/atmosphere/gnssro_k5.yaml.j2 create mode 100644 observations/atmosphere/gnssro_metop.yaml.j2 create mode 100644 observations/atmosphere/gnssro_paz.yaml.j2 create mode 100644 observations/atmosphere/gnssro_piq.yaml.j2 create mode 100644 observations/atmosphere/gnssro_s6.yaml.j2 create mode 100644 observations/atmosphere/gnssro_spire.yaml.j2 create mode 100644 observations/atmosphere/gnssro_tdm.yaml.j2 create mode 100644 observations/atmosphere/gnssro_tsx.yaml.j2 diff --git a/observations/atmosphere/gnssro_cosmic2.yaml.j2 b/observations/atmosphere/gnssro_cosmic2.yaml.j2 new file mode 100644 index 0000000..8995da2 --- /dev/null +++ b/observations/atmosphere/gnssro_cosmic2.yaml.j2 @@ -0,0 +1,94 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: gnssrobndnbam + obsdatain: + engine: + type: H5File + obsfile: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}" + obsgrouping: + group variables: [ 'sequenceNumber' ] + sort variable: 'impactHeightRO' + sort order: 'ascending' + obsdataout: + engine: + type: H5File + obsfile: "{{atmosphere_obsdataout_path}}/{{atmosphere_obsdataout_prefix}}{{observation_from_jcb}}{{atmosphere_obsdataout_suffix}}" + simulated variables: [bendingAngle] + + # Observation Operator + # -------------------- + obs operator: + name: GnssroBndNBAM + obs options: + use_compress: 1 + sr_steps: 2 + vertlayer: full + super_ref_qc: NBAM + output_diags: true + + # Observation Filters (QC) + # ------------------------ + obs filters: + # Apply gross check using pccf + # Step 0-A: Create Diagnostic Flags + # Diagnostic flag for pccf + - filter: Create Diagnostic Flags + filter variables: + - name: bendingAngle + flags: + - name: pccfCheckReject + initial value: false + force reinitialization: true + + # Step 0-B: pccf Check for CDACC data - good: 0.1-100, reject: 0 + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265-269,750-755,44,786,820,825 + test variables: + - name: MetaData/pccf + minvalue: 0.1 + maxvalue: 100.1 + actions: + - name: set + flag: pccfCheckReject + - name: reject + + #1. gpstop + - filter: Domain Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/impactHeightRO + minvalue: 0 + maxvalue: 55000.1 + action: + name: reject + #3. RONBAM cut off check + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + action: + name: reject + defer to post: true + #4. assign obs error + - filter: ROobserror + filter variables: + - name: bendingAngle + errmodel: NBAM + defer to post: true + #5. Obs error inflate + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: RONBAMErrInflate + defer to post: true + # -------------------------------- diff --git a/observations/atmosphere/gnssro_geoopt.yaml.j2 b/observations/atmosphere/gnssro_geoopt.yaml.j2 new file mode 100644 index 0000000..67504fa --- /dev/null +++ b/observations/atmosphere/gnssro_geoopt.yaml.j2 @@ -0,0 +1,96 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: gnssrobndnbam + obsdatain: + engine: + type: H5File + obsfile: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}" + obsgrouping: + group variables: [ 'sequenceNumber' ] + sort variable: 'impactHeightRO' + sort order: 'ascending' + obsdataout: + engine: + type: H5File + obsfile: "{{atmosphere_obsdataout_path}}/{{atmosphere_obsdataout_prefix}}{{observation_from_jcb}}{{atmosphere_obsdataout_suffix}}" + simulated variables: [bendingAngle] + + # Observation Operator + # -------------------- + obs operator: + name: GnssroBndNBAM + obs options: + use_compress: 1 + sr_steps: 2 + vertlayer: full + super_ref_qc: NBAM + output_diags: true + + # Observation Filters (QC) + # ------------------------ + obs filters: + # Apply gross check using pccf + # Step 0-A: Create Diagnostic Flags + # Diagnostic flag for pccf + - filter: Create Diagnostic Flags + filter variables: + - name: bendingAngle + flags: + - name: pccfCheckReject + initial value: false + force reinitialization: true + + # Step 0-B: pccf Check for CDACC data - good: 0.1-100, reject: 0 + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265-269,750-755,44,786,820,825 + test variables: + - name: MetaData/pccf + minvalue: 0.1 + maxvalue: 100.1 + actions: + - name: set + flag: pccfCheckReject + - name: reject + + #1. commgpstop + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265,266,267,268,269 + test variables: + - name: MetaData/impactHeightRO + maxvalue: 45000.1 + action: + name: reject + #3. RONBAM cut off check + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + action: + name: reject + defer to post: true + #4. assign obs error + - filter: ROobserror + filter variables: + - name: bendingAngle + errmodel: NBAM + defer to post: true + #5. Obs error inflate + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: RONBAMErrInflate + defer to post: true + # -------------------------------- diff --git a/observations/atmosphere/gnssro_grace.yaml.j2 b/observations/atmosphere/gnssro_grace.yaml.j2 new file mode 100644 index 0000000..8995da2 --- /dev/null +++ b/observations/atmosphere/gnssro_grace.yaml.j2 @@ -0,0 +1,94 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: gnssrobndnbam + obsdatain: + engine: + type: H5File + obsfile: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}" + obsgrouping: + group variables: [ 'sequenceNumber' ] + sort variable: 'impactHeightRO' + sort order: 'ascending' + obsdataout: + engine: + type: H5File + obsfile: "{{atmosphere_obsdataout_path}}/{{atmosphere_obsdataout_prefix}}{{observation_from_jcb}}{{atmosphere_obsdataout_suffix}}" + simulated variables: [bendingAngle] + + # Observation Operator + # -------------------- + obs operator: + name: GnssroBndNBAM + obs options: + use_compress: 1 + sr_steps: 2 + vertlayer: full + super_ref_qc: NBAM + output_diags: true + + # Observation Filters (QC) + # ------------------------ + obs filters: + # Apply gross check using pccf + # Step 0-A: Create Diagnostic Flags + # Diagnostic flag for pccf + - filter: Create Diagnostic Flags + filter variables: + - name: bendingAngle + flags: + - name: pccfCheckReject + initial value: false + force reinitialization: true + + # Step 0-B: pccf Check for CDACC data - good: 0.1-100, reject: 0 + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265-269,750-755,44,786,820,825 + test variables: + - name: MetaData/pccf + minvalue: 0.1 + maxvalue: 100.1 + actions: + - name: set + flag: pccfCheckReject + - name: reject + + #1. gpstop + - filter: Domain Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/impactHeightRO + minvalue: 0 + maxvalue: 55000.1 + action: + name: reject + #3. RONBAM cut off check + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + action: + name: reject + defer to post: true + #4. assign obs error + - filter: ROobserror + filter variables: + - name: bendingAngle + errmodel: NBAM + defer to post: true + #5. Obs error inflate + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: RONBAMErrInflate + defer to post: true + # -------------------------------- diff --git a/observations/atmosphere/gnssro_k5.yaml.j2 b/observations/atmosphere/gnssro_k5.yaml.j2 new file mode 100644 index 0000000..8995da2 --- /dev/null +++ b/observations/atmosphere/gnssro_k5.yaml.j2 @@ -0,0 +1,94 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: gnssrobndnbam + obsdatain: + engine: + type: H5File + obsfile: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}" + obsgrouping: + group variables: [ 'sequenceNumber' ] + sort variable: 'impactHeightRO' + sort order: 'ascending' + obsdataout: + engine: + type: H5File + obsfile: "{{atmosphere_obsdataout_path}}/{{atmosphere_obsdataout_prefix}}{{observation_from_jcb}}{{atmosphere_obsdataout_suffix}}" + simulated variables: [bendingAngle] + + # Observation Operator + # -------------------- + obs operator: + name: GnssroBndNBAM + obs options: + use_compress: 1 + sr_steps: 2 + vertlayer: full + super_ref_qc: NBAM + output_diags: true + + # Observation Filters (QC) + # ------------------------ + obs filters: + # Apply gross check using pccf + # Step 0-A: Create Diagnostic Flags + # Diagnostic flag for pccf + - filter: Create Diagnostic Flags + filter variables: + - name: bendingAngle + flags: + - name: pccfCheckReject + initial value: false + force reinitialization: true + + # Step 0-B: pccf Check for CDACC data - good: 0.1-100, reject: 0 + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265-269,750-755,44,786,820,825 + test variables: + - name: MetaData/pccf + minvalue: 0.1 + maxvalue: 100.1 + actions: + - name: set + flag: pccfCheckReject + - name: reject + + #1. gpstop + - filter: Domain Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/impactHeightRO + minvalue: 0 + maxvalue: 55000.1 + action: + name: reject + #3. RONBAM cut off check + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + action: + name: reject + defer to post: true + #4. assign obs error + - filter: ROobserror + filter variables: + - name: bendingAngle + errmodel: NBAM + defer to post: true + #5. Obs error inflate + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: RONBAMErrInflate + defer to post: true + # -------------------------------- diff --git a/observations/atmosphere/gnssro_metop.yaml.j2 b/observations/atmosphere/gnssro_metop.yaml.j2 new file mode 100644 index 0000000..7842fdc --- /dev/null +++ b/observations/atmosphere/gnssro_metop.yaml.j2 @@ -0,0 +1,107 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: gnssrobndnbam + obsdatain: + engine: + type: H5File + obsfile: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}" + obsgrouping: + group variables: [ 'sequenceNumber' ] + sort variable: 'impactHeightRO' + sort order: 'ascending' + obsdataout: + engine: + type: H5File + obsfile: "{{atmosphere_obsdataout_path}}/{{atmosphere_obsdataout_prefix}}{{observation_from_jcb}}{{atmosphere_obsdataout_suffix}}" + simulated variables: [bendingAngle] + + # Observation Operator + # -------------------- + obs operator: + name: GnssroBndNBAM + obs options: + use_compress: 1 + sr_steps: 2 + vertlayer: full + super_ref_qc: NBAM + output_diags: true + + # Observation Filters (QC) + # ------------------------ + obs filters: + # Apply gross check using qfro + # Step 0-A: Create Diagnostic Flags + # Diagnostic flag for qfro + - filter: Create Diagnostic Flags + filter variables: + - name: bendingAngle + flags: + - name: qfroCheckReject + initial value: false + force reinitialization: true + + # Step 0-B: qfro Check - good: 0, reject: 1 + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 3-5,421,440,821 + test variables: + - name: MetaData/qualityFlags + minvalue: -0.1 + maxvalue: 0.9 + actions: + - name: set + flag: qfroCheckReject + - name: reject + + #1. gpstop + - filter: Domain Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/impactHeightRO + minvalue: 0 + maxvalue: 55000.1 + action: + name: reject + #2. metop below 8 km + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 3-5 + test variables: + - name: MetaData/impactHeightRO + minvalue: 8000.1 + action: + name: reject + #3. RONBAM cut off check + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + action: + name: reject + defer to post: true + #4. assign obs error + - filter: ROobserror + filter variables: + - name: bendingAngle + errmodel: NBAM + defer to post: true + #5. Obs error inflate + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: RONBAMErrInflate + defer to post: true + # -------------------------------- diff --git a/observations/atmosphere/gnssro_paz.yaml.j2 b/observations/atmosphere/gnssro_paz.yaml.j2 new file mode 100644 index 0000000..8995da2 --- /dev/null +++ b/observations/atmosphere/gnssro_paz.yaml.j2 @@ -0,0 +1,94 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: gnssrobndnbam + obsdatain: + engine: + type: H5File + obsfile: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}" + obsgrouping: + group variables: [ 'sequenceNumber' ] + sort variable: 'impactHeightRO' + sort order: 'ascending' + obsdataout: + engine: + type: H5File + obsfile: "{{atmosphere_obsdataout_path}}/{{atmosphere_obsdataout_prefix}}{{observation_from_jcb}}{{atmosphere_obsdataout_suffix}}" + simulated variables: [bendingAngle] + + # Observation Operator + # -------------------- + obs operator: + name: GnssroBndNBAM + obs options: + use_compress: 1 + sr_steps: 2 + vertlayer: full + super_ref_qc: NBAM + output_diags: true + + # Observation Filters (QC) + # ------------------------ + obs filters: + # Apply gross check using pccf + # Step 0-A: Create Diagnostic Flags + # Diagnostic flag for pccf + - filter: Create Diagnostic Flags + filter variables: + - name: bendingAngle + flags: + - name: pccfCheckReject + initial value: false + force reinitialization: true + + # Step 0-B: pccf Check for CDACC data - good: 0.1-100, reject: 0 + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265-269,750-755,44,786,820,825 + test variables: + - name: MetaData/pccf + minvalue: 0.1 + maxvalue: 100.1 + actions: + - name: set + flag: pccfCheckReject + - name: reject + + #1. gpstop + - filter: Domain Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/impactHeightRO + minvalue: 0 + maxvalue: 55000.1 + action: + name: reject + #3. RONBAM cut off check + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + action: + name: reject + defer to post: true + #4. assign obs error + - filter: ROobserror + filter variables: + - name: bendingAngle + errmodel: NBAM + defer to post: true + #5. Obs error inflate + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: RONBAMErrInflate + defer to post: true + # -------------------------------- diff --git a/observations/atmosphere/gnssro_piq.yaml.j2 b/observations/atmosphere/gnssro_piq.yaml.j2 new file mode 100644 index 0000000..67504fa --- /dev/null +++ b/observations/atmosphere/gnssro_piq.yaml.j2 @@ -0,0 +1,96 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: gnssrobndnbam + obsdatain: + engine: + type: H5File + obsfile: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}" + obsgrouping: + group variables: [ 'sequenceNumber' ] + sort variable: 'impactHeightRO' + sort order: 'ascending' + obsdataout: + engine: + type: H5File + obsfile: "{{atmosphere_obsdataout_path}}/{{atmosphere_obsdataout_prefix}}{{observation_from_jcb}}{{atmosphere_obsdataout_suffix}}" + simulated variables: [bendingAngle] + + # Observation Operator + # -------------------- + obs operator: + name: GnssroBndNBAM + obs options: + use_compress: 1 + sr_steps: 2 + vertlayer: full + super_ref_qc: NBAM + output_diags: true + + # Observation Filters (QC) + # ------------------------ + obs filters: + # Apply gross check using pccf + # Step 0-A: Create Diagnostic Flags + # Diagnostic flag for pccf + - filter: Create Diagnostic Flags + filter variables: + - name: bendingAngle + flags: + - name: pccfCheckReject + initial value: false + force reinitialization: true + + # Step 0-B: pccf Check for CDACC data - good: 0.1-100, reject: 0 + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265-269,750-755,44,786,820,825 + test variables: + - name: MetaData/pccf + minvalue: 0.1 + maxvalue: 100.1 + actions: + - name: set + flag: pccfCheckReject + - name: reject + + #1. commgpstop + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265,266,267,268,269 + test variables: + - name: MetaData/impactHeightRO + maxvalue: 45000.1 + action: + name: reject + #3. RONBAM cut off check + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + action: + name: reject + defer to post: true + #4. assign obs error + - filter: ROobserror + filter variables: + - name: bendingAngle + errmodel: NBAM + defer to post: true + #5. Obs error inflate + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: RONBAMErrInflate + defer to post: true + # -------------------------------- diff --git a/observations/atmosphere/gnssro_s6.yaml.j2 b/observations/atmosphere/gnssro_s6.yaml.j2 new file mode 100644 index 0000000..8995da2 --- /dev/null +++ b/observations/atmosphere/gnssro_s6.yaml.j2 @@ -0,0 +1,94 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: gnssrobndnbam + obsdatain: + engine: + type: H5File + obsfile: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}" + obsgrouping: + group variables: [ 'sequenceNumber' ] + sort variable: 'impactHeightRO' + sort order: 'ascending' + obsdataout: + engine: + type: H5File + obsfile: "{{atmosphere_obsdataout_path}}/{{atmosphere_obsdataout_prefix}}{{observation_from_jcb}}{{atmosphere_obsdataout_suffix}}" + simulated variables: [bendingAngle] + + # Observation Operator + # -------------------- + obs operator: + name: GnssroBndNBAM + obs options: + use_compress: 1 + sr_steps: 2 + vertlayer: full + super_ref_qc: NBAM + output_diags: true + + # Observation Filters (QC) + # ------------------------ + obs filters: + # Apply gross check using pccf + # Step 0-A: Create Diagnostic Flags + # Diagnostic flag for pccf + - filter: Create Diagnostic Flags + filter variables: + - name: bendingAngle + flags: + - name: pccfCheckReject + initial value: false + force reinitialization: true + + # Step 0-B: pccf Check for CDACC data - good: 0.1-100, reject: 0 + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265-269,750-755,44,786,820,825 + test variables: + - name: MetaData/pccf + minvalue: 0.1 + maxvalue: 100.1 + actions: + - name: set + flag: pccfCheckReject + - name: reject + + #1. gpstop + - filter: Domain Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/impactHeightRO + minvalue: 0 + maxvalue: 55000.1 + action: + name: reject + #3. RONBAM cut off check + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + action: + name: reject + defer to post: true + #4. assign obs error + - filter: ROobserror + filter variables: + - name: bendingAngle + errmodel: NBAM + defer to post: true + #5. Obs error inflate + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: RONBAMErrInflate + defer to post: true + # -------------------------------- diff --git a/observations/atmosphere/gnssro_spire.yaml.j2 b/observations/atmosphere/gnssro_spire.yaml.j2 new file mode 100644 index 0000000..67504fa --- /dev/null +++ b/observations/atmosphere/gnssro_spire.yaml.j2 @@ -0,0 +1,96 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: gnssrobndnbam + obsdatain: + engine: + type: H5File + obsfile: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}" + obsgrouping: + group variables: [ 'sequenceNumber' ] + sort variable: 'impactHeightRO' + sort order: 'ascending' + obsdataout: + engine: + type: H5File + obsfile: "{{atmosphere_obsdataout_path}}/{{atmosphere_obsdataout_prefix}}{{observation_from_jcb}}{{atmosphere_obsdataout_suffix}}" + simulated variables: [bendingAngle] + + # Observation Operator + # -------------------- + obs operator: + name: GnssroBndNBAM + obs options: + use_compress: 1 + sr_steps: 2 + vertlayer: full + super_ref_qc: NBAM + output_diags: true + + # Observation Filters (QC) + # ------------------------ + obs filters: + # Apply gross check using pccf + # Step 0-A: Create Diagnostic Flags + # Diagnostic flag for pccf + - filter: Create Diagnostic Flags + filter variables: + - name: bendingAngle + flags: + - name: pccfCheckReject + initial value: false + force reinitialization: true + + # Step 0-B: pccf Check for CDACC data - good: 0.1-100, reject: 0 + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265-269,750-755,44,786,820,825 + test variables: + - name: MetaData/pccf + minvalue: 0.1 + maxvalue: 100.1 + actions: + - name: set + flag: pccfCheckReject + - name: reject + + #1. commgpstop + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265,266,267,268,269 + test variables: + - name: MetaData/impactHeightRO + maxvalue: 45000.1 + action: + name: reject + #3. RONBAM cut off check + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + action: + name: reject + defer to post: true + #4. assign obs error + - filter: ROobserror + filter variables: + - name: bendingAngle + errmodel: NBAM + defer to post: true + #5. Obs error inflate + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: RONBAMErrInflate + defer to post: true + # -------------------------------- diff --git a/observations/atmosphere/gnssro_tdm.yaml.j2 b/observations/atmosphere/gnssro_tdm.yaml.j2 new file mode 100644 index 0000000..8995da2 --- /dev/null +++ b/observations/atmosphere/gnssro_tdm.yaml.j2 @@ -0,0 +1,94 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: gnssrobndnbam + obsdatain: + engine: + type: H5File + obsfile: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}" + obsgrouping: + group variables: [ 'sequenceNumber' ] + sort variable: 'impactHeightRO' + sort order: 'ascending' + obsdataout: + engine: + type: H5File + obsfile: "{{atmosphere_obsdataout_path}}/{{atmosphere_obsdataout_prefix}}{{observation_from_jcb}}{{atmosphere_obsdataout_suffix}}" + simulated variables: [bendingAngle] + + # Observation Operator + # -------------------- + obs operator: + name: GnssroBndNBAM + obs options: + use_compress: 1 + sr_steps: 2 + vertlayer: full + super_ref_qc: NBAM + output_diags: true + + # Observation Filters (QC) + # ------------------------ + obs filters: + # Apply gross check using pccf + # Step 0-A: Create Diagnostic Flags + # Diagnostic flag for pccf + - filter: Create Diagnostic Flags + filter variables: + - name: bendingAngle + flags: + - name: pccfCheckReject + initial value: false + force reinitialization: true + + # Step 0-B: pccf Check for CDACC data - good: 0.1-100, reject: 0 + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265-269,750-755,44,786,820,825 + test variables: + - name: MetaData/pccf + minvalue: 0.1 + maxvalue: 100.1 + actions: + - name: set + flag: pccfCheckReject + - name: reject + + #1. gpstop + - filter: Domain Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/impactHeightRO + minvalue: 0 + maxvalue: 55000.1 + action: + name: reject + #3. RONBAM cut off check + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + action: + name: reject + defer to post: true + #4. assign obs error + - filter: ROobserror + filter variables: + - name: bendingAngle + errmodel: NBAM + defer to post: true + #5. Obs error inflate + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: RONBAMErrInflate + defer to post: true + # -------------------------------- diff --git a/observations/atmosphere/gnssro_tsx.yaml.j2 b/observations/atmosphere/gnssro_tsx.yaml.j2 new file mode 100644 index 0000000..8995da2 --- /dev/null +++ b/observations/atmosphere/gnssro_tsx.yaml.j2 @@ -0,0 +1,94 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: gnssrobndnbam + obsdatain: + engine: + type: H5File + obsfile: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}" + obsgrouping: + group variables: [ 'sequenceNumber' ] + sort variable: 'impactHeightRO' + sort order: 'ascending' + obsdataout: + engine: + type: H5File + obsfile: "{{atmosphere_obsdataout_path}}/{{atmosphere_obsdataout_prefix}}{{observation_from_jcb}}{{atmosphere_obsdataout_suffix}}" + simulated variables: [bendingAngle] + + # Observation Operator + # -------------------- + obs operator: + name: GnssroBndNBAM + obs options: + use_compress: 1 + sr_steps: 2 + vertlayer: full + super_ref_qc: NBAM + output_diags: true + + # Observation Filters (QC) + # ------------------------ + obs filters: + # Apply gross check using pccf + # Step 0-A: Create Diagnostic Flags + # Diagnostic flag for pccf + - filter: Create Diagnostic Flags + filter variables: + - name: bendingAngle + flags: + - name: pccfCheckReject + initial value: false + force reinitialization: true + + # Step 0-B: pccf Check for CDACC data - good: 0.1-100, reject: 0 + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265-269,750-755,44,786,820,825 + test variables: + - name: MetaData/pccf + minvalue: 0.1 + maxvalue: 100.1 + actions: + - name: set + flag: pccfCheckReject + - name: reject + + #1. gpstop + - filter: Domain Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/impactHeightRO + minvalue: 0 + maxvalue: 55000.1 + action: + name: reject + #3. RONBAM cut off check + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + action: + name: reject + defer to post: true + #4. assign obs error + - filter: ROobserror + filter variables: + - name: bendingAngle + errmodel: NBAM + defer to post: true + #5. Obs error inflate + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: RONBAMErrInflate + defer to post: true + # -------------------------------- From 5476b4dcaec29d0e4f1fb85cd8385281d6c473da Mon Sep 17 00:00:00 2001 From: Xuanli Li <101414760+XuanliLi-NOAA@users.noreply.github.com> Date: Fri, 24 Jan 2025 11:38:00 -0500 Subject: [PATCH 05/12] Feature/gnssroyaml (#69) Update the obs space name for the gnssro yaml files. --- observations/atmosphere/gnssro_cosmic2.yaml.j2 | 2 +- observations/atmosphere/gnssro_geoopt.yaml.j2 | 2 +- observations/atmosphere/gnssro_grace.yaml.j2 | 2 +- observations/atmosphere/gnssro_k5.yaml.j2 | 2 +- observations/atmosphere/gnssro_metop.yaml.j2 | 2 +- observations/atmosphere/gnssro_paz.yaml.j2 | 2 +- observations/atmosphere/gnssro_piq.yaml.j2 | 2 +- observations/atmosphere/gnssro_s6.yaml.j2 | 2 +- observations/atmosphere/gnssro_spire.yaml.j2 | 2 +- observations/atmosphere/gnssro_tdm.yaml.j2 | 2 +- observations/atmosphere/gnssro_tsx.yaml.j2 | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/observations/atmosphere/gnssro_cosmic2.yaml.j2 b/observations/atmosphere/gnssro_cosmic2.yaml.j2 index 8995da2..fa3f488 100644 --- a/observations/atmosphere/gnssro_cosmic2.yaml.j2 +++ b/observations/atmosphere/gnssro_cosmic2.yaml.j2 @@ -3,7 +3,7 @@ # Observation Space (I/O) # ----------------------- obs space: - name: gnssrobndnbam + name: gnssrobndnbam_cosmic2 obsdatain: engine: type: H5File diff --git a/observations/atmosphere/gnssro_geoopt.yaml.j2 b/observations/atmosphere/gnssro_geoopt.yaml.j2 index 67504fa..a7774c5 100644 --- a/observations/atmosphere/gnssro_geoopt.yaml.j2 +++ b/observations/atmosphere/gnssro_geoopt.yaml.j2 @@ -3,7 +3,7 @@ # Observation Space (I/O) # ----------------------- obs space: - name: gnssrobndnbam + name: gnssrobndnbam_geoopt obsdatain: engine: type: H5File diff --git a/observations/atmosphere/gnssro_grace.yaml.j2 b/observations/atmosphere/gnssro_grace.yaml.j2 index 8995da2..5a459de 100644 --- a/observations/atmosphere/gnssro_grace.yaml.j2 +++ b/observations/atmosphere/gnssro_grace.yaml.j2 @@ -3,7 +3,7 @@ # Observation Space (I/O) # ----------------------- obs space: - name: gnssrobndnbam + name: gnssrobndnbam_grace obsdatain: engine: type: H5File diff --git a/observations/atmosphere/gnssro_k5.yaml.j2 b/observations/atmosphere/gnssro_k5.yaml.j2 index 8995da2..72893f4 100644 --- a/observations/atmosphere/gnssro_k5.yaml.j2 +++ b/observations/atmosphere/gnssro_k5.yaml.j2 @@ -3,7 +3,7 @@ # Observation Space (I/O) # ----------------------- obs space: - name: gnssrobndnbam + name: gnssrobndnbam_k5 obsdatain: engine: type: H5File diff --git a/observations/atmosphere/gnssro_metop.yaml.j2 b/observations/atmosphere/gnssro_metop.yaml.j2 index 7842fdc..199424b 100644 --- a/observations/atmosphere/gnssro_metop.yaml.j2 +++ b/observations/atmosphere/gnssro_metop.yaml.j2 @@ -3,7 +3,7 @@ # Observation Space (I/O) # ----------------------- obs space: - name: gnssrobndnbam + name: gnssrobndnbam_metop obsdatain: engine: type: H5File diff --git a/observations/atmosphere/gnssro_paz.yaml.j2 b/observations/atmosphere/gnssro_paz.yaml.j2 index 8995da2..c36e892 100644 --- a/observations/atmosphere/gnssro_paz.yaml.j2 +++ b/observations/atmosphere/gnssro_paz.yaml.j2 @@ -3,7 +3,7 @@ # Observation Space (I/O) # ----------------------- obs space: - name: gnssrobndnbam + name: gnssrobndnbam_paz obsdatain: engine: type: H5File diff --git a/observations/atmosphere/gnssro_piq.yaml.j2 b/observations/atmosphere/gnssro_piq.yaml.j2 index 67504fa..ad4686b 100644 --- a/observations/atmosphere/gnssro_piq.yaml.j2 +++ b/observations/atmosphere/gnssro_piq.yaml.j2 @@ -3,7 +3,7 @@ # Observation Space (I/O) # ----------------------- obs space: - name: gnssrobndnbam + name: gnssrobndnbam_piq obsdatain: engine: type: H5File diff --git a/observations/atmosphere/gnssro_s6.yaml.j2 b/observations/atmosphere/gnssro_s6.yaml.j2 index 8995da2..bbb0698 100644 --- a/observations/atmosphere/gnssro_s6.yaml.j2 +++ b/observations/atmosphere/gnssro_s6.yaml.j2 @@ -3,7 +3,7 @@ # Observation Space (I/O) # ----------------------- obs space: - name: gnssrobndnbam + name: gnssrobndnbam_s6 obsdatain: engine: type: H5File diff --git a/observations/atmosphere/gnssro_spire.yaml.j2 b/observations/atmosphere/gnssro_spire.yaml.j2 index 67504fa..568da6d 100644 --- a/observations/atmosphere/gnssro_spire.yaml.j2 +++ b/observations/atmosphere/gnssro_spire.yaml.j2 @@ -3,7 +3,7 @@ # Observation Space (I/O) # ----------------------- obs space: - name: gnssrobndnbam + name: gnssrobndnbam_spire obsdatain: engine: type: H5File diff --git a/observations/atmosphere/gnssro_tdm.yaml.j2 b/observations/atmosphere/gnssro_tdm.yaml.j2 index 8995da2..fb38f01 100644 --- a/observations/atmosphere/gnssro_tdm.yaml.j2 +++ b/observations/atmosphere/gnssro_tdm.yaml.j2 @@ -3,7 +3,7 @@ # Observation Space (I/O) # ----------------------- obs space: - name: gnssrobndnbam + name: gnssrobndnbam_tdm obsdatain: engine: type: H5File diff --git a/observations/atmosphere/gnssro_tsx.yaml.j2 b/observations/atmosphere/gnssro_tsx.yaml.j2 index 8995da2..1073084 100644 --- a/observations/atmosphere/gnssro_tsx.yaml.j2 +++ b/observations/atmosphere/gnssro_tsx.yaml.j2 @@ -3,7 +3,7 @@ # Observation Space (I/O) # ----------------------- obs space: - name: gnssrobndnbam + name: gnssrobndnbam_tsx obsdatain: engine: type: H5File From 831ec0d0073ad9a86d675b12e4a19cbcaf8d247b Mon Sep 17 00:00:00 2001 From: jianjunj Date: Fri, 24 Jan 2025 13:39:27 -0500 Subject: [PATCH 06/12] Set channels without bias correction. (#59) Set channels without bias correction from chronicle tables. ATMS Channel 15 and AMSUA channel 14 are bias corrected. Set -999 in the channels list "without bc" to let UFO conduct bias correction for all channels. Co-authored-by: Dan Holdaway <27729500+danholdaway@users.noreply.github.com> --- observations/atmosphere/cris-fsr_n20.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/observations/atmosphere/cris-fsr_n20.yaml.j2 b/observations/atmosphere/cris-fsr_n20.yaml.j2 index 6bea2e3..e073716 100644 --- a/observations/atmosphere/cris-fsr_n20.yaml.j2 +++ b/observations/atmosphere/cris-fsr_n20.yaml.j2 @@ -34,6 +34,8 @@ obs bias: input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" + variables without bc: [brightnessTemperature] + channels: &{{observation_from_jcb}}_not_bias_corrected {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant From de8f25503e271ceeff9ad253310036351d14764b Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 28 Jan 2025 17:19:24 +0000 Subject: [PATCH 07/12] change aerosol gaussian increment from ges res to anl res (#71) OOPS chokes trying to interpolate and write a C1152 equivalent Gaussian increment, and we don't need that anyways for now, so instead write out an increment at the analysis resolution (C384) --- model/aero/aero_final_increment_gaussian.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/aero/aero_final_increment_gaussian.yaml.j2 b/model/aero/aero_final_increment_gaussian.yaml.j2 index 97ffc8d..72eabe0 100644 --- a/model/aero/aero_final_increment_gaussian.yaml.j2 +++ b/model/aero/aero_final_increment_gaussian.yaml.j2 @@ -2,7 +2,7 @@ grid type: regular gaussian local interpolator type: atlas interpolator interpolation method: type: finite-element -number of latitude gridpoints: {{ aero_npy_ges - 1 }} +number of latitude gridpoints: {{ aero_npy_anl - 1 }} variables to output: [mass_fraction_of_sulfate_in_air, mass_fraction_of_hydrophobic_black_carbon_in_air, mass_fraction_of_hydrophilic_black_carbon_in_air, From de1fd083ccdf637470f23157b3d694dc331f0ef8 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 3 Feb 2025 11:15:49 -0500 Subject: [PATCH 08/12] Configuration for the low res marine B (#66) description coming soon ... Resolves part of https://github.com/NOAA-EMC/GDASApp/issues/1375 --------- Co-authored-by: Dan Holdaway <27729500+danholdaway@users.noreply.github.com> --- algorithm/marine/soca_chgres.yaml.j2 | 33 +++++++++ algorithm/marine/soca_diagb.yaml.j2 | 5 ++ algorithm/marine/soca_ensb.yaml.j2 | 7 +- algorithm/marine/soca_ensweights.yaml.j2 | 24 +++--- .../soca_parameters_diffusion_hz.yaml.j2 | 11 ++- .../soca_parameters_diffusion_vt.yaml.j2 | 12 +-- algorithm/marine/soca_setcorscales.yaml.j2 | 7 +- algorithm/marine/soca_vtscales.yaml.j2 | 6 +- algorithm/marine/socaincr2mom6.yaml.j2 | 2 + .../marine/marine_3dfgat_outer_loop_1.yaml.j2 | 3 +- .../marine/marine_3dvar_outer_loop_1.yaml.j2 | 7 -- ...d_error_hybrid_diffusion_diffusion.yaml.j2 | 74 ++++++++++--------- ..._background_error_static_diffusion.yaml.j2 | 1 + .../gdas-marine-templates.yaml | 1 - 14 files changed, 124 insertions(+), 69 deletions(-) create mode 100644 algorithm/marine/soca_chgres.yaml.j2 delete mode 100644 model/marine/marine_3dvar_outer_loop_1.yaml.j2 diff --git a/algorithm/marine/soca_chgres.yaml.j2 b/algorithm/marine/soca_chgres.yaml.j2 new file mode 100644 index 0000000..2f417bc --- /dev/null +++ b/algorithm/marine/soca_chgres.yaml.j2 @@ -0,0 +1,33 @@ +input geometry: + mom6_input_nml: mom_input.nml + fields metadata: ./fields_metadata.yaml + +output geometry: + geom_grid_file: ./anl_geom/soca_gridspec.nc + mom6_input_nml: ./anl_geom/mom_input.nml + fields metadata: ./fields_metadata.yaml + +states: +- input: + date: '{{ marine_window_end_iso }}' + basename: ./bkg/ + ocn_filename: 'ocean.bkg.f009.nc' + ice_filename: 'ice.bkg.f009.nc' + read_from_file: 1 + + state variables: + - sea_surface_height_above_geoid + - sea_water_potential_temperature + - sea_water_salinity + - sea_water_cell_thickness + - sea_water_depth + - mom6_mld + - sea_ice_area_fraction + - sea_ice_thickness + - sea_ice_snow_thickness + + output: + datadir: anl_geom/ + exp: f009_anl_geom + type: fc + date: '{{ marine_window_end_iso }}' \ No newline at end of file diff --git a/algorithm/marine/soca_diagb.yaml.j2 b/algorithm/marine/soca_diagb.yaml.j2 index 788f674..1f7c0eb 100644 --- a/algorithm/marine/soca_diagb.yaml.j2 +++ b/algorithm/marine/soca_diagb.yaml.j2 @@ -2,6 +2,11 @@ geometry: mom6_input_nml: mom_input.nml fields metadata: ./fields_metadata.yaml +output geometry: + geom_grid_file: ./anl_geom/soca_gridspec.nc + mom6_input_nml: ./anl_geom/mom_input.nml + fields metadata: ./fields_metadata.yaml + date: '{{ marine_window_end_iso }}' background: diff --git a/algorithm/marine/soca_ensb.yaml.j2 b/algorithm/marine/soca_ensb.yaml.j2 index 852850f..ab437fa 100644 --- a/algorithm/marine/soca_ensb.yaml.j2 +++ b/algorithm/marine/soca_ensb.yaml.j2 @@ -3,6 +3,11 @@ geometry: mom6_input_nml: mom_input.nml fields metadata: ./fields_metadata.yaml +output geometry: + geom_grid_file: ./anl_geom/soca_gridspec.nc + mom6_input_nml: ./anl_geom/mom_input.nml + fields metadata: ./fields_metadata.yaml + date: '{{ marine_window_begin_iso }}' layers variable: [sea_water_cell_thickness] @@ -113,5 +118,5 @@ output increment: date: '{{ marine_window_begin_iso }}' exp: trash type: incr - output file: 'ocn.pert.steric.%mem%.nc' + output file: 'pert.%mem%.nc' pattern: '%mem%' diff --git a/algorithm/marine/soca_ensweights.yaml.j2 b/algorithm/marine/soca_ensweights.yaml.j2 index 9f818b3..00b2e1e 100644 --- a/algorithm/marine/soca_ensweights.yaml.j2 +++ b/algorithm/marine/soca_ensweights.yaml.j2 @@ -2,6 +2,11 @@ geometry: mom6_input_nml: mom_input.nml fields metadata: ./fields_metadata.yaml +output geometry: + geom_grid_file: ./anl_geom/soca_gridspec.nc + mom6_input_nml: ./anl_geom/mom_input.nml + fields metadata: ./fields_metadata.yaml + date: '{{ marine_window_middle_iso }}' variables: @@ -29,15 +34,16 @@ weights: ice: 0.0025 # 5% of original variance ocean: 0.0625 # 25% " " # Apply localized weights to the ocean ens. B - ocean local weights: - - lon: -172.0 - lat: 11.0 - amplitude: -1.0 - length scale: 700.0 - - lon: -160.0 - lat: 12.0 - amplitude: -1.0 - length scale: 700.0 + # Example below was for old ens. from Xiao + # ocean local weights: + # - lon: -172.0 + # lat: 11.0 + # amplitude: -1.0 + # length scale: 700.0 + # - lon: -160.0 + # lat: 12.0 + # amplitude: -1.0 + # length scale: 700.0 output: datadir: ./ diff --git a/algorithm/marine/soca_parameters_diffusion_hz.yaml.j2 b/algorithm/marine/soca_parameters_diffusion_hz.yaml.j2 index 8f3cdf7..cb57948 100644 --- a/algorithm/marine/soca_parameters_diffusion_hz.yaml.j2 +++ b/algorithm/marine/soca_parameters_diffusion_hz.yaml.j2 @@ -1,12 +1,15 @@ +# Analysis/B-mat resolution geometry: &geom - mom6_input_nml: mom_input.nml + geom_grid_file: ./anl_geom/soca_gridspec.nc + mom6_input_nml: ./anl_geom/mom_input.nml fields metadata: ./fields_metadata.yaml +# Background interpolated at the analysis resolution background: read_from_file: 1 - basename: ./INPUT/ - ocn_filename: MOM.res.nc - ice_filename: cice.res.nc + basename: ./anl_geom/ + ocn_filename: 'ocn.f009_anl_geom.fc.{{ marine_window_end_iso }}.PT0S.nc' + ice_filename: 'ice.f009_anl_geom.fc.{{ marine_window_end_iso }}.PT0S.nc' date: '{{ marine_window_end_iso }}' state variables: [sea_surface_height_above_geoid] diff --git a/algorithm/marine/soca_parameters_diffusion_vt.yaml.j2 b/algorithm/marine/soca_parameters_diffusion_vt.yaml.j2 index 8e23f2d..a5c5d46 100644 --- a/algorithm/marine/soca_parameters_diffusion_vt.yaml.j2 +++ b/algorithm/marine/soca_parameters_diffusion_vt.yaml.j2 @@ -1,13 +1,15 @@ +# Analysis/B-mat resolution geometry: &geom - mom6_input_nml: mom_input.nml + geom_grid_file: ./anl_geom/soca_gridspec.nc + mom6_input_nml: ./anl_geom/mom_input.nml fields metadata: ./fields_metadata.yaml +# Background interpolated at the analysis resolution background: read_from_file: 1 - basename: ./INPUT/ - ocn_filename: MOM.res.nc - ice_filename: cice.res.nc - date: '{{ marine_window_middle_iso }}' + basename: ./anl_geom/ + ocn_filename: 'ocn.f009_anl_geom.fc.{{ marine_window_end_iso }}.PT0S.nc' + date: '{{ marine_window_end_iso }}' state variables: [sea_water_potential_temperature] background error: diff --git a/algorithm/marine/soca_setcorscales.yaml.j2 b/algorithm/marine/soca_setcorscales.yaml.j2 index 0c2b9c0..9d07bef 100644 --- a/algorithm/marine/soca_setcorscales.yaml.j2 +++ b/algorithm/marine/soca_setcorscales.yaml.j2 @@ -1,5 +1,6 @@ resolution: - mom6_input_nml: mom_input.nml + geom_grid_file: ./anl_geom/soca_gridspec.nc + mom6_input_nml: ./anl_geom/mom_input.nml fields metadata: ./fields_metadata.yaml date: "0001-01-01T00:00:00Z" @@ -9,8 +10,8 @@ corr variables: [sea_surface_height_above_geoid] scales: vert layers: 5 # in units of layer sea_surface_height_above_geoid: - rossby mult: 2.00 - min grid mult: 6.0 + rossby mult: 1.50 + min grid mult: 2.0 rh output: datadir: ./ diff --git a/algorithm/marine/soca_vtscales.yaml.j2 b/algorithm/marine/soca_vtscales.yaml.j2 index 797af37..f0e68bc 100644 --- a/algorithm/marine/soca_vtscales.yaml.j2 +++ b/algorithm/marine/soca_vtscales.yaml.j2 @@ -1,6 +1,6 @@ -gridspec_filename: soca_gridspec.nc -restart_filename: ./INPUT/MOM.res.nc -mld_filename: './staticb/ocn.bkgerr_stddev.incr.{{ marine_window_end_iso }}.nc' +gridspec_filename: ./anl_geom/soca_gridspec.nc +restart_filename: './anl_geom/ocn.f009_anl_geom.fc.{{ marine_window_end_iso }}.PT0S.nc' +mld_filename: './anl_geom/ocn.f009_anl_geom.fc.{{ marine_window_end_iso }}.PT0S.nc' output_filename: ./vt_scales.nc output_variable_vt: Temp output_variable_hz: ave_ssh diff --git a/algorithm/marine/socaincr2mom6.yaml.j2 b/algorithm/marine/socaincr2mom6.yaml.j2 index 0790fc9..85b8f03 100644 --- a/algorithm/marine/socaincr2mom6.yaml.j2 +++ b/algorithm/marine/socaincr2mom6.yaml.j2 @@ -6,6 +6,8 @@ date: '{{ marine_window_begin }}' layers variable: [sea_water_cell_thickness] +domains: [ocn] + increment variables: - sea_water_potential_temperature - sea_water_salinity diff --git a/model/marine/marine_3dfgat_outer_loop_1.yaml.j2 b/model/marine/marine_3dfgat_outer_loop_1.yaml.j2 index 2931f62..e90d9dd 100644 --- a/model/marine/marine_3dfgat_outer_loop_1.yaml.j2 +++ b/model/marine/marine_3dfgat_outer_loop_1.yaml.j2 @@ -1,7 +1,8 @@ - ninner: {{marine_ninner_1}} gradient norm reduction: {{marine_grad_red_1}} geometry: - mom6_input_nml: mom_input.nml + geom_grid_file: ./anl_geom/soca_gridspec.nc + mom6_input_nml: ./anl_geom/mom_input.nml fields metadata: {{marine_soca_files_path}}/fields_metadata.yaml diagnostics: departures: ombg diff --git a/model/marine/marine_3dvar_outer_loop_1.yaml.j2 b/model/marine/marine_3dvar_outer_loop_1.yaml.j2 deleted file mode 100644 index 2931f62..0000000 --- a/model/marine/marine_3dvar_outer_loop_1.yaml.j2 +++ /dev/null @@ -1,7 +0,0 @@ -- ninner: {{marine_ninner_1}} - gradient norm reduction: {{marine_grad_red_1}} - geometry: - mom6_input_nml: mom_input.nml - fields metadata: {{marine_soca_files_path}}/fields_metadata.yaml - diagnostics: - departures: ombg diff --git a/model/marine/marine_background_error_hybrid_diffusion_diffusion.yaml.j2 b/model/marine/marine_background_error_hybrid_diffusion_diffusion.yaml.j2 index 5a5a1ac..7edddcb 100644 --- a/model/marine/marine_background_error_hybrid_diffusion_diffusion.yaml.j2 +++ b/model/marine/marine_background_error_hybrid_diffusion_diffusion.yaml.j2 @@ -2,6 +2,7 @@ covariance model: hybrid components: - covariance: covariance model: SABER + change background resolution: true saber central block: saber block name: diffusion read: @@ -57,41 +58,44 @@ components: value: 1.00 - covariance: - covariance model: ensemble - members from template: - template: - read_from_file: 1 - date: '{{marine_stddev_time}}' - basename: ../ensdata/ - ocn_filename: 'ocn.pert.steric.%mem%.nc' - ice_filename: 'ens/ice.%mem%.nc' - state variables: - - sea_ice_area_fraction - - sea_ice_thickness - - sea_ice_snow_thickness - - sea_water_salinity - - sea_water_potential_temperature - #- eastward_sea_water_velocity - #- northward_sea_water_velocity - - sea_surface_height_above_geoid - pattern: '%mem%' - nmembers: '{{marine_number_ensemble_members}}' - localization: - localization method: SABER - saber central block: - saber block name: diffusion - read: - groups: - - variables: - - sea_water_potential_temperature - - sea_water_salinity - - sea_surface_height_above_geoid - - sea_ice_area_fraction - horizontal: - filepath: ./staticb/hz_ocean - vertical: - levels: {{marine_vt_levels}} - filepath: ./staticb/vt_ocean + covariance model: SABER + ensemble: + members from template: + template: + read_from_file: 1 + date: '{{marine_stddev_time}}' + basename: ../ensdata/ + ocn_filename: ocn.pert.%mem%.nc + ice_filename: ice.pert.%mem%.nc + state variables: + - sea_water_potential_temperature + - sea_water_salinity + #- eastward_sea_water_velocity + #- northward_sea_water_velocity + - sea_surface_height_above_geoid + - sea_ice_area_fraction + - sea_ice_thickness + - sea_ice_snow_thickness + pattern: '%mem%' + nmembers: '{{marine_number_ensemble_members}}' + saber central block: + saber block name: Ensemble + localization: + saber central block: + saber block name: diffusion + read: + groups: + - variables: + - sea_water_potential_temperature + - sea_water_salinity + - sea_surface_height_above_geoid + - sea_ice_area_fraction + multivariate strategy: duplicated + horizontal: + filepath: ./staticb/hz_ocean + vertical: + strategy: duplicated + weight: read_from_file: 3 basename: ./ diff --git a/model/marine/marine_background_error_static_diffusion.yaml.j2 b/model/marine/marine_background_error_static_diffusion.yaml.j2 index 2a41011..1ac5820 100644 --- a/model/marine/marine_background_error_static_diffusion.yaml.j2 +++ b/model/marine/marine_background_error_static_diffusion.yaml.j2 @@ -1,4 +1,5 @@ covariance model: SABER +change background resolution: true saber central block: saber block name: diffusion read: diff --git a/test/client_integration/gdas-marine-templates.yaml b/test/client_integration/gdas-marine-templates.yaml index f258cc9..238b4a8 100644 --- a/test/client_integration/gdas-marine-templates.yaml +++ b/test/client_integration/gdas-marine-templates.yaml @@ -1,7 +1,6 @@ # This part is for testing only. Normally this would just show algorithm: # ----------------------------------------------------------------------------------- supported_algorithms: -- 3dvar - hofx3d - hofx4d - 3dfgat From 700d6a8036b363479fcc5ee061df92bc30ee78dc Mon Sep 17 00:00:00 2001 From: Kevin Dougherty <69815622+kevindougherty-noaa@users.noreply.github.com> Date: Mon, 10 Feb 2025 10:42:32 -0500 Subject: [PATCH 09/12] Update stat anlaysis job templates and add snow template (#70) The following pull request updates the existing aero stats algorithms and adds a new snow template. These have been tested successfully for initial tests. These templates should provide as guides for future developments of other observation spaces. --------- Co-authored-by: Dan Holdaway <27729500+danholdaway@users.noreply.github.com> --- .../obstats/aero/viirs_n20_template.yaml.j2 | 14 +++--- .../obstats/aero/viirs_npp_template.yaml.j2 | 14 +++--- .../obstats/snow/ims_snow_template.yaml.j2 | 47 +++++++++++++++++++ 3 files changed, 61 insertions(+), 14 deletions(-) create mode 100644 algorithm/obstats/snow/ims_snow_template.yaml.j2 diff --git a/algorithm/obstats/aero/viirs_n20_template.yaml.j2 b/algorithm/obstats/aero/viirs_n20_template.yaml.j2 index 9606dd9..50ad9c1 100644 --- a/algorithm/obstats/aero/viirs_n20_template.yaml.j2 +++ b/algorithm/obstats/aero/viirs_n20_template.yaml.j2 @@ -3,17 +3,17 @@ obsdatain: engine: type: H5File - obsfile: {{ aero_obsdatain_path }}/diag_{{ obspace }}_{{ stat_current_cycle_YMDH }}.nc4 - simulated variables: {{ aero_obsdatatin_simulated_variables }} - observed variables: {{ aero_obsdatain_observed_variables }} - variables: {{ aero_variables }} - groups to process: {{ aero_file_groups }} - qc groups: {{ aero_file_qc_groups }} + obsfile: {{ aero_obsdatain_path }}/aero/diag_{{ obspace }}_{{ stat_current_cycle_YMDH }}.nc4 + simulated variables: ['aerosolOpticalDepth'] + observed variables: ['aerosolOpticalDepth'] + variables: ['aerosolOpticalDepth'] + groups to process: ['ombg', 'oman'] + qc groups: ['EffectiveQC0', 'EffectiveQC1'] statistics to compute: ['mean', 'count', 'RMS'] output file: "{{ obspace }}_{{ stat_current_cycle_YMDH }}_output_aod.nc" domains to process: - domain: - name: "nowhere" + name: "Global" first mask variable: latitude first mask range: [-180,-100] - domain: diff --git a/algorithm/obstats/aero/viirs_npp_template.yaml.j2 b/algorithm/obstats/aero/viirs_npp_template.yaml.j2 index e0909bb..c46a47a 100644 --- a/algorithm/obstats/aero/viirs_npp_template.yaml.j2 +++ b/algorithm/obstats/aero/viirs_npp_template.yaml.j2 @@ -3,17 +3,17 @@ obsdatain: engine: type: H5File - obsfile: {{ aero_obsdatain_path }}/diag_{{ obspace }}_{{ stat_current_cycle_YMDH }}.nc4 - simulated variables: {{ aero_obsdatatin_simulated_variables }} - observed variables: {{ aero_obsdatain_observed_variables }} - variables: {{ aero_variables }} - groups to process: {{ aero_file_groups }} - qc groups: {{ aero_file_qc_groups }} + obsfile: {{ aero_obsdatain_path }}/aero/diag_{{ obspace }}_{{ stat_current_cycle_YMDH }}.nc4 + simulated variables: ['aerosolOpticalDepth'] + observed variables: ['aerosolOpticalDepth'] + variables: ['aerosolOpticalDepth'] + groups to process: ['ombg', 'oman'] + qc groups: ['EffectiveQC0', 'EffectiveQC1'] statistics to compute: ['mean', 'count', 'RMS'] output file: "{{ obspace }}_{{ stat_current_cycle_YMDH }}_output_aod.nc" domains to process: - domain: - name: "nowhere" + name: "Global" first mask variable: latitude first mask range: [-180,-100] - domain: diff --git a/algorithm/obstats/snow/ims_snow_template.yaml.j2 b/algorithm/obstats/snow/ims_snow_template.yaml.j2 new file mode 100644 index 0000000..7e98803 --- /dev/null +++ b/algorithm/obstats/snow/ims_snow_template.yaml.j2 @@ -0,0 +1,47 @@ +- obs space: + name: ims_snow + obsdatain: + engine: + type: H5File + obsfile: {{ snow_obsdatain_path }}/snow/diag_{{ obspace }}_{{ stat_current_cycle_YMDH }}.nc + simulated variables: ['totalSnowDepth'] + observed variables: ['totalSnowDepth'] + statistics to compute: ['mean', 'count', 'RMS'] + output file: "{{ obspace }}_{{ stat_current_cycle_YMDH }}_output_snow.nc" + domains to process: + - domain: + name: "Global" + first mask variable: latitude + first mask range: [-180,-100] + - domain: + name: "SH" + first mask variable: latitude + first mask range: [-90,0] + - domain: + name: "NH" + first mask variable: latitude + first mask range: [0,90] + - domain: + name: "CONUS" + first mask variable: latitude + first mask range: [25,49] + second mask variable: longitude + second mask range: [-125,-66] + - domain: + name: "Europe" + first mask variable: latitude + first mask range: [35,70] + second mask variable: longitude + second mask range: [-11,38] + - domain: + name: "Africa" + first mask variable: latitude + first mask range: [-35,37] + second mask variable: longitude + second mask range: [-17,52] + - domain: + name: "Asia" + first mask variable: latitude + first mask range: [0,70] + second mask variable: longitude + second mask range: [38, 180] \ No newline at end of file From c414b86306b6285b595048aa708600fe9df20651 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Fri, 21 Feb 2025 14:05:45 -0500 Subject: [PATCH 10/12] Remove the ens B from the marine hybrid B (#73) - fixes https://github.com/NOAA-EMC/GDASApp/issues/1491 --- algorithm/marine/soca_diagb.yaml.j2 | 5 +- ...d_error_hybrid_diffusion_diffusion.yaml.j2 | 3 + observations/marine/sst_abi_g16_l3c.yaml.j2 | 63 +------------------ observations/marine/sst_abi_g17_l3c.yaml.j2 | 63 +------------------ observations/marine/sst_ahi_h08_l3c.yaml.j2 | 63 +------------------ observations/marine/sst_avhrr_ma_l3u.yaml.j2 | 57 +---------------- observations/marine/sst_avhrr_mb_l3u.yaml.j2 | 57 +---------------- observations/marine/sst_avhrr_mc_l3u.yaml.j2 | 57 +---------------- observations/marine/sst_generic.yaml.j2 | 46 ++++++++++++++ observations/marine/sst_viirs_n20_l3u.yaml.j2 | 57 +---------------- observations/marine/sst_viirs_npp_l3u.yaml.j2 | 57 +---------------- 11 files changed, 60 insertions(+), 468 deletions(-) mode change 100644 => 120000 observations/marine/sst_abi_g16_l3c.yaml.j2 mode change 100644 => 120000 observations/marine/sst_abi_g17_l3c.yaml.j2 mode change 100644 => 120000 observations/marine/sst_ahi_h08_l3c.yaml.j2 mode change 100644 => 120000 observations/marine/sst_avhrr_ma_l3u.yaml.j2 mode change 100644 => 120000 observations/marine/sst_avhrr_mb_l3u.yaml.j2 mode change 100644 => 120000 observations/marine/sst_avhrr_mc_l3u.yaml.j2 create mode 100644 observations/marine/sst_generic.yaml.j2 mode change 100644 => 120000 observations/marine/sst_viirs_n20_l3u.yaml.j2 mode change 100644 => 120000 observations/marine/sst_viirs_npp_l3u.yaml.j2 diff --git a/algorithm/marine/soca_diagb.yaml.j2 b/algorithm/marine/soca_diagb.yaml.j2 index 1f7c0eb..17063d5 100644 --- a/algorithm/marine/soca_diagb.yaml.j2 +++ b/algorithm/marine/soca_diagb.yaml.j2 @@ -36,10 +36,11 @@ variables: - mom6_mld vertical e-folding scale: 500.0 +min efold depth ratio: 3.0 # keep the e-folding scale at most + # 1/(min efold depth ratio) of the local depth rescale: 1.0 # rescales the filtered std. dev. by "rescale" -min sst: 0.0 # Added to sst bkg. err. max ssh: 0.0 # Limits the amplitude of the unbalanced bkg err -min depth: 500.0 # zero out the bkg. error. at less than min depth +min depth: 50.0 # zero out the bkg. error. at less than min depth number of halo points: 4 number of neighbors: 16 diff --git a/model/marine/marine_background_error_hybrid_diffusion_diffusion.yaml.j2 b/model/marine/marine_background_error_hybrid_diffusion_diffusion.yaml.j2 index 7edddcb..30ff421 100644 --- a/model/marine/marine_background_error_hybrid_diffusion_diffusion.yaml.j2 +++ b/model/marine/marine_background_error_hybrid_diffusion_diffusion.yaml.j2 @@ -57,6 +57,7 @@ components: weight: value: 1.00 +{% if marine_ensemble_b | default(false) %} - covariance: covariance model: SABER ensemble: @@ -102,3 +103,5 @@ components: ocn_filename: 'ocean.ens_weights.nc' ice_filename: 'ice.ens_weights.nc' date: '{{marine_stddev_time}}' +{% endif %} + diff --git a/observations/marine/sst_abi_g16_l3c.yaml.j2 b/observations/marine/sst_abi_g16_l3c.yaml.j2 deleted file mode 100644 index 2d91234..0000000 --- a/observations/marine/sst_abi_g16_l3c.yaml.j2 +++ /dev/null @@ -1,62 +0,0 @@ -- obs space: - name: {{observation_from_jcb}} - obsdatain: - engine: - type: H5File - obsfile: "{{marine_obsdatain_path}}/{{marine_obsdatain_prefix}}{{observation_from_jcb}}{{marine_obsdatain_suffix}}" - obsdataout: - engine: - type: H5File - obsfile: "{{marine_obsdataout_path}}/{{marine_obsdataout_prefix}}{{observation_from_jcb}}{{marine_obsdataout_suffix}}" - simulated variables: [seaSurfaceTemperature] - io pool: - max pool size: 1 - get values: - time interpolation: linear - obs operator: - name: Identity - observation alias file: obsop_name_map.yaml - obs error: - covariance model: diagonal - obs filters: - - filter: Domain Check - action: - name: passivate - where: - - variable: {name: ObsValue/seaSurfaceTemperature} - maxvalue: -4.0 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/sea_area_fraction} - minvalue: 0.9 - - filter: Bounds Check - minvalue: 1.0 - maxvalue: 41.0 - - filter: Background Check - absolute threshold: 5.0 - - filter: Domain Check - where: - - variable: {name: ObsError/seaSurfaceTemperature} - minvalue: 1.0e-8 - - filter: Domain Check - where: - - variable: { name: GeoVaLs/sea_ice_area_fraction} - maxvalue: 1.0e-5 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/sea_surface_temperature} - minvalue: -1.0 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/distance_from_coast} - minvalue: 100e3 - - filter: Perform Action - action: - name: assign error - error function: - name: ObsFunction/LinearCombination - options: - variables: - - ObsError/seaSurfaceTemperature - coefs: - - 1.0 diff --git a/observations/marine/sst_abi_g16_l3c.yaml.j2 b/observations/marine/sst_abi_g16_l3c.yaml.j2 new file mode 120000 index 0000000..5298b9a --- /dev/null +++ b/observations/marine/sst_abi_g16_l3c.yaml.j2 @@ -0,0 +1 @@ +sst_generic.yaml.j2 \ No newline at end of file diff --git a/observations/marine/sst_abi_g17_l3c.yaml.j2 b/observations/marine/sst_abi_g17_l3c.yaml.j2 deleted file mode 100644 index 2d91234..0000000 --- a/observations/marine/sst_abi_g17_l3c.yaml.j2 +++ /dev/null @@ -1,62 +0,0 @@ -- obs space: - name: {{observation_from_jcb}} - obsdatain: - engine: - type: H5File - obsfile: "{{marine_obsdatain_path}}/{{marine_obsdatain_prefix}}{{observation_from_jcb}}{{marine_obsdatain_suffix}}" - obsdataout: - engine: - type: H5File - obsfile: "{{marine_obsdataout_path}}/{{marine_obsdataout_prefix}}{{observation_from_jcb}}{{marine_obsdataout_suffix}}" - simulated variables: [seaSurfaceTemperature] - io pool: - max pool size: 1 - get values: - time interpolation: linear - obs operator: - name: Identity - observation alias file: obsop_name_map.yaml - obs error: - covariance model: diagonal - obs filters: - - filter: Domain Check - action: - name: passivate - where: - - variable: {name: ObsValue/seaSurfaceTemperature} - maxvalue: -4.0 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/sea_area_fraction} - minvalue: 0.9 - - filter: Bounds Check - minvalue: 1.0 - maxvalue: 41.0 - - filter: Background Check - absolute threshold: 5.0 - - filter: Domain Check - where: - - variable: {name: ObsError/seaSurfaceTemperature} - minvalue: 1.0e-8 - - filter: Domain Check - where: - - variable: { name: GeoVaLs/sea_ice_area_fraction} - maxvalue: 1.0e-5 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/sea_surface_temperature} - minvalue: -1.0 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/distance_from_coast} - minvalue: 100e3 - - filter: Perform Action - action: - name: assign error - error function: - name: ObsFunction/LinearCombination - options: - variables: - - ObsError/seaSurfaceTemperature - coefs: - - 1.0 diff --git a/observations/marine/sst_abi_g17_l3c.yaml.j2 b/observations/marine/sst_abi_g17_l3c.yaml.j2 new file mode 120000 index 0000000..5298b9a --- /dev/null +++ b/observations/marine/sst_abi_g17_l3c.yaml.j2 @@ -0,0 +1 @@ +sst_generic.yaml.j2 \ No newline at end of file diff --git a/observations/marine/sst_ahi_h08_l3c.yaml.j2 b/observations/marine/sst_ahi_h08_l3c.yaml.j2 deleted file mode 100644 index 2d91234..0000000 --- a/observations/marine/sst_ahi_h08_l3c.yaml.j2 +++ /dev/null @@ -1,62 +0,0 @@ -- obs space: - name: {{observation_from_jcb}} - obsdatain: - engine: - type: H5File - obsfile: "{{marine_obsdatain_path}}/{{marine_obsdatain_prefix}}{{observation_from_jcb}}{{marine_obsdatain_suffix}}" - obsdataout: - engine: - type: H5File - obsfile: "{{marine_obsdataout_path}}/{{marine_obsdataout_prefix}}{{observation_from_jcb}}{{marine_obsdataout_suffix}}" - simulated variables: [seaSurfaceTemperature] - io pool: - max pool size: 1 - get values: - time interpolation: linear - obs operator: - name: Identity - observation alias file: obsop_name_map.yaml - obs error: - covariance model: diagonal - obs filters: - - filter: Domain Check - action: - name: passivate - where: - - variable: {name: ObsValue/seaSurfaceTemperature} - maxvalue: -4.0 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/sea_area_fraction} - minvalue: 0.9 - - filter: Bounds Check - minvalue: 1.0 - maxvalue: 41.0 - - filter: Background Check - absolute threshold: 5.0 - - filter: Domain Check - where: - - variable: {name: ObsError/seaSurfaceTemperature} - minvalue: 1.0e-8 - - filter: Domain Check - where: - - variable: { name: GeoVaLs/sea_ice_area_fraction} - maxvalue: 1.0e-5 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/sea_surface_temperature} - minvalue: -1.0 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/distance_from_coast} - minvalue: 100e3 - - filter: Perform Action - action: - name: assign error - error function: - name: ObsFunction/LinearCombination - options: - variables: - - ObsError/seaSurfaceTemperature - coefs: - - 1.0 diff --git a/observations/marine/sst_ahi_h08_l3c.yaml.j2 b/observations/marine/sst_ahi_h08_l3c.yaml.j2 new file mode 120000 index 0000000..5298b9a --- /dev/null +++ b/observations/marine/sst_ahi_h08_l3c.yaml.j2 @@ -0,0 +1 @@ +sst_generic.yaml.j2 \ No newline at end of file diff --git a/observations/marine/sst_avhrr_ma_l3u.yaml.j2 b/observations/marine/sst_avhrr_ma_l3u.yaml.j2 deleted file mode 100644 index 63f847a..0000000 --- a/observations/marine/sst_avhrr_ma_l3u.yaml.j2 +++ /dev/null @@ -1,56 +0,0 @@ -- obs space: - name: {{observation_from_jcb}} - obsdatain: - engine: - type: H5File - obsfile: "{{marine_obsdatain_path}}/{{marine_obsdatain_prefix}}{{observation_from_jcb}}{{marine_obsdatain_suffix}}" - obsdataout: - engine: - type: H5File - obsfile: "{{marine_obsdataout_path}}/{{marine_obsdataout_prefix}}{{observation_from_jcb}}{{marine_obsdataout_suffix}}" - simulated variables: [seaSurfaceTemperature] - io pool: - max pool size: 1 - get values: - time interpolation: linear - obs operator: - name: Identity - observation alias file: obsop_name_map.yaml - obs error: - covariance model: diagonal - obs filters: - - filter: Domain Check - where: - - variable: {name: GeoVaLs/sea_area_fraction} - minvalue: 0.9 - - filter: Bounds Check - minvalue: 1.0 - maxvalue: 41.0 - - filter: Background Check - absolute threshold: 5.0 - - filter: Domain Check - where: - - variable: {name: ObsError/seaSurfaceTemperature} - minvalue: 1.0e-8 - - filter: Domain Check - where: - - variable: { name: GeoVaLs/sea_ice_area_fraction} - maxvalue: 1.0e-5 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/sea_surface_temperature} - minvalue: -1.0 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/distance_from_coast} - minvalue: 100e3 - - filter: Perform Action - action: - name: assign error - error function: - name: ObsFunction/LinearCombination - options: - variables: - - ObsError/seaSurfaceTemperature - coefs: - - 1.0 diff --git a/observations/marine/sst_avhrr_ma_l3u.yaml.j2 b/observations/marine/sst_avhrr_ma_l3u.yaml.j2 new file mode 120000 index 0000000..5298b9a --- /dev/null +++ b/observations/marine/sst_avhrr_ma_l3u.yaml.j2 @@ -0,0 +1 @@ +sst_generic.yaml.j2 \ No newline at end of file diff --git a/observations/marine/sst_avhrr_mb_l3u.yaml.j2 b/observations/marine/sst_avhrr_mb_l3u.yaml.j2 deleted file mode 100644 index 63f847a..0000000 --- a/observations/marine/sst_avhrr_mb_l3u.yaml.j2 +++ /dev/null @@ -1,56 +0,0 @@ -- obs space: - name: {{observation_from_jcb}} - obsdatain: - engine: - type: H5File - obsfile: "{{marine_obsdatain_path}}/{{marine_obsdatain_prefix}}{{observation_from_jcb}}{{marine_obsdatain_suffix}}" - obsdataout: - engine: - type: H5File - obsfile: "{{marine_obsdataout_path}}/{{marine_obsdataout_prefix}}{{observation_from_jcb}}{{marine_obsdataout_suffix}}" - simulated variables: [seaSurfaceTemperature] - io pool: - max pool size: 1 - get values: - time interpolation: linear - obs operator: - name: Identity - observation alias file: obsop_name_map.yaml - obs error: - covariance model: diagonal - obs filters: - - filter: Domain Check - where: - - variable: {name: GeoVaLs/sea_area_fraction} - minvalue: 0.9 - - filter: Bounds Check - minvalue: 1.0 - maxvalue: 41.0 - - filter: Background Check - absolute threshold: 5.0 - - filter: Domain Check - where: - - variable: {name: ObsError/seaSurfaceTemperature} - minvalue: 1.0e-8 - - filter: Domain Check - where: - - variable: { name: GeoVaLs/sea_ice_area_fraction} - maxvalue: 1.0e-5 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/sea_surface_temperature} - minvalue: -1.0 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/distance_from_coast} - minvalue: 100e3 - - filter: Perform Action - action: - name: assign error - error function: - name: ObsFunction/LinearCombination - options: - variables: - - ObsError/seaSurfaceTemperature - coefs: - - 1.0 diff --git a/observations/marine/sst_avhrr_mb_l3u.yaml.j2 b/observations/marine/sst_avhrr_mb_l3u.yaml.j2 new file mode 120000 index 0000000..5298b9a --- /dev/null +++ b/observations/marine/sst_avhrr_mb_l3u.yaml.j2 @@ -0,0 +1 @@ +sst_generic.yaml.j2 \ No newline at end of file diff --git a/observations/marine/sst_avhrr_mc_l3u.yaml.j2 b/observations/marine/sst_avhrr_mc_l3u.yaml.j2 deleted file mode 100644 index 63f847a..0000000 --- a/observations/marine/sst_avhrr_mc_l3u.yaml.j2 +++ /dev/null @@ -1,56 +0,0 @@ -- obs space: - name: {{observation_from_jcb}} - obsdatain: - engine: - type: H5File - obsfile: "{{marine_obsdatain_path}}/{{marine_obsdatain_prefix}}{{observation_from_jcb}}{{marine_obsdatain_suffix}}" - obsdataout: - engine: - type: H5File - obsfile: "{{marine_obsdataout_path}}/{{marine_obsdataout_prefix}}{{observation_from_jcb}}{{marine_obsdataout_suffix}}" - simulated variables: [seaSurfaceTemperature] - io pool: - max pool size: 1 - get values: - time interpolation: linear - obs operator: - name: Identity - observation alias file: obsop_name_map.yaml - obs error: - covariance model: diagonal - obs filters: - - filter: Domain Check - where: - - variable: {name: GeoVaLs/sea_area_fraction} - minvalue: 0.9 - - filter: Bounds Check - minvalue: 1.0 - maxvalue: 41.0 - - filter: Background Check - absolute threshold: 5.0 - - filter: Domain Check - where: - - variable: {name: ObsError/seaSurfaceTemperature} - minvalue: 1.0e-8 - - filter: Domain Check - where: - - variable: { name: GeoVaLs/sea_ice_area_fraction} - maxvalue: 1.0e-5 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/sea_surface_temperature} - minvalue: -1.0 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/distance_from_coast} - minvalue: 100e3 - - filter: Perform Action - action: - name: assign error - error function: - name: ObsFunction/LinearCombination - options: - variables: - - ObsError/seaSurfaceTemperature - coefs: - - 1.0 diff --git a/observations/marine/sst_avhrr_mc_l3u.yaml.j2 b/observations/marine/sst_avhrr_mc_l3u.yaml.j2 new file mode 120000 index 0000000..5298b9a --- /dev/null +++ b/observations/marine/sst_avhrr_mc_l3u.yaml.j2 @@ -0,0 +1 @@ +sst_generic.yaml.j2 \ No newline at end of file diff --git a/observations/marine/sst_generic.yaml.j2 b/observations/marine/sst_generic.yaml.j2 new file mode 100644 index 0000000..30754a0 --- /dev/null +++ b/observations/marine/sst_generic.yaml.j2 @@ -0,0 +1,46 @@ +- obs space: + name: {{observation_from_jcb}} + obsdatain: + engine: + type: H5File + obsfile: "{{marine_obsdatain_path}}/{{marine_obsdatain_prefix}}{{observation_from_jcb}}{{marine_obsdatain_suffix}}" + obsdataout: + engine: + type: H5File + obsfile: "{{marine_obsdataout_path}}/{{marine_obsdataout_prefix}}{{observation_from_jcb}}{{marine_obsdataout_suffix}}" + simulated variables: [seaSurfaceTemperature] + io pool: + max pool size: 1 + get values: + time interpolation: linear + obs operator: + name: Identity + observation alias file: obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: {name: GeoVaLs/sea_area_fraction} + minvalue: 0.9 + - filter: Bounds Check + minvalue: -1.2 + maxvalue: 41.0 + - filter: Background Check + absolute threshold: 5.0 + - filter: Domain Check + where: + - variable: {name: ObsError/seaSurfaceTemperature} + minvalue: 1.0e-8 + - filter: Domain Check + where: + - variable: { name: GeoVaLs/sea_ice_area_fraction} + maxvalue: 1.0e-5 + - filter: Domain Check + where: + - variable: {name: GeoVaLs/sea_surface_temperature} + minvalue: -1.2 + - filter: Domain Check + where: + - variable: {name: GeoVaLs/distance_from_coast} + minvalue: 50e3 diff --git a/observations/marine/sst_viirs_n20_l3u.yaml.j2 b/observations/marine/sst_viirs_n20_l3u.yaml.j2 deleted file mode 100644 index 63f847a..0000000 --- a/observations/marine/sst_viirs_n20_l3u.yaml.j2 +++ /dev/null @@ -1,56 +0,0 @@ -- obs space: - name: {{observation_from_jcb}} - obsdatain: - engine: - type: H5File - obsfile: "{{marine_obsdatain_path}}/{{marine_obsdatain_prefix}}{{observation_from_jcb}}{{marine_obsdatain_suffix}}" - obsdataout: - engine: - type: H5File - obsfile: "{{marine_obsdataout_path}}/{{marine_obsdataout_prefix}}{{observation_from_jcb}}{{marine_obsdataout_suffix}}" - simulated variables: [seaSurfaceTemperature] - io pool: - max pool size: 1 - get values: - time interpolation: linear - obs operator: - name: Identity - observation alias file: obsop_name_map.yaml - obs error: - covariance model: diagonal - obs filters: - - filter: Domain Check - where: - - variable: {name: GeoVaLs/sea_area_fraction} - minvalue: 0.9 - - filter: Bounds Check - minvalue: 1.0 - maxvalue: 41.0 - - filter: Background Check - absolute threshold: 5.0 - - filter: Domain Check - where: - - variable: {name: ObsError/seaSurfaceTemperature} - minvalue: 1.0e-8 - - filter: Domain Check - where: - - variable: { name: GeoVaLs/sea_ice_area_fraction} - maxvalue: 1.0e-5 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/sea_surface_temperature} - minvalue: -1.0 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/distance_from_coast} - minvalue: 100e3 - - filter: Perform Action - action: - name: assign error - error function: - name: ObsFunction/LinearCombination - options: - variables: - - ObsError/seaSurfaceTemperature - coefs: - - 1.0 diff --git a/observations/marine/sst_viirs_n20_l3u.yaml.j2 b/observations/marine/sst_viirs_n20_l3u.yaml.j2 new file mode 120000 index 0000000..5298b9a --- /dev/null +++ b/observations/marine/sst_viirs_n20_l3u.yaml.j2 @@ -0,0 +1 @@ +sst_generic.yaml.j2 \ No newline at end of file diff --git a/observations/marine/sst_viirs_npp_l3u.yaml.j2 b/observations/marine/sst_viirs_npp_l3u.yaml.j2 deleted file mode 100644 index 63f847a..0000000 --- a/observations/marine/sst_viirs_npp_l3u.yaml.j2 +++ /dev/null @@ -1,56 +0,0 @@ -- obs space: - name: {{observation_from_jcb}} - obsdatain: - engine: - type: H5File - obsfile: "{{marine_obsdatain_path}}/{{marine_obsdatain_prefix}}{{observation_from_jcb}}{{marine_obsdatain_suffix}}" - obsdataout: - engine: - type: H5File - obsfile: "{{marine_obsdataout_path}}/{{marine_obsdataout_prefix}}{{observation_from_jcb}}{{marine_obsdataout_suffix}}" - simulated variables: [seaSurfaceTemperature] - io pool: - max pool size: 1 - get values: - time interpolation: linear - obs operator: - name: Identity - observation alias file: obsop_name_map.yaml - obs error: - covariance model: diagonal - obs filters: - - filter: Domain Check - where: - - variable: {name: GeoVaLs/sea_area_fraction} - minvalue: 0.9 - - filter: Bounds Check - minvalue: 1.0 - maxvalue: 41.0 - - filter: Background Check - absolute threshold: 5.0 - - filter: Domain Check - where: - - variable: {name: ObsError/seaSurfaceTemperature} - minvalue: 1.0e-8 - - filter: Domain Check - where: - - variable: { name: GeoVaLs/sea_ice_area_fraction} - maxvalue: 1.0e-5 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/sea_surface_temperature} - minvalue: -1.0 - - filter: Domain Check - where: - - variable: {name: GeoVaLs/distance_from_coast} - minvalue: 100e3 - - filter: Perform Action - action: - name: assign error - error function: - name: ObsFunction/LinearCombination - options: - variables: - - ObsError/seaSurfaceTemperature - coefs: - - 1.0 diff --git a/observations/marine/sst_viirs_npp_l3u.yaml.j2 b/observations/marine/sst_viirs_npp_l3u.yaml.j2 new file mode 120000 index 0000000..5298b9a --- /dev/null +++ b/observations/marine/sst_viirs_npp_l3u.yaml.j2 @@ -0,0 +1 @@ +sst_generic.yaml.j2 \ No newline at end of file From bfcff02b7d220bd8bbec03f833d75fada00b1cb7 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty <69815622+kevindougherty-noaa@users.noreply.github.com> Date: Tue, 25 Feb 2025 11:25:43 -0500 Subject: [PATCH 11/12] Quick Filename Fix for Stats YAMLs (#80) Quick filename fix for the YAMLs needed for the analysis stats job in the g-w. --- algorithm/obstats/aero/viirs_n20_template.yaml.j2 | 2 +- algorithm/obstats/aero/viirs_npp_template.yaml.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/algorithm/obstats/aero/viirs_n20_template.yaml.j2 b/algorithm/obstats/aero/viirs_n20_template.yaml.j2 index 50ad9c1..9994716 100644 --- a/algorithm/obstats/aero/viirs_n20_template.yaml.j2 +++ b/algorithm/obstats/aero/viirs_n20_template.yaml.j2 @@ -3,7 +3,7 @@ obsdatain: engine: type: H5File - obsfile: {{ aero_obsdatain_path }}/aero/diag_{{ obspace }}_{{ stat_current_cycle_YMDH }}.nc4 + obsfile: {{ aero_obsdatain_path }}/aero/diag_{{ obspace }}_aod_{{ stat_current_cycle_YMDH }}.nc simulated variables: ['aerosolOpticalDepth'] observed variables: ['aerosolOpticalDepth'] variables: ['aerosolOpticalDepth'] diff --git a/algorithm/obstats/aero/viirs_npp_template.yaml.j2 b/algorithm/obstats/aero/viirs_npp_template.yaml.j2 index c46a47a..3c4c5d7 100644 --- a/algorithm/obstats/aero/viirs_npp_template.yaml.j2 +++ b/algorithm/obstats/aero/viirs_npp_template.yaml.j2 @@ -3,7 +3,7 @@ obsdatain: engine: type: H5File - obsfile: {{ aero_obsdatain_path }}/aero/diag_{{ obspace }}_{{ stat_current_cycle_YMDH }}.nc4 + obsfile: {{ aero_obsdatain_path }}/aero/diag_{{ obspace }}_aod_{{ stat_current_cycle_YMDH }}.nc simulated variables: ['aerosolOpticalDepth'] observed variables: ['aerosolOpticalDepth'] variables: ['aerosolOpticalDepth'] From 7a418aa9c7dff6134dade1625c0041e5370b2e32 Mon Sep 17 00:00:00 2001 From: Travis Elless <113720457+TravisElless-NOAA@users.noreply.github.com> Date: Wed, 26 Feb 2025 09:52:56 -0500 Subject: [PATCH 12/12] Update for missing snow files (#76) This PR adds missing file actions to the snow observation yamls and also makes an additional change to the obs staging yaml copy statement. These modifications are one step to ensuring the snow DA does not cause an operational failure if snow observation files are missing. References [GDASApp Issue 1462 ](https://github.com/NOAA-EMC/GDASApp/issues/1462) --- algorithm/snow/snow_obs_staging.yaml.j2 | 2 +- observations/snow/adpsfc_snow.yaml.j2 | 1 + observations/snow/ghcn_snow.yaml.j2 | 1 + observations/snow/sfcsno.yaml.j2 | 1 + observations/snow/snocvr_snow.yaml.j2 | 1 + 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/algorithm/snow/snow_obs_staging.yaml.j2 b/algorithm/snow/snow_obs_staging.yaml.j2 index a8d3dc8..778cfd1 100644 --- a/algorithm/snow/snow_obs_staging.yaml.j2 +++ b/algorithm/snow/snow_obs_staging.yaml.j2 @@ -1,6 +1,6 @@ mkdir: - '{{snow_obsdatain_path}}' -copy: +copy_opt: {% for observation_from_jcb in observations %} {% if use_observer(observation_from_jcb) %} {% if not observation_from_jcb == 'ims_snow' %} diff --git a/observations/snow/adpsfc_snow.yaml.j2 b/observations/snow/adpsfc_snow.yaml.j2 index f0fe2fd..6298cc0 100644 --- a/observations/snow/adpsfc_snow.yaml.j2 +++ b/observations/snow/adpsfc_snow.yaml.j2 @@ -8,6 +8,7 @@ engine: type: H5File obsfile: "{{snow_obsdatain_path}}/{{snow_obsdatain_prefix}}{{observation_from_jcb}}{{snow_obsdatain_suffix}}" + missing file action: warn obsdataout: engine: type: H5File diff --git a/observations/snow/ghcn_snow.yaml.j2 b/observations/snow/ghcn_snow.yaml.j2 index ea7a956..b825e79 100644 --- a/observations/snow/ghcn_snow.yaml.j2 +++ b/observations/snow/ghcn_snow.yaml.j2 @@ -8,6 +8,7 @@ engine: type: H5File obsfile: "{{snow_obsdatain_path}}/{{snow_obsdatain_prefix}}{{observation_from_jcb}}.nc" + missing file action: warn obsdataout: engine: type: H5File diff --git a/observations/snow/sfcsno.yaml.j2 b/observations/snow/sfcsno.yaml.j2 index ce44417..31c8785 100644 --- a/observations/snow/sfcsno.yaml.j2 +++ b/observations/snow/sfcsno.yaml.j2 @@ -9,6 +9,7 @@ type: bufr obsfile: "{{snow_obsdatain_path}}/{{snow_obsdatain_prefix}}sfcsno.tm00.bufr_d" mapping file: "{{snow_obsdatain_path}}/bufr_sfcsno_mapping.yaml" + missing file action: warn obsdataout: engine: type: H5File diff --git a/observations/snow/snocvr_snow.yaml.j2 b/observations/snow/snocvr_snow.yaml.j2 index fe73587..c38da08 100644 --- a/observations/snow/snocvr_snow.yaml.j2 +++ b/observations/snow/snocvr_snow.yaml.j2 @@ -8,6 +8,7 @@ engine: type: H5File obsfile: "{{snow_obsdatain_path}}/{{snow_obsdatain_prefix}}{{observation_from_jcb}}.nc4" + missing file action: warn obsdataout: engine: type: H5File