diff --git a/CLAUDE.md b/CLAUDE.md index 3209c780..21dc4769 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -64,7 +64,16 @@ Copy `.env.example` to `.env`. Key variables: ## Testing -Tests live in `tests/unit/` and `tests/integration/`. Integration tests exercise the full pipeline with real (cached) data. Use `--cache` pytest fixtures to control data caching behaviour. +Tests live in `tests/unit/` and `tests/integration/`. Tests follow pytest +conventions and use pytest libraries. + +Integration tests exercise the full pipeline with real (cached) data. Use +`input_files` pytest fixture to use cached data. For integration tests which +should download DataSource assets from the internet, `input_files` should be +omitted. + +Unit tests should avoid calling methods which read from the filesystem, and +should construct example data to pass to methods. ## Code Style diff --git a/changelog/183.feature.md b/changelog/183.feature.md new file mode 100644 index 00000000..a9146210 --- /dev/null +++ b/changelog/183.feature.md @@ -0,0 +1 @@ +Update livestock sector implementation to use headcounts dataset diff --git a/docs/data-sources.md b/docs/data-sources.md index e58ff678..b2a3e6bb 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -140,12 +140,9 @@ Superpower Institute, with data synced using Some sectoral inventories are spatialised by identifying which Australian Land Use and Management (ALUM) Classification codes fall within those sectors, and then distributing inventory emissions to the areas where those ALUM codes are -featured in the Land Use of Australia geographical dataset provided by the -Australian Department of Agriculture. - -This is done using a combination of the official +featured in the [Land Use of Australia](https://www.agriculture.gov.au/abares/aclump/land-use/land-use-of-australia-2010-11-to-2020-21) -GeoTIFF raster, and a manual [mapping between ALUM codes and Open Methane sectors][4]. +geographical dataset provided by the Australian Department of Agriculture. Due to issues fetching the file directly from the agriculture.gov.au service in cloud-hosted services, the land use dataset is mirrored from the Open @@ -200,33 +197,25 @@ substances which can act as a proxy for methane emission. ## Sector: Livestock -Australian national dataset of CH4 flux estimates from enteric fermentation in livestock. +Australian national dataset of livestock headcounts for cattle and sheep, used +to distribute ANGA inventory emissions for enteric fermentation and manure for +those animals. ### Sources -Enteric fermentation emissions generated by CSIRO Ag. and Food using livestock -census data and UNFCCC emissions factors. Underlying livestock numbers -taken from: "Navarro, J. Marcos Martinez, R. (2021) Estimating long-term profits, fertiliser and pesticide use -baselines in Australian agricultural regions. User Guide. CSIRO, -Australia." +Livestock headcounts generated by CSIRO Agriculture and Food using livestock +census data. Attribution TBC -- Dataset: [EntericFermentation.nc][2] +- Dataset: TBC - Resolution: 0.01 degree -- Period: 2020, single annual average +- Period: Unknown - Updates: never ### Considerations -- Spatial distribution might change during the year since farmers are moving their cows around -- Doesn't appear to be available online - -### Alternative candidates - -- https://publications.csiro.au/publications/publication/PIcsiro:EP2022-1389/ - - Resolution: 0.1 degree - - Period: 1972 - 2019, annual averages - - Global -- Electronic ear tags might be a useful database +- Real spatial distribution likely to be highly variable during the year as + grazing herds move through available pastures. +- Dataset not yet publicly available. ## Sector: Termites @@ -238,7 +227,7 @@ Global dataset of CH4 flux estimates from termites. Termite emissions used in [Saunois et al. 2020](https://essd.copernicus.org/articles/12/1561/2020/) supplied by Simona Castaldi and Sergio Noce. -- Dataset: [termite_emissions_2010-2016.nc][5] +- Dataset: [termite_emissions_2010-2016.nc][4] - Resolution: 0.5 degree - Period: mean of 2010 – 2016 - Updates: never @@ -305,7 +294,7 @@ and described in Zhang et al. (2023 under review) (could this be https://essd.co ## Sector: Agriculture -Agricultural emissions (excluding livestock) reported in the +Agricultural emissions (excluding cattle and sheep) reported in the [Australian UNFCCC Inventory](#Australian-UNFCCC-Inventory) are spatialised according to the [Land Use of Australia](#Land-Use-of-Australia) dataset. @@ -475,5 +464,4 @@ spatialised according to the [Land Use of Australia](#Land-Use-of-Australia) dat [1]: https://openmethane.s3.amazonaws.com/prior/inputs/DLEM_totflux_CRU_diagnostic.nc [2]: https://openmethane.s3.amazonaws.com/prior/inputs/EntericFermentation.nc [3]: https://openmethane.s3.amazonaws.com/prior/inputs/ch4-electricity.csv -[4]: https://openmethane.s3.amazonaws.com/prior/inputs/landuse-sector-map.csv -[5]: https://openmethane.s3.amazonaws.com/prior/inputs/termite_emissions_2010-2016.nc +[4]: https://openmethane.s3.amazonaws.com/prior/inputs/termite_emissions_2010-2016.nc diff --git a/src/openmethane_prior/data_sources/landuse/__init__.py b/src/openmethane_prior/data_sources/landuse/__init__.py index 0c7d0dd8..f67db24f 100644 --- a/src/openmethane_prior/data_sources/landuse/__init__.py +++ b/src/openmethane_prior/data_sources/landuse/__init__.py @@ -1,2 +1 @@ -from .data import alum_sector_mapping_data_source, landuse_map_data_source -from .alum import alum_codes_for_sector \ No newline at end of file +from .data import landuse_map_data_source diff --git a/src/openmethane_prior/data_sources/landuse/alum.py b/src/openmethane_prior/data_sources/landuse/alum.py deleted file mode 100644 index 279a6604..00000000 --- a/src/openmethane_prior/data_sources/landuse/alum.py +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright 2023 The Superpower Institute Ltd. -# -# This file is part of OpenMethane. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import pandas as pd - -def alum_codes_for_sector(sector_name: str, alum_mapping: pd.DataFrame) -> list[int]: - alum_codes = [] - for alum, sector in alum_mapping.values: - if sector == sector_name: - alum_codes.append(int(alum)) - - return alum_codes diff --git a/src/openmethane_prior/data_sources/landuse/data.py b/src/openmethane_prior/data_sources/landuse/data.py index ef2e647b..b0548aef 100644 --- a/src/openmethane_prior/data_sources/landuse/data.py +++ b/src/openmethane_prior/data_sources/landuse/data.py @@ -23,11 +23,6 @@ from openmethane_prior.lib import ConfiguredDataSource, DataSource from openmethane_prior.lib.data_manager.parsers import parse_csv -alum_sector_mapping_data_source = DataSource( - name="alum-sector-mapping", - url="https://openmethane.s3.amazonaws.com/prior/inputs/landuse-sector-map.csv", - parse=parse_csv, -) # filename of the GeoTIFF file inside the official zip file source NLUM_GEOTIFF_FILENAME="NLUM_v7_250_ALUMV8_2020_21_alb.tif" diff --git a/src/openmethane_prior/lib/__init__.py b/src/openmethane_prior/lib/__init__.py index 46ae13be..0e208318 100644 --- a/src/openmethane_prior/lib/__init__.py +++ b/src/openmethane_prior/lib/__init__.py @@ -23,6 +23,7 @@ DataManager, DataSource, ) +from .grid.grid import Grid from .grid.domain import Domain from .grid.regrid import regrid_data from .regrid import regrid_data_array_conservative diff --git a/src/openmethane_prior/sectors/agriculture/sector.py b/src/openmethane_prior/sectors/agriculture/sector.py index 77622ccd..75cd48a2 100644 --- a/src/openmethane_prior/sectors/agriculture/sector.py +++ b/src/openmethane_prior/sectors/agriculture/sector.py @@ -26,11 +26,7 @@ inventory_data_source, inventory_domain_data_source, ) -from openmethane_prior.data_sources.landuse import ( - alum_codes_for_sector, - alum_sector_mapping_data_source, - landuse_map_data_source, -) +from openmethane_prior.data_sources.landuse import landuse_map_data_source from openmethane_prior.lib import ( kg_to_period_cell_flux, logger, @@ -42,6 +38,17 @@ logger = logger.get_logger(__name__) +# ALUM Classification Version 8 +# Source: https://www.agriculture.gov.au/abares/aclump/land-use/alum-classification +alum_codes_agriculture = [ + 520, # Intensive animal production + 523, # Poultry farms + 524, # Piggeries + 525, # Aquaculture + 526, # Horse studs + 527, # Saleyards/stockyards + 542, # Rural residential with agriculture +] def process_emissions( sector: PriorSector, @@ -50,11 +57,6 @@ def process_emissions( ): config = sector_config.prior_config - # Import a map of land use type numbers to emissions sectors - # make a dictionary of all landuse types corresponding to sectors in map - sector_mapping_asset = sector_config.data_manager.get_asset(alum_sector_mapping_data_source) - sector_alum_codes = alum_codes_for_sector(sector.name, sector_mapping_asset.data) - # load the national inventory data, ready to calculate sectoral totals emissions_inventory = sector_config.data_manager.get_asset(inventory_data_source).data @@ -80,7 +82,7 @@ def process_emissions( ) # create a mask of pixels which match the sector code - sector_mask = np.isin(dataBand, sector_alum_codes) + sector_mask = np.isin(dataBand, alum_codes_agriculture) sector_xr = xr.DataArray(sector_mask, coords={ 'y': lu_y, 'x': lu_x }) # now aggregate to coarser resolution of the domain grid @@ -111,8 +113,11 @@ def process_emissions( sector = PriorSector( name="agriculture", emission_category="anthropogenic", - unfccc_categories=[ # All Agriculture, except Enteric Fermentation - "3.B", # Manure Management + unfccc_categories=[ # All Agriculture, except cattle and sheep + "3.A.3", # Enteric Fermentation - Swine + "3.A.4", # Enteric Fermentation - Other Livestock + "3.B.3", # Manure Management - Swine + "3.B.4", # Manure Management - Other Livestock "3.C", # Rice Cultivation "3.D", # Agricultural Soils "3.E", # Prescribed Burning of Savannas diff --git a/src/openmethane_prior/sectors/livestock/sector.py b/src/openmethane_prior/sectors/livestock/sector.py index e1f814c5..66e98092 100644 --- a/src/openmethane_prior/sectors/livestock/sector.py +++ b/src/openmethane_prior/sectors/livestock/sector.py @@ -15,64 +15,124 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import numpy as np +import pandas as pd +import pyproj import xarray as xr +from openmethane_prior.data_sources.inventory import ( + inventory_data_source, + get_sector_emissions_by_code, +) +from openmethane_prior.lib.data_manager.parsers import parse_csv from openmethane_prior.lib import ( DataSource, - convert_to_timescale, + Grid, logger, PriorSector, PriorSectorConfig, ) +from openmethane_prior.lib.units import seconds_in_period logger = logger.get_logger(__name__) -livestock_data_source = DataSource( - name="enteric-fermentation", - url="https://openmethane.s3.amazonaws.com/prior/inputs/EntericFermentation.nc", +livestock_headcount_data_source = DataSource( + name="livestock-headcount", + url="https://openmethane.s3.amazonaws.com/prior/inputs/lmap_selected_cols_Nasimeh_171025.csv", + parse=parse_csv, ) +def gridded_livestock_headcounts( + livestock_df: pd.DataFrame, + domain_grid: Grid, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Given a list of coordinates for livestock areas with headcounts of beef + cattle, dairy cattle and sheep, calculate the annual methane emissions in + each area, and return as gridded emissions over the domain of interest.""" + # find the domain grid cell (ix, iy) each livestock area centre point is in + livestock_df["ix"], livestock_df["iy"], valid_cells = domain_grid.lonlat_to_cell_index( + lon=livestock_df["lon"], + lat=livestock_df["lat"], + ) + + # discard data outside the domain + livestock_df = livestock_df[valid_cells] + + logger.debug(f"Mapping livestock headcounts to domain grid") + beef_gridded = np.zeros(domain_grid.shape) + dairy_gridded = np.zeros(domain_grid.shape) + sheep_gridded = np.zeros(domain_grid.shape) + + row_idx = (livestock_df["iy"], livestock_df["ix"]) + np.add.at(beef_gridded, row_idx, livestock_df["heads_mapped_mix_beef"]) + np.add.at(dairy_gridded, row_idx, livestock_df["heads_mapped_dairy"]) + np.add.at(sheep_gridded, row_idx, livestock_df["heads_mapped_mix_sheep"]) + + return beef_gridded, dairy_gridded, sheep_gridded + + def process_emissions( sector: PriorSector, sector_config: PriorSectorConfig, prior_ds: xr.Dataset, ): config = sector_config.prior_config + domain_grid = config.domain().grid - livestock_asset = sector_config.data_manager.get_asset(livestock_data_source) - with xr.open_dataset(livestock_asset.path) as lss: - ls = lss.load() + livestock_headcount_da = sector_config.data_manager.get_asset(livestock_headcount_data_source) + livestock_df: pd.DataFrame = livestock_headcount_da.data - domain_grid = config.domain().grid + # dataset X/Y coords are ESPF:4283 (GDA94), convert to EPSG:4326 + transformer = pyproj.Transformer.from_crs(4283,4326) + livestock_df["lon"], livestock_df["lat"] = transformer.transform( + yy=livestock_df["Y"], xx=livestock_df["X"], + ) + + # place gridded headcounts on the grid + beef_np, dairy_np, sheep_np = gridded_livestock_headcounts(livestock_df, domain_grid) + + # get inventory totals using animal-specific UNFCCC sectors + emissions_inventory = sector_config.data_manager.get_asset(inventory_data_source).data + get_emissions_params = dict( + emissions_inventory=emissions_inventory, + start_date=config.start_date, + end_date=config.end_date, + ) + beef_total_emissions = get_sector_emissions_by_code( + category_codes=["3.A.1.b", "3.A.1.c", "3.B.1.b", "3.B.1.c"], **get_emissions_params, + ) + dairy_total_emissions = get_sector_emissions_by_code( + category_codes=["3.A.1.a", "3.B.1.a"], **get_emissions_params, + ) + sheep_total_emissions = get_sector_emissions_by_code( + category_codes=["3.A.2", "3.B.2"], **get_emissions_params, + ) + logger.debug(f"Total emissions from beef: {beef_total_emissions / 1e6:.2f} kt, dairy: {dairy_total_emissions / 1e6:.2f} kt, sheep: {sheep_total_emissions / 1e6:.2f} kt") - # Re-project into domain coordinates - # - create meshgrids of the lats and lons - lonmesh, latmesh = np.meshgrid(ls.lon, ls.lat) - cell_x, cell_y, cell_valid = domain_grid.lonlat_to_cell_index(lonmesh, latmesh) + beef_total_headcount = livestock_df["heads_mapped_mix_beef"].sum() + dairy_total_headcount = livestock_df["heads_mapped_dairy"].sum() + sheep_total_headcount = livestock_df["heads_mapped_mix_sheep"].sum() - enteric_as_array = lss.CH4_total.to_numpy() + # distribute the national emission totals by headcount proportion + beef_ch4_np = beef_total_emissions * beef_np / beef_total_headcount + dairy_ch4_np = dairy_total_emissions * dairy_np / dairy_total_headcount + sheep_ch4_np = sheep_total_emissions * sheep_np / sheep_total_headcount - livestockCH4 = np.zeros(domain_grid.shape) - logger.info("Distribute livestock CH4 (long process)") - # we're accumulating emissions from fine to coarse grid - # accumulate in mass units and divide by area at end - for j in range(ls.lat.size): - ix, iy = cell_x[j,:], cell_y[j,:] - # input domain is bigger so mask indices out of range - mask = cell_valid[j, :] - if mask.any(): - # the following needs to use .at method since iy,ix indices may be repeated and we need to acumulate - np.add.at(livestockCH4, (iy[mask], ix[mask]), enteric_as_array[j, mask]) + total_ch4_np = beef_ch4_np + dairy_ch4_np + sheep_ch4_np - return convert_to_timescale(livestockCH4, domain_grid.cell_area) + # convert CH4 kg to CH4 kg/s/m2 + return total_ch4_np / domain_grid.cell_area / seconds_in_period(config.start_date, config.end_date) sector = PriorSector( name="livestock", emission_category="anthropogenic", - unfccc_categories=["3.A"], # Enteric Fermentation + unfccc_categories=[ + "3.A.1", # Enteric Fermentation - Cattle + "3.A.2", # Enteric Fermentation - Sheep + "3.B.1", # Manure Management - Cattle + "3.B.2", # Manure Management - Sheep + ], cf_standard_name="domesticated_livestock", create_estimate=process_emissions, ) diff --git a/src/openmethane_prior/sectors/lulucf/sector.py b/src/openmethane_prior/sectors/lulucf/sector.py index 652f59ce..fc104a24 100644 --- a/src/openmethane_prior/sectors/lulucf/sector.py +++ b/src/openmethane_prior/sectors/lulucf/sector.py @@ -26,11 +26,7 @@ inventory_data_source, inventory_domain_data_source, ) -from openmethane_prior.data_sources.landuse import ( - alum_codes_for_sector, - alum_sector_mapping_data_source, - landuse_map_data_source, -) +from openmethane_prior.data_sources.landuse import landuse_map_data_source from openmethane_prior.lib import ( kg_to_period_cell_flux, logger, @@ -42,6 +38,21 @@ logger = logger.get_logger(__name__) +# ALUM Classification Version 8 +# Source: https://www.agriculture.gov.au/abares/aclump/land-use/alum-classification +alum_codes_lulucf = [ + 220, # Production native forests + 310, # Plantation forests + 311, # Hardwood plantation forestry + 312, # Softwood plantation forestry + 313, # Other forest plantation + 314, # Environmental forest plantation + 410, # Irrigated plantation forests + 411, # Irrigated hardwood plantation forestry + 412, # Irrigated softwood plantation forestry + 413, # Irrigated other forest plantation + 414, # Irrigated environmental forest plantation +] def process_emissions( sector: PriorSector, @@ -50,11 +61,6 @@ def process_emissions( ): config = sector_config.prior_config - # Import a map of land use type numbers to emissions sectors - # make a dictionary of all landuse types corresponding to sectors in map - sector_mapping_asset = sector_config.data_manager.get_asset(alum_sector_mapping_data_source) - sector_alum_codes = alum_codes_for_sector(sector.name, sector_mapping_asset.data) - # load the national inventory data, ready to calculate sectoral totals emissions_inventory = sector_config.data_manager.get_asset(inventory_data_source).data @@ -80,7 +86,7 @@ def process_emissions( ) # create a mask of pixels which match the sector code - sector_mask = np.isin(dataBand, sector_alum_codes) + sector_mask = np.isin(dataBand, alum_codes_lulucf) sector_xr = xr.DataArray(sector_mask, coords={ 'y': lu_y, 'x': lu_x }) # now aggregate to coarser resolution of the domain grid diff --git a/tests/integration/test_inventory_inputs.py b/tests/integration/test_inventory_inputs.py index 2259c868..d878cb7e 100644 --- a/tests/integration/test_inventory_inputs.py +++ b/tests/integration/test_inventory_inputs.py @@ -28,10 +28,11 @@ def all_sector_meta(): def test_inventory_get_sector_emissions_by_code(all_sector_meta, inventory_df): expected_annual_emissions = { - "agriculture": 153367168.6905247, + "agriculture": 98593325.56317905, "coal": 901455051.0124934, "electricity": 11164637.314332796, "industrial": 2937681.1201194106, + "livestock": 2174713044.926005, "lulucf": 661762727.6374531, "oil_gas": 238171271.587112, "stationary": 73354555.56013045, diff --git a/tests/integration/test_om_prior.py b/tests/integration/test_om_prior.py index 0d7b4ad7..1a8d9272 100644 --- a/tests/integration/test_om_prior.py +++ b/tests/integration/test_om_prior.py @@ -5,13 +5,13 @@ def test_009_prior_emissions_ds(prior_emissions_ds): assert dataset_metrics(prior_emissions_ds) == { 'max': { 'LANDMASK': 1.0, - 'OCH4_TOTAL': 8.352448105492377e-09, - 'ch4_sector_agriculture': 1.3196047170278112e-12, + 'OCH4_TOTAL': 8.360290000548561e-09, + 'ch4_sector_agriculture': 6.15621818800407e-13, 'ch4_sector_coal': 8.072982591959402e-09, 'ch4_sector_electricity': 1.2683449229281463e-11, 'ch4_sector_fire': 5.498670963000052e-11, 'ch4_sector_industrial': 5.351265777744653e-13, - 'ch4_sector_livestock': 7.545056510007907e-11, + 'ch4_sector_livestock': 6.329324205281718e-11, 'ch4_sector_lulucf': 1.4454168861914367e-10, 'ch4_sector_oil_gas': 5.671488137556794e-11, 'ch4_sector_stationary': 1.3362230506306242e-11, @@ -19,7 +19,7 @@ def test_009_prior_emissions_ds(prior_emissions_ds): 'ch4_sector_transport': 2.1317017815070317e-12, 'ch4_sector_waste': 3.530954573289664e-12, 'ch4_sector_wetlands': 2.5512281176531815e-10, - 'ch4_total': 8.352448105492377e-09, + 'ch4_total': 8.360290000548561e-09, 'lambert_conformal': 0, 'land_mask': 1, 'lat': -22.806066513061523, @@ -29,13 +29,13 @@ def test_009_prior_emissions_ds(prior_emissions_ds): }, 'mean': { 'LANDMASK': 1.0, - 'OCH4_TOTAL': 5.779904932204169e-10, - 'ch4_sector_agriculture': 1.0532989458203914e-12, + 'OCH4_TOTAL': 5.764039041672882e-10, + 'ch4_sector_agriculture': 3.078109094002035e-14, 'ch4_sector_coal': 3.979529404503476e-10, 'ch4_sector_electricity': 2.2548354185389268e-13, 'ch4_sector_fire': 3.6974516872713414e-13, 'ch4_sector_industrial': 5.344609974538503e-14, - 'ch4_sector_livestock': 4.0547154335803873e-11, + 'ch4_sector_livestock': 3.998308313755556e-11, 'ch4_sector_lulucf': 7.935998439039457e-12, 'ch4_sector_oil_gas': 5.671488137556793e-13, 'ch4_sector_stationary': 1.3345610816621784e-12, @@ -43,7 +43,7 @@ def test_009_prior_emissions_ds(prior_emissions_ds): 'ch4_sector_transport': 2.129050411132023e-13, 'ch4_sector_waste': 5.84814233122829e-14, 'ch4_sector_wetlands': 1.2524275058017897e-10, - 'ch4_total': 5.779904932204169e-10, + 'ch4_total': 5.764039041672882e-10, 'lambert_conformal': 0.0, 'land_mask': 1.0, 'lat': -23.267749786376953, @@ -53,13 +53,13 @@ def test_009_prior_emissions_ds(prior_emissions_ds): }, 'x_band': { 'LANDMASK': 10.0, - 'OCH4_TOTAL': 5.182458643835442e-09, - 'ch4_sector_agriculture': 2.4194872992444853e-11, + 'OCH4_TOTAL': 5.109405382549537e-09, + 'ch4_sector_agriculture': 0, 'ch4_sector_coal': 2.85589757264367e-09, 'ch4_sector_electricity': 0.0, 'ch4_sector_fire': 0.0, 'ch4_sector_industrial': 9.990360612431248e-13, - 'ch4_sector_livestock': 8.812359129868445e-10, + 'ch4_sector_livestock': 8.323775246933841e-10, 'ch4_sector_lulucf': 4.318257917252425e-11, 'ch4_sector_oil_gas': 0.0, 'ch4_sector_stationary': 2.494615421637521e-11, @@ -67,20 +67,20 @@ def test_009_prior_emissions_ds(prior_emissions_ds): 'ch4_sector_transport': 3.979706932888128e-12, 'ch4_sector_waste': 0.0, 'ch4_sector_wetlands': 1.3007149871069323e-09, - 'ch4_total': 5.182458643835442e-09, + 'ch4_total': 5.109405382549537e-09, 'land_mask': 10, 'lat': -232.6233367919922, 'lon': 1486.8963623046875, }, 'y_band': { 'LANDMASK': 10.0, - 'OCH4_TOTAL': 1.951880649183436e-08, - 'ch4_sector_agriculture': 2.3428548325447984e-11, + 'OCH4_TOTAL': 1.9487195537341483e-08, + 'ch4_sector_agriculture': 0, 'ch4_sector_coal': 1.6145965183918805e-08, 'ch4_sector_electricity': 0.0, 'ch4_sector_fire': 0.0, 'ch4_sector_industrial': 3.7006265826194363e-13, - 'ch4_sector_livestock': 1.0031707194660715e-09, + 'ch4_sector_livestock': 9.949883132986455e-10, 'ch4_sector_lulucf': 0.0, 'ch4_sector_oil_gas': 0.0, 'ch4_sector_stationary': 9.240547464560038e-12, @@ -88,7 +88,7 @@ def test_009_prior_emissions_ds(prior_emissions_ds): 'ch4_sector_transport': 1.4741619285048626e-12, 'ch4_sector_waste': 0.0, 'ch4_sector_wetlands': 2.2887601258680945e-09, - 'ch4_total': 1.951880649183436e-08, + 'ch4_total': 1.9487195537341483e-08, 'land_mask': 10, 'lat': -232.2213897705078, 'lon': 1486.336669921875, diff --git a/tests/integration/test_sector_livestock/test_sector_livestock.py b/tests/integration/test_sector_livestock/test_sector_livestock.py new file mode 100644 index 00000000..3bd5b793 --- /dev/null +++ b/tests/integration/test_sector_livestock/test_sector_livestock.py @@ -0,0 +1,11 @@ +from openmethane_prior.lib import create_prior +from openmethane_prior.sectors.livestock import sector + + +def test_sector_livestock(input_files, config): + """Run the full livestock sector over the au-test domain.""" + # run the prior and return the result + prior_livestock_ds = create_prior(config, [sector]) + + assert prior_livestock_ds['ch4_sector_livestock'].max().item() == 6.329324205281718e-11 + assert prior_livestock_ds['ch4_sector_livestock'].mean().item() == 3.998308313755556e-11 diff --git a/tests/unit/test_data/test_landuse.py b/tests/unit/test_data/test_landuse.py deleted file mode 100644 index 54c6e812..00000000 --- a/tests/unit/test_data/test_landuse.py +++ /dev/null @@ -1,9 +0,0 @@ -from openmethane_prior.data_sources.landuse import alum_codes_for_sector, alum_sector_mapping_data_source - - -def test_alum_codes_for_sector(data_manager): - sector_mapping_asset = data_manager.get_asset(alum_sector_mapping_data_source) - agriculture_codes = alum_codes_for_sector("agriculture", sector_mapping_asset.data) - assert agriculture_codes == [210, 320, 420, 520, 521, 522, 523, 524, 525, 526, 527, 542] - - assert alum_codes_for_sector("fake sector", sector_mapping_asset.data) == [] diff --git a/tests/unit/test_livestock/test_gridded_livestock_headcounts.py b/tests/unit/test_livestock/test_gridded_livestock_headcounts.py new file mode 100644 index 00000000..fa2dd61b --- /dev/null +++ b/tests/unit/test_livestock/test_gridded_livestock_headcounts.py @@ -0,0 +1,159 @@ +import pandas as pd + +from openmethane_prior.lib.grid.grid import Grid +from openmethane_prior.sectors.livestock.sector import ( + gridded_livestock_headcounts, +) + +# Simple 5x5 degree grid using default EPSG:4326 projection (lon/lat == x/y). +# origin at (130, -40), 1-degree cells. +# ix: 0→[130,131), 1→[131,132), 2→[132,133), 3→[133,134), 4→[134,135) +# iy: 0→[-40,-39), 1→[-39,-38), 2→[-38,-37), 3→[-37,-36), 4→[-36,-35) +TEST_GRID = Grid(dimensions=(5, 5), origin_xy=(130.0, -40.0), cell_size=(1.0, 1.0)) + + +def make_df(rows: list) -> pd.DataFrame: + return pd.DataFrame( + rows, + columns=["lon", "lat", "heads_mapped_mix_beef", "heads_mapped_dairy", "heads_mapped_mix_sheep"], + ) + + +def make_row(values) -> list: + return [values["lon"], values["lat"], values["beef"], values["dairy"], values["sheep"]] + + +# --- headcount placement --- + +def test_headcount_placement_beef(): + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": 1, "dairy": 0, "sheep": 0}), + ]) + + result_beef, result_dairy, result_sheep = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_beef[0, 0] == 1 + assert result_beef.sum() == 1 + assert result_dairy.sum() == 0 + assert result_sheep.sum() == 0 + + +def test_headcount_placement_dairy(): + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": 0, "dairy": 1, "sheep": 0}), + ]) + + result_beef, result_dairy, result_sheep = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_beef.sum() == 0 + assert result_dairy[0, 0] == 1 + assert result_dairy.sum() == 1 + assert result_sheep.sum() == 0 + + +def test_headcount_placement_sheep(): + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": 0, "dairy": 0, "sheep": 1}), + ]) + + result_beef, result_dairy, result_sheep = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_beef.sum() == 0 + assert result_dairy.sum() == 0 + assert result_sheep[0, 0] == 1 + assert result_sheep.sum() == 1 + + +def test_headcount_placement_mixed(): + beef, sheep, dairy = 3, 5, 7 + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": beef, "dairy": dairy, "sheep": sheep}), + ]) + result_beef, result_dairy, result_sheep = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_beef[0, 0] == beef + assert result_dairy[0, 0] == dairy + assert result_sheep[0, 0] == sheep + + +def test_zero_headcounts_produce_no_emissions(): + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": 0, "dairy": 0, "sheep": 0}), + ]) + result_beef, result_dairy, result_sheep = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_beef.sum() == 0.0 + assert result_dairy.sum() == 0.0 + assert result_sheep.sum() == 0.0 + + +# --- grid cell placement --- + +def test_emissions_placed_in_lower_left_cell(): + # lon=130.5, lat=-39.5 → ix=0, iy=0 + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": 1, "dairy": 0, "sheep": 0}), + ]) + + result_beef, _, _ = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_beef[0, 0] == 1 + assert result_beef.sum() == 1 + + +def test_emissions_placed_in_interior_cell(): + # lon=132.5, lat=-37.5 → ix=2, iy=2 + df = make_df([ + make_row({"lon": 132.5, "lat": -37.5, "beef": 0, "dairy": 1, "sheep": 0}), + ]) + + _, result_dairy, _ = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_dairy[2, 2] == 1 + assert result_dairy.sum() == 1 + + +def test_emissions_placed_in_upper_right_cell(): + # lon=134.5, lat=-35.5 → ix=4, iy=4 + df = make_df([ + make_row({"lon": 134.5, "lat": -35.5, "beef": 0, "dairy": 0, "sheep": 1}), + ]) + + _, _, result_sheep = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_sheep[4, 4] == 1 + assert result_sheep.sum() == 1 + + +def test_point_outside_domain_excluded(): + # lon=125.0 is west of the grid (origin_x=130), should be discarded + df = make_df([ + # outside TEST_GRID, should not be counted + make_row({"lon": 125.0, "lat": -39.5, "beef": 100, "dairy": 100, "sheep": 100}), + # inside TEST_GRID + make_row({"lon": 130.5, "lat": -39.5, "beef": 1, "dairy": 0, "sheep": 0}), + ]) + + result_beef, result_dairy, result_sheep = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_beef.sum() == 1 + assert result_dairy.sum() == 0 + assert result_sheep.sum() == 0 + + +# --- aggregation within a grid cell --- + +def test_two_rows_in_same_cell_are_aggregated(): + # lon=132.2 and lon=132.7, lat=-36.8 and lat=-36.3 both → ix=2, iy=3 + beef_1, dairy_1, sheep_1 = 10, 0, 0 + beef_2, dairy_2, sheep_2 = 0, 5, 20 + df = make_df([ + make_row({"lon": 132.2, "lat": -36.8, "beef": beef_1, "dairy": dairy_1, "sheep": sheep_1}), + make_row({"lon": 132.7, "lat": -36.3, "beef": beef_2, "dairy": dairy_2, "sheep": sheep_2}), + ]) + + result_beef, result_dairy, result_sheep = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_beef[3, 2] == beef_1 + beef_2 + assert result_dairy[3, 2] == dairy_1 + dairy_2 + assert result_sheep[3, 2] == sheep_1 + sheep_2