From 2cc71d82d6452a7de91cd73b97e07b371ccda299 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Wed, 6 May 2026 19:03:07 +1000 Subject: [PATCH 01/13] Update livestock sector implementation to use headcounts dataset Resolves #182 --- .../sectors/livestock/sector.py | 63 +++++++++++-------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/src/openmethane_prior/sectors/livestock/sector.py b/src/openmethane_prior/sectors/livestock/sector.py index e1f814c5..99d84142 100644 --- a/src/openmethane_prior/sectors/livestock/sector.py +++ b/src/openmethane_prior/sectors/livestock/sector.py @@ -15,8 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import numpy as np +import pandas as pd import xarray as xr from openmethane_prior.lib import ( @@ -26,47 +26,58 @@ PriorSector, PriorSectorConfig, ) +from openmethane_prior.lib.data_manager.parsers import parse_csv logger = logger.get_logger(__name__) -livestock_data_source = DataSource( - name="enteric-fermentation", - url="https://openmethane.s3.amazonaws.com/prior/inputs/EntericFermentation.nc", +livestock_headcount_data_source = DataSource( + name="livestock-headcount", + url="https://openmethane.s3.amazonaws.com/prior/inputs/lmap_selected_cols_Nasimeh_171025.csv", + parse=parse_csv, ) + +def livestock_to_ch4_kg_year(beef: float, sheep: float, dairy: float) -> float: + """Calculate the annual emissions created by a number of beef cattle, dairy + cattle and/or sheep.""" + # TODO: document where these emission factors come from + return beef * 51.0 + sheep * 6.8 + dairy * 93.0 + + def process_emissions( sector: PriorSector, sector_config: PriorSectorConfig, prior_ds: xr.Dataset, ): config = sector_config.prior_config - - livestock_asset = sector_config.data_manager.get_asset(livestock_data_source) - with xr.open_dataset(livestock_asset.path) as lss: - ls = lss.load() - domain_grid = config.domain().grid - # Re-project into domain coordinates - # - create meshgrids of the lats and lons - lonmesh, latmesh = np.meshgrid(ls.lon, ls.lat) - cell_x, cell_y, cell_valid = domain_grid.lonlat_to_cell_index(lonmesh, latmesh) + livestock_headcount_da = sector_config.data_manager.get_asset(livestock_headcount_data_source) + livestock_df: pd.DataFrame = livestock_headcount_da.data + livestock_df = livestock_df.rename(columns={"X": "lon", "Y": "lat"}) + + # find the domain grid cell (ix, iy) each livestock area centre point is in + livestock_df["ix"], livestock_df["iy"], valid_cells = domain_grid.lonlat_to_cell_index( + lon=livestock_df["lon"], + lat=livestock_df["lat"], + ) - enteric_as_array = lss.CH4_total.to_numpy() + # throw away any data outside the domain + livestock_df = livestock_df[valid_cells] - livestockCH4 = np.zeros(domain_grid.shape) - logger.info("Distribute livestock CH4 (long process)") - # we're accumulating emissions from fine to coarse grid - # accumulate in mass units and divide by area at end - for j in range(ls.lat.size): - ix, iy = cell_x[j,:], cell_y[j,:] - # input domain is bigger so mask indices out of range - mask = cell_valid[j, :] - if mask.any(): - # the following needs to use .at method since iy,ix indices may be repeated and we need to acumulate - np.add.at(livestockCH4, (iy[mask], ix[mask]), enteric_as_array[j, mask]) + ch4_gridded = np.zeros(domain_grid.shape) + for _, livestock_area in livestock_df.iterrows(): + # calculate the total annual CH4 emissions for this area + livestock_ch4 = livestock_to_ch4_kg_year( + beef=livestock_area["heads_mapped_mix_beef"], + sheep=livestock_area["heads_mapped_mix_sheep"], + dairy=livestock_area["heads_mapped_dairy"], + ) + # add the CH4 to the grid + ch4_gridded[livestock_area["iy"], livestock_area["ix"]] += livestock_ch4 - return convert_to_timescale(livestockCH4, domain_grid.cell_area) + # convert annual CH4 kg to CH4 kg/s/m3 + return convert_to_timescale(ch4_gridded, domain_grid.cell_area) sector = PriorSector( From 515cecfd408969a72918bcce2b447de5248e9f77 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Wed, 6 May 2026 19:05:18 +1000 Subject: [PATCH 02/13] Add changelog for #183 --- changelog/183.feature.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/183.feature.md diff --git a/changelog/183.feature.md b/changelog/183.feature.md new file mode 100644 index 00000000..a9146210 --- /dev/null +++ b/changelog/183.feature.md @@ -0,0 +1 @@ +Update livestock sector implementation to use headcounts dataset From b38074437977c7a9ce2da11bbfb8d5f8a5569c38 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 7 May 2026 11:07:45 +1000 Subject: [PATCH 03/13] Improve performance assigning livestock emissions to grid cells Iterating over rows in the livestock DataFrame isn't necessary once we've calculated the grid cell indexes. The emissions can be distributed to grid cells directly using `np.add.at((indices), values)`. --- src/openmethane_prior/lib/__init__.py | 1 + .../sectors/livestock/sector.py | 61 +++++++++++-------- 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/src/openmethane_prior/lib/__init__.py b/src/openmethane_prior/lib/__init__.py index 46ae13be..0e208318 100644 --- a/src/openmethane_prior/lib/__init__.py +++ b/src/openmethane_prior/lib/__init__.py @@ -23,6 +23,7 @@ DataManager, DataSource, ) +from .grid.grid import Grid from .grid.domain import Domain from .grid.regrid import regrid_data from .regrid import regrid_data_array_conservative diff --git a/src/openmethane_prior/sectors/livestock/sector.py b/src/openmethane_prior/sectors/livestock/sector.py index 99d84142..f656ee28 100644 --- a/src/openmethane_prior/sectors/livestock/sector.py +++ b/src/openmethane_prior/sectors/livestock/sector.py @@ -21,6 +21,7 @@ from openmethane_prior.lib import ( DataSource, + Grid, convert_to_timescale, logger, PriorSector, @@ -36,12 +37,42 @@ parse=parse_csv, ) +# Estimated kg CH4 emitted per animal, per year +# TODO: these figures came from example code, a reference should be added +# citing their origin +ENTERIC_ANNUAL_KG_CH4 = { + "beef_cattle": 51.0, + "dairy_cattle": 93.0, + "sheep": 6.8, +} -def livestock_to_ch4_kg_year(beef: float, sheep: float, dairy: float) -> float: - """Calculate the annual emissions created by a number of beef cattle, dairy - cattle and/or sheep.""" - # TODO: document where these emission factors come from - return beef * 51.0 + sheep * 6.8 + dairy * 93.0 +def gridded_livestock_emissions_by_headcount( + livestock_df: pd.DataFrame, + domain_grid: Grid, +) -> np.ndarray: + """Given a list of coordinates for livestock areas with headcounts of beef + cattle, dairy cattle and sheep, calculate the annual methane emissions in + each area, and return as gridded emissions over the domain of interest.""" + # find the domain grid cell (ix, iy) each livestock area centre point is in + livestock_df["ix"], livestock_df["iy"], valid_cells = domain_grid.lonlat_to_cell_index( + lon=livestock_df["lon"], + lat=livestock_df["lat"], + ) + + # discard data outside the domain + livestock_df = livestock_df[valid_cells] + + logger.debug(f"Calculating annual livestock emissions per headcount") + livestock_df["beef_ch4"] = livestock_df["heads_mapped_mix_beef"] * ENTERIC_ANNUAL_KG_CH4["beef_cattle"] + livestock_df["sheep_ch4"] = livestock_df["heads_mapped_mix_sheep"] * ENTERIC_ANNUAL_KG_CH4["sheep"] + livestock_df["dairy_ch4"] = livestock_df["heads_mapped_dairy"] * ENTERIC_ANNUAL_KG_CH4["dairy_cattle"] + livestock_df["total_ch4"] = livestock_df["beef_ch4"] + livestock_df["sheep_ch4"] + livestock_df["dairy_ch4"] + + logger.debug(f"Adding livestock emissions to domain grid") + ch4_gridded = np.zeros(domain_grid.shape) + np.add.at(ch4_gridded, (livestock_df["iy"], livestock_df["ix"]), livestock_df["total_ch4"]) + + return ch4_gridded def process_emissions( @@ -56,25 +87,7 @@ def process_emissions( livestock_df: pd.DataFrame = livestock_headcount_da.data livestock_df = livestock_df.rename(columns={"X": "lon", "Y": "lat"}) - # find the domain grid cell (ix, iy) each livestock area centre point is in - livestock_df["ix"], livestock_df["iy"], valid_cells = domain_grid.lonlat_to_cell_index( - lon=livestock_df["lon"], - lat=livestock_df["lat"], - ) - - # throw away any data outside the domain - livestock_df = livestock_df[valid_cells] - - ch4_gridded = np.zeros(domain_grid.shape) - for _, livestock_area in livestock_df.iterrows(): - # calculate the total annual CH4 emissions for this area - livestock_ch4 = livestock_to_ch4_kg_year( - beef=livestock_area["heads_mapped_mix_beef"], - sheep=livestock_area["heads_mapped_mix_sheep"], - dairy=livestock_area["heads_mapped_dairy"], - ) - # add the CH4 to the grid - ch4_gridded[livestock_area["iy"], livestock_area["ix"]] += livestock_ch4 + ch4_gridded = gridded_livestock_emissions_by_headcount(livestock_df, domain_grid) # convert annual CH4 kg to CH4 kg/s/m3 return convert_to_timescale(ch4_gridded, domain_grid.cell_area) From a3a3bb76b3fd4dd3c5034fd2bff18b6727708884 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 7 May 2026 11:55:39 +1000 Subject: [PATCH 04/13] Add unit tests for gridded_livestock_emissions_by_headcount --- ...ridded_livestock_emissions_by_headcount.py | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 tests/unit/test_livestock/test_gridded_livestock_emissions_by_headcount.py diff --git a/tests/unit/test_livestock/test_gridded_livestock_emissions_by_headcount.py b/tests/unit/test_livestock/test_gridded_livestock_emissions_by_headcount.py new file mode 100644 index 00000000..d04e8abd --- /dev/null +++ b/tests/unit/test_livestock/test_gridded_livestock_emissions_by_headcount.py @@ -0,0 +1,193 @@ +import pandas as pd +import pytest + +from openmethane_prior.lib.grid.grid import Grid +from openmethane_prior.sectors.livestock.sector import ( + ENTERIC_ANNUAL_KG_CH4, + gridded_livestock_emissions_by_headcount, +) + +# Simple 5x5 degree grid using default EPSG:4326 projection (lon/lat == x/y). +# origin at (130, -40), 1-degree cells. +# ix: 0→[130,131), 1→[131,132), 2→[132,133), 3→[133,134), 4→[134,135) +# iy: 0→[-40,-39), 1→[-39,-38), 2→[-38,-37), 3→[-37,-36), 4→[-36,-35) +TEST_GRID = Grid(dimensions=(5, 5), origin_xy=(130.0, -40.0), cell_size=(1.0, 1.0)) + + +def make_df(rows: list) -> pd.DataFrame: + return pd.DataFrame( + rows, + columns=["lon", "lat", "heads_mapped_mix_beef", "heads_mapped_mix_sheep", "heads_mapped_dairy"], + ) + + +def make_row(values) -> list: + return [values["lon"], values["lat"], values["beef"], values["sheep"], values["dairy"]] + + +# --- headcount scaling --- + +def test_headcount_scales_emissions(): + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": 1, "sheep": 0, "dairy": 0}), + ]) + + result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) + + assert result[0, 0] == pytest.approx(ENTERIC_ANNUAL_KG_CH4["beef_cattle"]) + + +def test_sheep_headcount_scales_emissions(): + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": 0, "sheep": 1, "dairy": 0}), + ]) + + result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) + + assert result[0, 0] == pytest.approx(ENTERIC_ANNUAL_KG_CH4["sheep"]) + + +def test_dairy_headcount_scales_emissions(): + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": 0, "sheep": 0, "dairy": 1}), + ]) + + result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) + + assert result[0, 0] == pytest.approx(ENTERIC_ANNUAL_KG_CH4["dairy_cattle"]) + + +def test_mixed_headcounts_sum_correctly(): + beef, sheep, dairy = 3, 5, 7 + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": beef, "sheep": sheep, "dairy": dairy}), + ]) + result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) + + expected = ( + beef * ENTERIC_ANNUAL_KG_CH4["beef_cattle"] + + sheep * ENTERIC_ANNUAL_KG_CH4["sheep"] + + dairy * ENTERIC_ANNUAL_KG_CH4["dairy_cattle"] + ) + assert result[0, 0] == pytest.approx(expected) + + +def test_zero_headcounts_produce_no_emissions(): + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": 0, "sheep": 0, "dairy": 0}), + ]) + result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) + + assert result.sum() == 0.0 + + +# --- grid cell placement --- + +def test_emissions_placed_in_lower_left_cell(): + # lon=130.5, lat=-39.5 → ix=0, iy=0 + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": 1, "sheep": 0, "dairy": 0}), + ]) + + result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) + + assert result[0, 0] == pytest.approx(ENTERIC_ANNUAL_KG_CH4["beef_cattle"]) + assert result.sum() == pytest.approx(ENTERIC_ANNUAL_KG_CH4["beef_cattle"]) + + +def test_emissions_placed_in_interior_cell(): + # lon=132.5, lat=-37.5 → ix=2, iy=2 + df = make_df([ + make_row({"lon": 132.5, "lat": -37.5, "beef": 0, "sheep": 0, "dairy": 1}), + ]) + + result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) + + assert result[2, 2] == pytest.approx(ENTERIC_ANNUAL_KG_CH4["dairy_cattle"]) + assert result.sum() == pytest.approx(ENTERIC_ANNUAL_KG_CH4["dairy_cattle"]) + + +def test_emissions_placed_in_upper_right_cell(): + # lon=134.5, lat=-35.5 → ix=4, iy=4 + df = make_df([ + make_row({"lon": 134.5, "lat": -35.5, "beef": 0, "sheep": 1, "dairy": 0}), + ]) + + result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) + + assert result[4, 4] == pytest.approx(ENTERIC_ANNUAL_KG_CH4["sheep"]) + assert result.sum() == pytest.approx(ENTERIC_ANNUAL_KG_CH4["sheep"]) + + +def test_point_outside_domain_excluded(): + # lon=125.0 is west of the grid (origin_x=130), should be discarded + df = make_df([ + # outside TEST_GRID, should not be counted + make_row({"lon": 125.0, "lat": -39.5, "beef": 100, "sheep": 100, "dairy": 100}), + # inside TEST_GRID + make_row({"lon": 130.5, "lat": -39.5, "beef": 1, "sheep": 0, "dairy": 0}), + ]) + + result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) + + assert result.sum() == pytest.approx(ENTERIC_ANNUAL_KG_CH4["beef_cattle"]) + + +# --- aggregation within a grid cell --- + +def test_two_rows_in_same_cell_are_aggregated(): + # lon=132.2 and lon=132.7, lat=-36.8 and lat=-36.3 both → ix=2, iy=3 + beef_1, sheep_1, dairy_1 = 10, 0, 0 + beef_2, sheep_2, dairy_2 = 0, 20, 5 + df = make_df([ + make_row({"lon": 132.2, "lat": -36.8, "beef": beef_1, "sheep": sheep_1, "dairy": dairy_1}), + make_row({"lon": 132.7, "lat": -36.3, "beef": beef_2, "sheep": sheep_2, "dairy": dairy_2}), + ]) + + result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) + + expected = ( + beef_1 * ENTERIC_ANNUAL_KG_CH4["beef_cattle"] + + sheep_1 * ENTERIC_ANNUAL_KG_CH4["sheep"] + + dairy_1 * ENTERIC_ANNUAL_KG_CH4["dairy_cattle"] + + beef_2 * ENTERIC_ANNUAL_KG_CH4["beef_cattle"] + + sheep_2 * ENTERIC_ANNUAL_KG_CH4["sheep"] + + dairy_2 * ENTERIC_ANNUAL_KG_CH4["dairy_cattle"] + ) + assert result[3, 2] == pytest.approx(expected) + assert result.sum() == pytest.approx(expected) + + +def test_three_rows_in_same_cell_are_aggregated(): + # all three → ix=1, iy=1 + df = make_df([ + make_row({"lon": 131.1, "lat": -38.9, "beef": 5, "sheep": 0, "dairy": 0}), + make_row({"lon": 131.5, "lat": -38.5, "beef": 0, "sheep": 10, "dairy": 0}), + make_row({"lon": 131.9, "lat": -38.1, "beef": 0, "sheep": 0, "dairy": 3}), + ]) + + result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) + + expected = ( + 5 * ENTERIC_ANNUAL_KG_CH4["beef_cattle"] + + 10 * ENTERIC_ANNUAL_KG_CH4["sheep"] + + 3 * ENTERIC_ANNUAL_KG_CH4["dairy_cattle"] + ) + assert result[1, 1] == pytest.approx(expected) + assert result.sum() == pytest.approx(expected) + + +def test_rows_in_different_cells_are_not_aggregated(): + # ix=0, iy=0 and ix=4, iy=4 are independent + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": 10, "sheep": 0, "dairy": 0}), + make_row({"lon": 134.5, "lat": -35.5, "beef": 0, "sheep": 0, "dairy": 7}), + ]) + + result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) + + assert result[0, 0] == pytest.approx(10 * ENTERIC_ANNUAL_KG_CH4["beef_cattle"]) + assert result[4, 4] == pytest.approx(7 * ENTERIC_ANNUAL_KG_CH4["dairy_cattle"]) + assert result.sum() == pytest.approx( + 10 * ENTERIC_ANNUAL_KG_CH4["beef_cattle"] + 7 * ENTERIC_ANNUAL_KG_CH4["dairy_cattle"] + ) From 76d41cc04abe82ceaa10c3142d24aa00e41b9942 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 7 May 2026 11:56:01 +1000 Subject: [PATCH 05/13] Claude should always write tests using pytest conventions --- CLAUDE.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 3209c780..21dc4769 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -64,7 +64,16 @@ Copy `.env.example` to `.env`. Key variables: ## Testing -Tests live in `tests/unit/` and `tests/integration/`. Integration tests exercise the full pipeline with real (cached) data. Use `--cache` pytest fixtures to control data caching behaviour. +Tests live in `tests/unit/` and `tests/integration/`. Tests follow pytest +conventions and use pytest libraries. + +Integration tests exercise the full pipeline with real (cached) data. Use +`input_files` pytest fixture to use cached data. For integration tests which +should download DataSource assets from the internet, `input_files` should be +omitted. + +Unit tests should avoid calling methods which read from the filesystem, and +should construct example data to pass to methods. ## Code Style From edc4ddd72c797cb52604c98801415d42ef385a4a Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 7 May 2026 12:18:17 +1000 Subject: [PATCH 06/13] Avoid mutating filtered DataFrame unnecessarily in livestock sector --- src/openmethane_prior/sectors/livestock/sector.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/openmethane_prior/sectors/livestock/sector.py b/src/openmethane_prior/sectors/livestock/sector.py index f656ee28..acbaca4c 100644 --- a/src/openmethane_prior/sectors/livestock/sector.py +++ b/src/openmethane_prior/sectors/livestock/sector.py @@ -63,14 +63,15 @@ def gridded_livestock_emissions_by_headcount( livestock_df = livestock_df[valid_cells] logger.debug(f"Calculating annual livestock emissions per headcount") - livestock_df["beef_ch4"] = livestock_df["heads_mapped_mix_beef"] * ENTERIC_ANNUAL_KG_CH4["beef_cattle"] - livestock_df["sheep_ch4"] = livestock_df["heads_mapped_mix_sheep"] * ENTERIC_ANNUAL_KG_CH4["sheep"] - livestock_df["dairy_ch4"] = livestock_df["heads_mapped_dairy"] * ENTERIC_ANNUAL_KG_CH4["dairy_cattle"] - livestock_df["total_ch4"] = livestock_df["beef_ch4"] + livestock_df["sheep_ch4"] + livestock_df["dairy_ch4"] + total_ch4 = ( + livestock_df["heads_mapped_mix_beef"] * ENTERIC_ANNUAL_KG_CH4["beef_cattle"] + + livestock_df["heads_mapped_mix_sheep"] * ENTERIC_ANNUAL_KG_CH4["sheep"] + + livestock_df["heads_mapped_dairy"] * ENTERIC_ANNUAL_KG_CH4["dairy_cattle"] + ) logger.debug(f"Adding livestock emissions to domain grid") ch4_gridded = np.zeros(domain_grid.shape) - np.add.at(ch4_gridded, (livestock_df["iy"], livestock_df["ix"]), livestock_df["total_ch4"]) + np.add.at(ch4_gridded, (livestock_df["iy"], livestock_df["ix"]), total_ch4) return ch4_gridded From edda8b07619d30fe3e9b17a4ec4a4fc3c1893f9e Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Tue, 26 May 2026 09:31:42 +1000 Subject: [PATCH 07/13] Convert livestock dataset coords from EPSG:4283 (GDA94) --- src/openmethane_prior/sectors/livestock/sector.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/openmethane_prior/sectors/livestock/sector.py b/src/openmethane_prior/sectors/livestock/sector.py index acbaca4c..4c5953a7 100644 --- a/src/openmethane_prior/sectors/livestock/sector.py +++ b/src/openmethane_prior/sectors/livestock/sector.py @@ -17,6 +17,7 @@ # import numpy as np import pandas as pd +import pyproj import xarray as xr from openmethane_prior.lib import ( @@ -86,7 +87,12 @@ def process_emissions( livestock_headcount_da = sector_config.data_manager.get_asset(livestock_headcount_data_source) livestock_df: pd.DataFrame = livestock_headcount_da.data - livestock_df = livestock_df.rename(columns={"X": "lon", "Y": "lat"}) + + # dataset X/Y coords are ESPF:4283 (GDA94), convert to EPSG:4326 + transformer = pyproj.Transformer.from_crs(4283,4326) + livestock_df["lon"], livestock_df["lat"] = transformer.transform( + yy=livestock_df["Y"], xx=livestock_df["X"], + ) ch4_gridded = gridded_livestock_emissions_by_headcount(livestock_df, domain_grid) From 4a55bd22b801bfd0a33ae1dc1645ae2ba65e28df Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 28 May 2026 10:59:08 +1000 Subject: [PATCH 08/13] Spatialise per-animal inventory emissions by animal headcounts Since we have per-animal headcounts, and the enteric fermentation inventory sector can be broken down by animal, we can scale the emissions independently for each animal based on where the animals are. It's worth noting, the livestock headcounts dataset includes beef and dairy cows and sheep, but does not include swine or other livestock. Based on this we've moved the UNFCCC categories for enteric fermentation from swine and other livestock into the agriculture sector, which is spatialised by land use. This causes a small increase in the agriculture emissions because their combined sector total will be increased by this inclusion. --- .../sectors/agriculture/sector.py | 4 +- .../sectors/livestock/sector.py | 81 +++++--- tests/integration/test_inventory_inputs.py | 3 +- tests/integration/test_om_prior.py | 32 +-- .../test_sector_livestock.py | 13 ++ ...ridded_livestock_emissions_by_headcount.py | 193 ------------------ .../test_gridded_livestock_headcounts.py | 159 +++++++++++++++ 7 files changed, 247 insertions(+), 238 deletions(-) create mode 100644 tests/integration/test_sector_livestock/test_sector_livestock.py delete mode 100644 tests/unit/test_livestock/test_gridded_livestock_emissions_by_headcount.py create mode 100644 tests/unit/test_livestock/test_gridded_livestock_headcounts.py diff --git a/src/openmethane_prior/sectors/agriculture/sector.py b/src/openmethane_prior/sectors/agriculture/sector.py index 77622ccd..07ad6ca2 100644 --- a/src/openmethane_prior/sectors/agriculture/sector.py +++ b/src/openmethane_prior/sectors/agriculture/sector.py @@ -111,7 +111,9 @@ def process_emissions( sector = PriorSector( name="agriculture", emission_category="anthropogenic", - unfccc_categories=[ # All Agriculture, except Enteric Fermentation + unfccc_categories=[ # All Agriculture, except cattle and sheep + "3.A.3", # Enteric Fermentation - Swine + "3.A.4", # Enteric Fermentation - Other Livestock "3.B", # Manure Management "3.C", # Rice Cultivation "3.D", # Agricultural Soils diff --git a/src/openmethane_prior/sectors/livestock/sector.py b/src/openmethane_prior/sectors/livestock/sector.py index 4c5953a7..776cfba2 100644 --- a/src/openmethane_prior/sectors/livestock/sector.py +++ b/src/openmethane_prior/sectors/livestock/sector.py @@ -20,15 +20,19 @@ import pyproj import xarray as xr +from openmethane_prior.data_sources.inventory import ( + inventory_data_source, + get_sector_emissions_by_code, +) +from openmethane_prior.lib.data_manager.parsers import parse_csv from openmethane_prior.lib import ( DataSource, Grid, - convert_to_timescale, logger, PriorSector, PriorSectorConfig, ) -from openmethane_prior.lib.data_manager.parsers import parse_csv +from openmethane_prior.lib.units import seconds_in_period logger = logger.get_logger(__name__) @@ -38,19 +42,10 @@ parse=parse_csv, ) -# Estimated kg CH4 emitted per animal, per year -# TODO: these figures came from example code, a reference should be added -# citing their origin -ENTERIC_ANNUAL_KG_CH4 = { - "beef_cattle": 51.0, - "dairy_cattle": 93.0, - "sheep": 6.8, -} - -def gridded_livestock_emissions_by_headcount( +def gridded_livestock_headcounts( livestock_df: pd.DataFrame, domain_grid: Grid, -) -> np.ndarray: +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: """Given a list of coordinates for livestock areas with headcounts of beef cattle, dairy cattle and sheep, calculate the annual methane emissions in each area, and return as gridded emissions over the domain of interest.""" @@ -63,18 +58,17 @@ def gridded_livestock_emissions_by_headcount( # discard data outside the domain livestock_df = livestock_df[valid_cells] - logger.debug(f"Calculating annual livestock emissions per headcount") - total_ch4 = ( - livestock_df["heads_mapped_mix_beef"] * ENTERIC_ANNUAL_KG_CH4["beef_cattle"] - + livestock_df["heads_mapped_mix_sheep"] * ENTERIC_ANNUAL_KG_CH4["sheep"] - + livestock_df["heads_mapped_dairy"] * ENTERIC_ANNUAL_KG_CH4["dairy_cattle"] - ) + logger.debug(f"Mapping livestock headcounts to domain grid") + beef_gridded = np.zeros(domain_grid.shape) + dairy_gridded = np.zeros(domain_grid.shape) + sheep_gridded = np.zeros(domain_grid.shape) - logger.debug(f"Adding livestock emissions to domain grid") - ch4_gridded = np.zeros(domain_grid.shape) - np.add.at(ch4_gridded, (livestock_df["iy"], livestock_df["ix"]), total_ch4) + row_idx = (livestock_df["iy"], livestock_df["ix"]) + np.add.at(beef_gridded, row_idx, livestock_df["heads_mapped_mix_beef"]) + np.add.at(dairy_gridded, row_idx, livestock_df["heads_mapped_dairy"]) + np.add.at(sheep_gridded, row_idx, livestock_df["heads_mapped_mix_sheep"]) - return ch4_gridded + return beef_gridded, dairy_gridded, sheep_gridded def process_emissions( @@ -94,16 +88,49 @@ def process_emissions( yy=livestock_df["Y"], xx=livestock_df["X"], ) - ch4_gridded = gridded_livestock_emissions_by_headcount(livestock_df, domain_grid) + # place gridded headcounts on the grid + beef_np, dairy_np, sheep_np = gridded_livestock_headcounts(livestock_df, domain_grid) + + # get inventory totals using animal-specific UNFCCC sectors + emissions_inventory = sector_config.data_manager.get_asset(inventory_data_source).data + get_emissions_params = dict( + emissions_inventory=emissions_inventory, + start_date=config.start_date, + end_date=config.end_date, + ) + beef_total_emissions = get_sector_emissions_by_code( + category_codes=["3.A.1.b", "3.A.1.c"], **get_emissions_params, + ) + dairy_total_emissions = get_sector_emissions_by_code( + category_codes=["3.A.1.a"], **get_emissions_params, + ) + sheep_total_emissions = get_sector_emissions_by_code( + category_codes=["3.A.2"], **get_emissions_params, + ) + logger.debug(f"Total emissions from beef: {beef_total_emissions / 1e6:.2f} kt, dairy: {dairy_total_emissions / 1e6:.2f} kt, sheep: {sheep_total_emissions / 1e6:.2f} kt") + + beef_total_headcount = livestock_df["heads_mapped_mix_beef"].sum() + dairy_total_headcount = livestock_df["heads_mapped_dairy"].sum() + sheep_total_headcount = livestock_df["heads_mapped_mix_sheep"].sum() + + # distribute the national emission totals by headcount proportion + beef_ch4_np = beef_total_emissions * beef_np / beef_total_headcount + dairy_ch4_np = dairy_total_emissions * dairy_np / dairy_total_headcount + sheep_ch4_np = sheep_total_emissions * sheep_np / sheep_total_headcount + + total_ch4_np = beef_ch4_np + dairy_ch4_np + sheep_ch4_np - # convert annual CH4 kg to CH4 kg/s/m3 - return convert_to_timescale(ch4_gridded, domain_grid.cell_area) + # convert CH4 kg to CH4 kg/s/m3 + return total_ch4_np / domain_grid.cell_area / seconds_in_period(config.start_date, config.end_date) sector = PriorSector( name="livestock", emission_category="anthropogenic", - unfccc_categories=["3.A"], # Enteric Fermentation + unfccc_categories=[ + "3.A.1", # Enteric Fermentation - Cattle + "3.A.2", # Enteric Fermentation - Sheep + ], cf_standard_name="domesticated_livestock", create_estimate=process_emissions, ) diff --git a/tests/integration/test_inventory_inputs.py b/tests/integration/test_inventory_inputs.py index 2259c868..de68b1c1 100644 --- a/tests/integration/test_inventory_inputs.py +++ b/tests/integration/test_inventory_inputs.py @@ -28,10 +28,11 @@ def all_sector_meta(): def test_inventory_get_sector_emissions_by_code(all_sector_meta, inventory_df): expected_annual_emissions = { - "agriculture": 153367168.6905247, + "agriculture": 166010041.1571281, "coal": 901455051.0124934, "electricity": 11164637.314332796, "industrial": 2937681.1201194106, + "livestock": 2107296329.3320558, "lulucf": 661762727.6374531, "oil_gas": 238171271.587112, "stationary": 73354555.56013045, diff --git a/tests/integration/test_om_prior.py b/tests/integration/test_om_prior.py index 0d7b4ad7..82353eea 100644 --- a/tests/integration/test_om_prior.py +++ b/tests/integration/test_om_prior.py @@ -5,13 +5,13 @@ def test_009_prior_emissions_ds(prior_emissions_ds): assert dataset_metrics(prior_emissions_ds) == { 'max': { 'LANDMASK': 1.0, - 'OCH4_TOTAL': 8.352448105492377e-09, - 'ch4_sector_agriculture': 1.3196047170278112e-12, + 'OCH4_TOTAL': 8.360644219903412e-09, + 'ch4_sector_agriculture': 1.4283867613607557e-12, 'ch4_sector_coal': 8.072982591959402e-09, 'ch4_sector_electricity': 1.2683449229281463e-11, 'ch4_sector_fire': 5.498670963000052e-11, 'ch4_sector_industrial': 5.351265777744653e-13, - 'ch4_sector_livestock': 7.545056510007907e-11, + 'ch4_sector_livestock': 6.201040469743898e-11, 'ch4_sector_lulucf': 1.4454168861914367e-10, 'ch4_sector_oil_gas': 5.671488137556794e-11, 'ch4_sector_stationary': 1.3362230506306242e-11, @@ -19,7 +19,7 @@ def test_009_prior_emissions_ds(prior_emissions_ds): 'ch4_sector_transport': 2.1317017815070317e-12, 'ch4_sector_waste': 3.530954573289664e-12, 'ch4_sector_wetlands': 2.5512281176531815e-10, - 'ch4_total': 8.352448105492377e-09, + 'ch4_total': 8.360644219903412e-09, 'lambert_conformal': 0, 'land_mask': 1, 'lat': -22.806066513061523, @@ -29,13 +29,13 @@ def test_009_prior_emissions_ds(prior_emissions_ds): }, 'mean': { 'LANDMASK': 1.0, - 'OCH4_TOTAL': 5.779904932204169e-10, - 'ch4_sector_agriculture': 1.0532989458203914e-12, + 'OCH4_TOTAL': 5.767005478563111e-10, + 'ch4_sector_agriculture': 1.1401279872307238e-12, 'ch4_sector_coal': 3.979529404503476e-10, 'ch4_sector_electricity': 2.2548354185389268e-13, 'ch4_sector_fire': 3.6974516872713414e-13, 'ch4_sector_industrial': 5.344609974538503e-14, - 'ch4_sector_livestock': 4.0547154335803873e-11, + 'ch4_sector_livestock': 3.9170379930287976e-11, 'ch4_sector_lulucf': 7.935998439039457e-12, 'ch4_sector_oil_gas': 5.671488137556793e-13, 'ch4_sector_stationary': 1.3345610816621784e-12, @@ -43,7 +43,7 @@ def test_009_prior_emissions_ds(prior_emissions_ds): 'ch4_sector_transport': 2.129050411132023e-13, 'ch4_sector_waste': 5.84814233122829e-14, 'ch4_sector_wetlands': 1.2524275058017897e-10, - 'ch4_total': 5.779904932204169e-10, + 'ch4_total': 5.767005478563111e-10, 'lambert_conformal': 0.0, 'land_mask': 1.0, 'lat': -23.267749786376953, @@ -53,13 +53,13 @@ def test_009_prior_emissions_ds(prior_emissions_ds): }, 'x_band': { 'LANDMASK': 10.0, - 'OCH4_TOTAL': 5.182458643835442e-09, - 'ch4_sector_agriculture': 2.4194872992444853e-11, + 'OCH4_TOTAL': 5.118724020185502e-09, + 'ch4_sector_agriculture': 2.61893852221542e-11, 'ch4_sector_coal': 2.85589757264367e-09, 'ch4_sector_electricity': 0.0, 'ch4_sector_fire': 0.0, 'ch4_sector_industrial': 9.990360612431248e-13, - 'ch4_sector_livestock': 8.812359129868445e-10, + 'ch4_sector_livestock': 8.155067771071957e-10, 'ch4_sector_lulucf': 4.318257917252425e-11, 'ch4_sector_oil_gas': 0.0, 'ch4_sector_stationary': 2.494615421637521e-11, @@ -67,20 +67,20 @@ def test_009_prior_emissions_ds(prior_emissions_ds): 'ch4_sector_transport': 3.979706932888128e-12, 'ch4_sector_waste': 0.0, 'ch4_sector_wetlands': 1.3007149871069323e-09, - 'ch4_total': 5.182458643835442e-09, + 'ch4_total': 5.118724020185502e-09, 'land_mask': 10, 'lat': -232.6233367919922, 'lon': 1486.8963623046875, }, 'y_band': { 'LANDMASK': 10.0, - 'OCH4_TOTAL': 1.951880649183436e-08, - 'ch4_sector_agriculture': 2.3428548325447984e-11, + 'OCH4_TOTAL': 1.949238884650836e-08, + 'ch4_sector_agriculture': 2.5359888331821807e-11, 'ch4_sector_coal': 1.6145965183918805e-08, 'ch4_sector_electricity': 0.0, 'ch4_sector_fire': 0.0, 'ch4_sector_industrial': 3.7006265826194363e-13, - 'ch4_sector_livestock': 1.0031707194660715e-09, + 'ch4_sector_livestock': 9.74821734133699e-10, 'ch4_sector_lulucf': 0.0, 'ch4_sector_oil_gas': 0.0, 'ch4_sector_stationary': 9.240547464560038e-12, @@ -88,7 +88,7 @@ def test_009_prior_emissions_ds(prior_emissions_ds): 'ch4_sector_transport': 1.4741619285048626e-12, 'ch4_sector_waste': 0.0, 'ch4_sector_wetlands': 2.2887601258680945e-09, - 'ch4_total': 1.951880649183436e-08, + 'ch4_total': 1.949238884650836e-08, 'land_mask': 10, 'lat': -232.2213897705078, 'lon': 1486.336669921875, diff --git a/tests/integration/test_sector_livestock/test_sector_livestock.py b/tests/integration/test_sector_livestock/test_sector_livestock.py new file mode 100644 index 00000000..1376a329 --- /dev/null +++ b/tests/integration/test_sector_livestock/test_sector_livestock.py @@ -0,0 +1,13 @@ +from openmethane_prior.lib import create_prior +from openmethane_prior.sectors.livestock import sector + + +def test_sector_livestock(input_files, config): + """Run the full livestock sector over the au-test domain.""" + # run the prior and return the result + prior_livestock_ds = create_prior(config, [sector]) + + print(prior_livestock_ds['ch4_sector_livestock'].max()) + + assert prior_livestock_ds['ch4_sector_livestock'].max().item() == 6.201040469743898e-11 + assert prior_livestock_ds['ch4_sector_livestock'].mean().item() == 3.9170379930287976e-11 diff --git a/tests/unit/test_livestock/test_gridded_livestock_emissions_by_headcount.py b/tests/unit/test_livestock/test_gridded_livestock_emissions_by_headcount.py deleted file mode 100644 index d04e8abd..00000000 --- a/tests/unit/test_livestock/test_gridded_livestock_emissions_by_headcount.py +++ /dev/null @@ -1,193 +0,0 @@ -import pandas as pd -import pytest - -from openmethane_prior.lib.grid.grid import Grid -from openmethane_prior.sectors.livestock.sector import ( - ENTERIC_ANNUAL_KG_CH4, - gridded_livestock_emissions_by_headcount, -) - -# Simple 5x5 degree grid using default EPSG:4326 projection (lon/lat == x/y). -# origin at (130, -40), 1-degree cells. -# ix: 0→[130,131), 1→[131,132), 2→[132,133), 3→[133,134), 4→[134,135) -# iy: 0→[-40,-39), 1→[-39,-38), 2→[-38,-37), 3→[-37,-36), 4→[-36,-35) -TEST_GRID = Grid(dimensions=(5, 5), origin_xy=(130.0, -40.0), cell_size=(1.0, 1.0)) - - -def make_df(rows: list) -> pd.DataFrame: - return pd.DataFrame( - rows, - columns=["lon", "lat", "heads_mapped_mix_beef", "heads_mapped_mix_sheep", "heads_mapped_dairy"], - ) - - -def make_row(values) -> list: - return [values["lon"], values["lat"], values["beef"], values["sheep"], values["dairy"]] - - -# --- headcount scaling --- - -def test_headcount_scales_emissions(): - df = make_df([ - make_row({"lon": 130.5, "lat": -39.5, "beef": 1, "sheep": 0, "dairy": 0}), - ]) - - result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) - - assert result[0, 0] == pytest.approx(ENTERIC_ANNUAL_KG_CH4["beef_cattle"]) - - -def test_sheep_headcount_scales_emissions(): - df = make_df([ - make_row({"lon": 130.5, "lat": -39.5, "beef": 0, "sheep": 1, "dairy": 0}), - ]) - - result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) - - assert result[0, 0] == pytest.approx(ENTERIC_ANNUAL_KG_CH4["sheep"]) - - -def test_dairy_headcount_scales_emissions(): - df = make_df([ - make_row({"lon": 130.5, "lat": -39.5, "beef": 0, "sheep": 0, "dairy": 1}), - ]) - - result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) - - assert result[0, 0] == pytest.approx(ENTERIC_ANNUAL_KG_CH4["dairy_cattle"]) - - -def test_mixed_headcounts_sum_correctly(): - beef, sheep, dairy = 3, 5, 7 - df = make_df([ - make_row({"lon": 130.5, "lat": -39.5, "beef": beef, "sheep": sheep, "dairy": dairy}), - ]) - result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) - - expected = ( - beef * ENTERIC_ANNUAL_KG_CH4["beef_cattle"] - + sheep * ENTERIC_ANNUAL_KG_CH4["sheep"] - + dairy * ENTERIC_ANNUAL_KG_CH4["dairy_cattle"] - ) - assert result[0, 0] == pytest.approx(expected) - - -def test_zero_headcounts_produce_no_emissions(): - df = make_df([ - make_row({"lon": 130.5, "lat": -39.5, "beef": 0, "sheep": 0, "dairy": 0}), - ]) - result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) - - assert result.sum() == 0.0 - - -# --- grid cell placement --- - -def test_emissions_placed_in_lower_left_cell(): - # lon=130.5, lat=-39.5 → ix=0, iy=0 - df = make_df([ - make_row({"lon": 130.5, "lat": -39.5, "beef": 1, "sheep": 0, "dairy": 0}), - ]) - - result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) - - assert result[0, 0] == pytest.approx(ENTERIC_ANNUAL_KG_CH4["beef_cattle"]) - assert result.sum() == pytest.approx(ENTERIC_ANNUAL_KG_CH4["beef_cattle"]) - - -def test_emissions_placed_in_interior_cell(): - # lon=132.5, lat=-37.5 → ix=2, iy=2 - df = make_df([ - make_row({"lon": 132.5, "lat": -37.5, "beef": 0, "sheep": 0, "dairy": 1}), - ]) - - result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) - - assert result[2, 2] == pytest.approx(ENTERIC_ANNUAL_KG_CH4["dairy_cattle"]) - assert result.sum() == pytest.approx(ENTERIC_ANNUAL_KG_CH4["dairy_cattle"]) - - -def test_emissions_placed_in_upper_right_cell(): - # lon=134.5, lat=-35.5 → ix=4, iy=4 - df = make_df([ - make_row({"lon": 134.5, "lat": -35.5, "beef": 0, "sheep": 1, "dairy": 0}), - ]) - - result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) - - assert result[4, 4] == pytest.approx(ENTERIC_ANNUAL_KG_CH4["sheep"]) - assert result.sum() == pytest.approx(ENTERIC_ANNUAL_KG_CH4["sheep"]) - - -def test_point_outside_domain_excluded(): - # lon=125.0 is west of the grid (origin_x=130), should be discarded - df = make_df([ - # outside TEST_GRID, should not be counted - make_row({"lon": 125.0, "lat": -39.5, "beef": 100, "sheep": 100, "dairy": 100}), - # inside TEST_GRID - make_row({"lon": 130.5, "lat": -39.5, "beef": 1, "sheep": 0, "dairy": 0}), - ]) - - result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) - - assert result.sum() == pytest.approx(ENTERIC_ANNUAL_KG_CH4["beef_cattle"]) - - -# --- aggregation within a grid cell --- - -def test_two_rows_in_same_cell_are_aggregated(): - # lon=132.2 and lon=132.7, lat=-36.8 and lat=-36.3 both → ix=2, iy=3 - beef_1, sheep_1, dairy_1 = 10, 0, 0 - beef_2, sheep_2, dairy_2 = 0, 20, 5 - df = make_df([ - make_row({"lon": 132.2, "lat": -36.8, "beef": beef_1, "sheep": sheep_1, "dairy": dairy_1}), - make_row({"lon": 132.7, "lat": -36.3, "beef": beef_2, "sheep": sheep_2, "dairy": dairy_2}), - ]) - - result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) - - expected = ( - beef_1 * ENTERIC_ANNUAL_KG_CH4["beef_cattle"] - + sheep_1 * ENTERIC_ANNUAL_KG_CH4["sheep"] - + dairy_1 * ENTERIC_ANNUAL_KG_CH4["dairy_cattle"] - + beef_2 * ENTERIC_ANNUAL_KG_CH4["beef_cattle"] - + sheep_2 * ENTERIC_ANNUAL_KG_CH4["sheep"] - + dairy_2 * ENTERIC_ANNUAL_KG_CH4["dairy_cattle"] - ) - assert result[3, 2] == pytest.approx(expected) - assert result.sum() == pytest.approx(expected) - - -def test_three_rows_in_same_cell_are_aggregated(): - # all three → ix=1, iy=1 - df = make_df([ - make_row({"lon": 131.1, "lat": -38.9, "beef": 5, "sheep": 0, "dairy": 0}), - make_row({"lon": 131.5, "lat": -38.5, "beef": 0, "sheep": 10, "dairy": 0}), - make_row({"lon": 131.9, "lat": -38.1, "beef": 0, "sheep": 0, "dairy": 3}), - ]) - - result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) - - expected = ( - 5 * ENTERIC_ANNUAL_KG_CH4["beef_cattle"] - + 10 * ENTERIC_ANNUAL_KG_CH4["sheep"] - + 3 * ENTERIC_ANNUAL_KG_CH4["dairy_cattle"] - ) - assert result[1, 1] == pytest.approx(expected) - assert result.sum() == pytest.approx(expected) - - -def test_rows_in_different_cells_are_not_aggregated(): - # ix=0, iy=0 and ix=4, iy=4 are independent - df = make_df([ - make_row({"lon": 130.5, "lat": -39.5, "beef": 10, "sheep": 0, "dairy": 0}), - make_row({"lon": 134.5, "lat": -35.5, "beef": 0, "sheep": 0, "dairy": 7}), - ]) - - result = gridded_livestock_emissions_by_headcount(df, TEST_GRID) - - assert result[0, 0] == pytest.approx(10 * ENTERIC_ANNUAL_KG_CH4["beef_cattle"]) - assert result[4, 4] == pytest.approx(7 * ENTERIC_ANNUAL_KG_CH4["dairy_cattle"]) - assert result.sum() == pytest.approx( - 10 * ENTERIC_ANNUAL_KG_CH4["beef_cattle"] + 7 * ENTERIC_ANNUAL_KG_CH4["dairy_cattle"] - ) diff --git a/tests/unit/test_livestock/test_gridded_livestock_headcounts.py b/tests/unit/test_livestock/test_gridded_livestock_headcounts.py new file mode 100644 index 00000000..fa2dd61b --- /dev/null +++ b/tests/unit/test_livestock/test_gridded_livestock_headcounts.py @@ -0,0 +1,159 @@ +import pandas as pd + +from openmethane_prior.lib.grid.grid import Grid +from openmethane_prior.sectors.livestock.sector import ( + gridded_livestock_headcounts, +) + +# Simple 5x5 degree grid using default EPSG:4326 projection (lon/lat == x/y). +# origin at (130, -40), 1-degree cells. +# ix: 0→[130,131), 1→[131,132), 2→[132,133), 3→[133,134), 4→[134,135) +# iy: 0→[-40,-39), 1→[-39,-38), 2→[-38,-37), 3→[-37,-36), 4→[-36,-35) +TEST_GRID = Grid(dimensions=(5, 5), origin_xy=(130.0, -40.0), cell_size=(1.0, 1.0)) + + +def make_df(rows: list) -> pd.DataFrame: + return pd.DataFrame( + rows, + columns=["lon", "lat", "heads_mapped_mix_beef", "heads_mapped_dairy", "heads_mapped_mix_sheep"], + ) + + +def make_row(values) -> list: + return [values["lon"], values["lat"], values["beef"], values["dairy"], values["sheep"]] + + +# --- headcount placement --- + +def test_headcount_placement_beef(): + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": 1, "dairy": 0, "sheep": 0}), + ]) + + result_beef, result_dairy, result_sheep = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_beef[0, 0] == 1 + assert result_beef.sum() == 1 + assert result_dairy.sum() == 0 + assert result_sheep.sum() == 0 + + +def test_headcount_placement_dairy(): + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": 0, "dairy": 1, "sheep": 0}), + ]) + + result_beef, result_dairy, result_sheep = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_beef.sum() == 0 + assert result_dairy[0, 0] == 1 + assert result_dairy.sum() == 1 + assert result_sheep.sum() == 0 + + +def test_headcount_placement_sheep(): + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": 0, "dairy": 0, "sheep": 1}), + ]) + + result_beef, result_dairy, result_sheep = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_beef.sum() == 0 + assert result_dairy.sum() == 0 + assert result_sheep[0, 0] == 1 + assert result_sheep.sum() == 1 + + +def test_headcount_placement_mixed(): + beef, sheep, dairy = 3, 5, 7 + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": beef, "dairy": dairy, "sheep": sheep}), + ]) + result_beef, result_dairy, result_sheep = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_beef[0, 0] == beef + assert result_dairy[0, 0] == dairy + assert result_sheep[0, 0] == sheep + + +def test_zero_headcounts_produce_no_emissions(): + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": 0, "dairy": 0, "sheep": 0}), + ]) + result_beef, result_dairy, result_sheep = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_beef.sum() == 0.0 + assert result_dairy.sum() == 0.0 + assert result_sheep.sum() == 0.0 + + +# --- grid cell placement --- + +def test_emissions_placed_in_lower_left_cell(): + # lon=130.5, lat=-39.5 → ix=0, iy=0 + df = make_df([ + make_row({"lon": 130.5, "lat": -39.5, "beef": 1, "dairy": 0, "sheep": 0}), + ]) + + result_beef, _, _ = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_beef[0, 0] == 1 + assert result_beef.sum() == 1 + + +def test_emissions_placed_in_interior_cell(): + # lon=132.5, lat=-37.5 → ix=2, iy=2 + df = make_df([ + make_row({"lon": 132.5, "lat": -37.5, "beef": 0, "dairy": 1, "sheep": 0}), + ]) + + _, result_dairy, _ = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_dairy[2, 2] == 1 + assert result_dairy.sum() == 1 + + +def test_emissions_placed_in_upper_right_cell(): + # lon=134.5, lat=-35.5 → ix=4, iy=4 + df = make_df([ + make_row({"lon": 134.5, "lat": -35.5, "beef": 0, "dairy": 0, "sheep": 1}), + ]) + + _, _, result_sheep = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_sheep[4, 4] == 1 + assert result_sheep.sum() == 1 + + +def test_point_outside_domain_excluded(): + # lon=125.0 is west of the grid (origin_x=130), should be discarded + df = make_df([ + # outside TEST_GRID, should not be counted + make_row({"lon": 125.0, "lat": -39.5, "beef": 100, "dairy": 100, "sheep": 100}), + # inside TEST_GRID + make_row({"lon": 130.5, "lat": -39.5, "beef": 1, "dairy": 0, "sheep": 0}), + ]) + + result_beef, result_dairy, result_sheep = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_beef.sum() == 1 + assert result_dairy.sum() == 0 + assert result_sheep.sum() == 0 + + +# --- aggregation within a grid cell --- + +def test_two_rows_in_same_cell_are_aggregated(): + # lon=132.2 and lon=132.7, lat=-36.8 and lat=-36.3 both → ix=2, iy=3 + beef_1, dairy_1, sheep_1 = 10, 0, 0 + beef_2, dairy_2, sheep_2 = 0, 5, 20 + df = make_df([ + make_row({"lon": 132.2, "lat": -36.8, "beef": beef_1, "dairy": dairy_1, "sheep": sheep_1}), + make_row({"lon": 132.7, "lat": -36.3, "beef": beef_2, "dairy": dairy_2, "sheep": sheep_2}), + ]) + + result_beef, result_dairy, result_sheep = gridded_livestock_headcounts(df, TEST_GRID) + + assert result_beef[3, 2] == beef_1 + beef_2 + assert result_dairy[3, 2] == dairy_1 + dairy_2 + assert result_sheep[3, 2] == sheep_1 + sheep_2 From 1dc9af89a405d1908e5ffc06f68ded4af2f72f46 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 28 May 2026 14:44:53 +1000 Subject: [PATCH 09/13] Hard code sector ALUM codes This removes the landuse-sector-map.csv file where prior sectors are associated with ALUM codes, which is how the NLUM Land Use GeoTIFF designates land use of each pixel. The landuse-sector-map.csv was already tightly coupled to the sector implementation, because one of the columns was the sector name. With this change we lose the ability to update which ALUM codes fall into each sector without updating the implementation. However, like UNFCCC and ANZSIC codes, the sector implementation is intimately bound to the types of activities and emissions it represents. It doesn't make sense to change the ALUM mapping for a sector without changing the implementation. This also makes the ALUM mapping more visible, so that if the implementation does change (for example, the recent change to include swine and other livestock in the agriculture sector), it forces the person making the change to evaluate whether the mapped ALUM codes are still accurate. --- docs/data-sources.md | 12 +++----- .../data_sources/landuse/__init__.py | 3 +- .../data_sources/landuse/alum.py | 27 ----------------- .../data_sources/landuse/data.py | 5 ---- .../sectors/agriculture/sector.py | 29 ++++++++++++------- .../sectors/lulucf/sector.py | 28 +++++++++++------- tests/unit/test_data/test_landuse.py | 9 ------ 7 files changed, 40 insertions(+), 73 deletions(-) delete mode 100644 src/openmethane_prior/data_sources/landuse/alum.py delete mode 100644 tests/unit/test_data/test_landuse.py diff --git a/docs/data-sources.md b/docs/data-sources.md index e58ff678..dc34392e 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -140,12 +140,9 @@ Superpower Institute, with data synced using Some sectoral inventories are spatialised by identifying which Australian Land Use and Management (ALUM) Classification codes fall within those sectors, and then distributing inventory emissions to the areas where those ALUM codes are -featured in the Land Use of Australia geographical dataset provided by the -Australian Department of Agriculture. - -This is done using a combination of the official +featured in the [Land Use of Australia](https://www.agriculture.gov.au/abares/aclump/land-use/land-use-of-australia-2010-11-to-2020-21) -GeoTIFF raster, and a manual [mapping between ALUM codes and Open Methane sectors][4]. +geographical dataset provided by the Australian Department of Agriculture. Due to issues fetching the file directly from the agriculture.gov.au service in cloud-hosted services, the land use dataset is mirrored from the Open @@ -238,7 +235,7 @@ Global dataset of CH4 flux estimates from termites. Termite emissions used in [Saunois et al. 2020](https://essd.copernicus.org/articles/12/1561/2020/) supplied by Simona Castaldi and Sergio Noce. -- Dataset: [termite_emissions_2010-2016.nc][5] +- Dataset: [termite_emissions_2010-2016.nc][4] - Resolution: 0.5 degree - Period: mean of 2010 – 2016 - Updates: never @@ -475,5 +472,4 @@ spatialised according to the [Land Use of Australia](#Land-Use-of-Australia) dat [1]: https://openmethane.s3.amazonaws.com/prior/inputs/DLEM_totflux_CRU_diagnostic.nc [2]: https://openmethane.s3.amazonaws.com/prior/inputs/EntericFermentation.nc [3]: https://openmethane.s3.amazonaws.com/prior/inputs/ch4-electricity.csv -[4]: https://openmethane.s3.amazonaws.com/prior/inputs/landuse-sector-map.csv -[5]: https://openmethane.s3.amazonaws.com/prior/inputs/termite_emissions_2010-2016.nc +[4]: https://openmethane.s3.amazonaws.com/prior/inputs/termite_emissions_2010-2016.nc diff --git a/src/openmethane_prior/data_sources/landuse/__init__.py b/src/openmethane_prior/data_sources/landuse/__init__.py index 0c7d0dd8..f67db24f 100644 --- a/src/openmethane_prior/data_sources/landuse/__init__.py +++ b/src/openmethane_prior/data_sources/landuse/__init__.py @@ -1,2 +1 @@ -from .data import alum_sector_mapping_data_source, landuse_map_data_source -from .alum import alum_codes_for_sector \ No newline at end of file +from .data import landuse_map_data_source diff --git a/src/openmethane_prior/data_sources/landuse/alum.py b/src/openmethane_prior/data_sources/landuse/alum.py deleted file mode 100644 index 279a6604..00000000 --- a/src/openmethane_prior/data_sources/landuse/alum.py +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright 2023 The Superpower Institute Ltd. -# -# This file is part of OpenMethane. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import pandas as pd - -def alum_codes_for_sector(sector_name: str, alum_mapping: pd.DataFrame) -> list[int]: - alum_codes = [] - for alum, sector in alum_mapping.values: - if sector == sector_name: - alum_codes.append(int(alum)) - - return alum_codes diff --git a/src/openmethane_prior/data_sources/landuse/data.py b/src/openmethane_prior/data_sources/landuse/data.py index ef2e647b..b0548aef 100644 --- a/src/openmethane_prior/data_sources/landuse/data.py +++ b/src/openmethane_prior/data_sources/landuse/data.py @@ -23,11 +23,6 @@ from openmethane_prior.lib import ConfiguredDataSource, DataSource from openmethane_prior.lib.data_manager.parsers import parse_csv -alum_sector_mapping_data_source = DataSource( - name="alum-sector-mapping", - url="https://openmethane.s3.amazonaws.com/prior/inputs/landuse-sector-map.csv", - parse=parse_csv, -) # filename of the GeoTIFF file inside the official zip file source NLUM_GEOTIFF_FILENAME="NLUM_v7_250_ALUMV8_2020_21_alb.tif" diff --git a/src/openmethane_prior/sectors/agriculture/sector.py b/src/openmethane_prior/sectors/agriculture/sector.py index 07ad6ca2..3acb3617 100644 --- a/src/openmethane_prior/sectors/agriculture/sector.py +++ b/src/openmethane_prior/sectors/agriculture/sector.py @@ -26,11 +26,7 @@ inventory_data_source, inventory_domain_data_source, ) -from openmethane_prior.data_sources.landuse import ( - alum_codes_for_sector, - alum_sector_mapping_data_source, - landuse_map_data_source, -) +from openmethane_prior.data_sources.landuse import landuse_map_data_source from openmethane_prior.lib import ( kg_to_period_cell_flux, logger, @@ -42,6 +38,22 @@ logger = logger.get_logger(__name__) +# ALUM Classification Version 8 +# Source: https://www.agriculture.gov.au/abares/aclump/land-use/alum-classification +alum_codes_agriculture = [ + 210, # Grazing native vegetation + 320, # Grazing modified pastures + 420, # Grazing irrigated modified pastures + 520, # Intensive animal production + 521, # Dairy sheds and yards + 522, # Feedlots + 523, # Poultry farms + 524, # Piggeries + 525, # Aquaculture + 526, # Horse studs + 527, # Saleyards/stockyards + 542, # Rural residential with agriculture +] def process_emissions( sector: PriorSector, @@ -50,11 +62,6 @@ def process_emissions( ): config = sector_config.prior_config - # Import a map of land use type numbers to emissions sectors - # make a dictionary of all landuse types corresponding to sectors in map - sector_mapping_asset = sector_config.data_manager.get_asset(alum_sector_mapping_data_source) - sector_alum_codes = alum_codes_for_sector(sector.name, sector_mapping_asset.data) - # load the national inventory data, ready to calculate sectoral totals emissions_inventory = sector_config.data_manager.get_asset(inventory_data_source).data @@ -80,7 +87,7 @@ def process_emissions( ) # create a mask of pixels which match the sector code - sector_mask = np.isin(dataBand, sector_alum_codes) + sector_mask = np.isin(dataBand, alum_codes_agriculture) sector_xr = xr.DataArray(sector_mask, coords={ 'y': lu_y, 'x': lu_x }) # now aggregate to coarser resolution of the domain grid diff --git a/src/openmethane_prior/sectors/lulucf/sector.py b/src/openmethane_prior/sectors/lulucf/sector.py index 652f59ce..fc104a24 100644 --- a/src/openmethane_prior/sectors/lulucf/sector.py +++ b/src/openmethane_prior/sectors/lulucf/sector.py @@ -26,11 +26,7 @@ inventory_data_source, inventory_domain_data_source, ) -from openmethane_prior.data_sources.landuse import ( - alum_codes_for_sector, - alum_sector_mapping_data_source, - landuse_map_data_source, -) +from openmethane_prior.data_sources.landuse import landuse_map_data_source from openmethane_prior.lib import ( kg_to_period_cell_flux, logger, @@ -42,6 +38,21 @@ logger = logger.get_logger(__name__) +# ALUM Classification Version 8 +# Source: https://www.agriculture.gov.au/abares/aclump/land-use/alum-classification +alum_codes_lulucf = [ + 220, # Production native forests + 310, # Plantation forests + 311, # Hardwood plantation forestry + 312, # Softwood plantation forestry + 313, # Other forest plantation + 314, # Environmental forest plantation + 410, # Irrigated plantation forests + 411, # Irrigated hardwood plantation forestry + 412, # Irrigated softwood plantation forestry + 413, # Irrigated other forest plantation + 414, # Irrigated environmental forest plantation +] def process_emissions( sector: PriorSector, @@ -50,11 +61,6 @@ def process_emissions( ): config = sector_config.prior_config - # Import a map of land use type numbers to emissions sectors - # make a dictionary of all landuse types corresponding to sectors in map - sector_mapping_asset = sector_config.data_manager.get_asset(alum_sector_mapping_data_source) - sector_alum_codes = alum_codes_for_sector(sector.name, sector_mapping_asset.data) - # load the national inventory data, ready to calculate sectoral totals emissions_inventory = sector_config.data_manager.get_asset(inventory_data_source).data @@ -80,7 +86,7 @@ def process_emissions( ) # create a mask of pixels which match the sector code - sector_mask = np.isin(dataBand, sector_alum_codes) + sector_mask = np.isin(dataBand, alum_codes_lulucf) sector_xr = xr.DataArray(sector_mask, coords={ 'y': lu_y, 'x': lu_x }) # now aggregate to coarser resolution of the domain grid diff --git a/tests/unit/test_data/test_landuse.py b/tests/unit/test_data/test_landuse.py deleted file mode 100644 index 54c6e812..00000000 --- a/tests/unit/test_data/test_landuse.py +++ /dev/null @@ -1,9 +0,0 @@ -from openmethane_prior.data_sources.landuse import alum_codes_for_sector, alum_sector_mapping_data_source - - -def test_alum_codes_for_sector(data_manager): - sector_mapping_asset = data_manager.get_asset(alum_sector_mapping_data_source) - agriculture_codes = alum_codes_for_sector("agriculture", sector_mapping_asset.data) - assert agriculture_codes == [210, 320, 420, 520, 521, 522, 523, 524, 525, 526, 527, 542] - - assert alum_codes_for_sector("fake sector", sector_mapping_asset.data) == [] From dcd68b914f73f38d7d2c1de1244051c422ba0020 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Fri, 5 Jun 2026 13:53:00 +1000 Subject: [PATCH 10/13] Add manure management to livestock sector Emissions from manure management are likely to occur where livestock are, so spatialise manure management alongside enteric fermentation according to livestock headcount locations. --- .../sectors/agriculture/sector.py | 3 +- .../sectors/livestock/sector.py | 8 +++-- tests/integration/test_inventory_inputs.py | 4 +-- tests/integration/test_om_prior.py | 32 +++++++++---------- .../test_sector_livestock.py | 6 ++-- 5 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/openmethane_prior/sectors/agriculture/sector.py b/src/openmethane_prior/sectors/agriculture/sector.py index 3acb3617..37a3ce78 100644 --- a/src/openmethane_prior/sectors/agriculture/sector.py +++ b/src/openmethane_prior/sectors/agriculture/sector.py @@ -121,7 +121,8 @@ def process_emissions( unfccc_categories=[ # All Agriculture, except cattle and sheep "3.A.3", # Enteric Fermentation - Swine "3.A.4", # Enteric Fermentation - Other Livestock - "3.B", # Manure Management + "3.B.3", # Manure Management - Swine + "3.B.4", # Manure Management - Other Livestock "3.C", # Rice Cultivation "3.D", # Agricultural Soils "3.E", # Prescribed Burning of Savannas diff --git a/src/openmethane_prior/sectors/livestock/sector.py b/src/openmethane_prior/sectors/livestock/sector.py index 776cfba2..4fe7cf34 100644 --- a/src/openmethane_prior/sectors/livestock/sector.py +++ b/src/openmethane_prior/sectors/livestock/sector.py @@ -99,13 +99,13 @@ def process_emissions( end_date=config.end_date, ) beef_total_emissions = get_sector_emissions_by_code( - category_codes=["3.A.1.b", "3.A.1.c"], **get_emissions_params, + category_codes=["3.A.1.b", "3.A.1.c", "3.B.1.b", "3.B.1.c"], **get_emissions_params, ) dairy_total_emissions = get_sector_emissions_by_code( - category_codes=["3.A.1.a"], **get_emissions_params, + category_codes=["3.A.1.a", "3.B.1.a"], **get_emissions_params, ) sheep_total_emissions = get_sector_emissions_by_code( - category_codes=["3.A.2"], **get_emissions_params, + category_codes=["3.A.2", "3.B.2"], **get_emissions_params, ) logger.debug(f"Total emissions from beef: {beef_total_emissions / 1e6:.2f} kt, dairy: {dairy_total_emissions / 1e6:.2f} kt, sheep: {sheep_total_emissions / 1e6:.2f} kt") @@ -130,6 +130,8 @@ def process_emissions( unfccc_categories=[ "3.A.1", # Enteric Fermentation - Cattle "3.A.2", # Enteric Fermentation - Sheep + "3.B.1", # Manure Management - Cattle + "3.B.2", # Manure Management - Sheep ], cf_standard_name="domesticated_livestock", create_estimate=process_emissions, diff --git a/tests/integration/test_inventory_inputs.py b/tests/integration/test_inventory_inputs.py index de68b1c1..d878cb7e 100644 --- a/tests/integration/test_inventory_inputs.py +++ b/tests/integration/test_inventory_inputs.py @@ -28,11 +28,11 @@ def all_sector_meta(): def test_inventory_get_sector_emissions_by_code(all_sector_meta, inventory_df): expected_annual_emissions = { - "agriculture": 166010041.1571281, + "agriculture": 98593325.56317905, "coal": 901455051.0124934, "electricity": 11164637.314332796, "industrial": 2937681.1201194106, - "livestock": 2107296329.3320558, + "livestock": 2174713044.926005, "lulucf": 661762727.6374531, "oil_gas": 238171271.587112, "stationary": 73354555.56013045, diff --git a/tests/integration/test_om_prior.py b/tests/integration/test_om_prior.py index 82353eea..b3117518 100644 --- a/tests/integration/test_om_prior.py +++ b/tests/integration/test_om_prior.py @@ -5,13 +5,13 @@ def test_009_prior_emissions_ds(prior_emissions_ds): assert dataset_metrics(prior_emissions_ds) == { 'max': { 'LANDMASK': 1.0, - 'OCH4_TOTAL': 8.360644219903412e-09, - 'ch4_sector_agriculture': 1.4283867613607557e-12, + 'OCH4_TOTAL': 8.360290000548561e-09, + 'ch4_sector_agriculture': 8.127365082234856e-13, 'ch4_sector_coal': 8.072982591959402e-09, 'ch4_sector_electricity': 1.2683449229281463e-11, 'ch4_sector_fire': 5.498670963000052e-11, 'ch4_sector_industrial': 5.351265777744653e-13, - 'ch4_sector_livestock': 6.201040469743898e-11, + 'ch4_sector_livestock': 6.329324205281718e-11, 'ch4_sector_lulucf': 1.4454168861914367e-10, 'ch4_sector_oil_gas': 5.671488137556794e-11, 'ch4_sector_stationary': 1.3362230506306242e-11, @@ -19,7 +19,7 @@ def test_009_prior_emissions_ds(prior_emissions_ds): 'ch4_sector_transport': 2.1317017815070317e-12, 'ch4_sector_waste': 3.530954573289664e-12, 'ch4_sector_wetlands': 2.5512281176531815e-10, - 'ch4_total': 8.360644219903412e-09, + 'ch4_total': 8.360290000548561e-09, 'lambert_conformal': 0, 'land_mask': 1, 'lat': -22.806066513061523, @@ -29,13 +29,13 @@ def test_009_prior_emissions_ds(prior_emissions_ds): }, 'mean': { 'LANDMASK': 1.0, - 'OCH4_TOTAL': 5.767005478563111e-10, - 'ch4_sector_agriculture': 1.1401279872307238e-12, + 'OCH4_TOTAL': 5.764117280604889e-10, + 'ch4_sector_agriculture': 3.860498414061556e-14, 'ch4_sector_coal': 3.979529404503476e-10, 'ch4_sector_electricity': 2.2548354185389268e-13, 'ch4_sector_fire': 3.6974516872713414e-13, 'ch4_sector_industrial': 5.344609974538503e-14, - 'ch4_sector_livestock': 3.9170379930287976e-11, + 'ch4_sector_livestock': 3.998308313755556e-11, 'ch4_sector_lulucf': 7.935998439039457e-12, 'ch4_sector_oil_gas': 5.671488137556793e-13, 'ch4_sector_stationary': 1.3345610816621784e-12, @@ -43,7 +43,7 @@ def test_009_prior_emissions_ds(prior_emissions_ds): 'ch4_sector_transport': 2.129050411132023e-13, 'ch4_sector_waste': 5.84814233122829e-14, 'ch4_sector_wetlands': 1.2524275058017897e-10, - 'ch4_total': 5.767005478563111e-10, + 'ch4_total': 5.764117280604889e-10, 'lambert_conformal': 0.0, 'land_mask': 1.0, 'lat': -23.267749786376953, @@ -53,13 +53,13 @@ def test_009_prior_emissions_ds(prior_emissions_ds): }, 'x_band': { 'LANDMASK': 10.0, - 'OCH4_TOTAL': 5.118724020185502e-09, - 'ch4_sector_agriculture': 2.61893852221542e-11, + 'OCH4_TOTAL': 5.1098117508036485e-09, + 'ch4_sector_agriculture': 4.063682541117428e-13, 'ch4_sector_coal': 2.85589757264367e-09, 'ch4_sector_electricity': 0.0, 'ch4_sector_fire': 0.0, 'ch4_sector_industrial': 9.990360612431248e-13, - 'ch4_sector_livestock': 8.155067771071957e-10, + 'ch4_sector_livestock': 8.323775246933841e-10, 'ch4_sector_lulucf': 4.318257917252425e-11, 'ch4_sector_oil_gas': 0.0, 'ch4_sector_stationary': 2.494615421637521e-11, @@ -67,20 +67,20 @@ def test_009_prior_emissions_ds(prior_emissions_ds): 'ch4_sector_transport': 3.979706932888128e-12, 'ch4_sector_waste': 0.0, 'ch4_sector_wetlands': 1.3007149871069323e-09, - 'ch4_total': 5.118724020185502e-09, + 'ch4_total': 5.1098117508036485e-09, 'land_mask': 10, 'lat': -232.6233367919922, 'lon': 1486.8963623046875, }, 'y_band': { 'LANDMASK': 10.0, - 'OCH4_TOTAL': 1.949238884650836e-08, - 'ch4_sector_agriculture': 2.5359888331821807e-11, + 'OCH4_TOTAL': 1.9487601905595595e-08, + 'ch4_sector_agriculture': 4.063682541117428e-13, 'ch4_sector_coal': 1.6145965183918805e-08, 'ch4_sector_electricity': 0.0, 'ch4_sector_fire': 0.0, 'ch4_sector_industrial': 3.7006265826194363e-13, - 'ch4_sector_livestock': 9.74821734133699e-10, + 'ch4_sector_livestock': 9.949883132986455e-10, 'ch4_sector_lulucf': 0.0, 'ch4_sector_oil_gas': 0.0, 'ch4_sector_stationary': 9.240547464560038e-12, @@ -88,7 +88,7 @@ def test_009_prior_emissions_ds(prior_emissions_ds): 'ch4_sector_transport': 1.4741619285048626e-12, 'ch4_sector_waste': 0.0, 'ch4_sector_wetlands': 2.2887601258680945e-09, - 'ch4_total': 1.949238884650836e-08, + 'ch4_total': 1.9487601905595595e-08, 'land_mask': 10, 'lat': -232.2213897705078, 'lon': 1486.336669921875, diff --git a/tests/integration/test_sector_livestock/test_sector_livestock.py b/tests/integration/test_sector_livestock/test_sector_livestock.py index 1376a329..3bd5b793 100644 --- a/tests/integration/test_sector_livestock/test_sector_livestock.py +++ b/tests/integration/test_sector_livestock/test_sector_livestock.py @@ -7,7 +7,5 @@ def test_sector_livestock(input_files, config): # run the prior and return the result prior_livestock_ds = create_prior(config, [sector]) - print(prior_livestock_ds['ch4_sector_livestock'].max()) - - assert prior_livestock_ds['ch4_sector_livestock'].max().item() == 6.201040469743898e-11 - assert prior_livestock_ds['ch4_sector_livestock'].mean().item() == 3.9170379930287976e-11 + assert prior_livestock_ds['ch4_sector_livestock'].max().item() == 6.329324205281718e-11 + assert prior_livestock_ds['ch4_sector_livestock'].mean().item() == 3.998308313755556e-11 From 0de8384b78fb20b4a18a6c93ddb44df8d5d50d14 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Fri, 5 Jun 2026 17:28:33 +1000 Subject: [PATCH 11/13] Add manure management to livestock sector Emissions from manure management are likely to occur where livestock are, so spatialise manure management alongside enteric fermentation according to livestock headcount locations. This also means removing some ALUM codes from the agriculture layer which are primarily related to beef and sheep management. --- .../sectors/agriculture/sector.py | 5 ----- tests/integration/test_om_prior.py | 20 +++++++++---------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/openmethane_prior/sectors/agriculture/sector.py b/src/openmethane_prior/sectors/agriculture/sector.py index 37a3ce78..75cd48a2 100644 --- a/src/openmethane_prior/sectors/agriculture/sector.py +++ b/src/openmethane_prior/sectors/agriculture/sector.py @@ -41,12 +41,7 @@ # ALUM Classification Version 8 # Source: https://www.agriculture.gov.au/abares/aclump/land-use/alum-classification alum_codes_agriculture = [ - 210, # Grazing native vegetation - 320, # Grazing modified pastures - 420, # Grazing irrigated modified pastures 520, # Intensive animal production - 521, # Dairy sheds and yards - 522, # Feedlots 523, # Poultry farms 524, # Piggeries 525, # Aquaculture diff --git a/tests/integration/test_om_prior.py b/tests/integration/test_om_prior.py index b3117518..1a8d9272 100644 --- a/tests/integration/test_om_prior.py +++ b/tests/integration/test_om_prior.py @@ -6,7 +6,7 @@ def test_009_prior_emissions_ds(prior_emissions_ds): 'max': { 'LANDMASK': 1.0, 'OCH4_TOTAL': 8.360290000548561e-09, - 'ch4_sector_agriculture': 8.127365082234856e-13, + 'ch4_sector_agriculture': 6.15621818800407e-13, 'ch4_sector_coal': 8.072982591959402e-09, 'ch4_sector_electricity': 1.2683449229281463e-11, 'ch4_sector_fire': 5.498670963000052e-11, @@ -29,8 +29,8 @@ def test_009_prior_emissions_ds(prior_emissions_ds): }, 'mean': { 'LANDMASK': 1.0, - 'OCH4_TOTAL': 5.764117280604889e-10, - 'ch4_sector_agriculture': 3.860498414061556e-14, + 'OCH4_TOTAL': 5.764039041672882e-10, + 'ch4_sector_agriculture': 3.078109094002035e-14, 'ch4_sector_coal': 3.979529404503476e-10, 'ch4_sector_electricity': 2.2548354185389268e-13, 'ch4_sector_fire': 3.6974516872713414e-13, @@ -43,7 +43,7 @@ def test_009_prior_emissions_ds(prior_emissions_ds): 'ch4_sector_transport': 2.129050411132023e-13, 'ch4_sector_waste': 5.84814233122829e-14, 'ch4_sector_wetlands': 1.2524275058017897e-10, - 'ch4_total': 5.764117280604889e-10, + 'ch4_total': 5.764039041672882e-10, 'lambert_conformal': 0.0, 'land_mask': 1.0, 'lat': -23.267749786376953, @@ -53,8 +53,8 @@ def test_009_prior_emissions_ds(prior_emissions_ds): }, 'x_band': { 'LANDMASK': 10.0, - 'OCH4_TOTAL': 5.1098117508036485e-09, - 'ch4_sector_agriculture': 4.063682541117428e-13, + 'OCH4_TOTAL': 5.109405382549537e-09, + 'ch4_sector_agriculture': 0, 'ch4_sector_coal': 2.85589757264367e-09, 'ch4_sector_electricity': 0.0, 'ch4_sector_fire': 0.0, @@ -67,15 +67,15 @@ def test_009_prior_emissions_ds(prior_emissions_ds): 'ch4_sector_transport': 3.979706932888128e-12, 'ch4_sector_waste': 0.0, 'ch4_sector_wetlands': 1.3007149871069323e-09, - 'ch4_total': 5.1098117508036485e-09, + 'ch4_total': 5.109405382549537e-09, 'land_mask': 10, 'lat': -232.6233367919922, 'lon': 1486.8963623046875, }, 'y_band': { 'LANDMASK': 10.0, - 'OCH4_TOTAL': 1.9487601905595595e-08, - 'ch4_sector_agriculture': 4.063682541117428e-13, + 'OCH4_TOTAL': 1.9487195537341483e-08, + 'ch4_sector_agriculture': 0, 'ch4_sector_coal': 1.6145965183918805e-08, 'ch4_sector_electricity': 0.0, 'ch4_sector_fire': 0.0, @@ -88,7 +88,7 @@ def test_009_prior_emissions_ds(prior_emissions_ds): 'ch4_sector_transport': 1.4741619285048626e-12, 'ch4_sector_waste': 0.0, 'ch4_sector_wetlands': 2.2887601258680945e-09, - 'ch4_total': 1.9487601905595595e-08, + 'ch4_total': 1.9487195537341483e-08, 'land_mask': 10, 'lat': -232.2213897705078, 'lon': 1486.336669921875, From 800d252c0789b73b03c803daa983035cc492c266 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Tue, 9 Jun 2026 15:30:15 +1000 Subject: [PATCH 12/13] Fix code comment specifying m3 rather than m2 --- src/openmethane_prior/sectors/livestock/sector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openmethane_prior/sectors/livestock/sector.py b/src/openmethane_prior/sectors/livestock/sector.py index 4fe7cf34..66e98092 100644 --- a/src/openmethane_prior/sectors/livestock/sector.py +++ b/src/openmethane_prior/sectors/livestock/sector.py @@ -120,7 +120,7 @@ def process_emissions( total_ch4_np = beef_ch4_np + dairy_ch4_np + sheep_ch4_np - # convert CH4 kg to CH4 kg/s/m3 + # convert CH4 kg to CH4 kg/s/m2 return total_ch4_np / domain_grid.cell_area / seconds_in_period(config.start_date, config.end_date) From 87b40e7ae402d3f15beb83ecacfdd354f9cd04d3 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Fri, 12 Jun 2026 10:02:40 +1000 Subject: [PATCH 13/13] Update data-sources.md to reflect new approach to livestock emissions --- docs/data-sources.md | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/docs/data-sources.md b/docs/data-sources.md index dc34392e..b2a3e6bb 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -197,33 +197,25 @@ substances which can act as a proxy for methane emission. ## Sector: Livestock -Australian national dataset of CH4 flux estimates from enteric fermentation in livestock. +Australian national dataset of livestock headcounts for cattle and sheep, used +to distribute ANGA inventory emissions for enteric fermentation and manure for +those animals. ### Sources -Enteric fermentation emissions generated by CSIRO Ag. and Food using livestock -census data and UNFCCC emissions factors. Underlying livestock numbers -taken from: "Navarro, J. Marcos Martinez, R. (2021) Estimating long-term profits, fertiliser and pesticide use -baselines in Australian agricultural regions. User Guide. CSIRO, -Australia." +Livestock headcounts generated by CSIRO Agriculture and Food using livestock +census data. Attribution TBC -- Dataset: [EntericFermentation.nc][2] +- Dataset: TBC - Resolution: 0.01 degree -- Period: 2020, single annual average +- Period: Unknown - Updates: never ### Considerations -- Spatial distribution might change during the year since farmers are moving their cows around -- Doesn't appear to be available online - -### Alternative candidates - -- https://publications.csiro.au/publications/publication/PIcsiro:EP2022-1389/ - - Resolution: 0.1 degree - - Period: 1972 - 2019, annual averages - - Global -- Electronic ear tags might be a useful database +- Real spatial distribution likely to be highly variable during the year as + grazing herds move through available pastures. +- Dataset not yet publicly available. ## Sector: Termites @@ -302,7 +294,7 @@ and described in Zhang et al. (2023 under review) (could this be https://essd.co ## Sector: Agriculture -Agricultural emissions (excluding livestock) reported in the +Agricultural emissions (excluding cattle and sheep) reported in the [Australian UNFCCC Inventory](#Australian-UNFCCC-Inventory) are spatialised according to the [Land Use of Australia](#Land-Use-of-Australia) dataset.