From ff418d8ddd5ae886a7757373e2fd4b375c1be71a Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Fri, 21 Nov 2025 14:39:14 +1100 Subject: [PATCH 01/24] Add parse_geo generic asset parser This reads the asset file using geopandas, and returns a GeoDataFrame, which is a pandas DataFrame that supports vector geometry fields. --- src/openmethane_prior/lib/data_manager/parsers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/openmethane_prior/lib/data_manager/parsers.py b/src/openmethane_prior/lib/data_manager/parsers.py index bb456d02..e9822e32 100644 --- a/src/openmethane_prior/lib/data_manager/parsers.py +++ b/src/openmethane_prior/lib/data_manager/parsers.py @@ -15,10 +15,19 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import geopandas as gpd import pandas as pd from openmethane_prior.lib.data_manager.source import ConfiguredDataSource def parse_csv(data_source: ConfiguredDataSource) -> pd.DataFrame: + """Read and parse a ConfiguredDataSource CSV asset as a pandas DataFrame.""" return pd.read_csv(data_source.asset_path) + + +def parse_geo(data_source: ConfiguredDataSource): + """Read and parse a file containing a collection of geometry vector data + into a geopandas GeoDataFrame. Asset file type can be anything supported by + pyogrio, which includes GeoJSON, GeoPackage, Shapefiles, etc.""" + return gpd.read_file(data_source.asset_path) From 723707d383431be0eb778a953696952e4afc1d30 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Wed, 21 Jan 2026 10:48:08 +1100 Subject: [PATCH 02/24] Add Victorian oil and gas fields data source This DataSource fetches oil and gas field shapes from the Victorian Open Data geoserver and saves them as GeoJSON. The data is then parsed as a GeoDataFrame using geopandas. --- pyproject.toml | 1 + src/openmethane_prior/sectors/oil_gas/data.py | 47 +++++++++++++++++++ uv.lock | 47 +++++++++++++++++++ 3 files changed, 95 insertions(+) create mode 100644 src/openmethane_prior/sectors/oil_gas/data.py diff --git a/pyproject.toml b/pyproject.toml index 78448824..d71a5cce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dependencies = [ "environs>=11.0.0,<12", "prettyprinter>=0.18.0,<0.19", "pytest-mock>=3.15.1,<4", + "owslib>=0.35.0", ] [dependency-groups] diff --git a/src/openmethane_prior/sectors/oil_gas/data.py b/src/openmethane_prior/sectors/oil_gas/data.py new file mode 100644 index 00000000..98e8537e --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data.py @@ -0,0 +1,47 @@ +# +# Copyright 2025 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from owslib.wfs import WebFeatureService + +from openmethane_prior.lib import DataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo +from openmethane_prior.lib.data_manager.source import ConfiguredDataSource + + +def fetch_oil_gas(data_source: ConfiguredDataSource): + geoserver_wfs = WebFeatureService("https://opendata.maps.vic.gov.au/geoserver/wfs", version="2.0.0") + + oil_gas_feature = geoserver_wfs.getfeature( + typename="open-data-platform:oilgas", + srsname="urn:ogc:def:crs:EPSG::4326", + outputFormat="application/json", + ) + + with open(data_source.asset_path, "wb") as asset_file: + asset_file.write(bytes(oil_gas_feature.read())) + return data_source.asset_path + + +# Locations and shapes of oil and gas fields in the Australian state of +# Victoria, via the Victorian Open Data provider. +# Source: https://discover.data.vic.gov.au/dataset/oil-and-gas-fields +vic_oil_gas_data_source = DataSource( + name="vic-oil-gas-fields", + file_path="VIC-oilgas.geojson", + fetch=fetch_oil_gas, + parse=parse_geo, +) diff --git a/uv.lock b/uv.lock index a10d9415..590d2908 100644 --- a/uv.lock +++ b/uv.lock @@ -920,6 +920,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/03/99102b3772bdc5d25fc7fe5f5fb862c54bb6e863991f50d02667999942c1/licenseheaders-0.8.8-py3-none-any.whl", hash = "sha256:3b159228b37bbba98bd01448c41a5eff773ab26ac5b14ac98c53d06dbc807696", size = 21272, upload-time = "2021-04-08T18:48:43.871Z" }, ] +[[package]] +name = "lxml" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/88/262177de60548e5a2bfc46ad28232c9e9cbde697bd94132aeb80364675cb/lxml-6.0.2.tar.gz", hash = "sha256:cd79f3367bd74b317dda655dc8fcfa304d9eb6e4fb06b7168c5cf27f96e0cd62", size = 4073426, upload-time = "2025-09-22T04:04:59.287Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/d5/becbe1e2569b474a23f0c672ead8a29ac50b2dc1d5b9de184831bda8d14c/lxml-6.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:13e35cbc684aadf05d8711a5d1b5857c92e5e580efa9a0d2be197199c8def607", size = 8634365, upload-time = "2025-09-22T04:00:45.672Z" }, + { url = "https://files.pythonhosted.org/packages/28/66/1ced58f12e804644426b85d0bb8a4478ca77bc1761455da310505f1a3526/lxml-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b1675e096e17c6fe9c0e8c81434f5736c0739ff9ac6123c87c2d452f48fc938", size = 4650793, upload-time = "2025-09-22T04:00:47.783Z" }, + { url = "https://files.pythonhosted.org/packages/11/84/549098ffea39dfd167e3f174b4ce983d0eed61f9d8d25b7bf2a57c3247fc/lxml-6.0.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8ac6e5811ae2870953390452e3476694196f98d447573234592d30488147404d", size = 4944362, upload-time = "2025-09-22T04:00:49.845Z" }, + { url = "https://files.pythonhosted.org/packages/ac/bd/f207f16abf9749d2037453d56b643a7471d8fde855a231a12d1e095c4f01/lxml-6.0.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5aa0fc67ae19d7a64c3fe725dc9a1bb11f80e01f78289d05c6f62545affec438", size = 5083152, upload-time = "2025-09-22T04:00:51.709Z" }, + { url = "https://files.pythonhosted.org/packages/15/ae/bd813e87d8941d52ad5b65071b1affb48da01c4ed3c9c99e40abb266fbff/lxml-6.0.2-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de496365750cc472b4e7902a485d3f152ecf57bd3ba03ddd5578ed8ceb4c5964", size = 5023539, upload-time = "2025-09-22T04:00:53.593Z" }, + { url = "https://files.pythonhosted.org/packages/02/cd/9bfef16bd1d874fbe0cb51afb00329540f30a3283beb9f0780adbb7eec03/lxml-6.0.2-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:200069a593c5e40b8f6fc0d84d86d970ba43138c3e68619ffa234bc9bb806a4d", size = 5344853, upload-time = "2025-09-22T04:00:55.524Z" }, + { url = "https://files.pythonhosted.org/packages/b8/89/ea8f91594bc5dbb879734d35a6f2b0ad50605d7fb419de2b63d4211765cc/lxml-6.0.2-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d2de809c2ee3b888b59f995625385f74629707c9355e0ff856445cdcae682b7", size = 5225133, upload-time = "2025-09-22T04:00:57.269Z" }, + { url = "https://files.pythonhosted.org/packages/b9/37/9c735274f5dbec726b2db99b98a43950395ba3d4a1043083dba2ad814170/lxml-6.0.2-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:b2c3da8d93cf5db60e8858c17684c47d01fee6405e554fb55018dd85fc23b178", size = 4677944, upload-time = "2025-09-22T04:00:59.052Z" }, + { url = "https://files.pythonhosted.org/packages/20/28/7dfe1ba3475d8bfca3878365075abe002e05d40dfaaeb7ec01b4c587d533/lxml-6.0.2-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:442de7530296ef5e188373a1ea5789a46ce90c4847e597856570439621d9c553", size = 5284535, upload-time = "2025-09-22T04:01:01.335Z" }, + { url = "https://files.pythonhosted.org/packages/e7/cf/5f14bc0de763498fc29510e3532bf2b4b3a1c1d5d0dff2e900c16ba021ef/lxml-6.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2593c77efde7bfea7f6389f1ab249b15ed4aa5bc5cb5131faa3b843c429fbedb", size = 5067343, upload-time = "2025-09-22T04:01:03.13Z" }, + { url = "https://files.pythonhosted.org/packages/1c/b0/bb8275ab5472f32b28cfbbcc6db7c9d092482d3439ca279d8d6fa02f7025/lxml-6.0.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3e3cb08855967a20f553ff32d147e14329b3ae70ced6edc2f282b94afbc74b2a", size = 4725419, upload-time = "2025-09-22T04:01:05.013Z" }, + { url = "https://files.pythonhosted.org/packages/25/4c/7c222753bc72edca3b99dbadba1b064209bc8ed4ad448af990e60dcce462/lxml-6.0.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2ed6c667fcbb8c19c6791bbf40b7268ef8ddf5a96940ba9404b9f9a304832f6c", size = 5275008, upload-time = "2025-09-22T04:01:07.327Z" }, + { url = "https://files.pythonhosted.org/packages/6c/8c/478a0dc6b6ed661451379447cdbec77c05741a75736d97e5b2b729687828/lxml-6.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b8f18914faec94132e5b91e69d76a5c1d7b0c73e2489ea8929c4aaa10b76bbf7", size = 5248906, upload-time = "2025-09-22T04:01:09.452Z" }, + { url = "https://files.pythonhosted.org/packages/2d/d9/5be3a6ab2784cdf9accb0703b65e1b64fcdd9311c9f007630c7db0cfcce1/lxml-6.0.2-cp311-cp311-win32.whl", hash = "sha256:6605c604e6daa9e0d7f0a2137bdc47a2e93b59c60a65466353e37f8272f47c46", size = 3610357, upload-time = "2025-09-22T04:01:11.102Z" }, + { url = "https://files.pythonhosted.org/packages/e2/7d/ca6fb13349b473d5732fb0ee3eec8f6c80fc0688e76b7d79c1008481bf1f/lxml-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e5867f2651016a3afd8dd2c8238baa66f1e2802f44bc17e236f547ace6647078", size = 4036583, upload-time = "2025-09-22T04:01:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/ab/a2/51363b5ecd3eab46563645f3a2c3836a2fc67d01a1b87c5017040f39f567/lxml-6.0.2-cp311-cp311-win_arm64.whl", hash = "sha256:4197fb2534ee05fd3e7afaab5d8bfd6c2e186f65ea7f9cd6a82809c887bd1285", size = 3680591, upload-time = "2025-09-22T04:01:14.874Z" }, + { url = "https://files.pythonhosted.org/packages/0b/11/29d08bc103a62c0eba8016e7ed5aeebbf1e4312e83b0b1648dd203b0e87d/lxml-6.0.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1c06035eafa8404b5cf475bb37a9f6088b0aca288d4ccc9d69389750d5543700", size = 3949829, upload-time = "2025-09-22T04:04:45.608Z" }, + { url = "https://files.pythonhosted.org/packages/12/b3/52ab9a3b31e5ab8238da241baa19eec44d2ab426532441ee607165aebb52/lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c7d13103045de1bdd6fe5d61802565f1a3537d70cd3abf596aa0af62761921ee", size = 4226277, upload-time = "2025-09-22T04:04:47.754Z" }, + { url = "https://files.pythonhosted.org/packages/a0/33/1eaf780c1baad88224611df13b1c2a9dfa460b526cacfe769103ff50d845/lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a3c150a95fbe5ac91de323aa756219ef9cf7fde5a3f00e2281e30f33fa5fa4f", size = 4330433, upload-time = "2025-09-22T04:04:49.907Z" }, + { url = "https://files.pythonhosted.org/packages/7a/c1/27428a2ff348e994ab4f8777d3a0ad510b6b92d37718e5887d2da99952a2/lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60fa43be34f78bebb27812ed90f1925ec99560b0fa1decdb7d12b84d857d31e9", size = 4272119, upload-time = "2025-09-22T04:04:51.801Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d0/3020fa12bcec4ab62f97aab026d57c2f0cfd480a558758d9ca233bb6a79d/lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21c73b476d3cfe836be731225ec3421fa2f048d84f6df6a8e70433dff1376d5a", size = 4417314, upload-time = "2025-09-22T04:04:55.024Z" }, + { url = "https://files.pythonhosted.org/packages/6c/77/d7f491cbc05303ac6801651aabeb262d43f319288c1ea96c66b1d2692ff3/lxml-6.0.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:27220da5be049e936c3aca06f174e8827ca6445a4353a1995584311487fc4e3e", size = 3518768, upload-time = "2025-09-22T04:04:57.097Z" }, +] + [[package]] name = "markdown-it-py" version = "4.0.0" @@ -1183,6 +1213,7 @@ dependencies = [ { name = "geopandas" }, { name = "netcdf4" }, { name = "numpy" }, + { name = "owslib" }, { name = "pandas" }, { name = "prettyprinter" }, { name = "pyproj" }, @@ -1217,6 +1248,7 @@ requires-dist = [ { name = "geopandas", specifier = ">=0.14.4,<0.15" }, { name = "netcdf4", specifier = ">=1.6.5,<2" }, { name = "numpy", specifier = ">=1.26.4,<2" }, + { name = "owslib", specifier = ">=0.35.0" }, { name = "pandas", specifier = ">=2.2.2,<3" }, { name = "prettyprinter", specifier = ">=0.18.0,<0.19" }, { name = "pyproj", specifier = ">=3.6.1,<4" }, @@ -1250,6 +1282,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/ab/fc8290c6a4c722e5514d80f62b2dc4c4df1a68a41d1364e625c35990fcf3/overrides-7.7.0-py3-none-any.whl", hash = "sha256:c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49", size = 17832, upload-time = "2024-01-27T21:01:31.393Z" }, ] +[[package]] +name = "owslib" +version = "0.35.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, + { name = "python-dateutil" }, + { name = "pyyaml" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bf/03/5df0b995a3a6eefc24cc2320e3fe830d95e76ee6b0b1c6c9d0f1d4040bbe/owslib-0.35.0.tar.gz", hash = "sha256:0182f377bb30d25b78284bbaf82a12dece97902ed844cee88791ff38665b9b00", size = 194282, upload-time = "2025-10-28T15:13:42.594Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/12/74c145b9e273b2b479f1ea578c7778d96d0c34f2d112fd0ec96c905bb792/owslib-0.35.0-py3-none-any.whl", hash = "sha256:01648ea9b2b86502f456ad68a8dd07131d336416c5637f34adb358aafc0ad380", size = 240487, upload-time = "2025-10-28T15:13:41.399Z" }, +] + [[package]] name = "packaging" version = "25.0" From 01553ece20b441c1a2fc343ae71fe0229f503c7f Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Tue, 25 Nov 2025 14:55:55 +1100 Subject: [PATCH 03/24] Add polygon_cell_intersection method for finding grid cells which intersect a polygon --- src/openmethane_prior/lib/__init__.py | 1 + src/openmethane_prior/lib/grid/geometry.py | 62 +++++++++++++++++++ tests/unit/test_grid/test_geometry.py | 71 ++++++++++++++++++++++ 3 files changed, 134 insertions(+) create mode 100644 src/openmethane_prior/lib/grid/geometry.py create mode 100644 tests/unit/test_grid/test_geometry.py diff --git a/src/openmethane_prior/lib/__init__.py b/src/openmethane_prior/lib/__init__.py index 28346421..274e5e56 100644 --- a/src/openmethane_prior/lib/__init__.py +++ b/src/openmethane_prior/lib/__init__.py @@ -23,6 +23,7 @@ DataManager, DataSource, ) +from .grid.geometry import polygon_cell_intersection from .grid.regrid import regrid_data from .outputs import add_sector, convert_to_timescale from .create_prior import create_prior diff --git a/src/openmethane_prior/lib/grid/geometry.py b/src/openmethane_prior/lib/grid/geometry.py new file mode 100644 index 00000000..b2c54535 --- /dev/null +++ b/src/openmethane_prior/lib/grid/geometry.py @@ -0,0 +1,62 @@ +# +# Copyright 2025 The Superpower Institute Ltd. +# +# This file is part of Open Methane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import numpy as np +from shapely import Polygon, MultiPolygon +import typing + +from .grid import Grid + +# a cell intersection is a tuple containing a coordinate tuple, and a fraction +# of intersection, representing how much of a shape falls within each cell. +CellIntersection = typing.TypeVar("CellIntersection", bound=tuple[tuple[float, float], float]) + +def polygon_cell_intersection( + geometry: Polygon | MultiPolygon, + grid: Grid, +) -> list[CellIntersection]: + """Determine which grid cells a polygon intersects with, returning a list + of grid cell indexes and their ratio of intersection with the polygon. + + Polygon shapes should have their coordinates specified in lat/lon in + the same CRS used by the Grid.""" + # find the smallest area of the grid to examine based on the bounding box + bounds_llc_lon, bounds_llc_lat, bounds_urc_lon, bounds_urc_lat = geometry.bounds + bbox_ix, bbox_iy, _ = grid.lonlat_to_cell_index( + np.array([bounds_llc_lon, bounds_urc_lon, bounds_urc_lon, bounds_llc_lon]), + np.array([bounds_llc_lat, bounds_llc_lat, bounds_urc_lat, bounds_urc_lat]), + ) + min_ix, max_ix = np.min(bbox_ix), np.max(bbox_ix) + min_iy, max_iy = np.min(bbox_iy), np.max(bbox_iy) + + cell_bounds_lon, cell_bounds_lat = grid.cell_bounds_lonlat() + + intersections: list[CellIntersection] = [] + for iy in range(min_iy, max_iy + 1): + for ix in range(min_ix, max_ix + 1): + # if part of the shape falls outside the grid, skip those areas + if not grid.valid_cell_coords(ix, iy): + continue + + # find the intersection between the geometry and the grid cell + cell_polygon = Polygon(zip(cell_bounds_lon[iy, ix], cell_bounds_lat[iy, ix])) + intersection = geometry.intersection(cell_polygon) + + if intersection is not None and intersection.area > 0: + intersections.append(((ix, iy), intersection.area / geometry.area)) + + return intersections diff --git a/tests/unit/test_grid/test_geometry.py b/tests/unit/test_grid/test_geometry.py new file mode 100644 index 00000000..50a4adaf --- /dev/null +++ b/tests/unit/test_grid/test_geometry.py @@ -0,0 +1,71 @@ +import math + +import numpy as np +import shapely +from shapely.geometry.polygon import Polygon + +from openmethane_prior.lib.grid.geometry import polygon_cell_intersection + + +def test_polygon_cell_intersection(aust10km_grid): + # Australian Capital Territory polygon, with a hole for Lake Burley Griffin + act_polygon = shapely.Polygon( + shell=[ + [149.399284, -35.319175], [149.352134, -35.351317], [149.336502, -35.339914], [149.254815, -35.330079], [149.207546, -35.345305], [149.146586, -35.414836], [149.139052, -35.432543], [149.155134, -35.436628], [149.135550, -35.454422], [149.151279, -35.506926], [149.131376, -35.554173], [149.142510, -35.592570], [149.084514, -35.580594], [149.078043, -35.586127], [149.087612, -35.639697], [149.097495, -35.647313], [149.095379, -35.679285], [149.109483, -35.696640], [149.090716, -35.765600], [149.101481, -35.803698], [149.093517, -35.824221], [149.095682, -35.845716], [149.064408, -35.874932], [149.048811, -35.920410], [149.012198, -35.899697], [148.959133, -35.895456], [148.909367, -35.853065], [148.907065, -35.829563], [148.886623, -35.810063], [148.897776, -35.794650], [148.894887, -35.771654], [148.903348, -35.757798], [148.894087, -35.751288], [148.886633, -35.719136], [148.877693, -35.714936], [148.872491, -35.721422], [148.855652, -35.760874], [148.856132, -35.753518], [148.835715, -35.741884], [148.822423, -35.720900], [148.791182, -35.703449], [148.798434, -35.666492], [148.767495, -35.647323], [148.783462, -35.628342], [148.768473, -35.603186], [148.788758, -35.588179], [148.773039, -35.568157], [148.778231, -35.558852], [148.769231, -35.544096], [148.772364, -35.529329], [148.762675, -35.495505], [148.774354, -35.486003], [148.767058, -35.465405], [148.788868, -35.426382], [148.785764, -35.408748], [148.796119, -35.406549], [148.795719, -35.392920], [148.808702, -35.382373], [148.793247, -35.339156], [148.807854, -35.309647], [149.120902, -35.124517], [149.138819, -35.128466], [149.138600, -35.135257], [149.149637, -35.138669], [149.146775, -35.144856], [149.164312, -35.142046], [149.167696, -35.159881], [149.189097, -35.165677], [149.183629, -35.175618], [149.197037, -35.185499], [149.189701, -35.203308], [149.208568, -35.211501], [149.204883, -35.229549], [149.214063, -35.219507], [149.238597, -35.222127], [149.246790, -35.229217], [149.234884, -35.242822], [149.273132, -35.259287], [149.272048, -35.273644], [149.315286, -35.276286], [149.322394, -35.286708], [149.341384, -35.286648], [149.361948, -35.308998], [149.394790, -35.303157], [149.399284, -35.319175] + ], + holes=[ + [[149.153176, -35.304976], [149.136510, -35.289256], [149.122879, -35.286560], [149.118459, -35.293608], [149.110274, -35.284577], [149.114443, -35.280492], [149.103116, -35.285130], [149.101406, -35.294277], [149.099252, -35.285600], [149.088030, -35.285150], [149.083012, -35.296961], [149.071123, -35.298567], [149.084355, -35.303898], [149.095399, -35.287391], [149.101389, -35.301479], [149.126110, -35.294247], [149.144619, -35.310156], [149.153176, -35.304976]] + ] + ) + + test_intersection = polygon_cell_intersection(act_polygon, aust10km_grid) + + assert len(test_intersection) == 41 # 41 cells intersecting + # spot test the first and last intersecting cells coordinates and coverage + assert test_intersection[0][0] == (366, 115) + assert math.isclose(test_intersection[0][1], 0.00018735184840114796) + assert test_intersection[-1][0] == (368, 124) + assert math.isclose(test_intersection[-1][1], 0.010287626279578469) + + # ACT sits entirely within the aust10km grid, so the sum of intersections + # should cover the entire shape of the polygon, adding up to 1 + test_coverage = np.sum([area_proportion for coords, area_proportion in test_intersection]) + assert math.isclose(test_coverage, 1) + + # Australian Capital Territory exterior polygon, without a hole + act_exterior = Polygon(act_polygon.exterior.coords) + test_exterior_intersection = polygon_cell_intersection(act_exterior, aust10km_grid) + + assert len(test_exterior_intersection) == 41 + + # proportion of intersection is different for these cells since the lake is + # no longer considered outside the geometry + assert test_exterior_intersection[0][0] == (366, 115) + assert math.isclose(test_exterior_intersection[0][1], 0.00018676566293762296) + + test_exterior_coverage = np.sum([area_proportion for coords, area_proportion in test_exterior_intersection]) + assert math.isclose(test_exterior_coverage, 1) + +def test_polygon_cell_intersection_multipolygon(aust10km_grid): + # 3 squares that roughly correspond to grid cells around Melbourne + test_multi_polygon = shapely.MultiPolygon([ + [[(144.7992, -37.8467), (144.9135, -37.8382), (144.9027, -37.7479), (144.7885, -37.7564)]], + [[(144.9135, -37.8382), (145.0278, -37.8296), (145.0169, -37.7394), (144.9027, -37.7479)]], + [[(144.9243, -37.9284), (145.0387, -37.9199), (145.0278, -37.8296), (144.9135, -37.8382)]], + ]) + + test_intersection = polygon_cell_intersection(test_multi_polygon, aust10km_grid) + + assert len(test_intersection) == 8 + + # spot test a few cells + assert test_intersection[0][0] == (327, 98) + assert math.isclose(test_intersection[0][1], 0.00015177599857517192) + assert test_intersection[1][0] == (328, 98) + assert math.isclose(test_intersection[1][1], 0.33307785884834573) + assert test_intersection[-1][0] == (328, 100) + assert math.isclose(test_intersection[-1][1], 0.00018626618857067368) + + # the entire shape fits in aust10km so should have 100% coverage + test_coverage = np.sum([area_proportion for coords, area_proportion in test_intersection]) + assert math.isclose(test_coverage, 1) From e1a8ad1a32586fe5a3083b05c79f62948b6e5bfd Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Wed, 21 Jan 2026 10:52:47 +1100 Subject: [PATCH 04/24] Bump geopandas from v0.14.4 to v1.1.2 --- pyproject.toml | 2 +- uv.lock | 48 ++++++++++++++++++++++++------------------------ 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d71a5cce..7a893e38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ "rioxarray>=0.15.5,<0.16", "pyproj>=3.6.1,<4", "pandas>=2.2.2,<3", - "geopandas>=0.14.4,<0.15", + "geopandas>=1.1.2", "python-dotenv>=1.0.1,<2", "colorama>=0.4.6,<0.5", "cdsapi>=0.7.3,<0.8", diff --git a/uv.lock b/uv.lock index 590d2908..cd4b2480 100644 --- a/uv.lock +++ b/uv.lock @@ -434,25 +434,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/a8/20d0723294217e47de6d9e2e40fd4a9d2f7c4b6ef974babd482a59743694/fastjsonschema-2.21.2-py3-none-any.whl", hash = "sha256:1c797122d0a86c5cace2e54bf4e819c36223b552017172f32c5c024a6b77e463", size = 24024, upload-time = "2025-08-14T18:49:34.776Z" }, ] -[[package]] -name = "fiona" -version = "1.10.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "certifi" }, - { name = "click" }, - { name = "click-plugins" }, - { name = "cligj" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/51/e0/71b63839cc609e1d62cea2fc9774aa605ece7ea78af823ff7a8f1c560e72/fiona-1.10.1.tar.gz", hash = "sha256:b00ae357669460c6491caba29c2022ff0acfcbde86a95361ea8ff5cd14a86b68", size = 444606, upload-time = "2024-09-16T20:15:47.074Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/b9/7a8356cfaff8ef162bad44283554d3171e13032635b4f8e10e694a9596ee/fiona-1.10.1-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:98fe556058b370da07a84f6537c286f87eb4af2343d155fbd3fba5d38ac17ed7", size = 16144293, upload-time = "2024-09-16T20:14:34.519Z" }, - { url = "https://files.pythonhosted.org/packages/65/0c/e8070b15c8303f60bd4444a120842597ccd6ed550548948e2e36cffbaa93/fiona-1.10.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:be29044d4aeebae92944b738160dc5f9afc4cdf04f551d59e803c5b910e17520", size = 14752213, upload-time = "2024-09-16T20:14:37.763Z" }, - { url = "https://files.pythonhosted.org/packages/7b/2e/3f80ba2fda9b8686681f0a1b18c8e95ad152ada1d6fb1d3f25281d9229fd/fiona-1.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94bd3d448f09f85439e4b77c38b9de1aebe3eef24acc72bd631f75171cdfde51", size = 17272183, upload-time = "2024-09-16T20:14:42.389Z" }, - { url = "https://files.pythonhosted.org/packages/95/32/c1d53b4d77926414ffdf5bd38344e900e378ae9ccb2a65754cdb6d5344c2/fiona-1.10.1-cp311-cp311-win_amd64.whl", hash = "sha256:30594c0cd8682c43fd01e7cdbe000f94540f8fa3b7cb5901e805c88c4ff2058b", size = 24489398, upload-time = "2024-09-16T20:14:46.233Z" }, -] - [[package]] name = "fonttools" version = "4.61.0" @@ -481,19 +462,19 @@ wheels = [ [[package]] name = "geopandas" -version = "0.14.4" +version = "1.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "fiona" }, { name = "numpy" }, { name = "packaging" }, { name = "pandas" }, + { name = "pyogrio" }, { name = "pyproj" }, { name = "shapely" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f6/79/79af2645d40d590a466f8329ab04c2d4fffc811e6713d1c1580dcfdf285c/geopandas-0.14.4.tar.gz", hash = "sha256:56765be9d58e2c743078085db3bd07dc6be7719f0dbe1dfdc1d705cb80be7c25", size = 1106304, upload-time = "2024-04-28T13:49:27.039Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8d/24/5eb5685d7bf89d64218919379f882d19a60f8219d66d833c83b1cf264c95/geopandas-1.1.2.tar.gz", hash = "sha256:33f7b33565c46a45b8459a2ab699ec943fdbb5716e58e251b3c413cf7783106c", size = 336037, upload-time = "2025-12-22T21:06:13.749Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3e/b0/69fa7a0f55122847506a42fea6988d03b34136938082f142151bc9d9f7e7/geopandas-0.14.4-py3-none-any.whl", hash = "sha256:3bb6473cb59d51e1a7fe2dbc24a1a063fb0ebdeddf3ce08ddbf8c7ddc99689aa", size = 1109913, upload-time = "2024-04-28T13:49:24.25Z" }, + { url = "https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl", hash = "sha256:2bb0b1052cb47378addb4ba54c47f8d4642dcbda9b61375638274f49d9f0bb0d", size = 341734, upload-time = "2025-12-22T21:06:12.498Z" }, ] [[package]] @@ -1245,7 +1226,7 @@ requires-dist = [ { name = "cdsapi", specifier = ">=0.7.3,<0.8" }, { name = "colorama", specifier = ">=0.4.6,<0.5" }, { name = "environs", specifier = ">=11.0.0,<12" }, - { name = "geopandas", specifier = ">=0.14.4,<0.15" }, + { name = "geopandas", specifier = ">=1.1.2" }, { name = "netcdf4", specifier = ">=1.6.5,<2" }, { name = "numpy", specifier = ">=1.26.4,<2" }, { name = "owslib", specifier = ">=0.35.0" }, @@ -1485,6 +1466,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, ] +[[package]] +name = "pyogrio" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "numpy" }, + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/d4/12f86b1ed09721363da4c09622464b604c851a9223fc0c6b393fb2012208/pyogrio-0.12.1.tar.gz", hash = "sha256:e548ab705bb3e5383693717de1e6c76da97f3762ab92522cb310f93128a75ff1", size = 303289, upload-time = "2025-11-28T19:04:53.341Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/46/b2c2dcdfd88759b56f103365905fffb85e8b08c1db1ec7c8f8b4c4c26016/pyogrio-0.12.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:01b322dac2a258d24b024d1028dcaa03c9bb6d9c3988b86d298a64873d10dc65", size = 23670744, upload-time = "2025-11-28T19:03:11.299Z" }, + { url = "https://files.pythonhosted.org/packages/d9/21/b69f1bc51d805c00dd7c484a18e1fd2e75b41da1d9f5b8591d7d9d4a7d2f/pyogrio-0.12.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:e10087abcbd6b7e8212560a7002984e5078ac7b3a969ddc2c9929044dbb0d403", size = 25246184, upload-time = "2025-11-28T19:03:13.997Z" }, + { url = "https://files.pythonhosted.org/packages/19/8c/b6aae08e8fcc4f2a903da5f6bd8f888d2b6d7290e54dde5abe15b4cca8df/pyogrio-0.12.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1f6c621972b09fd81a32317e742c69ff4a7763a803da211361a78317f9577765", size = 31434449, upload-time = "2025-11-28T19:03:16.777Z" }, + { url = "https://files.pythonhosted.org/packages/70/f9/9538fa893c29a3fdfeddf3b4c9f8db77f2d4134bc766587929fec8405ebf/pyogrio-0.12.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:c38253427b688464caad5316d4ebcec116b5e13f1f02cc4e3588502f136ca1b4", size = 30987586, upload-time = "2025-11-28T19:03:19.586Z" }, + { url = "https://files.pythonhosted.org/packages/89/a4/0aef5837b4e11840f501e48e01c31242838476c4f4aff9c05e228a083982/pyogrio-0.12.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:5f47787251de7ce13cc06038da93a1214dc283cbccf816be6e03c080358226c8", size = 32534386, upload-time = "2025-11-28T19:03:22.292Z" }, + { url = "https://files.pythonhosted.org/packages/34/97/e8f2ed8a339152b86f8403c258ae5d5f23ab32d690eeb0545bb3473d0c69/pyogrio-0.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:c1d756cf2da4cdf5609779f260d1e1e89be023184225855d6f3dcd33bbe17cb0", size = 22941718, upload-time = "2025-11-28T19:03:24.82Z" }, +] + [[package]] name = "pyparsing" version = "3.2.5" From 19a954c06c0a74fa77f02025f507ec720d7845a0 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 27 Nov 2025 15:44:48 +1100 Subject: [PATCH 05/24] Add method for allocating Safeguard facility emissions to oil field shapes The allocate_safeguard_facility_emissions method in the oil_gas sector module is similar to the implementation in the coal module, except it works on a data source which provides geometry shapes for locations instead of lat/lon points. In this naive implementation the emissions from a single facility are spread out equally over the area covered by its locations. Two grid cells fully covered by a facility location would be attributed equal emissions, while grid cells with only partial coverage would get a smaller share of the facility's reported total. --- .../data_sources/safeguard/data.py | 2 +- .../sectors/oil_gas/safeguard_oil_gas.py | 81 +++++++++++++++++++ .../test_allocate_safeguard.py | 70 ++++++++++++++++ 3 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 src/openmethane_prior/sectors/oil_gas/safeguard_oil_gas.py create mode 100644 tests/integration/test_sector_oil_gas/test_allocate_safeguard.py diff --git a/src/openmethane_prior/data_sources/safeguard/data.py b/src/openmethane_prior/data_sources/safeguard/data.py index 483679db..96cfbb3a 100644 --- a/src/openmethane_prior/data_sources/safeguard/data.py +++ b/src/openmethane_prior/data_sources/safeguard/data.py @@ -102,7 +102,7 @@ def parse_safeguard_csv(data_source: ConfiguredDataSource): safeguard_locations_data_source = DataSource( name="safeguard-locations", - url="https://openmethane.s3.amazonaws.com/prior/inputs/facility-locations-v1.csv", + url="https://openmethane.s3.amazonaws.com/prior/inputs/facility-locations-v1.1.csv", parse=parse_location_csv, ) diff --git a/src/openmethane_prior/sectors/oil_gas/safeguard_oil_gas.py b/src/openmethane_prior/sectors/oil_gas/safeguard_oil_gas.py new file mode 100644 index 00000000..a6ee1965 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/safeguard_oil_gas.py @@ -0,0 +1,81 @@ +# +# Copyright 2025 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import numpy as np + +from openmethane_prior.data_sources.safeguard import ( + filter_facilities, + filter_locations, + get_safeguard_facility_locations, +) +from openmethane_prior.lib import ( + convert_to_timescale, + DataAsset, + logger, + polygon_cell_intersection, + PriorConfig, +) + +logger = logger.get_logger(__name__) + +def allocate_safeguard_facility_emissions( + config: PriorConfig, + anzsic_codes: list[str], + safeguard_facilities_asset: DataAsset, + facility_locations_asset: DataAsset, + reference_data_asset: DataAsset, +): + domain_grid = config.domain_grid() + + sector_facilities = filter_facilities( + facility_df=safeguard_facilities_asset.data, + anzsic_codes=anzsic_codes, + period=(config.start_date.date(), config.end_date.date()), + ) + + oil_gas_facilities, oil_gas_locations_pivot = get_safeguard_facility_locations( + safeguard_facilities_df=sector_facilities, + locations_df=facility_locations_asset.data, + data_source_name=reference_data_asset.name, + ) + + oil_gas_locations = reference_data_asset.data + oil_gas_facilities_locations = oil_gas_locations.merge( + oil_gas_locations_pivot, + right_on="data_source_id", + left_on="id", + ) + + # make an empty grid to allocate emissions to + gridded_annual_emissions = np.zeros(config.domain_grid().shape) + + for _, facility in oil_gas_facilities.iterrows(): + facility_locations = filter_locations(oil_gas_facilities_locations, facility_id=facility["facility_name"]) + + # use the union of all facility location shapes, which will prevent + # overlapping shapes from being double-counted + facility_all_areas = facility_locations["geometry"].union_all() + facility_cells = polygon_cell_intersection(facility_all_areas, domain_grid) + + for cell_indexes, area_proportion in facility_cells: + # allocate a fraction of the facility emissions to each cell + # based on the portion of the total area in each cell + gridded_annual_emissions[cell_indexes[1], cell_indexes[0]] += facility["ch4_kg"] * area_proportion + + gridded_emissions = convert_to_timescale(gridded_annual_emissions, config.domain_grid().cell_area) + + return oil_gas_facilities, oil_gas_facilities_locations, gridded_emissions diff --git a/tests/integration/test_sector_oil_gas/test_allocate_safeguard.py b/tests/integration/test_sector_oil_gas/test_allocate_safeguard.py new file mode 100644 index 00000000..6158abfd --- /dev/null +++ b/tests/integration/test_sector_oil_gas/test_allocate_safeguard.py @@ -0,0 +1,70 @@ + +import attrs +import datetime + +import numpy as np + +from openmethane_prior.data_sources.safeguard import safeguard_mechanism_data_source, safeguard_locations_data_source +from openmethane_prior.lib import PriorConfig +from openmethane_prior.lib.grid.geometry import polygon_cell_intersection +from openmethane_prior.sectors.oil_gas.data import vic_oil_gas_data_source +from openmethane_prior.sectors.oil_gas.safeguard_oil_gas import allocate_safeguard_facility_emissions + + +def test_oil_gas_allocate_safeguard(config: PriorConfig, data_manager): + # we will need to test with configs in the SGM period and outside + config_params = attrs.asdict(config) + del config_params["start_date"] + del config_params["end_date"] + + # period within safeguard period, using the full domain since test fields + # are in Victoria, so not in au-test + config_2023 = PriorConfig( + **config_params, + domain_path="https://openmethane.s3.amazonaws.com/domains/aust10km/v1/domain.aust10km.nc", + start_date=datetime.datetime(2023, 7, 1), + end_date=datetime.datetime(2023, 7, 2), + ) + + safeguard_facilities_asset = data_manager.get_asset(safeguard_mechanism_data_source) + facility_locations_asset = data_manager.get_asset(safeguard_locations_data_source) + oil_gas_asset = data_manager.get_asset(vic_oil_gas_data_source) + + # verify facility emissions in safeguard data + otway_facility = safeguard_facilities_asset.data[safeguard_facilities_asset.data["facility_name"] == "Otway"] + assert len(otway_facility) == 1 + assert otway_facility.iloc[0]["ch4_kg"] == 257750.0 + + # verify facilities related to Otway lie inside the domain + oil_gas_otway_fields = oil_gas_asset.data[oil_gas_asset.data["tag"].isin(["geographe", "thylacine"])] + assert len(oil_gas_otway_fields) == 2 + oil_gas_otway_geometry = oil_gas_otway_fields["geometry"].union_all() + oil_gas_otway_cells = polygon_cell_intersection(oil_gas_otway_geometry, config_2023.domain_grid()) + assert oil_gas_otway_cells == [ + ((309, 84), 0.049298356711897674), + ((308, 85), 0.1448363451138687), + ((309, 85), 0.40146462148077633), + ((309, 86), 0.40120661630811183), + ((310, 86), 0.003194060385346512), + ] + + # run the test + facilities, locations, gridded_emissions = allocate_safeguard_facility_emissions( + config=config_2023, + anzsic_codes=["070"], + safeguard_facilities_asset=safeguard_facilities_asset, + facility_locations_asset=facility_locations_asset, + reference_data_asset=oil_gas_asset, + ) + + assert len(facilities) == 1 + assert len(locations) == 2 + + for cell_indexes, area_proportion in oil_gas_otway_cells: + ix, iy = cell_indexes + expected_emissions = (area_proportion * 257750.0 + / config_2023.domain_grid().cell_area + / (365 * 24 * 60 * 60)) + + # check the emissions are allocated to the right grid cell + assert np.isclose(gridded_emissions[iy, ix], expected_emissions) From 74c54a43a5393db55cb80b1f00ab77ea978d8314 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Fri, 28 Nov 2025 08:02:35 +1100 Subject: [PATCH 06/24] Add "Notes" column to SGM locations CSV, to record where information was found --- src/openmethane_prior/data_sources/safeguard/data.py | 2 ++ tests/unit/test_data/test_locations.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/openmethane_prior/data_sources/safeguard/data.py b/src/openmethane_prior/data_sources/safeguard/data.py index 96cfbb3a..f2c5e993 100644 --- a/src/openmethane_prior/data_sources/safeguard/data.py +++ b/src/openmethane_prior/data_sources/safeguard/data.py @@ -58,6 +58,7 @@ "safeguard_facility_name", # exact facility_name from Safeguard Mechanism "data_source_name", # DataSource name with facility details "data_source_id", # identifier in reference DataSource + "notes", # notes about how this location is related to the facility ] @@ -75,6 +76,7 @@ def parse_location_csv(data_source: ConfiguredDataSource): filepath_or_buffer=data_source.asset_path, header=0, names=safeguard_locations_csv_columns, + usecols=["safeguard_facility_name", "data_source_name", "data_source_id"], ) # filter out any rows with incomplete information diff --git a/tests/unit/test_data/test_locations.py b/tests/unit/test_data/test_locations.py index e11c66f2..2783874a 100644 --- a/tests/unit/test_data/test_locations.py +++ b/tests/unit/test_data/test_locations.py @@ -36,7 +36,7 @@ def location_rows_df(): ("Facility Name C", "test-ds", "id-003"), ("Facility Name E", "alternate-ds", "id-005"), ], - columns=safeguard_locations_csv_columns, + columns=safeguard_locations_csv_columns[:3], # ignore "notes" ) def test_locations_filter_locations(location_rows_df): From 59ffbbd75629f9bcf31a52f3f44495646058c4b5 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Fri, 28 Nov 2025 12:18:41 +1100 Subject: [PATCH 07/24] Update locations data source test to remove fixed values The test is meant to ensure there aren't any empty rows, and the shape of a single row, which it now does. --- .../test_safeguard_data_source.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_safeguard/test_safeguard_data_source.py b/tests/integration/test_safeguard/test_safeguard_data_source.py index 9cf8bed6..70c23979 100644 --- a/tests/integration/test_safeguard/test_safeguard_data_source.py +++ b/tests/integration/test_safeguard/test_safeguard_data_source.py @@ -27,10 +27,16 @@ def test_safeguard_data_source(data_manager): def test_safeguard_locations_data_source(data_manager): locations_df = data_manager.get_asset(safeguard_locations_data_source).data - assert len(locations_df) == 72 # 73 rows, but 72 have complete data - - locations_row_appin = locations_df.iloc[5] - - assert locations_row_appin.safeguard_facility_name == "Blackwater Mine" - assert locations_row_appin.data_source_name == "coal-facilities" - assert locations_row_appin.data_source_id == "Blackwater Coal Mine" + # only rows with complete data are read + assert len(locations_df) > 0 + assert len(locations_df[locations_df["safeguard_facility_name"] == ""]) == 0 + assert len(locations_df[locations_df["data_source_name"] == ""]) == 0 + assert len(locations_df[locations_df["data_source_id"] == ""]) == 0 + + # find a row with known values, check the values match expected + locations_row_blackwater = locations_df[locations_df["safeguard_facility_name"] == "Blackwater Mine"].iloc[0] + assert dict(locations_row_blackwater) == { + "safeguard_facility_name": "Blackwater Mine", + "data_source_name": "coal-facilities", + "data_source_id": "Blackwater Coal Mine", + } From 45a1776e108c14f10e23aff2ffedc7c58e596ecc Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Fri, 28 Nov 2025 16:06:26 +1100 Subject: [PATCH 08/24] Add Safeguard Mechanism facilities to oil & gas sector Currently, only facilities in Victoria are included, as this is the only location dataset that's been introduced. --- src/openmethane_prior/lib/__init__.py | 2 +- .../sectors/oil_gas/sector.py | 56 ++++++++++++++++--- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/src/openmethane_prior/lib/__init__.py b/src/openmethane_prior/lib/__init__.py index 274e5e56..9573a320 100644 --- a/src/openmethane_prior/lib/__init__.py +++ b/src/openmethane_prior/lib/__init__.py @@ -30,7 +30,7 @@ from .raster import remap_raster from .sector.config import PriorSectorConfig from .sector.sector import PriorSector -from .units import kg_to_period_cell_flux +from .units import kg_to_period_cell_flux, days_in_period from .utils import ( area_of_rectangle_m2, datetime64_to_datetime, diff --git a/src/openmethane_prior/sectors/oil_gas/sector.py b/src/openmethane_prior/sectors/oil_gas/sector.py index 261a33e7..42a4eae0 100644 --- a/src/openmethane_prior/sectors/oil_gas/sector.py +++ b/src/openmethane_prior/sectors/oil_gas/sector.py @@ -20,16 +20,21 @@ import pandas as pd import xarray as xr +from openmethane_prior.data_sources.inventory import get_sector_emissions_by_code, inventory_data_source +from openmethane_prior.data_sources.safeguard import safeguard_mechanism_data_source, safeguard_locations_data_source from openmethane_prior.lib.data_manager.parsers import parse_csv from openmethane_prior.lib import ( DataSource, + days_in_period, logger, PriorSectorConfig, kg_to_period_cell_flux, ) -from openmethane_prior.data_sources.inventory import get_sector_emissions_by_code, inventory_data_source from openmethane_prior.lib.sector.au_sector import AustraliaPriorSector +from .data import vic_oil_gas_data_source +from .safeguard_oil_gas import allocate_safeguard_facility_emissions + logger = logger.get_logger(__name__) oil_gas_facilities_data_source = DataSource( @@ -41,6 +46,25 @@ def process_emissions(sector: AustraliaPriorSector, sector_config: PriorSectorConfig, prior_ds: xr.Dataset): config = sector_config.prior_config + # prepare a grid to allocate emissions + domain_grid = config.domain_grid() + methane = np.zeros(domain_grid.shape) + + safeguard_mechanism_asset = sector_config.data_manager.get_asset(safeguard_mechanism_data_source) + facility_locations_asset = sector_config.data_manager.get_asset(safeguard_locations_data_source) + oil_gas_asset = sector_config.data_manager.get_asset(vic_oil_gas_data_source) + + safeguard_facilities, safeguard_locations, safeguard_gridded_ch4 = allocate_safeguard_facility_emissions( + config=config, + anzsic_codes=sector.anzsic_codes, + safeguard_facilities_asset=safeguard_mechanism_asset, + facility_locations_asset=facility_locations_asset, + reference_data_asset=oil_gas_asset, + ) + + # add safeguard emissions to our gridded sector emissions + methane += safeguard_gridded_ch4 + # read the total emissions over the sector (in kg) emissions_inventory = sector_config.data_manager.get_asset(inventory_data_source).data sector_total_emissions = get_sector_emissions_by_code( @@ -50,6 +74,10 @@ def process_emissions(sector: AustraliaPriorSector, sector_config: PriorSectorCo category_codes=sector.unfccc_categories, ) + # SGM emissions have been allocated, find the remaining inventory + safeguard_allocated_emissions = safeguard_facilities["ch4_kg"].sum() * (days_in_period(config.start_date, config.end_date) / 365) + sector_unallocated_emissions = sector_total_emissions - safeguard_allocated_emissions + # now read climate_trace facilities emissions for oil and gas oil_gas_facilities_asset = sector_config.data_manager.get_asset(oil_gas_facilities_data_source) @@ -64,22 +92,32 @@ def process_emissions(sector: AustraliaPriorSector, sector_config: PriorSectorCo years = np.array([x.year for x in oil_gas_ch4["start_time"]]) mask = years == target_date.year oil_gas_year = oil_gas_ch4.loc[mask, :] + + # remove facilities where emissions have already been allocated from + # another source, ie Safeguard Mechanism facilities + for _, facility in oil_gas_year.iterrows(): + cell_x, cell_y, cell_valid = domain_grid.lonlat_to_cell_index(facility["lon"], facility["lat"]) + if methane[cell_y, cell_x] > 0: + facility["emissions_quantity"] = 0 + # normalise emissions to match inventory total oil_gas_year.loc[:, "emissions_quantity"] *= ( - sector_total_emissions / oil_gas_year["emissions_quantity"].sum() + sector_unallocated_emissions / oil_gas_year["emissions_quantity"].sum() ) - domain_grid = config.domain_grid() - - methane = np.zeros(domain_grid.shape) - + # allocate the remaining locations emissions to the grid by allocating + # the remaining inventory emissions to locations, scaled by their relative + # fraction of the total emissions + facilities_gridded = np.zeros(domain_grid.shape) for _, facility in oil_gas_year.iterrows(): cell_x, cell_y, cell_valid = domain_grid.lonlat_to_cell_index(facility["lon"], facility["lat"]) - if cell_valid: - methane[cell_y, cell_x] += facility["emissions_quantity"] + facilities_gridded[cell_y, cell_x] += facility["emissions_quantity"] + + # convert to kg/m2/s before adding to the result + methane += kg_to_period_cell_flux(facilities_gridded, config) - return kg_to_period_cell_flux(methane, config) + return methane sector = AustraliaPriorSector( From a448b9117fe78043c7afc8459bb46fe95c4a1cb1 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Wed, 21 Jan 2026 14:38:32 +1100 Subject: [PATCH 09/24] Allocate oil & gas SGM emissions to domain and inventory grids For the output, we only need to allocate SGM facilities to the domain grid. However, we also need to remove SGM facilities from supplementary data sources so that these facilities aren't double-counted. The simplest way to do that is to remove any facilities which fall within a grid cell that already has emissions from the SGM, but we must do this across the entire inventory domain even when the domain of interest is a subdomain. --- src/openmethane_prior/lib/__init__.py | 1 + .../sectors/oil_gas/safeguard_oil_gas.py | 11 +++++++---- src/openmethane_prior/sectors/oil_gas/sector.py | 17 ++++++++++++++--- .../test_allocate_safeguard.py | 10 +++++++--- 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/openmethane_prior/lib/__init__.py b/src/openmethane_prior/lib/__init__.py index 9573a320..b225e58f 100644 --- a/src/openmethane_prior/lib/__init__.py +++ b/src/openmethane_prior/lib/__init__.py @@ -23,6 +23,7 @@ DataManager, DataSource, ) +from .grid.grid import Grid from .grid.geometry import polygon_cell_intersection from .grid.regrid import regrid_data from .outputs import add_sector, convert_to_timescale diff --git a/src/openmethane_prior/sectors/oil_gas/safeguard_oil_gas.py b/src/openmethane_prior/sectors/oil_gas/safeguard_oil_gas.py index a6ee1965..095ff4ac 100644 --- a/src/openmethane_prior/sectors/oil_gas/safeguard_oil_gas.py +++ b/src/openmethane_prior/sectors/oil_gas/safeguard_oil_gas.py @@ -25,6 +25,7 @@ from openmethane_prior.lib import ( convert_to_timescale, DataAsset, + Grid, logger, polygon_cell_intersection, PriorConfig, @@ -38,8 +39,10 @@ def allocate_safeguard_facility_emissions( safeguard_facilities_asset: DataAsset, facility_locations_asset: DataAsset, reference_data_asset: DataAsset, + grid: Grid = None, ): - domain_grid = config.domain_grid() + if grid is None: + grid = config.domain_grid() sector_facilities = filter_facilities( facility_df=safeguard_facilities_asset.data, @@ -61,7 +64,7 @@ def allocate_safeguard_facility_emissions( ) # make an empty grid to allocate emissions to - gridded_annual_emissions = np.zeros(config.domain_grid().shape) + gridded_annual_emissions = np.zeros(grid.shape) for _, facility in oil_gas_facilities.iterrows(): facility_locations = filter_locations(oil_gas_facilities_locations, facility_id=facility["facility_name"]) @@ -69,13 +72,13 @@ def allocate_safeguard_facility_emissions( # use the union of all facility location shapes, which will prevent # overlapping shapes from being double-counted facility_all_areas = facility_locations["geometry"].union_all() - facility_cells = polygon_cell_intersection(facility_all_areas, domain_grid) + facility_cells = polygon_cell_intersection(facility_all_areas, grid) for cell_indexes, area_proportion in facility_cells: # allocate a fraction of the facility emissions to each cell # based on the portion of the total area in each cell gridded_annual_emissions[cell_indexes[1], cell_indexes[0]] += facility["ch4_kg"] * area_proportion - gridded_emissions = convert_to_timescale(gridded_annual_emissions, config.domain_grid().cell_area) + gridded_emissions = convert_to_timescale(gridded_annual_emissions, grid.cell_area) return oil_gas_facilities, oil_gas_facilities_locations, gridded_emissions diff --git a/src/openmethane_prior/sectors/oil_gas/sector.py b/src/openmethane_prior/sectors/oil_gas/sector.py index 42a4eae0..72febcf2 100644 --- a/src/openmethane_prior/sectors/oil_gas/sector.py +++ b/src/openmethane_prior/sectors/oil_gas/sector.py @@ -93,11 +93,22 @@ def process_emissions(sector: AustraliaPriorSector, sector_config: PriorSectorCo mask = years == target_date.year oil_gas_year = oil_gas_ch4.loc[mask, :] - # remove facilities where emissions have already been allocated from - # another source, ie Safeguard Mechanism facilities + # allocate SGM facility emissions across the entire inventory domain, so + # we can locate facilities which already have emissions allocated and + # remove them from the secondary data source to prevent doubling up. + _, _, safeguard_inventory_ch4 = allocate_safeguard_facility_emissions( + config=config, + anzsic_codes=sector.anzsic_codes, + safeguard_facilities_asset=safeguard_mechanism_asset, + facility_locations_asset=facility_locations_asset, + reference_data_asset=oil_gas_asset, + grid=config.inventory_grid(), + ) for _, facility in oil_gas_year.iterrows(): cell_x, cell_y, cell_valid = domain_grid.lonlat_to_cell_index(facility["lon"], facility["lat"]) - if methane[cell_y, cell_x] > 0: + if safeguard_inventory_ch4[cell_y, cell_x] > 0: + # if the cell for this facility has emissions in the SGM, zero + # it in the secondary data source to prevent double counting facility["emissions_quantity"] = 0 # normalise emissions to match inventory total diff --git a/tests/integration/test_sector_oil_gas/test_allocate_safeguard.py b/tests/integration/test_sector_oil_gas/test_allocate_safeguard.py index 6158abfd..98fe362f 100644 --- a/tests/integration/test_sector_oil_gas/test_allocate_safeguard.py +++ b/tests/integration/test_sector_oil_gas/test_allocate_safeguard.py @@ -11,11 +11,12 @@ from openmethane_prior.sectors.oil_gas.safeguard_oil_gas import allocate_safeguard_facility_emissions -def test_oil_gas_allocate_safeguard(config: PriorConfig, data_manager): +def test_oil_gas_allocate_safeguard(config: PriorConfig, cache_dir, data_manager): # we will need to test with configs in the SGM period and outside config_params = attrs.asdict(config) del config_params["start_date"] del config_params["end_date"] + del config_params["domain_path"] # period within safeguard period, using the full domain since test fields # are in Victoria, so not in au-test @@ -25,6 +26,8 @@ def test_oil_gas_allocate_safeguard(config: PriorConfig, data_manager): start_date=datetime.datetime(2023, 7, 1), end_date=datetime.datetime(2023, 7, 2), ) + config_2023.prepare_paths() + config_2023.load_cached_inputs() safeguard_facilities_asset = data_manager.get_asset(safeguard_mechanism_data_source) facility_locations_asset = data_manager.get_asset(safeguard_locations_data_source) @@ -55,10 +58,11 @@ def test_oil_gas_allocate_safeguard(config: PriorConfig, data_manager): safeguard_facilities_asset=safeguard_facilities_asset, facility_locations_asset=facility_locations_asset, reference_data_asset=oil_gas_asset, + grid=config_2023.domain_grid(), ) - assert len(facilities) == 1 - assert len(locations) == 2 + assert len(facilities) == 2 + assert len(locations) == 34 for cell_indexes, area_proportion in oil_gas_otway_cells: ix, iy = cell_indexes From 0a541ceab2e516f03ea97fae2651f7b8ebb6786a Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Wed, 18 Feb 2026 12:31:24 +1100 Subject: [PATCH 10/24] Move vic-oil-gas-fields data source into its own file The oil and gas sector implementation is likely to use a number of data sources, this will help keep them a bit better organised than all existing in a single file. --- .../sectors/oil_gas/{data.py => data/vic_oil_gas_fields.py} | 0 src/openmethane_prior/sectors/oil_gas/sector.py | 2 +- .../integration/test_sector_oil_gas/test_allocate_safeguard.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/openmethane_prior/sectors/oil_gas/{data.py => data/vic_oil_gas_fields.py} (100%) diff --git a/src/openmethane_prior/sectors/oil_gas/data.py b/src/openmethane_prior/sectors/oil_gas/data/vic_oil_gas_fields.py similarity index 100% rename from src/openmethane_prior/sectors/oil_gas/data.py rename to src/openmethane_prior/sectors/oil_gas/data/vic_oil_gas_fields.py diff --git a/src/openmethane_prior/sectors/oil_gas/sector.py b/src/openmethane_prior/sectors/oil_gas/sector.py index 72febcf2..8334f192 100644 --- a/src/openmethane_prior/sectors/oil_gas/sector.py +++ b/src/openmethane_prior/sectors/oil_gas/sector.py @@ -32,7 +32,7 @@ ) from openmethane_prior.lib.sector.au_sector import AustraliaPriorSector -from .data import vic_oil_gas_data_source +from .data.vic_oil_gas_fields import vic_oil_gas_data_source from .safeguard_oil_gas import allocate_safeguard_facility_emissions logger = logger.get_logger(__name__) diff --git a/tests/integration/test_sector_oil_gas/test_allocate_safeguard.py b/tests/integration/test_sector_oil_gas/test_allocate_safeguard.py index 98fe362f..2b8a435b 100644 --- a/tests/integration/test_sector_oil_gas/test_allocate_safeguard.py +++ b/tests/integration/test_sector_oil_gas/test_allocate_safeguard.py @@ -7,7 +7,7 @@ from openmethane_prior.data_sources.safeguard import safeguard_mechanism_data_source, safeguard_locations_data_source from openmethane_prior.lib import PriorConfig from openmethane_prior.lib.grid.geometry import polygon_cell_intersection -from openmethane_prior.sectors.oil_gas.data import vic_oil_gas_data_source +from openmethane_prior.sectors.oil_gas.data.vic_oil_gas_fields import vic_oil_gas_data_source from openmethane_prior.sectors.oil_gas.safeguard_oil_gas import allocate_safeguard_facility_emissions From 5b98e40dee346aadebb70e47c6245a19d928e1f3 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Tue, 10 Mar 2026 16:29:34 +1100 Subject: [PATCH 11/24] Add QLD boreholes data source for mapping oil/gas fields --- pyproject.toml | 1 + .../sectors/oil_gas/data/qld_boreholes.py | 90 +++++++++++++++++++ uv.lock | 40 +++++++-- 3 files changed, 122 insertions(+), 9 deletions(-) create mode 100644 src/openmethane_prior/sectors/oil_gas/data/qld_boreholes.py diff --git a/pyproject.toml b/pyproject.toml index 7a893e38..4b368373 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "prettyprinter>=0.18.0,<0.19", "pytest-mock>=3.15.1,<4", "owslib>=0.35.0", + "bmi-arcgis-restapi>=2.4.15", ] [dependency-groups] diff --git a/src/openmethane_prior/sectors/oil_gas/data/qld_boreholes.py b/src/openmethane_prior/sectors/oil_gas/data/qld_boreholes.py new file mode 100644 index 00000000..b0224543 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/qld_boreholes.py @@ -0,0 +1,90 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import numpy as np +import restapi # https://github.com/Bolton-and-Menk-GIS/restapi +import json + +from openmethane_prior.lib import DataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo +from openmethane_prior.lib.data_manager.source import ConfiguredDataSource + + +# Queensland borehole series - REST Service (ArcGIS) +# https://www.data.qld.gov.au/dataset/queensland-borehole-series/resource/c206a53a-59de-48c2-9544-b7b7323ed5dd +def fetch_qld_boreholes(data_source: ConfiguredDataSource): + # qld_spatial_arcgis = restapi.ArcServer(url="https://spatial-gis.information.qld.gov.au/arcgis/rest/services") + qld_spatial_boreholes = restapi.MapService( + url="https://spatial-gis.information.qld.gov.au/arcgis/rest/services/GeoscientificInformation/Boreholes/MapServer" + ) + + boreholes_features = None + oil_gas_layers = ["Boreholes CSG", "Boreholes Gas or Gas Show", "Boreholes Oil or Oil Show", "Boreholes Petroleum"] + for layer_name in oil_gas_layers: + boreholes_layer = qld_spatial_boreholes.layer(layer_name) + # print(boreholes_layer.list_fields()) + + layer_features = boreholes_layer.query( + # TODO: filter out non-relevant results + # where="bore_type in ('COAL SEAM GAS','GREENHOUSE GAS STORAGE','PETROLEUM','UNCONVENTIONAL PETROLEUM')", + # this dataset has many fields, but we only need locations + fields=[ + "bore_name", + "bore_subtype", + "bore_type", + "borehole_pid", + "operator_name", + "result", + "rig_release_date", + "status", + "tenure_no", + "tenure_type", + ], + exceed_limit=True, + ) + + if boreholes_features is None: + boreholes_features = layer_features + else: + boreholes_features.features.extend(layer_features.features) + + with open(data_source.asset_path, "w") as asset_file: + json.dump(boreholes_features.json, asset_file) + return data_source.asset_path + + +def parse_qld_boreholes(data_source: ConfiguredDataSource): + boreholes_df = parse_geo(data_source=data_source) + + # lease/tenement name, i.e. "PL 100", is more useful than "PL" and 100.0, + # so combine tenure_no and tenure_type into a single string + boreholes_df["tenure"] = [ + None if t_no is np.nan else f"{t_type} {t_no:0.0f}" + for t_no, t_type in boreholes_df[["tenure_no", "tenure_type"]].values + ] + + return boreholes_df + +# Locations of all boreholes in the Australian state of Queensland, via the +# Queensland Open Data Portal. +# Source: https://www.data.qld.gov.au/dataset/queensland-borehole-series +qld_boreholes_data_source = DataSource( + name="qld-boreholes", + file_path="QLD-boreholes.geojson", + fetch=fetch_qld_boreholes, + parse=parse_qld_boreholes, +) diff --git a/uv.lock b/uv.lock index cd4b2480..16e0b46f 100644 --- a/uv.lock +++ b/uv.lock @@ -145,6 +145,17 @@ css = [ { name = "tinycss2" }, ] +[[package]] +name = "bmi-arcgis-restapi" +version = "2.4.15" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "munch" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/82/8cb4c901a9cf4304cffb65904a40fe6a55074d6b0c73b62fa392b3db2074/bmi_arcgis_restapi-2.4.15.tar.gz", hash = "sha256:8705e2a31deebc32390fc9a875fe152254df270744afcb30de90056388859e2f", size = 748077, upload-time = "2026-01-09T22:00:42.962Z" } + [[package]] name = "cartopy" version = "0.25.0" @@ -1057,6 +1068,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/93/cf/be4e93afbfa0def2cd6fac9302071db0bd6d0617999ecbf53f92b9398de3/multiurl-0.3.7-py3-none-any.whl", hash = "sha256:054f42974064f103be0ed55b43f0c32fc435a47dc7353a9adaffa643b99fa380", size = 21524, upload-time = "2025-07-29T11:57:03.191Z" }, ] +[[package]] +name = "munch" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/2b/45098135b5f9f13221820d90f9e0516e11a2a0f55012c13b081d202b782a/munch-4.0.0.tar.gz", hash = "sha256:542cb151461263216a4e37c3fd9afc425feeaf38aaa3025cd2a981fadb422235", size = 19089, upload-time = "2023-07-01T09:49:35.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/b3/7c69b37f03260a061883bec0e7b05be7117c1b1c85f5212c72c8c2bc3c8c/munch-4.0.0-py2.py3-none-any.whl", hash = "sha256:71033c45db9fb677a0b7eb517a4ce70ae09258490e419b0e7f00d1e386ecb1b4", size = 9950, upload-time = "2023-07-01T09:49:34.472Z" }, +] + [[package]] name = "nbclient" version = "0.10.2" @@ -1188,6 +1208,7 @@ name = "openmethane-prior" version = "1.3.1.dev1" source = { editable = "." } dependencies = [ + { name = "bmi-arcgis-restapi" }, { name = "cdsapi" }, { name = "colorama" }, { name = "environs" }, @@ -1223,6 +1244,7 @@ tests = [ [package.metadata] requires-dist = [ + { name = "bmi-arcgis-restapi", specifier = ">=2.4.15" }, { name = "cdsapi", specifier = ">=0.7.3,<0.8" }, { name = "colorama", specifier = ">=0.4.6,<0.5" }, { name = "environs", specifier = ">=11.0.0,<12" }, @@ -1289,7 +1311,7 @@ wheels = [ [[package]] name = "pandas" -version = "2.3.3" +version = "2.2.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, @@ -1297,15 +1319,15 @@ dependencies = [ { name = "pytz" }, { name = "tzdata" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/d6/9f8431bacc2e19dca897724cd097b1bb224a6ad5433784a44b587c7c13af/pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667", size = 4399213, upload-time = "2024-09-20T13:10:04.827Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/fa/7ac648108144a095b4fb6aa3de1954689f7af60a14cf25583f4960ecb878/pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523", size = 11578790, upload-time = "2025-09-29T23:18:30.065Z" }, - { url = "https://files.pythonhosted.org/packages/9b/35/74442388c6cf008882d4d4bdfc4109be87e9b8b7ccd097ad1e7f006e2e95/pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45", size = 10833831, upload-time = "2025-09-29T23:38:56.071Z" }, - { url = "https://files.pythonhosted.org/packages/fe/e4/de154cbfeee13383ad58d23017da99390b91d73f8c11856f2095e813201b/pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66", size = 12199267, upload-time = "2025-09-29T23:18:41.627Z" }, - { url = "https://files.pythonhosted.org/packages/bf/c9/63f8d545568d9ab91476b1818b4741f521646cbdd151c6efebf40d6de6f7/pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b", size = 12789281, upload-time = "2025-09-29T23:18:56.834Z" }, - { url = "https://files.pythonhosted.org/packages/f2/00/a5ac8c7a0e67fd1a6059e40aa08fa1c52cc00709077d2300e210c3ce0322/pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791", size = 13240453, upload-time = "2025-09-29T23:19:09.247Z" }, - { url = "https://files.pythonhosted.org/packages/27/4d/5c23a5bc7bd209231618dd9e606ce076272c9bc4f12023a70e03a86b4067/pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151", size = 13890361, upload-time = "2025-09-29T23:19:25.342Z" }, - { url = "https://files.pythonhosted.org/packages/8e/59/712db1d7040520de7a4965df15b774348980e6df45c129b8c64d0dbe74ef/pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c", size = 11348702, upload-time = "2025-09-29T23:19:38.296Z" }, + { url = "https://files.pythonhosted.org/packages/a8/44/d9502bf0ed197ba9bf1103c9867d5904ddcaf869e52329787fc54ed70cc8/pandas-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039", size = 12602222, upload-time = "2024-09-20T13:08:56.254Z" }, + { url = "https://files.pythonhosted.org/packages/52/11/9eac327a38834f162b8250aab32a6781339c69afe7574368fffe46387edf/pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd", size = 11321274, upload-time = "2024-09-20T13:08:58.645Z" }, + { url = "https://files.pythonhosted.org/packages/45/fb/c4beeb084718598ba19aa9f5abbc8aed8b42f90930da861fcb1acdb54c3a/pandas-2.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd8d0c3be0515c12fed0bdbae072551c8b54b7192c7b1fda0ba56059a0179698", size = 15579836, upload-time = "2024-09-20T19:01:57.571Z" }, + { url = "https://files.pythonhosted.org/packages/cd/5f/4dba1d39bb9c38d574a9a22548c540177f78ea47b32f99c0ff2ec499fac5/pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc", size = 13058505, upload-time = "2024-09-20T13:09:01.501Z" }, + { url = "https://files.pythonhosted.org/packages/b9/57/708135b90391995361636634df1f1130d03ba456e95bcf576fada459115a/pandas-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:63cc132e40a2e084cf01adf0775b15ac515ba905d7dcca47e9a251819c575ef3", size = 16744420, upload-time = "2024-09-20T19:02:00.678Z" }, + { url = "https://files.pythonhosted.org/packages/86/4a/03ed6b7ee323cf30404265c284cee9c65c56a212e0a08d9ee06984ba2240/pandas-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32", size = 14440457, upload-time = "2024-09-20T13:09:04.105Z" }, + { url = "https://files.pythonhosted.org/packages/ed/8c/87ddf1fcb55d11f9f847e3c69bb1c6f8e46e2f40ab1a2d2abadb2401b007/pandas-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5", size = 11617166, upload-time = "2024-09-20T13:09:06.917Z" }, ] [[package]] From ddc96f14fe6c858c624f0919f8aea3997e3b39f0 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Wed, 11 Mar 2026 09:32:12 +1100 Subject: [PATCH 12/24] Ensure parse_geo GeoDataFrames always use EPSG:4326 When multiple GeoDataFrame objects are used in a layer and often combined, they must share a common CRS. The CRS of the domain projection may not be known at the time that parse_geo is run, so for now just normalise to a standard WGS84 projection. --- src/openmethane_prior/lib/data_manager/parsers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/openmethane_prior/lib/data_manager/parsers.py b/src/openmethane_prior/lib/data_manager/parsers.py index e9822e32..f47609c0 100644 --- a/src/openmethane_prior/lib/data_manager/parsers.py +++ b/src/openmethane_prior/lib/data_manager/parsers.py @@ -30,4 +30,7 @@ def parse_geo(data_source: ConfiguredDataSource): """Read and parse a file containing a collection of geometry vector data into a geopandas GeoDataFrame. Asset file type can be anything supported by pyogrio, which includes GeoJSON, GeoPackage, Shapefiles, etc.""" - return gpd.read_file(data_source.asset_path) + geo_df = gpd.read_file(data_source.asset_path) + if geo_df.crs is not None and geo_df.crs != "EPSG:4326": + geo_df = geo_df.to_crs(epsg=4326) + return geo_df From 301de5c06794cdd414b63fb90ded8357b602b436 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Wed, 18 Mar 2026 11:49:51 +1100 Subject: [PATCH 13/24] Add oil and gas sector README.md This sector implementation is complex enough to warrant its own documentation where we can explain methodology, data sources, and decisions. --- .../sectors/oil_gas/README.md | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 src/openmethane_prior/sectors/oil_gas/README.md diff --git a/src/openmethane_prior/sectors/oil_gas/README.md b/src/openmethane_prior/sectors/oil_gas/README.md new file mode 100644 index 00000000..46f64fd4 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/README.md @@ -0,0 +1,94 @@ + +# Oil and Gas Sector + +The oil and gas sector is the most complicated sector implementation in the +Australia prior. There's no pre-existing high quality dataset that can be +used to both locate and scale emission sources from the sector, and no clear +spatial proxy. + +Oil and gas extraction occurs in "fields" of wells or bores, somtimes located +offshore. Extracted resources are transported through pipelines to various +pieces of infrastructure to be compressed or combined, and eventually to +refinery facilities. Methane can be lost to the atmosphere for various reasons +at any of these locations, resulting in physically dispersed infrastructure +which is not well documented by the private companies which operate it. + +## Methodology + +Emissions are likely to occur in three possible parts of the oil and gas +infrastructure: +- wells or boreholes where oil or gas are extracted +- processing facilities +- pipelines connecting wells and other infrastructure + +### Wells and boreholes + +#### Locations + +Oil and gas extraction are covered by petroleum mining laws in Australia, and +most Australian states provide public datasets of petroleum mining titles +(aka leases/licenses/tenements). Some states also provide public datasets +of drill/bore/well locations. + +Drillhole datasets typically include enough detail to determine which bores +or wells are used for petroleum production, giving us precise locations for +emission sources. + +#### Activity period + +Drillhole datasets typically include the approximate date a bore was drilled +(sometimes just the year), but don't include the date a bore was capped or +depleted. Although some bore datasets do include the bore status, ie CAPPED, +this is the status **at the time the dataset was last updated**, and not +necessarily the status in the period of interest for the prior. + +We use a very rough method to determine whether a single bore may have been +producing emissions during a target period (`start_date` to `end_date`). +- if the bore drill date is later than `end_date`, no emissions are possible +- correlate the bore coordinates with all petroleum production titles it + is contained by +- if the range from `start_date` to `end_date` overlaps with the range from + title grant date to title expiry date, emissions are possible + +This method has obvious problems: + +1. emissions are unlikely to start immediately after drilling, however without + a more accurate start date, this is at least a lower bound +2. most bores/wells will be depleted or capped well before the petroleum title + expires, however without an accurate capping date, this is at least an upper + bound. +3. there is potentially a long period (years) where an entire field / title + has stopped producing, but the title is still active. + +On a coarse grid such as our 10x10km grid, 1. and 2. probably aren't serious, +because emissions will be dispersed amongst a number of points within each grid +cell. + +The biggest issue is 3., as it will place emissions in some fields, possibly +for years after production has ceased. Assuming that capped wells don't +continue to emit methane, this will misallocate potentially significant +emissions. Unfortunately we currently don't have a better solution. + +### Processing facilities + +Not yet implemented. + +### Pipelines + +Public datasets do exist which record the shape and status of oil and gas +pipelines in Australia, however there are several issues using these shape +files to estimate emissions. + +1. Lack of temporal data + +Pipeline datasets list the current status of pipelines, but don't necessarily +include the dates pipelines actually began operating. + +2. Lack of leakage estimates + +We're not currently aware of any research or data on how much methane leaks +from pipelines, or where leakage occurs. This makes it hard to attribute a +specific volume of emission to specific locations along any given pipeline. + +For now, pipelines are not included in our bottom up estimate. If better data +or research becomes available in the future, this would be a welcome addition. \ No newline at end of file From e002702c76ade51275e0c625775e1469bf548796 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 19 Mar 2026 09:04:39 +1100 Subject: [PATCH 14/24] Add emission source concept in oil & gas sector Since emission sources for this sector will come from many different datasets, this defines a common structure of data as an interchange format. --- .../emission_sources/emission_source.py | 73 +++++++++++++++++++ .../unit/test_oil_gas/test_emission_source.py | 33 +++++++++ 2 files changed, 106 insertions(+) create mode 100644 src/openmethane_prior/sectors/oil_gas/emission_sources/emission_source.py create mode 100644 tests/unit/test_oil_gas/test_emission_source.py diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/emission_source.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/emission_source.py new file mode 100644 index 00000000..5bca4975 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/emission_source.py @@ -0,0 +1,73 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import numpy as np +import geopandas as gpd +import pyproj + +""" +An "emission source" in the oil and gas sector is any location that forms part +of oil and gas extraction or production infrastructure where methane emission +might occur. Emission sources are gathered from a variety of datasets, but must +satisfy several required fields. +""" + +emission_source_dtypes = { + # The location of the site (typically a POINT or POLYGON). This column + # is typically provided by GeoDataFrame. + "geometry": "geometry", + + # A string which can be used to determine what type of site/facility is at + # this location. Valid values are documented in emission_source_site_types. + "site_type": str, + + # The earliest date emissions may have occurred at this site. + "activity_start": np.datetime64, + + # The latest date emissions may have occurred at this site. + "activity_end": np.datetime64, + + # (Optional) the "name" value of the data source where this site was + # included. + "data_source": str, + + # (Optional) a unique id for this site in the specified data source. + "data_source_id": str, + + # (Optional) a string identifying a group of locations this site is part + # of. A group could be a title/license area, a field, etc. + "group_id": str, +} + +emission_source_site_types = [ + "drillhole-unknown", + "drillhole-csg", + "drillhole-petroleum", + "drillhole-gas", +] + + +def normalise_emission_source_df( + df: gpd.GeoDataFrame, + crs: pyproj.CRS = pyproj.CRS.from_epsg(4326), +) -> gpd.GeoDataFrame: + # select only columns which are present in emission_source_dtypes + normalised_df = df[list(emission_source_dtypes.keys())] + + normalised_df = normalised_df.to_crs(crs) + + return normalised_df \ No newline at end of file diff --git a/tests/unit/test_oil_gas/test_emission_source.py b/tests/unit/test_oil_gas/test_emission_source.py new file mode 100644 index 00000000..78cdde8d --- /dev/null +++ b/tests/unit/test_oil_gas/test_emission_source.py @@ -0,0 +1,33 @@ +import datetime +import geopandas as gpd +import numpy as np +import shapely + +from openmethane_prior.sectors.oil_gas.emission_sources.emission_source import normalise_emission_source_df + + +def test_normalise_emission_source(): + test_df = gpd.GeoDataFrame({ + "geometry": [shapely.Point(1.0, 2.0)], + "site_type": ["drillhole-csg"], + "activity_start": [np.datetime64(datetime.datetime(2022, 1, 1, 0, 0))], + "activity_end": [np.datetime64(datetime.datetime(2024, 1, 1, 0, 0))], + "data_source": ["nsw-drillholes"], + "data_source_id": ["012345"], + "group_id": ["xyz"], + "extra_column": [0], + }, crs="EPSG:7844") + + result_df = normalise_emission_source_df(test_df) + + assert list(result_df.columns) == [ + "geometry", + "site_type", + "activity_start", + "activity_end", + "data_source", + "data_source_id", + "group_id", + ] + + assert result_df.crs == "EPSG:4326" From 3480b261c98692a3ae3f258b036af8eeefaa314d Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Wed, 11 Mar 2026 10:29:03 +1100 Subject: [PATCH 15/24] Add NOPTA offshore oil and gas data sources NOPTA is the National Offshore Petroleum Title Administrator who manages Australian offshore resource titles. Using their titles and wells datasets we can identify likely offshore locations where oil and gas are being extracted within a prior period. --- .../sectors/oil_gas/data/esri_types.py | 39 ++++++ .../sectors/oil_gas/data/nopta_titles.py | 118 ++++++++++++++++++ .../sectors/oil_gas/data/nopta_wells.py | 110 ++++++++++++++++ 3 files changed, 267 insertions(+) create mode 100644 src/openmethane_prior/sectors/oil_gas/data/esri_types.py create mode 100644 src/openmethane_prior/sectors/oil_gas/data/nopta_titles.py create mode 100644 src/openmethane_prior/sectors/oil_gas/data/nopta_wells.py diff --git a/src/openmethane_prior/sectors/oil_gas/data/esri_types.py b/src/openmethane_prior/sectors/oil_gas/data/esri_types.py new file mode 100644 index 00000000..75bc12af --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/esri_types.py @@ -0,0 +1,39 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import numpy as np + + +def map_esri_date_to_date(esri_date_milliseconds) -> datetime.date | None: + """Convert an esriFieldTypeDate value to UTC datetime.date.""" + if esri_date_milliseconds is None \ + or np.isnan(esri_date_milliseconds) \ + or esri_date_milliseconds <= 0: + return None + + date_value = datetime.datetime.fromtimestamp( + esri_date_milliseconds / 1000, + tz=datetime.timezone.utc, + ) + return date_value.date() + + +def map_esri_date_to_str(esri_date_milliseconds) -> str | None: + """Convert an esriFieldTypeDate value to RFC3339 date string.""" + date = map_esri_date_to_date(esri_date_milliseconds) + return date.isoformat() if date is not None else None diff --git a/src/openmethane_prior/sectors/oil_gas/data/nopta_titles.py b/src/openmethane_prior/sectors/oil_gas/data/nopta_titles.py new file mode 100644 index 00000000..98ee6281 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/nopta_titles.py @@ -0,0 +1,118 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json +import restapi # https://github.com/Bolton-and-Menk-GIS/restapi + +from openmethane_prior.lib import DataSource, ConfiguredDataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo + +from .esri_types import map_esri_date_to_str + + +def fetch_nopta_titles(data_source: ConfiguredDataSource): + nopta_titles = restapi.MapService( + url="https://arcgis.nopta.gov.au/arcgis/rest/services/Public/TitlesCompany_NOPTA/MapServer" + ) + + titles_layer = nopta_titles.layer("Titles and Permits Current") + + layer_features = titles_layer.query( + # where="Type in ('Petroleum','Mineral or Coal') AND Purpose in ('Development','Appraisal', 'Exploration')", + # descriptions of available fields + # https://www.nopta.gov.au/maps-and-public-data/documents/DataDescription_OffshorePetroleumWells.docx + fields=[ + "OBJECTID", + "Title", + "RelTitle", + "TitleType", + "ExpiryDate", + "GrantDate", + "LastReDate", + "NoOfRenews", + "EndDate", + "Status", + "FieldName", + "BasinName", + "SubBasin", + "OffShoreAr", + "TitleOprat", + "TitleHold", + "NoOfBlocks", + "AreaKM2", + "NEATS_Links", + "TITLE_NUMBER_NEATS", + ], + exceed_limit=True, + ) + + for feature in layer_features["features"]: + # convert esriFieldTypeDate to RFC3339 date + for feature_key in feature["properties"]: + if feature_key.endswith("Date"): + feature["properties"][feature_key] = map_esri_date_to_str(feature["properties"][feature_key]) + + with open(data_source.asset_path, "w") as asset_file: + json.dump(layer_features.json, asset_file) + return data_source.asset_path + + +offshore_area_state_mapping = { + "Western Australia": "WA", + "Victoria": "VIC", + "Northern Territory": "NT", + "Queensland": "QLD", + "South Australia": "SA", + "Tasmania": "TAS", + "New South Wales": "NSW", + "ACT": "ACT", + # https://www.infrastructure.gov.au/territories-regions/territories/ashmore-and-cartier-islands + "Territory of Ashmore and Cartier Islands": "NT", + "Outside of Australia": None, + "UNK": None, +} +def map_offshore_area_to_state(offshore_area: str) -> str | None: + if offshore_area in offshore_area_state_mapping: + return offshore_area_state_mapping[offshore_area] + return None + + +def parse_nopta_titles(data_source: ConfiguredDataSource): + df = parse_geo(data_source=data_source) + + # # NOPIMS dataset may record multiple boreholes for a single well, all + # # with identical WellName and location. This will remove duplicate rows + # # so that only a single location is present for each well, keeping the + # # earliest "RigReleaseDate" when the first bore was drilled at the well. + # wells_df = wells_df.sort_values(by="RigReleaseDate") + # wells_df.drop_duplicates(subset="WellName", keep="first", inplace=True) + # + df["state"] = df["OffShoreAr"].map(map_offshore_area_to_state) + + return df + + +# Locations of all offshore petroleum and gas titles (licenses) administered +# by the National Offshore Petroleum Titles Administrator (NOPTA), via the +# National Electronic Approvals Tracking System (NEATS). +# Source: https://www.nopta.gov.au/maps-and-public-data/neats-info.html +nopta_titles_data_source = DataSource( + name="NOPTA-titles", + file_path="NOPTA-titles.geojson", + fetch=fetch_nopta_titles, + parse=parse_nopta_titles, +) diff --git a/src/openmethane_prior/sectors/oil_gas/data/nopta_wells.py b/src/openmethane_prior/sectors/oil_gas/data/nopta_wells.py new file mode 100644 index 00000000..6de3ca1b --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/nopta_wells.py @@ -0,0 +1,110 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json +import restapi # https://github.com/Bolton-and-Menk-GIS/restapi + +from openmethane_prior.lib import DataSource, ConfiguredDataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo + +from .esri_types import map_esri_date_to_str + + +def fetch_nopta_wells(data_source: ConfiguredDataSource): + # nopta_wells_arcgis = restapi.ArcServer(url="https://arcgis.nopta.gov.au/arcgis/rest/services") + nopta_wells = restapi.MapService( + url="https://arcgis.nopta.gov.au/arcgis/rest/services/Public/Petroleum_Wells/MapServer" + ) + + petroleum_wells_layer = nopta_wells.layer("Petroleum Wells") + + layer_features = petroleum_wells_layer.query( + where="Type in ('Petroleum','Mineral or Coal') AND Purpose in ('Development','Appraisal', 'Exploration')", + # descriptions of available fields + # https://www.nopta.gov.au/maps-and-public-data/documents/DataDescription_OffshorePetroleumWells.docx + fields=[ + "WellName", + "OffshoreArea", + "Jurisdiction", + "TitleNumber", + "Type", + "Purpose", + # Date when the rig finished drilling and was moved off the site, + # we can use this as a guess for when emissions may have started + # if no other info is available. + "RigReleaseDate", + ], + exceed_limit=True, + ) + + for feature in layer_features["features"]: + # convert esriFieldTypeDate to RFC3339 date + feature["properties"]["RigReleaseDate"] = map_esri_date_to_str(feature["properties"]["RigReleaseDate"]) + + with open(data_source.asset_path, "w") as asset_file: + json.dump(layer_features.json, asset_file) + return data_source.asset_path + + +offshore_area_state_mapping = { + "Western Australia": "WA", + "Victoria": "VIC", + "Northern Territory": "NT", + "Queensland": "QLD", + "South Australia": "SA", + "Tasmania": "TAS", + "New South Wales": "NSW", + "ACT": "ACT", + # https://www.infrastructure.gov.au/territories-regions/territories/ashmore-and-cartier-islands + "Territory of Ashmore and Cartier Islands": "NT", + "Outside of Australia": None, + "UNK": None, +} +def map_offshore_area_to_state(offshore_area: str) -> str | None: + if offshore_area in offshore_area_state_mapping: + return offshore_area_state_mapping[offshore_area] + return None + + +def parse_nopta_wells(data_source: ConfiguredDataSource): + wells_df = parse_geo(data_source=data_source) + + # NOPIMS dataset may record multiple boreholes for a single well, all + # with identical WellName and location. This will remove duplicate rows + # so that only a single location is present for each well, keeping the + # earliest "RigReleaseDate" when the first bore was drilled at the well. + wells_df = wells_df.sort_values(by="RigReleaseDate") + wells_df.drop_duplicates(subset="WellName", keep="first", inplace=True) + + wells_df["state"] = wells_df["OffshoreArea"].map(map_offshore_area_to_state) + + # 3D points provided by NOPIMS are unnecessary for our purposes + wells_df["geometry"] = wells_df["geometry"].force_2d() + + return wells_df + + +# Locations of all wells administered by the National Offshore Petroleum +# Titles Administrator (NOPTA), via the National Offshore Petroleum Information +# Management Systems (NOPIMS). +# Source: https://www.nopta.gov.au/maps-and-public-data/nopims-info.html +nopta_wells_data_source = DataSource( + name="NOPTA-wells", + file_path="NOPTA-wells.geojson", + fetch=fetch_nopta_wells, + parse=parse_nopta_wells, +) From 73edb433e3870d9806193302cb0ef258307b528a Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Mon, 16 Mar 2026 08:54:17 +1100 Subject: [PATCH 16/24] Add NSW oil & gas emission sources from petroleum drillholes Combine NSW drillholes dataset with petroleum leases dataset to extract producing wells with an activity period matching the period of interest. --- src/openmethane_prior/lib/utils.py | 22 +++++ .../sectors/oil_gas/data/nsw_drillholes.py | 84 +++++++++++++++++ .../sectors/oil_gas/data/nsw_titles.py | 69 ++++++++++++++ .../oil_gas/emission_sources/nsw_sources.py | 94 +++++++++++++++++++ 4 files changed, 269 insertions(+) create mode 100644 src/openmethane_prior/sectors/oil_gas/data/nsw_drillholes.py create mode 100644 src/openmethane_prior/sectors/oil_gas/data/nsw_titles.py create mode 100644 src/openmethane_prior/sectors/oil_gas/emission_sources/nsw_sources.py diff --git a/src/openmethane_prior/lib/utils.py b/src/openmethane_prior/lib/utils.py index 3404b325..1661eee5 100644 --- a/src/openmethane_prior/lib/utils.py +++ b/src/openmethane_prior/lib/utils.py @@ -29,6 +29,8 @@ import typing import numpy as np +import pandas as pd +import geopandas as gpd from numpy.typing import ArrayLike from urllib.parse import urlparse import xarray as xr @@ -196,3 +198,23 @@ def is_url(maybe_url: str) -> bool: """ parsed = urlparse(maybe_url) return parsed.scheme != "" and parsed.netloc != "" + + +DF = typing.TypeVar("DF", bound=pd.DataFrame | gpd.GeoDataFrame) +def rows_in_period( + df: DF, + start_date: datetime.date, + end_date: datetime.date, + start_field: str = "start_date", + end_field: str = "end_date", +) -> DF: + """Return the rows of the DataFrame df where the period between df[start_field] + and df[end_field] overlaps with the period between start_date and end_date.""" + midnight = datetime.time(0, 0) + period_start = datetime.datetime.combine(date=start_date, time=midnight) + period_end = datetime.datetime.combine(date=end_date + datetime.timedelta(days=1), time=midnight) + + return df[ + (df[start_field] <= np.datetime64(period_end)) + & ((np.isnat(df[end_field])) | (df[end_field] >= np.datetime64(period_start))) + ] diff --git a/src/openmethane_prior/sectors/oil_gas/data/nsw_drillholes.py b/src/openmethane_prior/sectors/oil_gas/data/nsw_drillholes.py new file mode 100644 index 00000000..8103eff4 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/nsw_drillholes.py @@ -0,0 +1,84 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import geopandas as gpd +import json +import pandas as pd +from owslib.wfs import WebFeatureService + +from openmethane_prior.lib import DataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo +from openmethane_prior.lib.data_manager.source import ConfiguredDataSource + + +def fetch_nsw_drillholes(data_source: ConfiguredDataSource): + geoserver_wfs = WebFeatureService("https://public-gs.geoscience.nsw.gov.au/geoserver/wfs", version="2.0.0") + + desired_properties = [ + "program", + "hole_name", + "title", + "reports", + "year_drilled", + "licence_holder", + "operator", + "business_purpose", + "hole_purpose", + "well_status", + "project", + "site_id", + "geom", + ] + + nsw_drillholes_feature_csg = geoserver_wfs.getfeature( + typename="drilling:drilling_drillholes_csg", + srsname="urn:ogc:def:crs:EPSG::4326", + propertyname=desired_properties, + outputFormat="application/json", + ) + + nsw_drillholes_feature_petroleum = geoserver_wfs.getfeature( + typename="drilling:drilling_drillholes_petroleum", + srsname="urn:ogc:def:crs:EPSG::4326", + propertyname=desired_properties, + outputFormat="application/json", + ) + + features_df = pd.concat([ + gpd.GeoDataFrame.from_features(json.loads(nsw_drillholes_feature_csg.read())), + gpd.GeoDataFrame.from_features(json.loads(nsw_drillholes_feature_petroleum.read())), + ]) + + # only include drillholes relevant to CSG and petroleum production + features_df = features_df[features_df["business_purpose"].isin(["Coal seam methane", "Petroleum"])] + features_df = features_df[features_df["hole_purpose"].isin(["Production"])] + + with open(data_source.asset_path, "w") as asset_file: + asset_file.write(features_df.to_json()) + return data_source.asset_path + + +# Locations of coal seam gas and petroleum production drillholes in the +# Australian state of New South Wales, via Geosciences NSW. +# Source: https://data.nsw.gov.au/data/dataset/coal-seam-gas-borehole +# Source: https://data.nsw.gov.au/data/dataset/nsw-drillholes-petroleum +nsw_drillholes_data_source = DataSource( + name="nsw-drillholes-csg-petroleum", + file_path="NSW-drillholes-csg-petroleum.geojson", + fetch=fetch_nsw_drillholes, + parse=parse_geo, +) diff --git a/src/openmethane_prior/sectors/oil_gas/data/nsw_titles.py b/src/openmethane_prior/sectors/oil_gas/data/nsw_titles.py new file mode 100644 index 00000000..0e3f959f --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/nsw_titles.py @@ -0,0 +1,69 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import geopandas as gpd +import json +from owslib.wfs import WebFeatureService + +from openmethane_prior.lib import DataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo +from openmethane_prior.lib.data_manager.source import ConfiguredDataSource + + +def fetch_nsw_titles(data_source: ConfiguredDataSource): + geoserver_wfs = WebFeatureService("https://public-gs.geoscience.nsw.gov.au/geoserver/wfs", version="2.0.0") + + desired_properties = [ + "tas_id", + "title", + "holder", + "company", + "grant_date", + "expiry_date", + "minerals", + "resource", + "operation", + "geom", + ] + + nsw_titles_feature = geoserver_wfs.getfeature( + typename="mining-and-exploration:titles_title_granted", + srsname="urn:ogc:def:crs:EPSG::4326", + propertyname=desired_properties, + outputFormat="application/json", + ) + + features_df = gpd.GeoDataFrame.from_features(json.loads(nsw_titles_feature.read())) + + # only include titles relevant to petroleum production + features_df = features_df[features_df["resource"] == "PETROLEUM"] + features_df = features_df[features_df["operation"] == "MINING"] + + with open(data_source.asset_path, "w") as asset_file: + asset_file.write(features_df.to_json()) + return data_source.asset_path + + +# Locations of coal seam gas and petroleum production titles in the +# Australian state of New South Wales, via Geosciences NSW. +# Source: https://data.nsw.gov.au/data/dataset/nsw-mining-titles +nsw_titles_data_source = DataSource( + name="nsw-titles-csg-petroleum", + file_path="NSW-titles-csg-petroleum.geojson", + fetch=fetch_nsw_titles, + parse=parse_geo, +) diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/nsw_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/nsw_sources.py new file mode 100644 index 00000000..59d7a83a --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/nsw_sources.py @@ -0,0 +1,94 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import geopandas as gpd + +from openmethane_prior.lib.data_manager.asset import DataAsset +from openmethane_prior.lib.utils import rows_in_period +from openmethane_prior.sectors.oil_gas.emission_sources.emission_source import normalise_emission_source_df + +nsw_drillhole_purpose_map = { + "Coal seam methane": "drillhole-csg", + "Petroleum": "drillhole-petroleum", +} + +def nsw_emission_sources( + start_date: datetime.date, + end_date: datetime.date, + nsw_drillholes_da: DataAsset, + nsw_titles_da: DataAsset, +) -> gpd.GeoDataFrame: + """Create normalised emission source DataFrame by combining NSW petroleum + drillhole dataset for locations, with land title dataset for production + start/end dates.""" + nsw_drillholes_df: gpd.GeoDataFrame = nsw_drillholes_da.data + nsw_titles_df: gpd.GeoDataFrame = nsw_titles_da.data + + drillholes_df = nsw_drillholes_df[["geometry", "gsnsw_drill_id", "year_drilled", "business_purpose"]] + + # no specific date, so we'll guess from Jan 1st of the specified year + drillholes_df["drilled_date"] = drillholes_df["year_drilled"].map( + lambda year: datetime.datetime(int(year), 1, 1, 0, 0, 0) + ) + del drillholes_df["year_drilled"] + + # map from PPL4 to PPL0004 to match formatting in drillholes dataset + nsw_titles_df["title"] = nsw_titles_df["title"].map( + lambda title: f"{title[:3]}{int(title[3:]):04d}" + ) + + # join drillholes with titles to use the title dates as start/end dates + sources_df = gpd.sjoin(drillholes_df, nsw_titles_df, how="inner", predicate="within") + + # start date of emissions must be after hole is drilled and after the title + # is granted, so choose the later of the two dates + sources_df["start_date"] = [ + granted if granted > drilled else drilled + for granted, drilled in sources_df[["grant_date", "drilled_date"]].values + ] + del sources_df["drilled_date"] + del sources_df["grant_date"] + + # NSW drill holes are either coal seam methane or petroleum + sources_df["site_type"] = sources_df["business_purpose"].map( + lambda purpose: nsw_drillhole_purpose_map[purpose] + ) + del sources_df["business_purpose"] + + # Best guess at an end date for a drillhole is the expiry of the petroleum + # title. Although this is very inaccurate, we don't have any better public + # data source that includes the date when wells are capped/expended. + sources_df = sources_df.rename(columns={"expiry_date": "end_date"}) + + # exclude any emission sources that would not have been emitting during + # the period between start_date and end_date + sources_df = rows_in_period(sources_df, start_date=start_date, end_date=end_date) + + # drop duplicates so there is only one entry for each location. + sources_df = sources_df.drop_duplicates(["gsnsw_drill_id"]) + + # normalise output to match emission sources format + sources_df = sources_df.rename(columns={ + "gsnsw_drill_id": "data_source_id", + "title": "group_id", + "start_date": "activity_start", + "end_date": "activity_end", + }) + sources_df["data_source"] = nsw_drillholes_da.name + + return normalise_emission_source_df(sources_df) From b043d2883a17ce31e37b54a1861451f9502306eb Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Mon, 16 Mar 2026 11:48:10 +1100 Subject: [PATCH 17/24] Add WA oil & gas emission sources from petroleum wells Combine WA petroleum wells dataset with petroleum titles dataset to extract producing wells with an activity period matching the title period. --- .../sectors/oil_gas/data/wa_titles.py | 96 +++++++++++++++++++ .../sectors/oil_gas/data/wa_wells.py | 65 +++++++++++++ .../oil_gas/emission_sources/wa_sources.py | 76 +++++++++++++++ 3 files changed, 237 insertions(+) create mode 100644 src/openmethane_prior/sectors/oil_gas/data/wa_titles.py create mode 100644 src/openmethane_prior/sectors/oil_gas/data/wa_wells.py create mode 100644 src/openmethane_prior/sectors/oil_gas/emission_sources/wa_sources.py diff --git a/src/openmethane_prior/sectors/oil_gas/data/wa_titles.py b/src/openmethane_prior/sectors/oil_gas/data/wa_titles.py new file mode 100644 index 00000000..efe60ce1 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/wa_titles.py @@ -0,0 +1,96 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import geopandas as gpd +import pandas as pd +import restapi # https://github.com/Bolton-and-Menk-GIS/restapi + +from openmethane_prior.lib import DataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo +from openmethane_prior.lib.data_manager.source import ConfiguredDataSource +from .esri_types import map_esri_date_to_str + + +# WA Petroleum Titles (DMIRS-011) - REST Service (ArcGIS) +# https://catalogue.data.wa.gov.au/dataset/wa-petroleum-titles-dmirs-011/resource/f5bb4e83-241e-4dcf-8efe-41707866af4e +def fetch_wa_titles(data_source: ConfiguredDataSource): + # wa_arcgis = restapi.ArcServer(url="https://public-services.slip.wa.gov.au/public/rest/services") + wa_arcgis_mining = restapi.MapService( + url="https://public-services.slip.wa.gov.au/public/rest/services/SLIP_Public_Services/Industry_and_Mining/MapServer" + ) + + # DMIRS-011 contains current active petroleum titles, but has expired + # titles periodically removed. Our interest is only in "Production License" + # titles, as we wouldn't expect significant emissions during exploration, + # or if the parcel of land is on a "retention" lease. + titles_layer = wa_arcgis_mining.layer("WA Petroleum Titles (DMIRS-011)") + titles_features = titles_layer.query( + where="type in ('Production Licence')", + fields=[ + "title_id", + "type", + "issued", + "expiry", + ], + exceed_limit=True, + ) + titles_df = gpd.GeoDataFrame.from_features(titles_features.features) + + # Rename some fields so that they match DMIRS-051 + titles_df = titles_df.rename(columns={ + "issued": "issued_date", + "expiry": "end_date", + }) + + # DMIRS-051 contains the record of previous titles. One parcel of land + # can appear in this dataset multiple times as titles change. Sometimes a + # single parcel can have multiple titles with the same dates or overlapping + # dates. No attempt is made to filter or de-duplicate these at this stage. + historic_titles_layer = wa_arcgis_mining.layer("WA Petroleum - Historical Titles (DMIRS-051)") + historic_titles_features = historic_titles_layer.query( + where="type in ('Production Licence')", + fields=[ + "title_id", + "type", + "issued_date", + "end_date", + ], + exceed_limit=True, + ) + historic_df = gpd.GeoDataFrame.from_features(historic_titles_features.features) + + # combine current and historic title datasets + df = pd.concat([titles_df, historic_df]) + + # convert date fields to RFC3339 + df["issued_date"] = df["issued_date"].map(map_esri_date_to_str) + df["end_date"] = df["end_date"].map(map_esri_date_to_str) + + with open(data_source.asset_path, "w") as asset_file: + asset_file.write(df.to_json()) + return data_source.asset_path + + +# Locations of all petroleum titles in the Australian state of Western +# Australia via the Data WA Portal. +# Source: https://catalogue.data.wa.gov.au/dataset/wa-petroleum-titles-dmirs-011 +wa_titles_data_source = DataSource( + name="wa-petroleum-titles", + file_path="WA-petroleum-titles.geojson", + fetch=fetch_wa_titles, + parse=parse_geo, +) diff --git a/src/openmethane_prior/sectors/oil_gas/data/wa_wells.py b/src/openmethane_prior/sectors/oil_gas/data/wa_wells.py new file mode 100644 index 00000000..c0b4b135 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/wa_wells.py @@ -0,0 +1,65 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import geopandas as gpd +import restapi # https://github.com/Bolton-and-Menk-GIS/restapi + +from openmethane_prior.lib import DataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo +from openmethane_prior.lib.data_manager.source import ConfiguredDataSource +from .esri_types import map_esri_date_to_str + + +# WA Petroleum Wells (DMIRS-025) - REST Service (ArcGIS) +# https://catalogue.data.wa.gov.au/dataset/mineral-exploration-drillholes-open-file/resource/1c61171a-3b23-4f2b-baae-04615c5bb39e +def fetch_wa_wells(data_source: ConfiguredDataSource): + # wa_arcgis = restapi.ArcServer(url="https://public-services.slip.wa.gov.au/public/rest/services") + wa_arcgis_mining = restapi.MapService( + url="https://public-services.slip.wa.gov.au/public/rest/services/SLIP_Public_Services/Industry_and_Mining/MapServer" + ) + + wells_layer = wa_arcgis_mining.layer("WA Onshore Petroleum Wells (DMIRS-025)") + wells_features = wells_layer.query( + fields=[ + "well_name", + "uwi", + "lease_no", + "operator", + "class", + "status", + "rig_release_date", + ], + exceed_limit=True, + ) + + df = gpd.GeoDataFrame.from_features(wells_features.features) + df["rig_release_date"] = df["rig_release_date"].map(map_esri_date_to_str) + + with open(data_source.asset_path, "w") as asset_file: + asset_file.write(df.to_json()) + return data_source.asset_path + + +# Locations of all petroleum wells in the Australian state of Western Australia +# via the Data WA Portal. +# Source: https://catalogue.data.wa.gov.au/dataset/wa-onshore-petroleum-wells-dmirs-025 +wa_wells_data_source = DataSource( + name="wa-petroleum-wells", + file_path="WA-petroleum-wells.geojson", + fetch=fetch_wa_wells, + parse=parse_geo, +) diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/wa_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/wa_sources.py new file mode 100644 index 00000000..0ab81cd2 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/wa_sources.py @@ -0,0 +1,76 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import geopandas as gpd +import numpy as np + +from openmethane_prior.lib.data_manager.asset import DataAsset +from openmethane_prior.lib.utils import rows_in_period +from openmethane_prior.sectors.oil_gas.emission_sources.emission_source import normalise_emission_source_df + +def wa_emission_sources( + start_date: datetime.date, + end_date: datetime.date, + wa_wells_da: DataAsset, + wa_titles_da: DataAsset, +) -> gpd.GeoDataFrame: + """Create normalised emission source DataFrame by combining WA petroleum + wells dataset for locations, with land title dataset for production + start/end dates.""" + wa_wells_df: gpd.GeoDataFrame = wa_wells_da.data + wa_titles_df: gpd.GeoDataFrame = wa_titles_da.data + + # filter out wells that aren't actively used for production + wa_wells_df = wa_wells_df[wa_wells_df["class"] == "DEV"] + + wells_df = wa_wells_df[["uwi", "rig_release_date", "geometry"]] + + # join drillholes with titles to use the title dates as start/end dates + titles_df = wa_titles_df[["title_id", "issued_date", "end_date", "geometry"]] + sources_df = gpd.sjoin(wells_df, titles_df, how="inner", predicate="within") + + # start date of emissions must be after hole is drilled and after the title + # is granted, so choose the latter of the two dates + sources_df["start_date"] = [ + issued if not np.isnat(issued) and (np.isnat(drilled) or issued > drilled) else drilled + for issued, drilled in sources_df[["issued_date", "rig_release_date"]].values + ] + del sources_df["issued_date"] + del sources_df["rig_release_date"] + + # exclude any emission sources that would not have been emitting during + # the period between start_date and end_date + sources_df = rows_in_period(sources_df, start_date=start_date, end_date=end_date) + + # after the join with titles, there may be multiple rows for each well. + # drop duplicates so there is only one entry for each location. + sources_df = sources_df.drop_duplicates(["uwi"]) + + # normalise output to match emission sources format + sources_df = sources_df.rename(columns={ + "uwi": "data_source_id", + "title_id": "group_id", + "start_date": "activity_start", + "end_date": "activity_end", + }) + sources_df["data_source"] = wa_wells_da.name + # not enough detail in the data source to determine the type of resource + # being extracted by the well + sources_df["site_type"] = "drillhole-unknown" + + return normalise_emission_source_df(sources_df) From 6b04862869d1ae99219893f6984de8adfbde8028 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 19 Mar 2026 14:46:01 +1100 Subject: [PATCH 18/24] Add QLD oil & gas emission sources from boreholes and petroleum titles Combine QLD boreholes dataset with petroleum leases dataset to extract producing wells with an activity period matching the lease period. --- .../sectors/oil_gas/data/qld_boreholes.py | 10 +- .../sectors/oil_gas/data/qld_leases.py | 94 +++++++++++++++++++ .../oil_gas/emission_sources/qld_sources.py | 85 +++++++++++++++++ 3 files changed, 187 insertions(+), 2 deletions(-) create mode 100644 src/openmethane_prior/sectors/oil_gas/data/qld_leases.py create mode 100644 src/openmethane_prior/sectors/oil_gas/emission_sources/qld_sources.py diff --git a/src/openmethane_prior/sectors/oil_gas/data/qld_boreholes.py b/src/openmethane_prior/sectors/oil_gas/data/qld_boreholes.py index b0224543..c5b32c9f 100644 --- a/src/openmethane_prior/sectors/oil_gas/data/qld_boreholes.py +++ b/src/openmethane_prior/sectors/oil_gas/data/qld_boreholes.py @@ -15,13 +15,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import geopandas as gpd import numpy as np import restapi # https://github.com/Bolton-and-Menk-GIS/restapi -import json from openmethane_prior.lib import DataSource from openmethane_prior.lib.data_manager.parsers import parse_geo from openmethane_prior.lib.data_manager.source import ConfiguredDataSource +from openmethane_prior.sectors.oil_gas.data.esri_types import map_esri_date_to_str # Queensland borehole series - REST Service (ArcGIS) @@ -62,14 +63,19 @@ def fetch_qld_boreholes(data_source: ConfiguredDataSource): else: boreholes_features.features.extend(layer_features.features) + df = gpd.GeoDataFrame.from_features(boreholes_features.features) + df["rig_release_date"] = df["rig_release_date"].map(map_esri_date_to_str) + with open(data_source.asset_path, "w") as asset_file: - json.dump(boreholes_features.json, asset_file) + asset_file.write(df.to_json()) return data_source.asset_path def parse_qld_boreholes(data_source: ConfiguredDataSource): boreholes_df = parse_geo(data_source=data_source) + # + # lease/tenement name, i.e. "PL 100", is more useful than "PL" and 100.0, # so combine tenure_no and tenure_type into a single string boreholes_df["tenure"] = [ diff --git a/src/openmethane_prior/sectors/oil_gas/data/qld_leases.py b/src/openmethane_prior/sectors/oil_gas/data/qld_leases.py new file mode 100644 index 00000000..c3236685 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/qld_leases.py @@ -0,0 +1,94 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import geopandas as gpd +import pandas as pd +import restapi # https://github.com/Bolton-and-Menk-GIS/restapi + +from openmethane_prior.lib import DataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo +from openmethane_prior.lib.data_manager.source import ConfiguredDataSource +from openmethane_prior.sectors.oil_gas.data.esri_types import map_esri_date_to_str + + +# Petroleum leases - Queensland - REST Service (ArcGIS) +# https://www.data.qld.gov.au/dataset/queensland-mining-and-exploration-tenure-series/resource/544ecb39-9c25-42f9-9928-b44c7ff05b30 +def fetch_qld_leases(data_source: ConfiguredDataSource): + # qld_spatial_arcgis = restapi.ArcServer(url="https://spatial-gis.information.qld.gov.au/arcgis/rest/services") + qld_current_leases = restapi.MapService( + url="https://spatial-gis.information.qld.gov.au/arcgis/rest/services/Economy/MinesPermitsCurrent/MapServer" + ) + qld_historic_leases = restapi.MapService( + url="https://spatial-gis.information.qld.gov.au/arcgis/rest/services/Economy/MinesPermitsHistoric/MapServer" + ) + + fields = [ + "displayname", + "permittype", + "permitstatus", + "approvedate", + "expirydate", + "permitminerals", + "permitpurpose", + ] + + leases_layer = qld_current_leases.layer("PL granted") + leases_features = leases_layer.query( + fields=fields, + exceed_limit=True, + ) + historic_layer = qld_historic_leases.layer("Historical petroleum lease") + historic_features = historic_layer.query( + fields=fields, + exceed_limit=True, + ) + + df = pd.concat([ + gpd.GeoDataFrame.from_features(leases_features.features), + gpd.GeoDataFrame.from_features(historic_features.features), + ]) + + df["approvedate"] = df["approvedate"].map(map_esri_date_to_str) + df["expirydate"] = df["expirydate"].map(map_esri_date_to_str) + + with open(data_source.asset_path, "w") as asset_file: + asset_file.write(df.to_json()) + return data_source.asset_path + + + +def parse_qld_leases(data_source: ConfiguredDataSource): + leases_df = parse_geo(data_source=data_source) + + # lease/tenement name, i.e. "PL 100", is more useful than "PL" and 100.0, + # so combine tenure_no and tenure_type into a single string + # leases_df["tenure"] = [ + # None if t_no is np.nan else f"{t_type} {t_no:0.0f}" + # for t_no, t_type in leases_df[["tenure_no", "tenure_type"]].values + # ] + + return leases_df + +# Locations of all mining leases in the Australian state of Queensland, via the +# Queensland Open Data Portal. +# Source: https://www.data.qld.gov.au/dataset/queensland-mining-and-exploration-tenure-series +qld_leases_data_source = DataSource( + name="qld-leases", + file_path="QLD-leases.geojson", + fetch=fetch_qld_leases, + parse=parse_qld_leases, +) diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/qld_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/qld_sources.py new file mode 100644 index 00000000..130e4244 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/qld_sources.py @@ -0,0 +1,85 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import geopandas as gpd +import numpy as np + +from openmethane_prior.lib.data_manager.asset import DataAsset +from openmethane_prior.lib.utils import rows_in_period +from openmethane_prior.sectors.oil_gas.emission_sources.emission_source import normalise_emission_source_df + +bore_type_map = { + "COAL SEAM GAS": "drillhole-csg", + "PETROLEUM": "drillhole-petroleum", + "UNCONVENTIONAL PETROLEUM": "drillhole-unknown", +} + +def qld_emission_sources( + start_date: datetime.date, + end_date: datetime.date, + qld_boreholes_da: DataAsset, + qld_leases_da: DataAsset, +) -> gpd.GeoDataFrame: + """Create normalised emission source DataFrame by combining WA petroleum + wells dataset for locations, with land title dataset for production + start/end dates.""" + qld_boreholes_df: gpd.GeoDataFrame = qld_boreholes_da.data + qld_leases_df: gpd.GeoDataFrame = qld_leases_da.data + + # filter non-production and storage wells + emitting_bore_types = ["COAL SEAM GAS", "PETROLEUM", "UNCONVENTIONAL PETROLEUM", "GREENHOUSE GAS STORAGE"] + qld_boreholes_df = qld_boreholes_df[qld_boreholes_df["bore_type"].isin(emitting_bore_types)] + emitting_bore_subtypes = ["DEVELOPMENT WELL", "COAL SEAM GAS INJECTION WELL", "PETROLEUM INJECTION WELL"] + qld_boreholes_df = qld_boreholes_df[qld_boreholes_df["bore_subtype"].isin(emitting_bore_subtypes)] + + boreholes_df = qld_boreholes_df[["borehole_pid", "bore_type", "rig_release_date", "geometry"]] + + # join boreholes with titles to use the title dates as start/end dates + titles_df = qld_leases_df[["displayname", "approvedate", "expirydate", "geometry"]] + sources_df = gpd.sjoin(boreholes_df, titles_df, how="inner", predicate="within") + + # start date of emissions must be after hole is drilled and after the title + # is granted, so choose the latter of the two dates + sources_df["start_date"] = [ + issued if not np.isnat(issued) and (np.isnat(drilled) or issued > drilled) else drilled + for issued, drilled in sources_df[["approvedate", "rig_release_date"]].values + ] + del sources_df["approvedate"] + del sources_df["rig_release_date"] + + # exclude any emission sources that would not have been emitting during + # the period between start_date and end_date + sources_df = rows_in_period(sources_df, start_date=start_date, end_date=end_date, end_field="expirydate") + + # after the join with titles, there may be multiple rows for each well. + # drop duplicates so there is only one entry for each location. + sources_df = sources_df.drop_duplicates(["borehole_pid"]) + + # normalise output to match emission sources format + sources_df = sources_df.rename(columns={ + "borehole_pid": "data_source_id", + "displayname": "group_id", + "start_date": "activity_start", + "expirydate": "activity_end", + }) + sources_df["data_source"] = qld_boreholes_da.name + sources_df["site_type"] = sources_df["bore_type"].map( + lambda bore_type: bore_type_map[bore_type] if bore_type in bore_type_map else "drillhole-unknown" + ) + + return normalise_emission_source_df(sources_df) From 0df254dff35c3c3c3df780e1702a733bcd08ad64 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 19 Mar 2026 11:41:14 +1100 Subject: [PATCH 19/24] Add notebook to visualise oil and gas emission sources --- notebooks/oil_gas_locations.py | 36 ++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 notebooks/oil_gas_locations.py diff --git a/notebooks/oil_gas_locations.py b/notebooks/oil_gas_locations.py new file mode 100644 index 00000000..694fd671 --- /dev/null +++ b/notebooks/oil_gas_locations.py @@ -0,0 +1,36 @@ +import datetime +import geopandas as gpd +import matplotlib.pyplot as plt +import os +import pathlib +from openmethane_prior.lib import PriorConfig +from openmethane_prior.lib.data_manager.manager import DataManager +from openmethane_prior.sectors.oil_gas.data.wa_titles import wa_titles_data_source +from openmethane_prior.sectors.oil_gas.data.wa_wells import wa_wells_data_source +from openmethane_prior.sectors.oil_gas.emission_sources.wa_sources import wa_emission_sources +os.environ["START_DATE"] = "2023-07-01" +os.environ["END_DATE"] = "2023-07-01" +prior_config = PriorConfig.from_env() +data_manager = DataManager( + data_path=pathlib.Path("../data"), + prior_config=prior_config, +) + +wa_wells_df = data_manager.get_asset(wa_wells_data_source) +wa_titles_df = data_manager.get_asset(wa_titles_data_source) +df = wa_emission_sources( + start_date=datetime.date(2021, 1, 1), + end_date=datetime.date(2023, 1, 2), + wa_wells_df=wa_wells_df.data, + wa_titles_df=wa_titles_df.data, +) +df + +au_shp = gpd.GeoDataFrame.from_file("~/Downloads/STE_2021_AUST_SHP_GDA2020/STE_2021_AUST_GDA2020.shx") +au_map = au_shp.plot(color='white', edgecolor='#CCCCCC', figsize=(32,24)) +wa_titles_df.data.plot(ax=au_map, alpha=0.5) +# wa_wells_df.data.plot(ax=au_map, markersize=3, column="class", legend=True) +df.plot(ax=au_map, markersize=3, column="class", legend=True) +au_map + + From 8041684c487d03f349bcce747a0961926e935691 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 19 Mar 2026 16:10:19 +1100 Subject: [PATCH 20/24] Visualise oil and gas emission sources --- notebooks/oil_gas_locations.py | 46 ++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/notebooks/oil_gas_locations.py b/notebooks/oil_gas_locations.py index 694fd671..d1f7285f 100644 --- a/notebooks/oil_gas_locations.py +++ b/notebooks/oil_gas_locations.py @@ -1,36 +1,60 @@ import datetime import geopandas as gpd -import matplotlib.pyplot as plt import os +import pandas as pd import pathlib from openmethane_prior.lib import PriorConfig from openmethane_prior.lib.data_manager.manager import DataManager +from openmethane_prior.sectors.oil_gas.data.nsw_drillholes import nsw_drillholes_data_source +from openmethane_prior.sectors.oil_gas.data.nsw_titles import nsw_titles_data_source +from openmethane_prior.sectors.oil_gas.data.qld_boreholes import qld_boreholes_data_source +from openmethane_prior.sectors.oil_gas.data.qld_leases import qld_leases_data_source from openmethane_prior.sectors.oil_gas.data.wa_titles import wa_titles_data_source from openmethane_prior.sectors.oil_gas.data.wa_wells import wa_wells_data_source +from openmethane_prior.sectors.oil_gas.emission_sources.nsw_sources import nsw_emission_sources +from openmethane_prior.sectors.oil_gas.emission_sources.qld_sources import qld_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.wa_sources import wa_emission_sources os.environ["START_DATE"] = "2023-07-01" os.environ["END_DATE"] = "2023-07-01" prior_config = PriorConfig.from_env() data_manager = DataManager( - data_path=pathlib.Path("../data"), + data_path=pathlib.Path("../data/inputs"), prior_config=prior_config, ) -wa_wells_df = data_manager.get_asset(wa_wells_data_source) -wa_titles_df = data_manager.get_asset(wa_titles_data_source) -df = wa_emission_sources( +nsw_drillholes_da = data_manager.get_asset(nsw_drillholes_data_source) +nsw_titles_da = data_manager.get_asset(nsw_titles_data_source) +nsw_df = nsw_emission_sources( start_date=datetime.date(2021, 1, 1), end_date=datetime.date(2023, 1, 2), - wa_wells_df=wa_wells_df.data, - wa_titles_df=wa_titles_df.data, + nsw_drillholes_da=nsw_drillholes_da, + nsw_titles_da=nsw_titles_da, ) -df +qld_boreholes_da = data_manager.get_asset(qld_boreholes_data_source) +qld_leases_da = data_manager.get_asset(qld_leases_data_source) +qld_df = qld_emission_sources( + start_date=datetime.date(2021, 1, 1), + end_date=datetime.date(2023, 1, 2), + qld_boreholes_da=qld_boreholes_da, + qld_leases_da=qld_leases_da, +) +wa_wells_da = data_manager.get_asset(wa_wells_data_source) +wa_titles_da = data_manager.get_asset(wa_titles_data_source) +wa_df = wa_emission_sources( + start_date=datetime.date(2021, 1, 1), + end_date=datetime.date(2023, 1, 2), + wa_wells_da=wa_wells_da, + wa_titles_da=wa_titles_da, +) +all_df = pd.concat([nsw_df, qld_df, wa_df]) au_shp = gpd.GeoDataFrame.from_file("~/Downloads/STE_2021_AUST_SHP_GDA2020/STE_2021_AUST_GDA2020.shx") au_map = au_shp.plot(color='white', edgecolor='#CCCCCC', figsize=(32,24)) -wa_titles_df.data.plot(ax=au_map, alpha=0.5) -# wa_wells_df.data.plot(ax=au_map, markersize=3, column="class", legend=True) -df.plot(ax=au_map, markersize=3, column="class", legend=True) +nsw_titles_da.data.plot(ax=au_map, alpha=0.5) +qld_leases_da.data.plot(ax=au_map, alpha=0.5) +# wa_wells_da.data.plot(ax=au_map, markersize=3, column="class", legend=True) +wa_titles_da.data.plot(ax=au_map, alpha=0.5) +all_df.plot(ax=au_map, markersize=3, column="site_type", legend=True) au_map From 8d7d6996511f0e46913154db5eae7f4f81167175 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 19 Mar 2026 17:19:29 +1100 Subject: [PATCH 21/24] Add offshore sources --- .../emission_sources/offshore_sources.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 src/openmethane_prior/sectors/oil_gas/emission_sources/offshore_sources.py diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/offshore_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/offshore_sources.py new file mode 100644 index 00000000..871b16ef --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/offshore_sources.py @@ -0,0 +1,77 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import geopandas as gpd +import numpy as np + +from openmethane_prior.lib.data_manager.asset import DataAsset +from openmethane_prior.lib.utils import rows_in_period +from openmethane_prior.sectors.oil_gas.emission_sources.emission_source import normalise_emission_source_df + +def offshore_emission_sources( + start_date: datetime.date, + end_date: datetime.date, + offshore_wells_da: DataAsset, + offshore_titles_da: DataAsset, +) -> gpd.GeoDataFrame: + """Create normalised emission source DataFrame by combining WA petroleum + wells dataset for locations, with land title dataset for production + start/end dates.""" + offshore_wells_df: gpd.GeoDataFrame = offshore_wells_da.data + offshore_titles_df: gpd.GeoDataFrame = offshore_titles_da.data + + # # filter out wells that aren't actively used for production + # offshore_wells_df = offshore_wells_df[offshore_wells_df["class"] == "DEV"] + wells_df = offshore_wells_df[["WellName", "Type", "Purpose", "RigReleaseDate", "geometry"]] + + # emissions are expected where production is ongoing, unlike exploration or retention + titles_df = offshore_titles_df[offshore_titles_df["TitleType"] == "Production Licence"] + + # join drillholes with titles to use the title dates as start/end dates + sources_df = gpd.sjoin(wells_df, titles_df, how="inner", predicate="within") + + # start date of emissions must be after hole is drilled and after the title + # is granted, so choose the latter of the two dates + sources_df["start_date"] = [ + issued if not np.isnat(issued) and (np.isnat(drilled) or issued > drilled) else drilled + for issued, drilled in sources_df[["GrantDate", "RigReleaseDate"]].values + ] + del sources_df["GrantDate"] + del sources_df["RigReleaseDate"] + + # exclude any emission sources that would not have been emitting during + # the period between start_date and end_date + sources_df = rows_in_period(sources_df, start_date=start_date, end_date=end_date, end_field="ExpiryDate") + + # after the join with titles, there may be multiple rows for each well. + # drop duplicates so there is only one entry for each location. + sources_df = sources_df.drop_duplicates(["WellName"]) + + # normalise output to match emission sources format + sources_df = sources_df.rename(columns={ + "WellName": "data_source_id", + "Title": "group_id", + "start_date": "activity_start", + "ExpiryDate": "activity_end", + }) + sources_df["data_source"] = offshore_wells_da.name + # not enough detail in the data source to determine the type of resource + # being extracted by the well + sources_df["site_type"] = "drillhole-unknown" + + return normalise_emission_source_df(sources_df) From 0e58a67471a4ffcad33ac2c4af3b26c8ca64a062 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 19 Mar 2026 17:23:12 +1100 Subject: [PATCH 22/24] Add all sources --- .../oil_gas/emission_sources/all_sources.py | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py new file mode 100644 index 00000000..e01bf2e9 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py @@ -0,0 +1,89 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import geopandas as gpd +import pandas as pd + +from openmethane_prior.lib.data_manager.manager import DataManager +from openmethane_prior.sectors.oil_gas.data.nsw_drillholes import nsw_drillholes_data_source +from openmethane_prior.sectors.oil_gas.data.nsw_titles import nsw_titles_data_source + +from .nsw_sources import nsw_emission_sources +from .offshore_sources import offshore_emission_sources +from .qld_sources import qld_emission_sources +from .wa_sources import wa_emission_sources +from ..data.nopta_titles import nopta_titles_data_source +from ..data.nopta_wells import nopta_wells_data_source +from ..data.qld_boreholes import qld_boreholes_data_source +from ..data.qld_leases import qld_leases_data_source +from ..data.wa_titles import wa_titles_data_source +from ..data.wa_wells import wa_wells_data_source + + +def all_emission_sources( + data_manager: DataManager, + start_date: datetime.date, + end_date: datetime.date, +) -> gpd.GeoDataFrame: + """Assemble a single DataFrame from multiple data sources, where each row + represents a possible location of emissions in the oil and gas sector.""" + nsw_drillholes_da = data_manager.get_asset(nsw_drillholes_data_source) + nsw_titles_da = data_manager.get_asset(nsw_titles_data_source) + nsw_df = nsw_emission_sources( + start_date=start_date, + end_date=end_date, + nsw_drillholes_da=nsw_drillholes_da, + nsw_titles_da=nsw_titles_da, + ) + + qld_boreholes_da = data_manager.get_asset(qld_boreholes_data_source) + qld_leases_da = data_manager.get_asset(qld_leases_data_source) + qld_df = qld_emission_sources( + start_date=start_date, + end_date=end_date, + qld_boreholes_da=qld_boreholes_da, + qld_leases_da=qld_leases_da, + ) + + wa_wells_da = data_manager.get_asset(wa_wells_data_source) + wa_titles_da = data_manager.get_asset(wa_titles_data_source) + wa_df = wa_emission_sources( + start_date=start_date, + end_date=end_date, + wa_wells_da=wa_wells_da, + wa_titles_da=wa_titles_da, + ) + + offshore_wells_da = data_manager.get_asset(nopta_wells_data_source) + offshore_titles_da = data_manager.get_asset(nopta_titles_data_source) + offshore_df = offshore_emission_sources( + start_date=start_date, + end_date=end_date, + offshore_wells_da=offshore_wells_da, + offshore_titles_da=offshore_titles_da, + ) + + all_df: gpd.GeoDataFrame = pd.concat([ + nsw_df, + qld_df, + wa_df, + offshore_df, + ]) + + return all_df + From 89c31099ae5b28675f31e67091a877c32e2ed80f Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 19 Mar 2026 17:23:29 +1100 Subject: [PATCH 23/24] Add offshore sources to oil and gas notebook --- notebooks/oil_gas_locations.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/notebooks/oil_gas_locations.py b/notebooks/oil_gas_locations.py index d1f7285f..56694f67 100644 --- a/notebooks/oil_gas_locations.py +++ b/notebooks/oil_gas_locations.py @@ -5,6 +5,8 @@ import pathlib from openmethane_prior.lib import PriorConfig from openmethane_prior.lib.data_manager.manager import DataManager +from openmethane_prior.sectors.oil_gas.data.nopta_titles import nopta_titles_data_source +from openmethane_prior.sectors.oil_gas.data.nopta_wells import nopta_wells_data_source from openmethane_prior.sectors.oil_gas.data.nsw_drillholes import nsw_drillholes_data_source from openmethane_prior.sectors.oil_gas.data.nsw_titles import nsw_titles_data_source from openmethane_prior.sectors.oil_gas.data.qld_boreholes import qld_boreholes_data_source @@ -12,6 +14,7 @@ from openmethane_prior.sectors.oil_gas.data.wa_titles import wa_titles_data_source from openmethane_prior.sectors.oil_gas.data.wa_wells import wa_wells_data_source from openmethane_prior.sectors.oil_gas.emission_sources.nsw_sources import nsw_emission_sources +from openmethane_prior.sectors.oil_gas.emission_sources.offshore_sources import offshore_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.qld_sources import qld_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.wa_sources import wa_emission_sources os.environ["START_DATE"] = "2023-07-01" @@ -46,15 +49,30 @@ wa_wells_da=wa_wells_da, wa_titles_da=wa_titles_da, ) -all_df = pd.concat([nsw_df, qld_df, wa_df]) +offshore_wells_da = data_manager.get_asset(nopta_wells_data_source) +offshore_titles_da = data_manager.get_asset(nopta_titles_data_source) +offshore_df = offshore_emission_sources( + start_date=datetime.date(2021, 1, 1), + end_date=datetime.date(2023, 1, 2), + offshore_wells_da=offshore_wells_da, + offshore_titles_da=offshore_titles_da, +) +all_df = pd.concat([ + nsw_df, + qld_df, + wa_df, + offshore_df, +]) au_shp = gpd.GeoDataFrame.from_file("~/Downloads/STE_2021_AUST_SHP_GDA2020/STE_2021_AUST_GDA2020.shx") au_map = au_shp.plot(color='white', edgecolor='#CCCCCC', figsize=(32,24)) +au_map.set_xlim(110, 130) +au_map.set_ylim(-35, -10) nsw_titles_da.data.plot(ax=au_map, alpha=0.5) qld_leases_da.data.plot(ax=au_map, alpha=0.5) # wa_wells_da.data.plot(ax=au_map, markersize=3, column="class", legend=True) wa_titles_da.data.plot(ax=au_map, alpha=0.5) -all_df.plot(ax=au_map, markersize=3, column="site_type", legend=True) +all_df.plot(ax=au_map, markersize=3, column="data_source", legend=True) au_map From 64dcf18b920f9e06b027cae987152a6ef63df22c Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Fri, 20 Mar 2026 13:24:47 +1100 Subject: [PATCH 24/24] Ensure offshore wells aren't included in emissions sources twice The NOPTA dataset has significant overlap with WA and VIC datasets, with many wells occurring in both. In the case of VIC fields dataset, the NOPTA dataset has more detail and includes all the same sources, so the simplest solution is not to use the VIC data. In the case of WA, the WA dataset does include some wells that are not found in NOPTA, so we want to combine both datasets but remove any duplicates. The two datasets store coords with slightly different precision, so we allow for a small amount of distance between "duplicate" points. --- .../oil_gas/emission_sources/all_sources.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py index e01bf2e9..3c928e58 100644 --- a/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py @@ -69,6 +69,12 @@ def all_emission_sources( wa_titles_da=wa_titles_da, ) + states_df: gpd.GeoDataFrame = pd.concat([ + nsw_df, + qld_df, + wa_df, + ]) + offshore_wells_da = data_manager.get_asset(nopta_wells_data_source) offshore_titles_da = data_manager.get_asset(nopta_titles_data_source) offshore_df = offshore_emission_sources( @@ -77,13 +83,12 @@ def all_emission_sources( offshore_wells_da=offshore_wells_da, offshore_titles_da=offshore_titles_da, ) + # NOPTA will have some wells that are already provided by state datasets, + # which we must avoid "double counting" + offshore_existing = states_df.sjoin_nearest(offshore_df, how="inner", max_distance=0.00005) + offshore_new = offshore_df[~offshore_df["data_source_id"].isin(offshore_existing["data_source_id_right"])] - all_df: gpd.GeoDataFrame = pd.concat([ - nsw_df, - qld_df, - wa_df, - offshore_df, - ]) + all_df: gpd.GeoDataFrame = pd.concat([states_df, offshore_new]) return all_df