From 89ca87390555b526678b88657ef18c77c11b1be2 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Wed, 18 Mar 2026 11:49:51 +1100 Subject: [PATCH 01/29] Add oil and gas sector README.md This sector implementation is complex enough to warrant its own documentation where we can explain methodology, data sources, and decisions. --- .../sectors/oil_gas/README.md | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 src/openmethane_prior/sectors/oil_gas/README.md diff --git a/src/openmethane_prior/sectors/oil_gas/README.md b/src/openmethane_prior/sectors/oil_gas/README.md new file mode 100644 index 00000000..46f64fd4 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/README.md @@ -0,0 +1,94 @@ + +# Oil and Gas Sector + +The oil and gas sector is the most complicated sector implementation in the +Australia prior. There's no pre-existing high quality dataset that can be +used to both locate and scale emission sources from the sector, and no clear +spatial proxy. + +Oil and gas extraction occurs in "fields" of wells or bores, somtimes located +offshore. Extracted resources are transported through pipelines to various +pieces of infrastructure to be compressed or combined, and eventually to +refinery facilities. Methane can be lost to the atmosphere for various reasons +at any of these locations, resulting in physically dispersed infrastructure +which is not well documented by the private companies which operate it. + +## Methodology + +Emissions are likely to occur in three possible parts of the oil and gas +infrastructure: +- wells or boreholes where oil or gas are extracted +- processing facilities +- pipelines connecting wells and other infrastructure + +### Wells and boreholes + +#### Locations + +Oil and gas extraction are covered by petroleum mining laws in Australia, and +most Australian states provide public datasets of petroleum mining titles +(aka leases/licenses/tenements). Some states also provide public datasets +of drill/bore/well locations. + +Drillhole datasets typically include enough detail to determine which bores +or wells are used for petroleum production, giving us precise locations for +emission sources. + +#### Activity period + +Drillhole datasets typically include the approximate date a bore was drilled +(sometimes just the year), but don't include the date a bore was capped or +depleted. Although some bore datasets do include the bore status, ie CAPPED, +this is the status **at the time the dataset was last updated**, and not +necessarily the status in the period of interest for the prior. + +We use a very rough method to determine whether a single bore may have been +producing emissions during a target period (`start_date` to `end_date`). +- if the bore drill date is later than `end_date`, no emissions are possible +- correlate the bore coordinates with all petroleum production titles it + is contained by +- if the range from `start_date` to `end_date` overlaps with the range from + title grant date to title expiry date, emissions are possible + +This method has obvious problems: + +1. emissions are unlikely to start immediately after drilling, however without + a more accurate start date, this is at least a lower bound +2. most bores/wells will be depleted or capped well before the petroleum title + expires, however without an accurate capping date, this is at least an upper + bound. +3. there is potentially a long period (years) where an entire field / title + has stopped producing, but the title is still active. + +On a coarse grid such as our 10x10km grid, 1. and 2. probably aren't serious, +because emissions will be dispersed amongst a number of points within each grid +cell. + +The biggest issue is 3., as it will place emissions in some fields, possibly +for years after production has ceased. Assuming that capped wells don't +continue to emit methane, this will misallocate potentially significant +emissions. Unfortunately we currently don't have a better solution. + +### Processing facilities + +Not yet implemented. + +### Pipelines + +Public datasets do exist which record the shape and status of oil and gas +pipelines in Australia, however there are several issues using these shape +files to estimate emissions. + +1. Lack of temporal data + +Pipeline datasets list the current status of pipelines, but don't necessarily +include the dates pipelines actually began operating. + +2. Lack of leakage estimates + +We're not currently aware of any research or data on how much methane leaks +from pipelines, or where leakage occurs. This makes it hard to attribute a +specific volume of emission to specific locations along any given pipeline. + +For now, pipelines are not included in our bottom up estimate. If better data +or research becomes available in the future, this would be a welcome addition. \ No newline at end of file From 7273b7464a334d44972c4d97f9a284396eab6899 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 19 Mar 2026 09:04:39 +1100 Subject: [PATCH 02/29] Add emission source concept in oil & gas sector Since emission sources for this sector will come from many different datasets, this defines a common structure of data as an interchange format. --- .../emission_sources/emission_source.py | 73 +++++++++++++++++++ .../unit/test_oil_gas/test_emission_source.py | 33 +++++++++ 2 files changed, 106 insertions(+) create mode 100644 src/openmethane_prior/sectors/oil_gas/emission_sources/emission_source.py create mode 100644 tests/unit/test_oil_gas/test_emission_source.py diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/emission_source.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/emission_source.py new file mode 100644 index 00000000..5bca4975 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/emission_source.py @@ -0,0 +1,73 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import numpy as np +import geopandas as gpd +import pyproj + +""" +An "emission source" in the oil and gas sector is any location that forms part +of oil and gas extraction or production infrastructure where methane emission +might occur. Emission sources are gathered from a variety of datasets, but must +satisfy several required fields. +""" + +emission_source_dtypes = { + # The location of the site (typically a POINT or POLYGON). This column + # is typically provided by GeoDataFrame. + "geometry": "geometry", + + # A string which can be used to determine what type of site/facility is at + # this location. Valid values are documented in emission_source_site_types. + "site_type": str, + + # The earliest date emissions may have occurred at this site. + "activity_start": np.datetime64, + + # The latest date emissions may have occurred at this site. + "activity_end": np.datetime64, + + # (Optional) the "name" value of the data source where this site was + # included. + "data_source": str, + + # (Optional) a unique id for this site in the specified data source. + "data_source_id": str, + + # (Optional) a string identifying a group of locations this site is part + # of. A group could be a title/license area, a field, etc. + "group_id": str, +} + +emission_source_site_types = [ + "drillhole-unknown", + "drillhole-csg", + "drillhole-petroleum", + "drillhole-gas", +] + + +def normalise_emission_source_df( + df: gpd.GeoDataFrame, + crs: pyproj.CRS = pyproj.CRS.from_epsg(4326), +) -> gpd.GeoDataFrame: + # select only columns which are present in emission_source_dtypes + normalised_df = df[list(emission_source_dtypes.keys())] + + normalised_df = normalised_df.to_crs(crs) + + return normalised_df \ No newline at end of file diff --git a/tests/unit/test_oil_gas/test_emission_source.py b/tests/unit/test_oil_gas/test_emission_source.py new file mode 100644 index 00000000..78cdde8d --- /dev/null +++ b/tests/unit/test_oil_gas/test_emission_source.py @@ -0,0 +1,33 @@ +import datetime +import geopandas as gpd +import numpy as np +import shapely + +from openmethane_prior.sectors.oil_gas.emission_sources.emission_source import normalise_emission_source_df + + +def test_normalise_emission_source(): + test_df = gpd.GeoDataFrame({ + "geometry": [shapely.Point(1.0, 2.0)], + "site_type": ["drillhole-csg"], + "activity_start": [np.datetime64(datetime.datetime(2022, 1, 1, 0, 0))], + "activity_end": [np.datetime64(datetime.datetime(2024, 1, 1, 0, 0))], + "data_source": ["nsw-drillholes"], + "data_source_id": ["012345"], + "group_id": ["xyz"], + "extra_column": [0], + }, crs="EPSG:7844") + + result_df = normalise_emission_source_df(test_df) + + assert list(result_df.columns) == [ + "geometry", + "site_type", + "activity_start", + "activity_end", + "data_source", + "data_source_id", + "group_id", + ] + + assert result_df.crs == "EPSG:4326" From 54efa3586174b53fd40c10802d6f595923665b88 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Fri, 20 Mar 2026 14:16:12 +1100 Subject: [PATCH 03/29] Add parse_geo generic asset parser This reads the asset file using geopandas, and returns a GeoDataFrame, which is a pandas DataFrame that supports vector geometry fields. --- pyproject.toml | 2 +- .../lib/data_manager/parsers.py | 9 ++++ uv.lock | 48 +++++++++---------- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9a08f925..f5ca147c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ "rioxarray>=0.15.5,<0.16", "pyproj>=3.6.1,<4", "pandas>=2.2.2,<3", - "geopandas>=0.14.4,<0.15", + "geopandas>=1.1.3,<2", "python-dotenv>=1.0.1,<2", "colorama>=0.4.6,<0.5", "cdsapi>=0.7.3,<0.8", diff --git a/src/openmethane_prior/lib/data_manager/parsers.py b/src/openmethane_prior/lib/data_manager/parsers.py index bb456d02..e9822e32 100644 --- a/src/openmethane_prior/lib/data_manager/parsers.py +++ b/src/openmethane_prior/lib/data_manager/parsers.py @@ -15,10 +15,19 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import geopandas as gpd import pandas as pd from openmethane_prior.lib.data_manager.source import ConfiguredDataSource def parse_csv(data_source: ConfiguredDataSource) -> pd.DataFrame: + """Read and parse a ConfiguredDataSource CSV asset as a pandas DataFrame.""" return pd.read_csv(data_source.asset_path) + + +def parse_geo(data_source: ConfiguredDataSource): + """Read and parse a file containing a collection of geometry vector data + into a geopandas GeoDataFrame. Asset file type can be anything supported by + pyogrio, which includes GeoJSON, GeoPackage, Shapefiles, etc.""" + return gpd.read_file(data_source.asset_path) diff --git a/uv.lock b/uv.lock index e36f0824..5672b9d3 100644 --- a/uv.lock +++ b/uv.lock @@ -434,25 +434,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/a8/20d0723294217e47de6d9e2e40fd4a9d2f7c4b6ef974babd482a59743694/fastjsonschema-2.21.2-py3-none-any.whl", hash = "sha256:1c797122d0a86c5cace2e54bf4e819c36223b552017172f32c5c024a6b77e463", size = 24024, upload-time = "2025-08-14T18:49:34.776Z" }, ] -[[package]] -name = "fiona" -version = "1.10.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "certifi" }, - { name = "click" }, - { name = "click-plugins" }, - { name = "cligj" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/51/e0/71b63839cc609e1d62cea2fc9774aa605ece7ea78af823ff7a8f1c560e72/fiona-1.10.1.tar.gz", hash = "sha256:b00ae357669460c6491caba29c2022ff0acfcbde86a95361ea8ff5cd14a86b68", size = 444606, upload-time = "2024-09-16T20:15:47.074Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/b9/7a8356cfaff8ef162bad44283554d3171e13032635b4f8e10e694a9596ee/fiona-1.10.1-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:98fe556058b370da07a84f6537c286f87eb4af2343d155fbd3fba5d38ac17ed7", size = 16144293, upload-time = "2024-09-16T20:14:34.519Z" }, - { url = "https://files.pythonhosted.org/packages/65/0c/e8070b15c8303f60bd4444a120842597ccd6ed550548948e2e36cffbaa93/fiona-1.10.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:be29044d4aeebae92944b738160dc5f9afc4cdf04f551d59e803c5b910e17520", size = 14752213, upload-time = "2024-09-16T20:14:37.763Z" }, - { url = "https://files.pythonhosted.org/packages/7b/2e/3f80ba2fda9b8686681f0a1b18c8e95ad152ada1d6fb1d3f25281d9229fd/fiona-1.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94bd3d448f09f85439e4b77c38b9de1aebe3eef24acc72bd631f75171cdfde51", size = 17272183, upload-time = "2024-09-16T20:14:42.389Z" }, - { url = "https://files.pythonhosted.org/packages/95/32/c1d53b4d77926414ffdf5bd38344e900e378ae9ccb2a65754cdb6d5344c2/fiona-1.10.1-cp311-cp311-win_amd64.whl", hash = "sha256:30594c0cd8682c43fd01e7cdbe000f94540f8fa3b7cb5901e805c88c4ff2058b", size = 24489398, upload-time = "2024-09-16T20:14:46.233Z" }, -] - [[package]] name = "fonttools" version = "4.61.0" @@ -481,19 +462,19 @@ wheels = [ [[package]] name = "geopandas" -version = "0.14.4" +version = "1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "fiona" }, { name = "numpy" }, { name = "packaging" }, { name = "pandas" }, + { name = "pyogrio" }, { name = "pyproj" }, { name = "shapely" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f6/79/79af2645d40d590a466f8329ab04c2d4fffc811e6713d1c1580dcfdf285c/geopandas-0.14.4.tar.gz", hash = "sha256:56765be9d58e2c743078085db3bd07dc6be7719f0dbe1dfdc1d705cb80be7c25", size = 1106304, upload-time = "2024-04-28T13:49:27.039Z" } +sdist = { url = "https://files.pythonhosted.org/packages/85/ba/8e6b2091878e99e86a36a814dcaeff652ed48bdb03d53e78e15aaa63a914/geopandas-1.1.3.tar.gz", hash = "sha256:91a31989b6f566012838d21d5f8033f37dce882079ccb7cfdc40d5ccce7f284f", size = 336718, upload-time = "2026-03-09T21:49:09.545Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3e/b0/69fa7a0f55122847506a42fea6988d03b34136938082f142151bc9d9f7e7/geopandas-0.14.4-py3-none-any.whl", hash = "sha256:3bb6473cb59d51e1a7fe2dbc24a1a063fb0ebdeddf3ce08ddbf8c7ddc99689aa", size = 1109913, upload-time = "2024-04-28T13:49:24.25Z" }, + { url = "https://files.pythonhosted.org/packages/3c/78/6a04792ace63a93e162f1305392d500ae8ddcb620e7eb88a22fd622b35bb/geopandas-1.1.3-py3-none-any.whl", hash = "sha256:90d62a64f95eaa3be2ccc115c5f3d6e24208bb11983b390fdc0621a3eccd0230", size = 342514, upload-time = "2026-03-09T21:49:07.973Z" }, ] [[package]] @@ -1224,7 +1205,7 @@ requires-dist = [ { name = "cdsapi", specifier = ">=0.7.3,<0.8" }, { name = "colorama", specifier = ">=0.4.6,<0.5" }, { name = "environs", specifier = ">=11.0.0,<12" }, - { name = "geopandas", specifier = ">=0.14.4,<0.15" }, + { name = "geopandas", specifier = ">=1.1.3,<2" }, { name = "netcdf4", specifier = ">=1.6.5,<2" }, { name = "numpy", specifier = ">=2.4.3,<3" }, { name = "pandas", specifier = ">=2.2.2,<3" }, @@ -1448,6 +1429,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, ] +[[package]] +name = "pyogrio" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "numpy" }, + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/d4/12f86b1ed09721363da4c09622464b604c851a9223fc0c6b393fb2012208/pyogrio-0.12.1.tar.gz", hash = "sha256:e548ab705bb3e5383693717de1e6c76da97f3762ab92522cb310f93128a75ff1", size = 303289, upload-time = "2025-11-28T19:04:53.341Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/46/b2c2dcdfd88759b56f103365905fffb85e8b08c1db1ec7c8f8b4c4c26016/pyogrio-0.12.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:01b322dac2a258d24b024d1028dcaa03c9bb6d9c3988b86d298a64873d10dc65", size = 23670744, upload-time = "2025-11-28T19:03:11.299Z" }, + { url = "https://files.pythonhosted.org/packages/d9/21/b69f1bc51d805c00dd7c484a18e1fd2e75b41da1d9f5b8591d7d9d4a7d2f/pyogrio-0.12.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:e10087abcbd6b7e8212560a7002984e5078ac7b3a969ddc2c9929044dbb0d403", size = 25246184, upload-time = "2025-11-28T19:03:13.997Z" }, + { url = "https://files.pythonhosted.org/packages/19/8c/b6aae08e8fcc4f2a903da5f6bd8f888d2b6d7290e54dde5abe15b4cca8df/pyogrio-0.12.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1f6c621972b09fd81a32317e742c69ff4a7763a803da211361a78317f9577765", size = 31434449, upload-time = "2025-11-28T19:03:16.777Z" }, + { url = "https://files.pythonhosted.org/packages/70/f9/9538fa893c29a3fdfeddf3b4c9f8db77f2d4134bc766587929fec8405ebf/pyogrio-0.12.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:c38253427b688464caad5316d4ebcec116b5e13f1f02cc4e3588502f136ca1b4", size = 30987586, upload-time = "2025-11-28T19:03:19.586Z" }, + { url = "https://files.pythonhosted.org/packages/89/a4/0aef5837b4e11840f501e48e01c31242838476c4f4aff9c05e228a083982/pyogrio-0.12.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:5f47787251de7ce13cc06038da93a1214dc283cbccf816be6e03c080358226c8", size = 32534386, upload-time = "2025-11-28T19:03:22.292Z" }, + { url = "https://files.pythonhosted.org/packages/34/97/e8f2ed8a339152b86f8403c258ae5d5f23ab32d690eeb0545bb3473d0c69/pyogrio-0.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:c1d756cf2da4cdf5609779f260d1e1e89be023184225855d6f3dcd33bbe17cb0", size = 22941718, upload-time = "2025-11-28T19:03:24.82Z" }, +] + [[package]] name = "pyparsing" version = "3.2.5" From 0290ac62d4e716b94850f6da2a6efa2564545638 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Wed, 11 Mar 2026 09:32:12 +1100 Subject: [PATCH 04/29] Ensure parse_geo GeoDataFrames always use EPSG:4326 When multiple GeoDataFrame objects are used in a layer and often combined, they must share a common CRS. The CRS of the domain projection may not be known at the time that parse_geo is run, so for now just normalise to a standard WGS84 projection. --- src/openmethane_prior/lib/data_manager/parsers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/openmethane_prior/lib/data_manager/parsers.py b/src/openmethane_prior/lib/data_manager/parsers.py index e9822e32..f47609c0 100644 --- a/src/openmethane_prior/lib/data_manager/parsers.py +++ b/src/openmethane_prior/lib/data_manager/parsers.py @@ -30,4 +30,7 @@ def parse_geo(data_source: ConfiguredDataSource): """Read and parse a file containing a collection of geometry vector data into a geopandas GeoDataFrame. Asset file type can be anything supported by pyogrio, which includes GeoJSON, GeoPackage, Shapefiles, etc.""" - return gpd.read_file(data_source.asset_path) + geo_df = gpd.read_file(data_source.asset_path) + if geo_df.crs is not None and geo_df.crs != "EPSG:4326": + geo_df = geo_df.to_crs(epsg=4326) + return geo_df From 0c9d2977f9a67473674d94e5c20b051d67bc8b78 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Fri, 20 Mar 2026 14:17:53 +1100 Subject: [PATCH 05/29] Add QLD oil & gas emission sources from boreholes and petroleum titles Combine QLD boreholes dataset with petroleum leases dataset to extract producing wells with an activity period matching the lease period. --- pyproject.toml | 1 + src/openmethane_prior/lib/__init__.py | 1 + src/openmethane_prior/lib/utils.py | 22 +++++ .../sectors/oil_gas/data/qld_boreholes.py | 96 +++++++++++++++++++ .../sectors/oil_gas/data/qld_leases.py | 94 ++++++++++++++++++ .../oil_gas/emission_sources/qld_sources.py | 93 ++++++++++++++++++ .../test_emission_sources.py | 50 ++++++++++ tests/unit/test_utils.py | 49 +++++++++- uv.lock | 40 ++++++-- 9 files changed, 434 insertions(+), 12 deletions(-) create mode 100644 src/openmethane_prior/sectors/oil_gas/data/qld_boreholes.py create mode 100644 src/openmethane_prior/sectors/oil_gas/data/qld_leases.py create mode 100644 src/openmethane_prior/sectors/oil_gas/emission_sources/qld_sources.py create mode 100644 tests/integration/test_sector_oil_gas/test_emission_sources.py diff --git a/pyproject.toml b/pyproject.toml index f5ca147c..6c53373c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dependencies = [ "environs>=11.0.0,<12", "prettyprinter>=0.18.0,<0.19", "pytest-mock>=3.15.1,<4", + "bmi-arcgis-restapi>=2.4.15", ] [dependency-groups] diff --git a/src/openmethane_prior/lib/__init__.py b/src/openmethane_prior/lib/__init__.py index 28346421..2e282fd4 100644 --- a/src/openmethane_prior/lib/__init__.py +++ b/src/openmethane_prior/lib/__init__.py @@ -39,6 +39,7 @@ load_zipped_pickle, redistribute_spatially, save_zipped_pickle, + rows_in_period, ) import openmethane_prior.lib.logger as logger diff --git a/src/openmethane_prior/lib/utils.py b/src/openmethane_prior/lib/utils.py index 3404b325..1661eee5 100644 --- a/src/openmethane_prior/lib/utils.py +++ b/src/openmethane_prior/lib/utils.py @@ -29,6 +29,8 @@ import typing import numpy as np +import pandas as pd +import geopandas as gpd from numpy.typing import ArrayLike from urllib.parse import urlparse import xarray as xr @@ -196,3 +198,23 @@ def is_url(maybe_url: str) -> bool: """ parsed = urlparse(maybe_url) return parsed.scheme != "" and parsed.netloc != "" + + +DF = typing.TypeVar("DF", bound=pd.DataFrame | gpd.GeoDataFrame) +def rows_in_period( + df: DF, + start_date: datetime.date, + end_date: datetime.date, + start_field: str = "start_date", + end_field: str = "end_date", +) -> DF: + """Return the rows of the DataFrame df where the period between df[start_field] + and df[end_field] overlaps with the period between start_date and end_date.""" + midnight = datetime.time(0, 0) + period_start = datetime.datetime.combine(date=start_date, time=midnight) + period_end = datetime.datetime.combine(date=end_date + datetime.timedelta(days=1), time=midnight) + + return df[ + (df[start_field] <= np.datetime64(period_end)) + & ((np.isnat(df[end_field])) | (df[end_field] >= np.datetime64(period_start))) + ] diff --git a/src/openmethane_prior/sectors/oil_gas/data/qld_boreholes.py b/src/openmethane_prior/sectors/oil_gas/data/qld_boreholes.py new file mode 100644 index 00000000..c5b32c9f --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/qld_boreholes.py @@ -0,0 +1,96 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import geopandas as gpd +import numpy as np +import restapi # https://github.com/Bolton-and-Menk-GIS/restapi + +from openmethane_prior.lib import DataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo +from openmethane_prior.lib.data_manager.source import ConfiguredDataSource +from openmethane_prior.sectors.oil_gas.data.esri_types import map_esri_date_to_str + + +# Queensland borehole series - REST Service (ArcGIS) +# https://www.data.qld.gov.au/dataset/queensland-borehole-series/resource/c206a53a-59de-48c2-9544-b7b7323ed5dd +def fetch_qld_boreholes(data_source: ConfiguredDataSource): + # qld_spatial_arcgis = restapi.ArcServer(url="https://spatial-gis.information.qld.gov.au/arcgis/rest/services") + qld_spatial_boreholes = restapi.MapService( + url="https://spatial-gis.information.qld.gov.au/arcgis/rest/services/GeoscientificInformation/Boreholes/MapServer" + ) + + boreholes_features = None + oil_gas_layers = ["Boreholes CSG", "Boreholes Gas or Gas Show", "Boreholes Oil or Oil Show", "Boreholes Petroleum"] + for layer_name in oil_gas_layers: + boreholes_layer = qld_spatial_boreholes.layer(layer_name) + # print(boreholes_layer.list_fields()) + + layer_features = boreholes_layer.query( + # TODO: filter out non-relevant results + # where="bore_type in ('COAL SEAM GAS','GREENHOUSE GAS STORAGE','PETROLEUM','UNCONVENTIONAL PETROLEUM')", + # this dataset has many fields, but we only need locations + fields=[ + "bore_name", + "bore_subtype", + "bore_type", + "borehole_pid", + "operator_name", + "result", + "rig_release_date", + "status", + "tenure_no", + "tenure_type", + ], + exceed_limit=True, + ) + + if boreholes_features is None: + boreholes_features = layer_features + else: + boreholes_features.features.extend(layer_features.features) + + df = gpd.GeoDataFrame.from_features(boreholes_features.features) + df["rig_release_date"] = df["rig_release_date"].map(map_esri_date_to_str) + + with open(data_source.asset_path, "w") as asset_file: + asset_file.write(df.to_json()) + return data_source.asset_path + + +def parse_qld_boreholes(data_source: ConfiguredDataSource): + boreholes_df = parse_geo(data_source=data_source) + + # + + # lease/tenement name, i.e. "PL 100", is more useful than "PL" and 100.0, + # so combine tenure_no and tenure_type into a single string + boreholes_df["tenure"] = [ + None if t_no is np.nan else f"{t_type} {t_no:0.0f}" + for t_no, t_type in boreholes_df[["tenure_no", "tenure_type"]].values + ] + + return boreholes_df + +# Locations of all boreholes in the Australian state of Queensland, via the +# Queensland Open Data Portal. +# Source: https://www.data.qld.gov.au/dataset/queensland-borehole-series +qld_boreholes_data_source = DataSource( + name="qld-boreholes", + file_path="QLD-boreholes.geojson", + fetch=fetch_qld_boreholes, + parse=parse_qld_boreholes, +) diff --git a/src/openmethane_prior/sectors/oil_gas/data/qld_leases.py b/src/openmethane_prior/sectors/oil_gas/data/qld_leases.py new file mode 100644 index 00000000..c3236685 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/qld_leases.py @@ -0,0 +1,94 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import geopandas as gpd +import pandas as pd +import restapi # https://github.com/Bolton-and-Menk-GIS/restapi + +from openmethane_prior.lib import DataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo +from openmethane_prior.lib.data_manager.source import ConfiguredDataSource +from openmethane_prior.sectors.oil_gas.data.esri_types import map_esri_date_to_str + + +# Petroleum leases - Queensland - REST Service (ArcGIS) +# https://www.data.qld.gov.au/dataset/queensland-mining-and-exploration-tenure-series/resource/544ecb39-9c25-42f9-9928-b44c7ff05b30 +def fetch_qld_leases(data_source: ConfiguredDataSource): + # qld_spatial_arcgis = restapi.ArcServer(url="https://spatial-gis.information.qld.gov.au/arcgis/rest/services") + qld_current_leases = restapi.MapService( + url="https://spatial-gis.information.qld.gov.au/arcgis/rest/services/Economy/MinesPermitsCurrent/MapServer" + ) + qld_historic_leases = restapi.MapService( + url="https://spatial-gis.information.qld.gov.au/arcgis/rest/services/Economy/MinesPermitsHistoric/MapServer" + ) + + fields = [ + "displayname", + "permittype", + "permitstatus", + "approvedate", + "expirydate", + "permitminerals", + "permitpurpose", + ] + + leases_layer = qld_current_leases.layer("PL granted") + leases_features = leases_layer.query( + fields=fields, + exceed_limit=True, + ) + historic_layer = qld_historic_leases.layer("Historical petroleum lease") + historic_features = historic_layer.query( + fields=fields, + exceed_limit=True, + ) + + df = pd.concat([ + gpd.GeoDataFrame.from_features(leases_features.features), + gpd.GeoDataFrame.from_features(historic_features.features), + ]) + + df["approvedate"] = df["approvedate"].map(map_esri_date_to_str) + df["expirydate"] = df["expirydate"].map(map_esri_date_to_str) + + with open(data_source.asset_path, "w") as asset_file: + asset_file.write(df.to_json()) + return data_source.asset_path + + + +def parse_qld_leases(data_source: ConfiguredDataSource): + leases_df = parse_geo(data_source=data_source) + + # lease/tenement name, i.e. "PL 100", is more useful than "PL" and 100.0, + # so combine tenure_no and tenure_type into a single string + # leases_df["tenure"] = [ + # None if t_no is np.nan else f"{t_type} {t_no:0.0f}" + # for t_no, t_type in leases_df[["tenure_no", "tenure_type"]].values + # ] + + return leases_df + +# Locations of all mining leases in the Australian state of Queensland, via the +# Queensland Open Data Portal. +# Source: https://www.data.qld.gov.au/dataset/queensland-mining-and-exploration-tenure-series +qld_leases_data_source = DataSource( + name="qld-leases", + file_path="QLD-leases.geojson", + fetch=fetch_qld_leases, + parse=parse_qld_leases, +) diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/qld_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/qld_sources.py new file mode 100644 index 00000000..0bec51d6 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/qld_sources.py @@ -0,0 +1,93 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import geopandas as gpd +import numpy as np + +from openmethane_prior.lib.data_manager.asset import DataAsset +from openmethane_prior.lib.utils import rows_in_period +from openmethane_prior.sectors.oil_gas.emission_sources.emission_source import normalise_emission_source_df + +bore_type_map = { + "COAL SEAM GAS": "drillhole-csg", + "PETROLEUM": "drillhole-petroleum", + "UNCONVENTIONAL PETROLEUM": "drillhole-unknown", +} + +def qld_emission_sources( + start_date: datetime.date, + end_date: datetime.date, + qld_boreholes_da: DataAsset, + qld_leases_da: DataAsset, +) -> gpd.GeoDataFrame: + """Create normalised emission source DataFrame by combining WA petroleum + wells dataset for locations, with land title dataset for production + start/end dates.""" + qld_boreholes_df: gpd.GeoDataFrame = qld_boreholes_da.data + qld_leases_df: gpd.GeoDataFrame = qld_leases_da.data + + # filter non-production and storage wells + emitting_bore_types = ["COAL SEAM GAS", "PETROLEUM", "UNCONVENTIONAL PETROLEUM", "GREENHOUSE GAS STORAGE"] + qld_boreholes_df = qld_boreholes_df[qld_boreholes_df["bore_type"].isin(emitting_bore_types)] + emitting_bore_subtypes = ["DEVELOPMENT WELL", "COAL SEAM GAS INJECTION WELL", "PETROLEUM INJECTION WELL"] + qld_boreholes_df = qld_boreholes_df[qld_boreholes_df["bore_subtype"].isin(emitting_bore_subtypes)] + + # filter bores without hydrocarbons + non_emitting_bore_result = ['NO HYDROCARBONS', 'UNKNOWN', 'COAL', 'WATER', 'NO COAL INTERSECTED'] + qld_boreholes_df = qld_boreholes_df[~qld_boreholes_df["result"].isin(non_emitting_bore_result)] + non_emitting_bore_status = ['WATER BORE', 'UNKNOWN'] + qld_boreholes_df = qld_boreholes_df[~qld_boreholes_df["status"].isin(non_emitting_bore_status)] + + # bore datasets may have duplicate rows for a single location due to + # further drilling at a site to deepen/extend an existing hole + qld_boreholes_df = qld_boreholes_df.sort_values(by="rig_release_date") + qld_boreholes_df.drop_duplicates(subset="geometry", keep="first", inplace=True) + + # join boreholes with titles to use the title dates as start/end dates + sources_df = gpd.sjoin(qld_boreholes_df, qld_leases_df, how="inner", predicate="within") + + # start date of emissions must be after hole is drilled and after the title + # is granted, so choose the latter of the two dates + sources_df["start_date"] = [ + issued if not np.isnat(issued) and (np.isnat(drilled) or issued > drilled) else drilled + for issued, drilled in sources_df[["approvedate", "rig_release_date"]].values + ] + del sources_df["approvedate"] + del sources_df["rig_release_date"] + + # exclude any emission sources that would not have been emitting during + # the period between start_date and end_date + sources_df = rows_in_period(sources_df, start_date=start_date, end_date=end_date, end_field="expirydate") + + # after the join with titles, there may be multiple rows for each well. + # drop duplicates so there is only one entry for each location. + sources_df = sources_df.drop_duplicates(["borehole_pid"]) + + # normalise output to match emission sources format + sources_df = sources_df.rename(columns={ + "borehole_pid": "data_source_id", + "displayname": "group_id", + "start_date": "activity_start", + "expirydate": "activity_end", + }) + sources_df["data_source"] = qld_boreholes_da.name + sources_df["site_type"] = sources_df["bore_type"].map( + lambda bore_type: bore_type_map[bore_type] if bore_type in bore_type_map else "drillhole-unknown" + ) + + return normalise_emission_source_df(sources_df) diff --git a/tests/integration/test_sector_oil_gas/test_emission_sources.py b/tests/integration/test_sector_oil_gas/test_emission_sources.py new file mode 100644 index 00000000..ce5a630f --- /dev/null +++ b/tests/integration/test_sector_oil_gas/test_emission_sources.py @@ -0,0 +1,50 @@ +import datetime + +from openmethane_prior.sectors.oil_gas.data.qld_boreholes import qld_boreholes_data_source +from openmethane_prior.sectors.oil_gas.data.qld_leases import qld_leases_data_source +from openmethane_prior.sectors.oil_gas.emission_sources.qld_sources import qld_emission_sources + + +def test_qld_emission_sources(input_files, data_manager): + start_date = datetime.datetime(2023, 1, 1, 0, 0) + start_date_end = datetime.datetime(2023, 1, 2, 0, 0) + boreholes_da = data_manager.get_asset(qld_boreholes_data_source) + df = qld_emission_sources( + start_date=start_date.date(), + end_date=start_date.date(), + qld_boreholes_da=boreholes_da, + qld_leases_da=data_manager.get_asset(qld_leases_data_source), + ) + + # original boreholes dataset has been filtered down + assert len(boreholes_da.data) == 23908 + assert len(df) == 8670 + + # no sources where activity period doesn't intersect date period + assert len(df[(df["activity_end"] < start_date) & (df["activity_start"] > start_date_end)]) == 0 + + # no sources which aren't related to hydrocarbon production + allowed_bore_types = { + "COAL SEAM GAS", "PETROLEUM", "UNCONVENTIONAL PETROLEUM", "GREENHOUSE GAS STORAGE", + } + assert set(df["bore_type"].unique()) - allowed_bore_types == set() + + allowed_bore_subtypes = { + "DEVELOPMENT WELL", "COAL SEAM GAS INJECTION WELL", "PETROLEUM INJECTION WELL", + } + assert set(df["bore_subtype"].unique()) - allowed_bore_subtypes == set() + + allowed_results = { + "GAS", "OIL AND GAS", "GAS PLUS CONDENSATE SHOW", "OIL", "DRY PLUS GAS SHOW", + "DRY PLUS OIL SHOW", "OIL PLUS GAS SHOW", "GAS AND CONDENSATE", + "GAS PLUS OIL SHOW", "COAL SEAM GAS", "DRY PLUS OIL AND GAS SHOW", + } + assert set(df["result"].unique()) - allowed_results == set() + + allowed_status = { + "PLUGGED AND ABANDONED", "SUSPENDED/CAPPED/SHUT-IN", "PRODUCING HYDROCARBONS", + } + assert set(df["status"].unique()) - allowed_status == set() + + # no duplicate entries for the same location + assert len(df["geometry"].unique()) == len(df) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 545db09f..6272eaca 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1,11 +1,17 @@ import datetime - import numpy as np +import pandas as pd import sys import xarray as xr -from openmethane_prior.lib.utils import get_command, get_timestamped_command, time_bounds, bounds_from_cell_edges, is_url - +from openmethane_prior.lib.utils import ( + get_command, + get_timestamped_command, + time_bounds, + bounds_from_cell_edges, + is_url, + rows_in_period, +) def test_get_command(): command = get_command() @@ -69,3 +75,40 @@ def test_is_url(): for test_url, expected in cases: assert is_url(test_url) == expected + + +def test_rows_in_period(): + dt = lambda st: np.datetime64(datetime.datetime.fromisoformat(st)) + + test_df = pd.DataFrame.from_records([ + (dt("2022-12-31T00:00:00Z"), dt("2022-12-31T23:59:59Z"), "a"), + (dt("2022-12-31T00:00:00Z"), dt("2023-01-01T00:00:00Z"), "b"), + (dt("2023-01-01T00:00:01Z"), dt("2023-01-01T23:59:59Z"), "c"), + (dt("2023-01-02T00:00:00Z"), dt("2023-02-02T23:59:59Z"), "d"), + (dt("2023-01-02T00:00:01Z"), dt("2023-02-02T23:59:59Z"), "e"), + ], columns=["start_date", "end_date", "test"]) + + result_df = rows_in_period( + test_df, + start_date=datetime.date(2023, 1, 1), + end_date=datetime.date(2023, 1, 1), + ) + + assert list(result_df["test"]) == ["b", "c", "d"] + + # works with arbitrary start/end field names "start_test" and "end_test" + test_df = pd.DataFrame.from_records([ + (dt("2022-12-31T00:00:00Z"), dt("2022-12-31T23:59:59Z"), "f"), + (dt("2022-12-31T00:00:00Z"), dt("2023-01-01T00:00:00Z"), "g"), + (dt("2023-01-02T00:00:01Z"), dt("2023-02-02T23:59:59Z"), "h"), + ], columns=["start_test", "end_test", "test"]) + + result_df = rows_in_period( + test_df, + start_date=datetime.date(2023, 1, 1), + end_date=datetime.date(2023, 1, 1), + start_field="start_test", + end_field="end_test" + ) + + assert list(result_df["test"]) == ["g"] diff --git a/uv.lock b/uv.lock index 5672b9d3..4ea4dc5c 100644 --- a/uv.lock +++ b/uv.lock @@ -145,6 +145,17 @@ css = [ { name = "tinycss2" }, ] +[[package]] +name = "bmi-arcgis-restapi" +version = "2.4.15" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "munch" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/82/8cb4c901a9cf4304cffb65904a40fe6a55074d6b0c73b62fa392b3db2074/bmi_arcgis_restapi-2.4.15.tar.gz", hash = "sha256:8705e2a31deebc32390fc9a875fe152254df270744afcb30de90056388859e2f", size = 748077, upload-time = "2026-01-09T22:00:42.962Z" } + [[package]] name = "cartopy" version = "0.25.0" @@ -1027,6 +1038,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/93/cf/be4e93afbfa0def2cd6fac9302071db0bd6d0617999ecbf53f92b9398de3/multiurl-0.3.7-py3-none-any.whl", hash = "sha256:054f42974064f103be0ed55b43f0c32fc435a47dc7353a9adaffa643b99fa380", size = 21524, upload-time = "2025-07-29T11:57:03.191Z" }, ] +[[package]] +name = "munch" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/2b/45098135b5f9f13221820d90f9e0516e11a2a0f55012c13b081d202b782a/munch-4.0.0.tar.gz", hash = "sha256:542cb151461263216a4e37c3fd9afc425feeaf38aaa3025cd2a981fadb422235", size = 19089, upload-time = "2023-07-01T09:49:35.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/b3/7c69b37f03260a061883bec0e7b05be7117c1b1c85f5212c72c8c2bc3c8c/munch-4.0.0-py2.py3-none-any.whl", hash = "sha256:71033c45db9fb677a0b7eb517a4ce70ae09258490e419b0e7f00d1e386ecb1b4", size = 9950, upload-time = "2023-07-01T09:49:34.472Z" }, +] + [[package]] name = "nbclient" version = "0.10.2" @@ -1168,6 +1188,7 @@ name = "openmethane-prior" version = "1.4.2.dev1" source = { editable = "." } dependencies = [ + { name = "bmi-arcgis-restapi" }, { name = "cdsapi" }, { name = "colorama" }, { name = "environs" }, @@ -1202,6 +1223,7 @@ tests = [ [package.metadata] requires-dist = [ + { name = "bmi-arcgis-restapi", specifier = ">=2.4.15" }, { name = "cdsapi", specifier = ">=0.7.3,<0.8" }, { name = "colorama", specifier = ">=0.4.6,<0.5" }, { name = "environs", specifier = ">=11.0.0,<12" }, @@ -1252,7 +1274,7 @@ wheels = [ [[package]] name = "pandas" -version = "2.3.3" +version = "2.2.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, @@ -1260,15 +1282,15 @@ dependencies = [ { name = "pytz" }, { name = "tzdata" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/d6/9f8431bacc2e19dca897724cd097b1bb224a6ad5433784a44b587c7c13af/pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667", size = 4399213, upload-time = "2024-09-20T13:10:04.827Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/fa/7ac648108144a095b4fb6aa3de1954689f7af60a14cf25583f4960ecb878/pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523", size = 11578790, upload-time = "2025-09-29T23:18:30.065Z" }, - { url = "https://files.pythonhosted.org/packages/9b/35/74442388c6cf008882d4d4bdfc4109be87e9b8b7ccd097ad1e7f006e2e95/pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45", size = 10833831, upload-time = "2025-09-29T23:38:56.071Z" }, - { url = "https://files.pythonhosted.org/packages/fe/e4/de154cbfeee13383ad58d23017da99390b91d73f8c11856f2095e813201b/pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66", size = 12199267, upload-time = "2025-09-29T23:18:41.627Z" }, - { url = "https://files.pythonhosted.org/packages/bf/c9/63f8d545568d9ab91476b1818b4741f521646cbdd151c6efebf40d6de6f7/pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b", size = 12789281, upload-time = "2025-09-29T23:18:56.834Z" }, - { url = "https://files.pythonhosted.org/packages/f2/00/a5ac8c7a0e67fd1a6059e40aa08fa1c52cc00709077d2300e210c3ce0322/pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791", size = 13240453, upload-time = "2025-09-29T23:19:09.247Z" }, - { url = "https://files.pythonhosted.org/packages/27/4d/5c23a5bc7bd209231618dd9e606ce076272c9bc4f12023a70e03a86b4067/pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151", size = 13890361, upload-time = "2025-09-29T23:19:25.342Z" }, - { url = "https://files.pythonhosted.org/packages/8e/59/712db1d7040520de7a4965df15b774348980e6df45c129b8c64d0dbe74ef/pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c", size = 11348702, upload-time = "2025-09-29T23:19:38.296Z" }, + { url = "https://files.pythonhosted.org/packages/a8/44/d9502bf0ed197ba9bf1103c9867d5904ddcaf869e52329787fc54ed70cc8/pandas-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039", size = 12602222, upload-time = "2024-09-20T13:08:56.254Z" }, + { url = "https://files.pythonhosted.org/packages/52/11/9eac327a38834f162b8250aab32a6781339c69afe7574368fffe46387edf/pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd", size = 11321274, upload-time = "2024-09-20T13:08:58.645Z" }, + { url = "https://files.pythonhosted.org/packages/45/fb/c4beeb084718598ba19aa9f5abbc8aed8b42f90930da861fcb1acdb54c3a/pandas-2.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd8d0c3be0515c12fed0bdbae072551c8b54b7192c7b1fda0ba56059a0179698", size = 15579836, upload-time = "2024-09-20T19:01:57.571Z" }, + { url = "https://files.pythonhosted.org/packages/cd/5f/4dba1d39bb9c38d574a9a22548c540177f78ea47b32f99c0ff2ec499fac5/pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc", size = 13058505, upload-time = "2024-09-20T13:09:01.501Z" }, + { url = "https://files.pythonhosted.org/packages/b9/57/708135b90391995361636634df1f1130d03ba456e95bcf576fada459115a/pandas-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:63cc132e40a2e084cf01adf0775b15ac515ba905d7dcca47e9a251819c575ef3", size = 16744420, upload-time = "2024-09-20T19:02:00.678Z" }, + { url = "https://files.pythonhosted.org/packages/86/4a/03ed6b7ee323cf30404265c284cee9c65c56a212e0a08d9ee06984ba2240/pandas-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32", size = 14440457, upload-time = "2024-09-20T13:09:04.105Z" }, + { url = "https://files.pythonhosted.org/packages/ed/8c/87ddf1fcb55d11f9f847e3c69bb1c6f8e46e2f40ab1a2d2abadb2401b007/pandas-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5", size = 11617166, upload-time = "2024-09-20T13:09:06.917Z" }, ] [[package]] From a10c64c96d9078b45b17fc6a49ab93a6edbfd9b0 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Wed, 11 Mar 2026 10:29:03 +1100 Subject: [PATCH 06/29] Add NOPTA offshore oil and gas emission sources NOPTA is the National Offshore Petroleum Title Administrator who manages Australian offshore resource titles. Using their titles and wells datasets we can identify likely offshore locations where oil and gas are being extracted within a prior period. --- .../sectors/oil_gas/data/esri_types.py | 39 ++++++ .../sectors/oil_gas/data/nopta_titles.py | 118 ++++++++++++++++++ .../sectors/oil_gas/data/nopta_wells.py | 110 ++++++++++++++++ .../emission_sources/offshore_sources.py | 80 ++++++++++++ .../test_emission_sources.py | 29 ++++- 5 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 src/openmethane_prior/sectors/oil_gas/data/esri_types.py create mode 100644 src/openmethane_prior/sectors/oil_gas/data/nopta_titles.py create mode 100644 src/openmethane_prior/sectors/oil_gas/data/nopta_wells.py create mode 100644 src/openmethane_prior/sectors/oil_gas/emission_sources/offshore_sources.py diff --git a/src/openmethane_prior/sectors/oil_gas/data/esri_types.py b/src/openmethane_prior/sectors/oil_gas/data/esri_types.py new file mode 100644 index 00000000..75bc12af --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/esri_types.py @@ -0,0 +1,39 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import numpy as np + + +def map_esri_date_to_date(esri_date_milliseconds) -> datetime.date | None: + """Convert an esriFieldTypeDate value to UTC datetime.date.""" + if esri_date_milliseconds is None \ + or np.isnan(esri_date_milliseconds) \ + or esri_date_milliseconds <= 0: + return None + + date_value = datetime.datetime.fromtimestamp( + esri_date_milliseconds / 1000, + tz=datetime.timezone.utc, + ) + return date_value.date() + + +def map_esri_date_to_str(esri_date_milliseconds) -> str | None: + """Convert an esriFieldTypeDate value to RFC3339 date string.""" + date = map_esri_date_to_date(esri_date_milliseconds) + return date.isoformat() if date is not None else None diff --git a/src/openmethane_prior/sectors/oil_gas/data/nopta_titles.py b/src/openmethane_prior/sectors/oil_gas/data/nopta_titles.py new file mode 100644 index 00000000..98ee6281 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/nopta_titles.py @@ -0,0 +1,118 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json +import restapi # https://github.com/Bolton-and-Menk-GIS/restapi + +from openmethane_prior.lib import DataSource, ConfiguredDataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo + +from .esri_types import map_esri_date_to_str + + +def fetch_nopta_titles(data_source: ConfiguredDataSource): + nopta_titles = restapi.MapService( + url="https://arcgis.nopta.gov.au/arcgis/rest/services/Public/TitlesCompany_NOPTA/MapServer" + ) + + titles_layer = nopta_titles.layer("Titles and Permits Current") + + layer_features = titles_layer.query( + # where="Type in ('Petroleum','Mineral or Coal') AND Purpose in ('Development','Appraisal', 'Exploration')", + # descriptions of available fields + # https://www.nopta.gov.au/maps-and-public-data/documents/DataDescription_OffshorePetroleumWells.docx + fields=[ + "OBJECTID", + "Title", + "RelTitle", + "TitleType", + "ExpiryDate", + "GrantDate", + "LastReDate", + "NoOfRenews", + "EndDate", + "Status", + "FieldName", + "BasinName", + "SubBasin", + "OffShoreAr", + "TitleOprat", + "TitleHold", + "NoOfBlocks", + "AreaKM2", + "NEATS_Links", + "TITLE_NUMBER_NEATS", + ], + exceed_limit=True, + ) + + for feature in layer_features["features"]: + # convert esriFieldTypeDate to RFC3339 date + for feature_key in feature["properties"]: + if feature_key.endswith("Date"): + feature["properties"][feature_key] = map_esri_date_to_str(feature["properties"][feature_key]) + + with open(data_source.asset_path, "w") as asset_file: + json.dump(layer_features.json, asset_file) + return data_source.asset_path + + +offshore_area_state_mapping = { + "Western Australia": "WA", + "Victoria": "VIC", + "Northern Territory": "NT", + "Queensland": "QLD", + "South Australia": "SA", + "Tasmania": "TAS", + "New South Wales": "NSW", + "ACT": "ACT", + # https://www.infrastructure.gov.au/territories-regions/territories/ashmore-and-cartier-islands + "Territory of Ashmore and Cartier Islands": "NT", + "Outside of Australia": None, + "UNK": None, +} +def map_offshore_area_to_state(offshore_area: str) -> str | None: + if offshore_area in offshore_area_state_mapping: + return offshore_area_state_mapping[offshore_area] + return None + + +def parse_nopta_titles(data_source: ConfiguredDataSource): + df = parse_geo(data_source=data_source) + + # # NOPIMS dataset may record multiple boreholes for a single well, all + # # with identical WellName and location. This will remove duplicate rows + # # so that only a single location is present for each well, keeping the + # # earliest "RigReleaseDate" when the first bore was drilled at the well. + # wells_df = wells_df.sort_values(by="RigReleaseDate") + # wells_df.drop_duplicates(subset="WellName", keep="first", inplace=True) + # + df["state"] = df["OffShoreAr"].map(map_offshore_area_to_state) + + return df + + +# Locations of all offshore petroleum and gas titles (licenses) administered +# by the National Offshore Petroleum Titles Administrator (NOPTA), via the +# National Electronic Approvals Tracking System (NEATS). +# Source: https://www.nopta.gov.au/maps-and-public-data/neats-info.html +nopta_titles_data_source = DataSource( + name="NOPTA-titles", + file_path="NOPTA-titles.geojson", + fetch=fetch_nopta_titles, + parse=parse_nopta_titles, +) diff --git a/src/openmethane_prior/sectors/oil_gas/data/nopta_wells.py b/src/openmethane_prior/sectors/oil_gas/data/nopta_wells.py new file mode 100644 index 00000000..6de3ca1b --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/nopta_wells.py @@ -0,0 +1,110 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json +import restapi # https://github.com/Bolton-and-Menk-GIS/restapi + +from openmethane_prior.lib import DataSource, ConfiguredDataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo + +from .esri_types import map_esri_date_to_str + + +def fetch_nopta_wells(data_source: ConfiguredDataSource): + # nopta_wells_arcgis = restapi.ArcServer(url="https://arcgis.nopta.gov.au/arcgis/rest/services") + nopta_wells = restapi.MapService( + url="https://arcgis.nopta.gov.au/arcgis/rest/services/Public/Petroleum_Wells/MapServer" + ) + + petroleum_wells_layer = nopta_wells.layer("Petroleum Wells") + + layer_features = petroleum_wells_layer.query( + where="Type in ('Petroleum','Mineral or Coal') AND Purpose in ('Development','Appraisal', 'Exploration')", + # descriptions of available fields + # https://www.nopta.gov.au/maps-and-public-data/documents/DataDescription_OffshorePetroleumWells.docx + fields=[ + "WellName", + "OffshoreArea", + "Jurisdiction", + "TitleNumber", + "Type", + "Purpose", + # Date when the rig finished drilling and was moved off the site, + # we can use this as a guess for when emissions may have started + # if no other info is available. + "RigReleaseDate", + ], + exceed_limit=True, + ) + + for feature in layer_features["features"]: + # convert esriFieldTypeDate to RFC3339 date + feature["properties"]["RigReleaseDate"] = map_esri_date_to_str(feature["properties"]["RigReleaseDate"]) + + with open(data_source.asset_path, "w") as asset_file: + json.dump(layer_features.json, asset_file) + return data_source.asset_path + + +offshore_area_state_mapping = { + "Western Australia": "WA", + "Victoria": "VIC", + "Northern Territory": "NT", + "Queensland": "QLD", + "South Australia": "SA", + "Tasmania": "TAS", + "New South Wales": "NSW", + "ACT": "ACT", + # https://www.infrastructure.gov.au/territories-regions/territories/ashmore-and-cartier-islands + "Territory of Ashmore and Cartier Islands": "NT", + "Outside of Australia": None, + "UNK": None, +} +def map_offshore_area_to_state(offshore_area: str) -> str | None: + if offshore_area in offshore_area_state_mapping: + return offshore_area_state_mapping[offshore_area] + return None + + +def parse_nopta_wells(data_source: ConfiguredDataSource): + wells_df = parse_geo(data_source=data_source) + + # NOPIMS dataset may record multiple boreholes for a single well, all + # with identical WellName and location. This will remove duplicate rows + # so that only a single location is present for each well, keeping the + # earliest "RigReleaseDate" when the first bore was drilled at the well. + wells_df = wells_df.sort_values(by="RigReleaseDate") + wells_df.drop_duplicates(subset="WellName", keep="first", inplace=True) + + wells_df["state"] = wells_df["OffshoreArea"].map(map_offshore_area_to_state) + + # 3D points provided by NOPIMS are unnecessary for our purposes + wells_df["geometry"] = wells_df["geometry"].force_2d() + + return wells_df + + +# Locations of all wells administered by the National Offshore Petroleum +# Titles Administrator (NOPTA), via the National Offshore Petroleum Information +# Management Systems (NOPIMS). +# Source: https://www.nopta.gov.au/maps-and-public-data/nopims-info.html +nopta_wells_data_source = DataSource( + name="NOPTA-wells", + file_path="NOPTA-wells.geojson", + fetch=fetch_nopta_wells, + parse=parse_nopta_wells, +) diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/offshore_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/offshore_sources.py new file mode 100644 index 00000000..9ce01446 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/offshore_sources.py @@ -0,0 +1,80 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import geopandas as gpd +import numpy as np + +from openmethane_prior.lib.data_manager.asset import DataAsset +from openmethane_prior.lib.utils import rows_in_period +from openmethane_prior.sectors.oil_gas.emission_sources.emission_source import normalise_emission_source_df + +def offshore_emission_sources( + start_date: datetime.date, + end_date: datetime.date, + offshore_wells_da: DataAsset, + offshore_titles_da: DataAsset, +) -> gpd.GeoDataFrame: + """Create normalised emission source DataFrame by combining WA petroleum + wells dataset for locations, with land title dataset for production + start/end dates.""" + offshore_wells_df: gpd.GeoDataFrame = offshore_wells_da.data + offshore_titles_df: gpd.GeoDataFrame = offshore_titles_da.data + + # filter out wells that aren't actively used for production + wells_df = offshore_wells_df[offshore_wells_df["Purpose"] == "Development"] + + # emissions are expected where production is ongoing, unlike exploration or retention + titles_df = offshore_titles_df[offshore_titles_df["TitleType"] == "Production Licence"] + + # NOPTA wells dataset appears to re-use lat/lon coords for multiple wells + # within the same field. Since these contain unique values for WellName and + # RigReleaseDate, we treat these as unique wells and don't de-duplicate + + # join drillholes with titles to use the title dates as start/end dates + sources_df = gpd.sjoin(wells_df, titles_df, how="inner", predicate="within") + + # start date of emissions must be after hole is drilled and after the title + # is granted, so choose the latter of the two dates + sources_df["start_date"] = [ + issued if not np.isnat(issued) and (np.isnat(drilled) or issued > drilled) else drilled + for issued, drilled in sources_df[["GrantDate", "RigReleaseDate"]].values + ] + del sources_df["GrantDate"] + del sources_df["RigReleaseDate"] + + # exclude any emission sources that would not have been emitting during + # the period between start_date and end_date + sources_df = rows_in_period(sources_df, start_date=start_date, end_date=end_date, end_field="ExpiryDate") + + # after the join with titles, there may be multiple rows for each well. + # drop duplicates so there is only one entry for each location. + sources_df = sources_df.drop_duplicates(["WellName"]) + + # normalise output to match emission sources format + sources_df = sources_df.rename(columns={ + "WellName": "data_source_id", + "Title": "group_id", + "start_date": "activity_start", + "ExpiryDate": "activity_end", + }) + sources_df["data_source"] = offshore_wells_da.name + # not enough detail in the data source to determine the type of resource + # being extracted by the well + sources_df["site_type"] = "drillhole-unknown" + + return normalise_emission_source_df(sources_df) diff --git a/tests/integration/test_sector_oil_gas/test_emission_sources.py b/tests/integration/test_sector_oil_gas/test_emission_sources.py index ce5a630f..a436c849 100644 --- a/tests/integration/test_sector_oil_gas/test_emission_sources.py +++ b/tests/integration/test_sector_oil_gas/test_emission_sources.py @@ -1,7 +1,10 @@ import datetime +from openmethane_prior.sectors.oil_gas.data.nopta_titles import nopta_titles_data_source +from openmethane_prior.sectors.oil_gas.data.nopta_wells import nopta_wells_data_source from openmethane_prior.sectors.oil_gas.data.qld_boreholes import qld_boreholes_data_source from openmethane_prior.sectors.oil_gas.data.qld_leases import qld_leases_data_source +from openmethane_prior.sectors.oil_gas.emission_sources.offshore_sources import offshore_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.qld_sources import qld_emission_sources @@ -46,5 +49,29 @@ def test_qld_emission_sources(input_files, data_manager): } assert set(df["status"].unique()) - allowed_status == set() - # no duplicate entries for the same location assert len(df["geometry"].unique()) == len(df) + + +def test_offshore_emission_sources(input_files, data_manager): + start_date = datetime.datetime(2023, 1, 1, 0, 0) + start_date_end = datetime.datetime(2023, 1, 2, 0, 0) + wells_da = data_manager.get_asset(nopta_wells_data_source) + df = offshore_emission_sources( + start_date=start_date.date(), + end_date=start_date.date(), + offshore_wells_da=wells_da, + offshore_titles_da=data_manager.get_asset(nopta_titles_data_source), + ) + + # original wells dataset has been filtered down + assert len(wells_da.data) == 3024 + assert len(df) == 900 + + # no sources where activity period doesn't intersect date period + assert len(df[(df["activity_end"] < start_date) & (df["activity_start"] > start_date_end)]) == 0 + + # no sources which aren't related to production + assert set(df["Purpose"].unique()) == {"Development"} + assert set(df["TitleType"].unique()) == {"Production Licence"} + + # no duplicate check, as we allow duplicate geometries from NOPTA dataset From 17049b376405bf7e990d4e1844e3cd904ff89670 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 23 Apr 2026 09:08:08 +1000 Subject: [PATCH 07/29] Add NSW oil & gas emission sources from petroleum drillholes Combine NSW drillholes dataset with petroleum leases dataset to extract producing wells with an activity period matching the period of interest. --- pyproject.toml | 1 + .../sectors/oil_gas/data/nsw_drillholes.py | 84 ++++++++++++++++ .../sectors/oil_gas/data/nsw_titles.py | 69 ++++++++++++++ .../oil_gas/emission_sources/nsw_sources.py | 95 +++++++++++++++++++ .../test_emission_sources.py | 29 ++++++ uv.lock | 47 +++++++++ 6 files changed, 325 insertions(+) create mode 100644 src/openmethane_prior/sectors/oil_gas/data/nsw_drillholes.py create mode 100644 src/openmethane_prior/sectors/oil_gas/data/nsw_titles.py create mode 100644 src/openmethane_prior/sectors/oil_gas/emission_sources/nsw_sources.py diff --git a/pyproject.toml b/pyproject.toml index 6c53373c..edb9cf5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "prettyprinter>=0.18.0,<0.19", "pytest-mock>=3.15.1,<4", "bmi-arcgis-restapi>=2.4.15", + "owslib>=0.35.0", ] [dependency-groups] diff --git a/src/openmethane_prior/sectors/oil_gas/data/nsw_drillholes.py b/src/openmethane_prior/sectors/oil_gas/data/nsw_drillholes.py new file mode 100644 index 00000000..8103eff4 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/nsw_drillholes.py @@ -0,0 +1,84 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import geopandas as gpd +import json +import pandas as pd +from owslib.wfs import WebFeatureService + +from openmethane_prior.lib import DataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo +from openmethane_prior.lib.data_manager.source import ConfiguredDataSource + + +def fetch_nsw_drillholes(data_source: ConfiguredDataSource): + geoserver_wfs = WebFeatureService("https://public-gs.geoscience.nsw.gov.au/geoserver/wfs", version="2.0.0") + + desired_properties = [ + "program", + "hole_name", + "title", + "reports", + "year_drilled", + "licence_holder", + "operator", + "business_purpose", + "hole_purpose", + "well_status", + "project", + "site_id", + "geom", + ] + + nsw_drillholes_feature_csg = geoserver_wfs.getfeature( + typename="drilling:drilling_drillholes_csg", + srsname="urn:ogc:def:crs:EPSG::4326", + propertyname=desired_properties, + outputFormat="application/json", + ) + + nsw_drillholes_feature_petroleum = geoserver_wfs.getfeature( + typename="drilling:drilling_drillholes_petroleum", + srsname="urn:ogc:def:crs:EPSG::4326", + propertyname=desired_properties, + outputFormat="application/json", + ) + + features_df = pd.concat([ + gpd.GeoDataFrame.from_features(json.loads(nsw_drillholes_feature_csg.read())), + gpd.GeoDataFrame.from_features(json.loads(nsw_drillholes_feature_petroleum.read())), + ]) + + # only include drillholes relevant to CSG and petroleum production + features_df = features_df[features_df["business_purpose"].isin(["Coal seam methane", "Petroleum"])] + features_df = features_df[features_df["hole_purpose"].isin(["Production"])] + + with open(data_source.asset_path, "w") as asset_file: + asset_file.write(features_df.to_json()) + return data_source.asset_path + + +# Locations of coal seam gas and petroleum production drillholes in the +# Australian state of New South Wales, via Geosciences NSW. +# Source: https://data.nsw.gov.au/data/dataset/coal-seam-gas-borehole +# Source: https://data.nsw.gov.au/data/dataset/nsw-drillholes-petroleum +nsw_drillholes_data_source = DataSource( + name="nsw-drillholes-csg-petroleum", + file_path="NSW-drillholes-csg-petroleum.geojson", + fetch=fetch_nsw_drillholes, + parse=parse_geo, +) diff --git a/src/openmethane_prior/sectors/oil_gas/data/nsw_titles.py b/src/openmethane_prior/sectors/oil_gas/data/nsw_titles.py new file mode 100644 index 00000000..0e3f959f --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/nsw_titles.py @@ -0,0 +1,69 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import geopandas as gpd +import json +from owslib.wfs import WebFeatureService + +from openmethane_prior.lib import DataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo +from openmethane_prior.lib.data_manager.source import ConfiguredDataSource + + +def fetch_nsw_titles(data_source: ConfiguredDataSource): + geoserver_wfs = WebFeatureService("https://public-gs.geoscience.nsw.gov.au/geoserver/wfs", version="2.0.0") + + desired_properties = [ + "tas_id", + "title", + "holder", + "company", + "grant_date", + "expiry_date", + "minerals", + "resource", + "operation", + "geom", + ] + + nsw_titles_feature = geoserver_wfs.getfeature( + typename="mining-and-exploration:titles_title_granted", + srsname="urn:ogc:def:crs:EPSG::4326", + propertyname=desired_properties, + outputFormat="application/json", + ) + + features_df = gpd.GeoDataFrame.from_features(json.loads(nsw_titles_feature.read())) + + # only include titles relevant to petroleum production + features_df = features_df[features_df["resource"] == "PETROLEUM"] + features_df = features_df[features_df["operation"] == "MINING"] + + with open(data_source.asset_path, "w") as asset_file: + asset_file.write(features_df.to_json()) + return data_source.asset_path + + +# Locations of coal seam gas and petroleum production titles in the +# Australian state of New South Wales, via Geosciences NSW. +# Source: https://data.nsw.gov.au/data/dataset/nsw-mining-titles +nsw_titles_data_source = DataSource( + name="nsw-titles-csg-petroleum", + file_path="NSW-titles-csg-petroleum.geojson", + fetch=fetch_nsw_titles, + parse=parse_geo, +) diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/nsw_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/nsw_sources.py new file mode 100644 index 00000000..e19bfff3 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/nsw_sources.py @@ -0,0 +1,95 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import geopandas as gpd + +from openmethane_prior.lib.data_manager.asset import DataAsset +from openmethane_prior.lib.utils import rows_in_period +from openmethane_prior.sectors.oil_gas.emission_sources.emission_source import normalise_emission_source_df + +nsw_drillhole_purpose_map = { + "Coal seam methane": "drillhole-csg", + "Petroleum": "drillhole-petroleum", +} + +def nsw_emission_sources( + start_date: datetime.date, + end_date: datetime.date, + nsw_drillholes_da: DataAsset, + nsw_titles_da: DataAsset, +) -> gpd.GeoDataFrame: + """Create normalised emission source DataFrame by combining NSW petroleum + drillhole dataset for locations, with land title dataset for production + start/end dates.""" + nsw_drillholes_df: gpd.GeoDataFrame = nsw_drillholes_da.data + nsw_titles_df: gpd.GeoDataFrame = nsw_titles_da.data + + # no specific date, so we'll guess from Jan 1st of the specified year + nsw_drillholes_df["drilled_date"] = nsw_drillholes_df["year_drilled"].map( + lambda year: datetime.datetime(int(year), 1, 1, 0, 0, 0) + ) + + # NSW drillhole dataset can contain multiple entries for a single exit + # point, due to branches in a drill hole or extensions. Remove all but + # the earliest record of the drilling at that location. + nsw_drillholes_df = nsw_drillholes_df.sort_values(by="drilled_date") + nsw_drillholes_df.drop_duplicates(subset="geometry", keep="first", inplace=True) + + # ignore the well title, we'll locate it spatially in a title instead + del nsw_drillholes_df["title"] + + # map from PPL4 to PPL0004 to match formatting in drillholes dataset + nsw_titles_df["title"] = nsw_titles_df["title"].map( + lambda title: f"{title[:3]}{int(title[3:]):04d}" + ) + + # join drillholes with titles to use the title dates as start/end dates + sources_df = gpd.sjoin(nsw_drillholes_df, nsw_titles_df, how="inner", predicate="within") + + # start date of emissions must be after hole is drilled and after the title + # is granted, so choose the later of the two dates + sources_df["start_date"] = [ + granted if granted > drilled else drilled + for granted, drilled in sources_df[["grant_date", "drilled_date"]].values + ] + del sources_df["drilled_date"] + del sources_df["grant_date"] + + # NSW drill holes are either coal seam methane or petroleum + sources_df["site_type"] = sources_df["business_purpose"].map( + lambda purpose: nsw_drillhole_purpose_map[purpose] + ) + del sources_df["business_purpose"] + + # exclude any emission sources that would not have been emitting during + # the period between start_date and end_date + sources_df = rows_in_period(sources_df, start_date=start_date, end_date=end_date, end_field="expiry_date") + + # drop duplicates so there is only one entry for each location. + sources_df = sources_df.drop_duplicates(["gsnsw_drill_id"]) + + # normalise output to match emission sources format + sources_df = sources_df.rename(columns={ + "gsnsw_drill_id": "data_source_id", + "title": "group_id", + "start_date": "activity_start", + "expiry_date": "activity_end", + }) + sources_df["data_source"] = nsw_drillholes_da.name + + return normalise_emission_source_df(sources_df) diff --git a/tests/integration/test_sector_oil_gas/test_emission_sources.py b/tests/integration/test_sector_oil_gas/test_emission_sources.py index a436c849..dce3fbf3 100644 --- a/tests/integration/test_sector_oil_gas/test_emission_sources.py +++ b/tests/integration/test_sector_oil_gas/test_emission_sources.py @@ -2,12 +2,41 @@ from openmethane_prior.sectors.oil_gas.data.nopta_titles import nopta_titles_data_source from openmethane_prior.sectors.oil_gas.data.nopta_wells import nopta_wells_data_source +from openmethane_prior.sectors.oil_gas.data.nsw_drillholes import nsw_drillholes_data_source +from openmethane_prior.sectors.oil_gas.data.nsw_titles import nsw_titles_data_source from openmethane_prior.sectors.oil_gas.data.qld_boreholes import qld_boreholes_data_source from openmethane_prior.sectors.oil_gas.data.qld_leases import qld_leases_data_source +from openmethane_prior.sectors.oil_gas.emission_sources.nsw_sources import nsw_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.offshore_sources import offshore_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.qld_sources import qld_emission_sources +def test_nsw_emission_sources(input_files, data_manager): + start_date = datetime.datetime(2023, 1, 1, 0, 0) + start_date_end = datetime.datetime(2023, 1, 2, 0, 0) + drillholes_da = data_manager.get_asset(nsw_drillholes_data_source) + df = nsw_emission_sources( + start_date=start_date.date(), + end_date=start_date.date(), + nsw_drillholes_da=drillholes_da, + nsw_titles_da=data_manager.get_asset(nsw_titles_data_source), + ) + + # original drillholes dataset has been filtered down + assert len(drillholes_da.data) == 775 + assert len(df) == 92 + + # no sources where activity period doesn't intersect date period + assert len(df[(df["activity_end"] < start_date) & (df["activity_start"] > start_date_end)]) == 0 + + # no sources which aren't related to production + assert list(df["hole_purpose"].unique()) == ["Production"] + assert list(df["resource"].unique()) == ["PETROLEUM"] + + # no duplicate entries for the same location + assert len(df["geometry"].unique()) == len(df) + + def test_qld_emission_sources(input_files, data_manager): start_date = datetime.datetime(2023, 1, 1, 0, 0) start_date_end = datetime.datetime(2023, 1, 2, 0, 0) diff --git a/uv.lock b/uv.lock index 4ea4dc5c..05e354b5 100644 --- a/uv.lock +++ b/uv.lock @@ -912,6 +912,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/03/99102b3772bdc5d25fc7fe5f5fb862c54bb6e863991f50d02667999942c1/licenseheaders-0.8.8-py3-none-any.whl", hash = "sha256:3b159228b37bbba98bd01448c41a5eff773ab26ac5b14ac98c53d06dbc807696", size = 21272, upload-time = "2021-04-08T18:48:43.871Z" }, ] +[[package]] +name = "lxml" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/28/30/9abc9e34c657c33834eaf6cd02124c61bdf5944d802aa48e69be8da3585d/lxml-6.1.0.tar.gz", hash = "sha256:bfd57d8008c4965709a919c3e9a98f76c2c7cb319086b3d26858250620023b13", size = 4197006, upload-time = "2026-04-18T04:32:51.613Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/5d/3bccad330292946f97962df9d5f2d3ae129cce6e212732a781e856b91e07/lxml-6.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cec05be8c876f92a5aa07b01d60bbb4d11cfbdd654cad0561c0d7b5c043a61b9", size = 8526232, upload-time = "2026-04-18T04:27:40.389Z" }, + { url = "https://files.pythonhosted.org/packages/a7/51/adc8826570a112f83bb4ddb3a2ab510bbc2ccd62c1b9fe1f34fae2d90b57/lxml-6.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9c03e048b6ce8e77b09c734e931584894ecd58d08296804ca2d0b184c933ce50", size = 4595448, upload-time = "2026-04-18T04:27:44.208Z" }, + { url = "https://files.pythonhosted.org/packages/54/84/5a9ec07cbe1d2334a6465f863b949a520d2699a755738986dcd3b6b89e3f/lxml-6.1.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:942454ff253da14218f972b23dc72fa4edf6c943f37edd19cd697618b626fac5", size = 4923771, upload-time = "2026-04-18T04:32:17.402Z" }, + { url = "https://files.pythonhosted.org/packages/a7/23/851cfa33b6b38adb628e45ad51fb27105fa34b2b3ba9d1d4aa7a9428dfe0/lxml-6.1.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d036ee7b99d5148072ac7c9b847193decdfeac633db350363f7bce4fff108f0e", size = 5068101, upload-time = "2026-04-18T04:32:21.437Z" }, + { url = "https://files.pythonhosted.org/packages/b0/38/41bf99c2023c6b79916ba057d83e9db21d642f473cac210201222882d38b/lxml-6.1.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ae5d8d5427f3cc317e7950f2da7ad276df0cfa37b8de2f5658959e618ea8512", size = 5002573, upload-time = "2026-04-18T04:32:25.373Z" }, + { url = "https://files.pythonhosted.org/packages/c2/20/053aa10bdc39747e1e923ce2d45413075e84f70a136045bb09e5eaca41d3/lxml-6.1.0-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:363e47283bde87051b821826e71dde47f107e08614e1aa312ba0c5711e77738c", size = 5202816, upload-time = "2026-04-18T04:32:29.393Z" }, + { url = "https://files.pythonhosted.org/packages/9a/da/bc710fad8bf04b93baee752c192eaa2210cd3a84f969d0be7830fea55802/lxml-6.1.0-cp311-cp311-manylinux_2_28_i686.whl", hash = "sha256:f504d861d9f2a8f94020130adac88d66de93841707a23a86244263d1e54682f5", size = 5329999, upload-time = "2026-04-18T04:32:34.019Z" }, + { url = "https://files.pythonhosted.org/packages/b3/cb/bf035dedbdf7fab49411aa52e4236f3445e98d38647d85419e6c0d2806b9/lxml-6.1.0-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:23a5dc68e08ed13331d61815c08f260f46b4a60fdd1640bbeb82cf89a9d90289", size = 4659643, upload-time = "2026-04-18T04:32:37.932Z" }, + { url = "https://files.pythonhosted.org/packages/5c/4f/22be31f33727a5e4c7b01b0a874503026e50329b259d3587e0b923cf964b/lxml-6.1.0-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f15401d8d3dbf239e23c818afc10c7207f7b95f9a307e092122b6f86dd43209a", size = 5265963, upload-time = "2026-04-18T04:32:41.881Z" }, + { url = "https://files.pythonhosted.org/packages/c8/2b/d44d0e5c79226017f4ab8c87a802ebe4f89f97e6585a8e4166dffcdd7b6e/lxml-6.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fcf3da95e93349e0647d48d4b36a12783105bcc74cb0c416952f9988410846a3", size = 5045444, upload-time = "2026-04-18T04:32:44.512Z" }, + { url = "https://files.pythonhosted.org/packages/d3/c3/3f034fec1594c331a6dbf9491238fdcc9d66f68cc529e109ec75b97197e1/lxml-6.1.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:0d082495c5fcf426e425a6e28daaba1fcb6d8f854a4ff01effb1f1f381203eb9", size = 4712703, upload-time = "2026-04-18T04:32:47.16Z" }, + { url = "https://files.pythonhosted.org/packages/12/16/0b83fccc158218aca75a7aa33e97441df737950734246b9fffa39301603d/lxml-6.1.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:e3c4f84b24a1fcba435157d111c4b755099c6ff00a3daee1ad281817de75ed11", size = 5252745, upload-time = "2026-04-18T04:32:50.427Z" }, + { url = "https://files.pythonhosted.org/packages/dd/ee/12e6c1b39a77666c02eaa77f94a870aaf63c4ac3a497b2d52319448b01c6/lxml-6.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:976a6b39b1b13e8c354ad8d3f261f3a4ac6609518af91bdb5094760a08f132c4", size = 5226822, upload-time = "2026-04-18T04:32:53.437Z" }, + { url = "https://files.pythonhosted.org/packages/34/20/c7852904858b4723af01d2fc14b5d38ff57cb92f01934a127ebd9a9e51aa/lxml-6.1.0-cp311-cp311-win32.whl", hash = "sha256:857efde87d365706590847b916baff69c0bc9252dc5af030e378c9800c0b10e3", size = 3594026, upload-time = "2026-04-18T04:27:31.903Z" }, + { url = "https://files.pythonhosted.org/packages/02/05/d60c732b56da5085175c07c74b2df4e6d181b0c9a61e1691474f06ef4b39/lxml-6.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:183bfb45a493081943be7ea2b5adfc2b611e1cf377cefa8b8a8be404f45ef9a7", size = 4025114, upload-time = "2026-04-18T04:27:34.077Z" }, + { url = "https://files.pythonhosted.org/packages/c2/df/c84dcc175fd690823436d15b41cb920cd5ba5e14cd8bfb00949d5903b320/lxml-6.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:19f4164243fc206d12ed3d866e80e74f5bc3627966520da1a5f97e42c32a3f39", size = 3667742, upload-time = "2026-04-18T04:27:38.45Z" }, + { url = "https://files.pythonhosted.org/packages/f2/88/55143966481409b1740a3ac669e611055f49efd68087a5ce41582325db3e/lxml-6.1.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:546b66c0dd1bb8d9fa89d7123e5fa19a8aff3a1f2141eb22df96112afb17b842", size = 3930134, upload-time = "2026-04-18T04:32:35.008Z" }, + { url = "https://files.pythonhosted.org/packages/b5/97/28b985c2983938d3cb696dd5501423afb90a8c3e869ef5d3c62569282c0f/lxml-6.1.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5cfa1a34df366d9dc0d5eaf420f4cf2bb1e1bebe1066d1c2fc28c179f8a4004c", size = 4210749, upload-time = "2026-04-18T04:36:03.626Z" }, + { url = "https://files.pythonhosted.org/packages/29/67/dfab2b7d58214921935ccea7ce9b3df9b7d46f305d12f0f532ac7cf6b804/lxml-6.1.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db88156fcf544cdbf0d95588051515cfdfd4c876fc66444eb98bceb5d6db76de", size = 4318463, upload-time = "2026-04-18T04:36:06.309Z" }, + { url = "https://files.pythonhosted.org/packages/32/a2/4ac7eb32a4d997dd352c32c32399aae27b3f268d440e6f9cfa405b575d2f/lxml-6.1.0-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:07f98f5496f96bf724b1e3c933c107f0cbf2745db18c03d2e13a291c3afd2635", size = 4251124, upload-time = "2026-04-18T04:36:09.056Z" }, + { url = "https://files.pythonhosted.org/packages/33/ef/d6abd850bb4822f9b720cfe36b547a558e694881010ff7d012191e8769c6/lxml-6.1.0-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4642e04449a1e164b5ff71ffd901ddb772dfabf5c9adf1b7be5dffe1212bc037", size = 4401758, upload-time = "2026-04-18T04:36:11.803Z" }, + { url = "https://files.pythonhosted.org/packages/40/44/3ee09a5b60cb44c4f2fbc1c9015cfd6ff5afc08f991cab295d3024dcbf2d/lxml-6.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:7da13bb6fbadfafb474e0226a30570a3445cfd47c86296f2446dafbd77079ace", size = 3508860, upload-time = "2026-04-18T04:32:48.619Z" }, +] + [[package]] name = "markdown-it-py" version = "4.0.0" @@ -1195,6 +1225,7 @@ dependencies = [ { name = "geopandas" }, { name = "netcdf4" }, { name = "numpy" }, + { name = "owslib" }, { name = "pandas" }, { name = "prettyprinter" }, { name = "pyproj" }, @@ -1230,6 +1261,7 @@ requires-dist = [ { name = "geopandas", specifier = ">=1.1.3,<2" }, { name = "netcdf4", specifier = ">=1.6.5,<2" }, { name = "numpy", specifier = ">=2.4.3,<3" }, + { name = "owslib", specifier = ">=0.35.0" }, { name = "pandas", specifier = ">=2.2.2,<3" }, { name = "prettyprinter", specifier = ">=0.18.0,<0.19" }, { name = "pyproj", specifier = ">=3.6.1,<4" }, @@ -1263,6 +1295,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/ab/fc8290c6a4c722e5514d80f62b2dc4c4df1a68a41d1364e625c35990fcf3/overrides-7.7.0-py3-none-any.whl", hash = "sha256:c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49", size = 17832, upload-time = "2024-01-27T21:01:31.393Z" }, ] +[[package]] +name = "owslib" +version = "0.35.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, + { name = "python-dateutil" }, + { name = "pyyaml" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bf/03/5df0b995a3a6eefc24cc2320e3fe830d95e76ee6b0b1c6c9d0f1d4040bbe/owslib-0.35.0.tar.gz", hash = "sha256:0182f377bb30d25b78284bbaf82a12dece97902ed844cee88791ff38665b9b00", size = 194282, upload-time = "2025-10-28T15:13:42.594Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/12/74c145b9e273b2b479f1ea578c7778d96d0c34f2d112fd0ec96c905bb792/owslib-0.35.0-py3-none-any.whl", hash = "sha256:01648ea9b2b86502f456ad68a8dd07131d336416c5637f34adb358aafc0ad380", size = 240487, upload-time = "2025-10-28T15:13:41.399Z" }, +] + [[package]] name = "packaging" version = "25.0" From 56c4b9fd908509a80a2d3e0850505eb46a72a479 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Mon, 16 Mar 2026 11:48:10 +1100 Subject: [PATCH 08/29] Add WA oil & gas emission sources from petroleum wells Combine WA petroleum wells dataset with petroleum titles dataset to extract producing wells with an activity period matching the title period. --- .../sectors/oil_gas/data/wa_titles.py | 96 +++++++++++++++++++ .../sectors/oil_gas/data/wa_wells.py | 65 +++++++++++++ .../oil_gas/emission_sources/wa_sources.py | 77 +++++++++++++++ .../test_emission_sources.py | 27 ++++++ 4 files changed, 265 insertions(+) create mode 100644 src/openmethane_prior/sectors/oil_gas/data/wa_titles.py create mode 100644 src/openmethane_prior/sectors/oil_gas/data/wa_wells.py create mode 100644 src/openmethane_prior/sectors/oil_gas/emission_sources/wa_sources.py diff --git a/src/openmethane_prior/sectors/oil_gas/data/wa_titles.py b/src/openmethane_prior/sectors/oil_gas/data/wa_titles.py new file mode 100644 index 00000000..efe60ce1 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/wa_titles.py @@ -0,0 +1,96 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import geopandas as gpd +import pandas as pd +import restapi # https://github.com/Bolton-and-Menk-GIS/restapi + +from openmethane_prior.lib import DataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo +from openmethane_prior.lib.data_manager.source import ConfiguredDataSource +from .esri_types import map_esri_date_to_str + + +# WA Petroleum Titles (DMIRS-011) - REST Service (ArcGIS) +# https://catalogue.data.wa.gov.au/dataset/wa-petroleum-titles-dmirs-011/resource/f5bb4e83-241e-4dcf-8efe-41707866af4e +def fetch_wa_titles(data_source: ConfiguredDataSource): + # wa_arcgis = restapi.ArcServer(url="https://public-services.slip.wa.gov.au/public/rest/services") + wa_arcgis_mining = restapi.MapService( + url="https://public-services.slip.wa.gov.au/public/rest/services/SLIP_Public_Services/Industry_and_Mining/MapServer" + ) + + # DMIRS-011 contains current active petroleum titles, but has expired + # titles periodically removed. Our interest is only in "Production License" + # titles, as we wouldn't expect significant emissions during exploration, + # or if the parcel of land is on a "retention" lease. + titles_layer = wa_arcgis_mining.layer("WA Petroleum Titles (DMIRS-011)") + titles_features = titles_layer.query( + where="type in ('Production Licence')", + fields=[ + "title_id", + "type", + "issued", + "expiry", + ], + exceed_limit=True, + ) + titles_df = gpd.GeoDataFrame.from_features(titles_features.features) + + # Rename some fields so that they match DMIRS-051 + titles_df = titles_df.rename(columns={ + "issued": "issued_date", + "expiry": "end_date", + }) + + # DMIRS-051 contains the record of previous titles. One parcel of land + # can appear in this dataset multiple times as titles change. Sometimes a + # single parcel can have multiple titles with the same dates or overlapping + # dates. No attempt is made to filter or de-duplicate these at this stage. + historic_titles_layer = wa_arcgis_mining.layer("WA Petroleum - Historical Titles (DMIRS-051)") + historic_titles_features = historic_titles_layer.query( + where="type in ('Production Licence')", + fields=[ + "title_id", + "type", + "issued_date", + "end_date", + ], + exceed_limit=True, + ) + historic_df = gpd.GeoDataFrame.from_features(historic_titles_features.features) + + # combine current and historic title datasets + df = pd.concat([titles_df, historic_df]) + + # convert date fields to RFC3339 + df["issued_date"] = df["issued_date"].map(map_esri_date_to_str) + df["end_date"] = df["end_date"].map(map_esri_date_to_str) + + with open(data_source.asset_path, "w") as asset_file: + asset_file.write(df.to_json()) + return data_source.asset_path + + +# Locations of all petroleum titles in the Australian state of Western +# Australia via the Data WA Portal. +# Source: https://catalogue.data.wa.gov.au/dataset/wa-petroleum-titles-dmirs-011 +wa_titles_data_source = DataSource( + name="wa-petroleum-titles", + file_path="WA-petroleum-titles.geojson", + fetch=fetch_wa_titles, + parse=parse_geo, +) diff --git a/src/openmethane_prior/sectors/oil_gas/data/wa_wells.py b/src/openmethane_prior/sectors/oil_gas/data/wa_wells.py new file mode 100644 index 00000000..c0b4b135 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/wa_wells.py @@ -0,0 +1,65 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import geopandas as gpd +import restapi # https://github.com/Bolton-and-Menk-GIS/restapi + +from openmethane_prior.lib import DataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo +from openmethane_prior.lib.data_manager.source import ConfiguredDataSource +from .esri_types import map_esri_date_to_str + + +# WA Petroleum Wells (DMIRS-025) - REST Service (ArcGIS) +# https://catalogue.data.wa.gov.au/dataset/mineral-exploration-drillholes-open-file/resource/1c61171a-3b23-4f2b-baae-04615c5bb39e +def fetch_wa_wells(data_source: ConfiguredDataSource): + # wa_arcgis = restapi.ArcServer(url="https://public-services.slip.wa.gov.au/public/rest/services") + wa_arcgis_mining = restapi.MapService( + url="https://public-services.slip.wa.gov.au/public/rest/services/SLIP_Public_Services/Industry_and_Mining/MapServer" + ) + + wells_layer = wa_arcgis_mining.layer("WA Onshore Petroleum Wells (DMIRS-025)") + wells_features = wells_layer.query( + fields=[ + "well_name", + "uwi", + "lease_no", + "operator", + "class", + "status", + "rig_release_date", + ], + exceed_limit=True, + ) + + df = gpd.GeoDataFrame.from_features(wells_features.features) + df["rig_release_date"] = df["rig_release_date"].map(map_esri_date_to_str) + + with open(data_source.asset_path, "w") as asset_file: + asset_file.write(df.to_json()) + return data_source.asset_path + + +# Locations of all petroleum wells in the Australian state of Western Australia +# via the Data WA Portal. +# Source: https://catalogue.data.wa.gov.au/dataset/wa-onshore-petroleum-wells-dmirs-025 +wa_wells_data_source = DataSource( + name="wa-petroleum-wells", + file_path="WA-petroleum-wells.geojson", + fetch=fetch_wa_wells, + parse=parse_geo, +) diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/wa_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/wa_sources.py new file mode 100644 index 00000000..22efdaef --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/wa_sources.py @@ -0,0 +1,77 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import geopandas as gpd +import numpy as np + +from openmethane_prior.lib.data_manager.asset import DataAsset +from openmethane_prior.lib.utils import rows_in_period +from openmethane_prior.sectors.oil_gas.emission_sources.emission_source import normalise_emission_source_df + +def wa_emission_sources( + start_date: datetime.date, + end_date: datetime.date, + wa_wells_da: DataAsset, + wa_titles_da: DataAsset, +) -> gpd.GeoDataFrame: + """Create normalised emission source DataFrame by combining WA petroleum + wells dataset for locations, with land title dataset for production + start/end dates.""" + wa_wells_df: gpd.GeoDataFrame = wa_wells_da.data + wa_titles_df: gpd.GeoDataFrame = wa_titles_da.data + + # filter out wells that aren't actively used for production + wells_df = wa_wells_df[wa_wells_df["class"] == "DEV"] + + # WA wells dataset appears to re-use lat/lon coords for multiple wells + # within the same field. Since these contain unique uwi/well_name values + # and rig_release_date, we treat these as unique wells + + # join drillholes with titles to use the title dates as start/end dates + sources_df = gpd.sjoin(wells_df, wa_titles_df, how="inner", predicate="within") + + # start date of emissions must be after hole is drilled and after the title + # is granted, so choose the latter of the two dates + sources_df["start_date"] = [ + issued if not np.isnat(issued) and (np.isnat(drilled) or issued > drilled) else drilled + for issued, drilled in sources_df[["issued_date", "rig_release_date"]].values + ] + del sources_df["issued_date"] + del sources_df["rig_release_date"] + + # exclude any emission sources that would not have been emitting during + # the period between start_date and end_date + sources_df = rows_in_period(sources_df, start_date=start_date, end_date=end_date) + + # after the join with titles, there may be multiple rows for each well. + # drop duplicates so there is only one entry for each location. + sources_df = sources_df.drop_duplicates(["uwi"]) + + # normalise output to match emission sources format + sources_df = sources_df.rename(columns={ + "uwi": "data_source_id", + "title_id": "group_id", + "start_date": "activity_start", + "end_date": "activity_end", + }) + sources_df["data_source"] = wa_wells_da.name + # not enough detail in the data source to determine the type of resource + # being extracted by the well + sources_df["site_type"] = "drillhole-unknown" + + return normalise_emission_source_df(sources_df) diff --git a/tests/integration/test_sector_oil_gas/test_emission_sources.py b/tests/integration/test_sector_oil_gas/test_emission_sources.py index dce3fbf3..75695483 100644 --- a/tests/integration/test_sector_oil_gas/test_emission_sources.py +++ b/tests/integration/test_sector_oil_gas/test_emission_sources.py @@ -7,8 +7,11 @@ from openmethane_prior.sectors.oil_gas.data.qld_boreholes import qld_boreholes_data_source from openmethane_prior.sectors.oil_gas.data.qld_leases import qld_leases_data_source from openmethane_prior.sectors.oil_gas.emission_sources.nsw_sources import nsw_emission_sources +from openmethane_prior.sectors.oil_gas.data.wa_titles import wa_titles_data_source +from openmethane_prior.sectors.oil_gas.data.wa_wells import wa_wells_data_source from openmethane_prior.sectors.oil_gas.emission_sources.offshore_sources import offshore_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.qld_sources import qld_emission_sources +from openmethane_prior.sectors.oil_gas.emission_sources.wa_sources import wa_emission_sources def test_nsw_emission_sources(input_files, data_manager): @@ -81,6 +84,30 @@ def test_qld_emission_sources(input_files, data_manager): assert len(df["geometry"].unique()) == len(df) +def test_wa_emission_sources(input_files, data_manager): + start_date = datetime.datetime(2023, 1, 1, 0, 0) + start_date_end = datetime.datetime(2023, 1, 2, 0, 0) + wells_da = data_manager.get_asset(wa_wells_data_source) + df = wa_emission_sources( + start_date=start_date.date(), + end_date=start_date.date(), + wa_wells_da=wells_da, + wa_titles_da=data_manager.get_asset(wa_titles_data_source), + ) + + # original wells dataset has been filtered down + assert len(wells_da.data) == 4363 + assert len(df) == 413 + + # no sources where activity period doesn't intersect date period + assert len(df[(df["activity_end"] < start_date) & (df["activity_start"] > start_date_end)]) == 0 + + # no sources which aren't related to production + assert set(df["class"].unique()) == {"DEV"} + + # no duplicate check, as we allow duplicate geometries from WA dataset + + def test_offshore_emission_sources(input_files, data_manager): start_date = datetime.datetime(2023, 1, 1, 0, 0) start_date_end = datetime.datetime(2023, 1, 2, 0, 0) From b6ef532ecdf9e4f691200f66c7da0689922682af Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 19 Mar 2026 11:41:14 +1100 Subject: [PATCH 09/29] Add notebook to visualise oil and gas emission sources --- notebooks/oil_gas_locations.py | 78 ++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 notebooks/oil_gas_locations.py diff --git a/notebooks/oil_gas_locations.py b/notebooks/oil_gas_locations.py new file mode 100644 index 00000000..56694f67 --- /dev/null +++ b/notebooks/oil_gas_locations.py @@ -0,0 +1,78 @@ +import datetime +import geopandas as gpd +import os +import pandas as pd +import pathlib +from openmethane_prior.lib import PriorConfig +from openmethane_prior.lib.data_manager.manager import DataManager +from openmethane_prior.sectors.oil_gas.data.nopta_titles import nopta_titles_data_source +from openmethane_prior.sectors.oil_gas.data.nopta_wells import nopta_wells_data_source +from openmethane_prior.sectors.oil_gas.data.nsw_drillholes import nsw_drillholes_data_source +from openmethane_prior.sectors.oil_gas.data.nsw_titles import nsw_titles_data_source +from openmethane_prior.sectors.oil_gas.data.qld_boreholes import qld_boreholes_data_source +from openmethane_prior.sectors.oil_gas.data.qld_leases import qld_leases_data_source +from openmethane_prior.sectors.oil_gas.data.wa_titles import wa_titles_data_source +from openmethane_prior.sectors.oil_gas.data.wa_wells import wa_wells_data_source +from openmethane_prior.sectors.oil_gas.emission_sources.nsw_sources import nsw_emission_sources +from openmethane_prior.sectors.oil_gas.emission_sources.offshore_sources import offshore_emission_sources +from openmethane_prior.sectors.oil_gas.emission_sources.qld_sources import qld_emission_sources +from openmethane_prior.sectors.oil_gas.emission_sources.wa_sources import wa_emission_sources +os.environ["START_DATE"] = "2023-07-01" +os.environ["END_DATE"] = "2023-07-01" +prior_config = PriorConfig.from_env() +data_manager = DataManager( + data_path=pathlib.Path("../data/inputs"), + prior_config=prior_config, +) + +nsw_drillholes_da = data_manager.get_asset(nsw_drillholes_data_source) +nsw_titles_da = data_manager.get_asset(nsw_titles_data_source) +nsw_df = nsw_emission_sources( + start_date=datetime.date(2021, 1, 1), + end_date=datetime.date(2023, 1, 2), + nsw_drillholes_da=nsw_drillholes_da, + nsw_titles_da=nsw_titles_da, +) +qld_boreholes_da = data_manager.get_asset(qld_boreholes_data_source) +qld_leases_da = data_manager.get_asset(qld_leases_data_source) +qld_df = qld_emission_sources( + start_date=datetime.date(2021, 1, 1), + end_date=datetime.date(2023, 1, 2), + qld_boreholes_da=qld_boreholes_da, + qld_leases_da=qld_leases_da, +) +wa_wells_da = data_manager.get_asset(wa_wells_data_source) +wa_titles_da = data_manager.get_asset(wa_titles_data_source) +wa_df = wa_emission_sources( + start_date=datetime.date(2021, 1, 1), + end_date=datetime.date(2023, 1, 2), + wa_wells_da=wa_wells_da, + wa_titles_da=wa_titles_da, +) +offshore_wells_da = data_manager.get_asset(nopta_wells_data_source) +offshore_titles_da = data_manager.get_asset(nopta_titles_data_source) +offshore_df = offshore_emission_sources( + start_date=datetime.date(2021, 1, 1), + end_date=datetime.date(2023, 1, 2), + offshore_wells_da=offshore_wells_da, + offshore_titles_da=offshore_titles_da, +) +all_df = pd.concat([ + nsw_df, + qld_df, + wa_df, + offshore_df, +]) + +au_shp = gpd.GeoDataFrame.from_file("~/Downloads/STE_2021_AUST_SHP_GDA2020/STE_2021_AUST_GDA2020.shx") +au_map = au_shp.plot(color='white', edgecolor='#CCCCCC', figsize=(32,24)) +au_map.set_xlim(110, 130) +au_map.set_ylim(-35, -10) +nsw_titles_da.data.plot(ax=au_map, alpha=0.5) +qld_leases_da.data.plot(ax=au_map, alpha=0.5) +# wa_wells_da.data.plot(ax=au_map, markersize=3, column="class", legend=True) +wa_titles_da.data.plot(ax=au_map, alpha=0.5) +all_df.plot(ax=au_map, markersize=3, column="data_source", legend=True) +au_map + + From 4eb77468caf126fddad078ee0cdcf9ec1e85ea40 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 19 Mar 2026 17:23:12 +1100 Subject: [PATCH 10/29] Combine all emission sources into a single output Emission sources outputs from each region are normalised into a single list which shares the emissions source structure. The offshore (NOPTA) dataset has significant overlap with WA and VIC datasets, with many wells occurring in both. In the case of VIC fields dataset, the NOPTA dataset has more detail and includes all the same sources, so the simplest solution is not to use the VIC data. In the case of WA, the WA dataset does include some wells that are not found in NOPTA, so we want to combine both datasets but remove any duplicates. The two datasets store coords with slightly different precision, so we allow for a small amount of distance between "duplicate" points. --- .../oil_gas/emission_sources/all_sources.py | 94 +++++++++++++++++++ .../test_emission_sources.py | 15 +++ 2 files changed, 109 insertions(+) create mode 100644 src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py new file mode 100644 index 00000000..3c928e58 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py @@ -0,0 +1,94 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import geopandas as gpd +import pandas as pd + +from openmethane_prior.lib.data_manager.manager import DataManager +from openmethane_prior.sectors.oil_gas.data.nsw_drillholes import nsw_drillholes_data_source +from openmethane_prior.sectors.oil_gas.data.nsw_titles import nsw_titles_data_source + +from .nsw_sources import nsw_emission_sources +from .offshore_sources import offshore_emission_sources +from .qld_sources import qld_emission_sources +from .wa_sources import wa_emission_sources +from ..data.nopta_titles import nopta_titles_data_source +from ..data.nopta_wells import nopta_wells_data_source +from ..data.qld_boreholes import qld_boreholes_data_source +from ..data.qld_leases import qld_leases_data_source +from ..data.wa_titles import wa_titles_data_source +from ..data.wa_wells import wa_wells_data_source + + +def all_emission_sources( + data_manager: DataManager, + start_date: datetime.date, + end_date: datetime.date, +) -> gpd.GeoDataFrame: + """Assemble a single DataFrame from multiple data sources, where each row + represents a possible location of emissions in the oil and gas sector.""" + nsw_drillholes_da = data_manager.get_asset(nsw_drillholes_data_source) + nsw_titles_da = data_manager.get_asset(nsw_titles_data_source) + nsw_df = nsw_emission_sources( + start_date=start_date, + end_date=end_date, + nsw_drillholes_da=nsw_drillholes_da, + nsw_titles_da=nsw_titles_da, + ) + + qld_boreholes_da = data_manager.get_asset(qld_boreholes_data_source) + qld_leases_da = data_manager.get_asset(qld_leases_data_source) + qld_df = qld_emission_sources( + start_date=start_date, + end_date=end_date, + qld_boreholes_da=qld_boreholes_da, + qld_leases_da=qld_leases_da, + ) + + wa_wells_da = data_manager.get_asset(wa_wells_data_source) + wa_titles_da = data_manager.get_asset(wa_titles_data_source) + wa_df = wa_emission_sources( + start_date=start_date, + end_date=end_date, + wa_wells_da=wa_wells_da, + wa_titles_da=wa_titles_da, + ) + + states_df: gpd.GeoDataFrame = pd.concat([ + nsw_df, + qld_df, + wa_df, + ]) + + offshore_wells_da = data_manager.get_asset(nopta_wells_data_source) + offshore_titles_da = data_manager.get_asset(nopta_titles_data_source) + offshore_df = offshore_emission_sources( + start_date=start_date, + end_date=end_date, + offshore_wells_da=offshore_wells_da, + offshore_titles_da=offshore_titles_da, + ) + # NOPTA will have some wells that are already provided by state datasets, + # which we must avoid "double counting" + offshore_existing = states_df.sjoin_nearest(offshore_df, how="inner", max_distance=0.00005) + offshore_new = offshore_df[~offshore_df["data_source_id"].isin(offshore_existing["data_source_id_right"])] + + all_df: gpd.GeoDataFrame = pd.concat([states_df, offshore_new]) + + return all_df + diff --git a/tests/integration/test_sector_oil_gas/test_emission_sources.py b/tests/integration/test_sector_oil_gas/test_emission_sources.py index 75695483..596c360c 100644 --- a/tests/integration/test_sector_oil_gas/test_emission_sources.py +++ b/tests/integration/test_sector_oil_gas/test_emission_sources.py @@ -6,6 +6,7 @@ from openmethane_prior.sectors.oil_gas.data.nsw_titles import nsw_titles_data_source from openmethane_prior.sectors.oil_gas.data.qld_boreholes import qld_boreholes_data_source from openmethane_prior.sectors.oil_gas.data.qld_leases import qld_leases_data_source +from openmethane_prior.sectors.oil_gas.emission_sources.all_sources import all_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.nsw_sources import nsw_emission_sources from openmethane_prior.sectors.oil_gas.data.wa_titles import wa_titles_data_source from openmethane_prior.sectors.oil_gas.data.wa_wells import wa_wells_data_source @@ -131,3 +132,17 @@ def test_offshore_emission_sources(input_files, data_manager): assert set(df["TitleType"].unique()) == {"Production Licence"} # no duplicate check, as we allow duplicate geometries from NOPTA dataset + + +def test_all_emission_sources(input_files, data_manager, config): + end_date_end = config.end_date + datetime.timedelta(days=1) + df = all_emission_sources( + data_manager=data_manager, + prior_config=config, + ) + + assert len(df) == 9882 + + # no sources where activity period doesn't intersect config period + assert len(df[(df["activity_end"] < config.start_date) & (df["activity_start"] > end_date_end)]) == 0 + From 2e705160af33901e4456e9797081dca40541b2fe Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Mon, 23 Mar 2026 10:08:59 +1100 Subject: [PATCH 11/29] Normalise state emission sources in integration method Originally the idea behind normalise_emission_sources was to ensure each function returned a function which was already in the right format, or raise an error if that failed. However, from a functional standpoint it doesn't matter if sources are normalised until they're combined, which happens in the calling function. If we want to ensure each sources function returns the right columns, we can do this with a unit test instead of a runtime test, as we don't expect the format to change underneath us. --- notebooks/oil_gas_locations.py | 28 +++++++++++++------ .../oil_gas/emission_sources/all_sources.py | 19 ++++++++++--- .../emission_sources/emission_source.py | 6 ++-- .../oil_gas/emission_sources/nsw_sources.py | 3 +- .../emission_sources/offshore_sources.py | 4 +-- .../oil_gas/emission_sources/qld_sources.py | 4 +-- .../oil_gas/emission_sources/wa_sources.py | 4 +-- .../unit/test_oil_gas/test_emission_source.py | 2 +- 8 files changed, 45 insertions(+), 25 deletions(-) diff --git a/notebooks/oil_gas_locations.py b/notebooks/oil_gas_locations.py index 56694f67..f146cbc1 100644 --- a/notebooks/oil_gas_locations.py +++ b/notebooks/oil_gas_locations.py @@ -17,9 +17,14 @@ from openmethane_prior.sectors.oil_gas.emission_sources.offshore_sources import offshore_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.qld_sources import qld_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.wa_sources import wa_emission_sources +os.environ["DOMAIN_FILE"] = "https://openmethane.s3.amazonaws.com/domains/aust10km/v1/domain.aust10km.nc" os.environ["START_DATE"] = "2023-07-01" os.environ["END_DATE"] = "2023-07-01" +os.environ["INPUTS"] = "../data/inputs" +os.environ["INPUT_CACHE"] = "../data/.cache" prior_config = PriorConfig.from_env() +prior_config.prepare_paths() +prior_config.load_cached_inputs() data_manager = DataManager( data_path=pathlib.Path("../data/inputs"), prior_config=prior_config, @@ -57,22 +62,27 @@ offshore_wells_da=offshore_wells_da, offshore_titles_da=offshore_titles_da, ) +print(f"{nsw_df.crs=}") +print(f"{qld_df.crs=}") +print(f"{wa_df.crs=}") +print(f"{offshore_df.crs=}") all_df = pd.concat([ nsw_df, qld_df, wa_df, offshore_df, -]) +]).to_crs("EPSG:4326") # for plotting -au_shp = gpd.GeoDataFrame.from_file("~/Downloads/STE_2021_AUST_SHP_GDA2020/STE_2021_AUST_GDA2020.shx") +au_shp = gpd.GeoDataFrame.from_file("~/Downloads/STE_2021_AUST_SHP_GDA2020/STE_2021_AUST_GDA2020.shx").to_crs("EPSG:4326") au_map = au_shp.plot(color='white', edgecolor='#CCCCCC', figsize=(32,24)) -au_map.set_xlim(110, 130) -au_map.set_ylim(-35, -10) -nsw_titles_da.data.plot(ax=au_map, alpha=0.5) -qld_leases_da.data.plot(ax=au_map, alpha=0.5) -# wa_wells_da.data.plot(ax=au_map, markersize=3, column="class", legend=True) -wa_titles_da.data.plot(ax=au_map, alpha=0.5) -all_df.plot(ax=au_map, markersize=3, column="data_source", legend=True) +# uncomment to show a smaller extent around QLD +# au_map.set_xlim(110, 130) +# au_map.set_ylim(-35, -10) +nsw_titles_da.data.to_crs("EPSG:4326").plot(ax=au_map, alpha=0.5) +qld_leases_da.data.to_crs("EPSG:4326").plot(ax=au_map, alpha=0.5) +# # wa_wells_da.data.to_crs("EPSG:4326").plot(ax=au_map, markersize=3, column="class", legend=True) +wa_titles_da.data.to_crs("EPSG:4326").plot(ax=au_map, alpha=0.5) +all_df.plot(ax=au_map, markersize=3, column="site_type", legend=True) au_map diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py index 3c928e58..1d6ca2bf 100644 --- a/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py @@ -15,13 +15,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import datetime import geopandas as gpd import pandas as pd +from openmethane_prior.lib import PriorConfig from openmethane_prior.lib.data_manager.manager import DataManager from openmethane_prior.sectors.oil_gas.data.nsw_drillholes import nsw_drillholes_data_source from openmethane_prior.sectors.oil_gas.data.nsw_titles import nsw_titles_data_source +from .emission_source import normalise_emission_source_df from .nsw_sources import nsw_emission_sources from .offshore_sources import offshore_emission_sources @@ -37,11 +38,13 @@ def all_emission_sources( data_manager: DataManager, - start_date: datetime.date, - end_date: datetime.date, + prior_config: PriorConfig, ) -> gpd.GeoDataFrame: """Assemble a single DataFrame from multiple data sources, where each row represents a possible location of emissions in the oil and gas sector.""" + start_date = prior_config.start_date.date() + end_date = prior_config.end_date.date() + nsw_drillholes_da = data_manager.get_asset(nsw_drillholes_data_source) nsw_titles_da = data_manager.get_asset(nsw_titles_data_source) nsw_df = nsw_emission_sources( @@ -50,6 +53,7 @@ def all_emission_sources( nsw_drillholes_da=nsw_drillholes_da, nsw_titles_da=nsw_titles_da, ) + nsw_df = normalise_emission_source_df(nsw_df, prior_config.crs) qld_boreholes_da = data_manager.get_asset(qld_boreholes_data_source) qld_leases_da = data_manager.get_asset(qld_leases_data_source) @@ -59,6 +63,7 @@ def all_emission_sources( qld_boreholes_da=qld_boreholes_da, qld_leases_da=qld_leases_da, ) + qld_df = normalise_emission_source_df(qld_df, prior_config.crs) wa_wells_da = data_manager.get_asset(wa_wells_data_source) wa_titles_da = data_manager.get_asset(wa_titles_data_source) @@ -68,6 +73,7 @@ def all_emission_sources( wa_wells_da=wa_wells_da, wa_titles_da=wa_titles_da, ) + wa_df = normalise_emission_source_df(wa_df, prior_config.crs) states_df: gpd.GeoDataFrame = pd.concat([ nsw_df, @@ -83,12 +89,17 @@ def all_emission_sources( offshore_wells_da=offshore_wells_da, offshore_titles_da=offshore_titles_da, ) + offshore_df = normalise_emission_source_df(offshore_df, prior_config.crs) + # NOPTA will have some wells that are already provided by state datasets, # which we must avoid "double counting" offshore_existing = states_df.sjoin_nearest(offshore_df, how="inner", max_distance=0.00005) offshore_new = offshore_df[~offshore_df["data_source_id"].isin(offshore_existing["data_source_id_right"])] - all_df: gpd.GeoDataFrame = pd.concat([states_df, offshore_new]) + all_df: gpd.GeoDataFrame = pd.concat([ + states_df, + offshore_new, + ]) return all_df diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/emission_source.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/emission_source.py index 5bca4975..9bc05127 100644 --- a/src/openmethane_prior/sectors/oil_gas/emission_sources/emission_source.py +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/emission_source.py @@ -17,7 +17,7 @@ # import numpy as np import geopandas as gpd -import pyproj +from typing import Any """ An "emission source" in the oil and gas sector is any location that forms part @@ -63,10 +63,10 @@ def normalise_emission_source_df( df: gpd.GeoDataFrame, - crs: pyproj.CRS = pyproj.CRS.from_epsg(4326), + crs: Any, # anything supported by GeoDataFrame.to_crs ) -> gpd.GeoDataFrame: # select only columns which are present in emission_source_dtypes - normalised_df = df[list(emission_source_dtypes.keys())] + normalised_df = df[list(emission_source_dtypes.keys())].set_crs(df.crs) normalised_df = normalised_df.to_crs(crs) diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/nsw_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/nsw_sources.py index e19bfff3..ff4286e7 100644 --- a/src/openmethane_prior/sectors/oil_gas/emission_sources/nsw_sources.py +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/nsw_sources.py @@ -20,7 +20,6 @@ from openmethane_prior.lib.data_manager.asset import DataAsset from openmethane_prior.lib.utils import rows_in_period -from openmethane_prior.sectors.oil_gas.emission_sources.emission_source import normalise_emission_source_df nsw_drillhole_purpose_map = { "Coal seam methane": "drillhole-csg", @@ -92,4 +91,4 @@ def nsw_emission_sources( }) sources_df["data_source"] = nsw_drillholes_da.name - return normalise_emission_source_df(sources_df) + return sources_df diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/offshore_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/offshore_sources.py index 9ce01446..e1ee274a 100644 --- a/src/openmethane_prior/sectors/oil_gas/emission_sources/offshore_sources.py +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/offshore_sources.py @@ -21,7 +21,7 @@ from openmethane_prior.lib.data_manager.asset import DataAsset from openmethane_prior.lib.utils import rows_in_period -from openmethane_prior.sectors.oil_gas.emission_sources.emission_source import normalise_emission_source_df + def offshore_emission_sources( start_date: datetime.date, @@ -77,4 +77,4 @@ def offshore_emission_sources( # being extracted by the well sources_df["site_type"] = "drillhole-unknown" - return normalise_emission_source_df(sources_df) + return sources_df diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/qld_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/qld_sources.py index 0bec51d6..ca9186c0 100644 --- a/src/openmethane_prior/sectors/oil_gas/emission_sources/qld_sources.py +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/qld_sources.py @@ -21,7 +21,7 @@ from openmethane_prior.lib.data_manager.asset import DataAsset from openmethane_prior.lib.utils import rows_in_period -from openmethane_prior.sectors.oil_gas.emission_sources.emission_source import normalise_emission_source_df + bore_type_map = { "COAL SEAM GAS": "drillhole-csg", @@ -90,4 +90,4 @@ def qld_emission_sources( lambda bore_type: bore_type_map[bore_type] if bore_type in bore_type_map else "drillhole-unknown" ) - return normalise_emission_source_df(sources_df) + return sources_df diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/wa_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/wa_sources.py index 22efdaef..8bc403d8 100644 --- a/src/openmethane_prior/sectors/oil_gas/emission_sources/wa_sources.py +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/wa_sources.py @@ -21,7 +21,7 @@ from openmethane_prior.lib.data_manager.asset import DataAsset from openmethane_prior.lib.utils import rows_in_period -from openmethane_prior.sectors.oil_gas.emission_sources.emission_source import normalise_emission_source_df + def wa_emission_sources( start_date: datetime.date, @@ -74,4 +74,4 @@ def wa_emission_sources( # being extracted by the well sources_df["site_type"] = "drillhole-unknown" - return normalise_emission_source_df(sources_df) + return sources_df diff --git a/tests/unit/test_oil_gas/test_emission_source.py b/tests/unit/test_oil_gas/test_emission_source.py index 78cdde8d..30e30950 100644 --- a/tests/unit/test_oil_gas/test_emission_source.py +++ b/tests/unit/test_oil_gas/test_emission_source.py @@ -18,7 +18,7 @@ def test_normalise_emission_source(): "extra_column": [0], }, crs="EPSG:7844") - result_df = normalise_emission_source_df(test_df) + result_df = normalise_emission_source_df(test_df, "EPSG:4326") assert list(result_df.columns) == [ "geometry", From c21891326bbc53a7c800d90fbf70103f9f6325c8 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Mon, 23 Mar 2026 10:15:53 +1100 Subject: [PATCH 12/29] Convert GeoDataFrame to prior CRS in parse_geo Since we have the config available to transform the coordinate system during parsing, we can avoid downstream complexity and errors by already using the same CRS as the rest of the data. --- .../lib/data_manager/parsers.py | 16 +++++++++++++--- .../oil_gas/emission_sources/all_sources.py | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/openmethane_prior/lib/data_manager/parsers.py b/src/openmethane_prior/lib/data_manager/parsers.py index f47609c0..1b9b9e09 100644 --- a/src/openmethane_prior/lib/data_manager/parsers.py +++ b/src/openmethane_prior/lib/data_manager/parsers.py @@ -17,6 +17,7 @@ # import geopandas as gpd import pandas as pd +import pyproj from openmethane_prior.lib.data_manager.source import ConfiguredDataSource @@ -26,11 +27,20 @@ def parse_csv(data_source: ConfiguredDataSource) -> pd.DataFrame: return pd.read_csv(data_source.asset_path) -def parse_geo(data_source: ConfiguredDataSource): +def parse_geo(data_source: ConfiguredDataSource, source_crs: pyproj.CRS = None): """Read and parse a file containing a collection of geometry vector data into a geopandas GeoDataFrame. Asset file type can be anything supported by pyogrio, which includes GeoJSON, GeoPackage, Shapefiles, etc.""" geo_df = gpd.read_file(data_source.asset_path) - if geo_df.crs is not None and geo_df.crs != "EPSG:4326": - geo_df = geo_df.to_crs(epsg=4326) + + if geo_df.crs is None: + if source_crs is not None: + geo_df = geo_df.set_crs(source_crs) + else: + raise ValueError("parse_geo could not determine CRS, must be called manually with source_crs parameter") + + # convert the geometries into the prior projection to ensure downstream + # comparisons are done using the same coordinate system + geo_df = geo_df.to_crs(data_source.prior_config.crs) + return geo_df diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py index 1d6ca2bf..e2f9555a 100644 --- a/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py @@ -93,7 +93,7 @@ def all_emission_sources( # NOPTA will have some wells that are already provided by state datasets, # which we must avoid "double counting" - offshore_existing = states_df.sjoin_nearest(offshore_df, how="inner", max_distance=0.00005) + offshore_existing = states_df.sjoin_nearest(offshore_df, how="inner", max_distance=50) offshore_new = offshore_df[~offshore_df["data_source_id"].isin(offshore_existing["data_source_id_right"])] all_df: gpd.GeoDataFrame = pd.concat([ From d5c6104af9ea4ccf8482fc6c000aa5e68a9bcfb6 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Mon, 23 Mar 2026 10:40:59 +1100 Subject: [PATCH 13/29] Add debug logging to oil and gas sector emission sources --- .../oil_gas/emission_sources/all_sources.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py index e2f9555a..92884da4 100644 --- a/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py @@ -18,23 +18,27 @@ import geopandas as gpd import pandas as pd -from openmethane_prior.lib import PriorConfig -from openmethane_prior.lib.data_manager.manager import DataManager -from openmethane_prior.sectors.oil_gas.data.nsw_drillholes import nsw_drillholes_data_source -from openmethane_prior.sectors.oil_gas.data.nsw_titles import nsw_titles_data_source -from .emission_source import normalise_emission_source_df +from openmethane_prior.lib import ( + logger, + DataManager, + PriorConfig, +) +from .emission_source import normalise_emission_source_df from .nsw_sources import nsw_emission_sources from .offshore_sources import offshore_emission_sources from .qld_sources import qld_emission_sources from .wa_sources import wa_emission_sources from ..data.nopta_titles import nopta_titles_data_source from ..data.nopta_wells import nopta_wells_data_source +from ..data.nsw_drillholes import nsw_drillholes_data_source +from ..data.nsw_titles import nsw_titles_data_source from ..data.qld_boreholes import qld_boreholes_data_source from ..data.qld_leases import qld_leases_data_source from ..data.wa_titles import wa_titles_data_source from ..data.wa_wells import wa_wells_data_source +logger = logger.get_logger(__name__) def all_emission_sources( data_manager: DataManager, @@ -54,6 +58,7 @@ def all_emission_sources( nsw_titles_da=nsw_titles_da, ) nsw_df = normalise_emission_source_df(nsw_df, prior_config.crs) + logger.debug(f"found {len(nsw_df)} NSW sources in {len(nsw_df['group_id'].unique())} titles") qld_boreholes_da = data_manager.get_asset(qld_boreholes_data_source) qld_leases_da = data_manager.get_asset(qld_leases_data_source) @@ -64,6 +69,7 @@ def all_emission_sources( qld_leases_da=qld_leases_da, ) qld_df = normalise_emission_source_df(qld_df, prior_config.crs) + logger.debug(f"found {len(qld_df)} QLD sources in {len(qld_df['group_id'].unique())} titles") wa_wells_da = data_manager.get_asset(wa_wells_data_source) wa_titles_da = data_manager.get_asset(wa_titles_data_source) @@ -74,6 +80,7 @@ def all_emission_sources( wa_titles_da=wa_titles_da, ) wa_df = normalise_emission_source_df(wa_df, prior_config.crs) + logger.debug(f"found {len(wa_df)} WA sources in {len(wa_df['group_id'].unique())} titles") states_df: gpd.GeoDataFrame = pd.concat([ nsw_df, @@ -95,6 +102,7 @@ def all_emission_sources( # which we must avoid "double counting" offshore_existing = states_df.sjoin_nearest(offshore_df, how="inner", max_distance=50) offshore_new = offshore_df[~offshore_df["data_source_id"].isin(offshore_existing["data_source_id_right"])] + logger.debug(f"found {len(offshore_new)} offshore sources in {len(offshore_new['group_id'].unique())} titles") all_df: gpd.GeoDataFrame = pd.concat([ states_df, From 7952764fa663b773ae0b018871b705783641470c Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Mon, 23 Mar 2026 11:08:35 +1100 Subject: [PATCH 14/29] Replace oil and gas production dataset with bores and titles datasets This replaces an outdated Climate TRACE dataset with only 18 approximate locations with the locations of all petroleum-related drillholes/boreholes/wells available in public datasets from Australian states. This approach has several drawbacks, first of all a high rate of false positives due to many of the wells being capped and abandoned. The lack of concrete data about when a well was capped leads us to continue allocating emissions to it during any period that falls within that well's petroleum title. A second weakness is the naive approach which allocates equal emissions to every borehole, which is certainly far from accurate. Even with these downsides, this approach will still likely result in more accurate dispersal of emissions from this sector than allocating all emissions to 18 points. Furthermore, many of these issues will be minimised by sourcing actual emissions per-facility or per-title from Safeguard Mechanism data sources, which will come in a future commit. --- .../sectors/oil_gas/sector.py | 51 ++++++++----------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/src/openmethane_prior/sectors/oil_gas/sector.py b/src/openmethane_prior/sectors/oil_gas/sector.py index 261a33e7..487798de 100644 --- a/src/openmethane_prior/sectors/oil_gas/sector.py +++ b/src/openmethane_prior/sectors/oil_gas/sector.py @@ -15,14 +15,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import numpy as np -import pandas as pd import xarray as xr -from openmethane_prior.lib.data_manager.parsers import parse_csv from openmethane_prior.lib import ( - DataSource, logger, PriorSectorConfig, kg_to_period_cell_flux, @@ -30,13 +26,9 @@ from openmethane_prior.data_sources.inventory import get_sector_emissions_by_code, inventory_data_source from openmethane_prior.lib.sector.au_sector import AustraliaPriorSector -logger = logger.get_logger(__name__) +from .emission_sources.all_sources import all_emission_sources -oil_gas_facilities_data_source = DataSource( - name="oil-gas-facilities", - url="https://openmethane.s3.amazonaws.com/prior/inputs/oil-and-gas-production-and-transport_emissions-sources.csv", - parse=parse_csv, -) +logger = logger.get_logger(__name__) def process_emissions(sector: AustraliaPriorSector, sector_config: PriorSectorConfig, prior_ds: xr.Dataset): config = sector_config.prior_config @@ -51,33 +43,30 @@ def process_emissions(sector: AustraliaPriorSector, sector_config: PriorSectorCo ) # now read climate_trace facilities emissions for oil and gas - oil_gas_facilities_asset = sector_config.data_manager.get_asset(oil_gas_facilities_data_source) - - # select gas and year - oil_gas_ch4 = oil_gas_facilities_asset.data.loc[oil_gas_facilities_asset.data["gas"] == "ch4"] - oil_gas_ch4.loc[:, "start_time"] = pd.to_datetime(oil_gas_ch4["start_time"]) - target_date = ( - config.start_date - if config.start_date <= oil_gas_ch4["start_time"].max() - else oil_gas_ch4["start_time"].max() - ) # start date or latest date in data - years = np.array([x.year for x in oil_gas_ch4["start_time"]]) - mask = years == target_date.year - oil_gas_year = oil_gas_ch4.loc[mask, :] - # normalise emissions to match inventory total - oil_gas_year.loc[:, "emissions_quantity"] *= ( - sector_total_emissions / oil_gas_year["emissions_quantity"].sum() + emission_sources_df = all_emission_sources( + data_manager=sector_config.data_manager, + prior_config=config, ) + # emission sources don't include a methane quantity or proxy, so naively + # distribute sector emissions evenly to each active petroleum well + emission_sources_df["emissions_quantity"] = sector_total_emissions / len(emission_sources_df) + domain_grid = config.domain_grid() methane = np.zeros(domain_grid.shape) - for _, facility in oil_gas_year.iterrows(): - cell_x, cell_y, cell_valid = domain_grid.lonlat_to_cell_index(facility["lon"], facility["lat"]) - - if cell_valid: - methane[cell_y, cell_x] += facility["emissions_quantity"] + for index, site in emission_sources_df.iterrows(): + # geometry type for each site will determine how to allocate emissions + # to the grid + geom_type = emission_sources_df.geom_type[index] if type(emission_sources_df.geom_type[index]) == str \ + else emission_sources_df.geom_type[index].iloc[0] + if geom_type == "Point": + cell_x, cell_y, cell_valid = domain_grid.xy_to_cell_index(site["geometry"].x, site["geometry"].y) + if cell_valid: + methane[cell_y, cell_x] += site["emissions_quantity"] + else: + raise NotImplementedError("Allocating emissions to non-point geometry is not implemented") return kg_to_period_cell_flux(methane, config) From 83f96b46c999dfe4274f7e4ce62fcd17fdfee394 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Mon, 23 Mar 2026 16:08:27 +1100 Subject: [PATCH 15/29] Remove hardcoded item counts from emission sources tests These datasets do update frequently, and we don't want tests to break arbitrarily based on updates to source data. --- .../test_emission_sources.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_sector_oil_gas/test_emission_sources.py b/tests/integration/test_sector_oil_gas/test_emission_sources.py index 596c360c..7c035196 100644 --- a/tests/integration/test_sector_oil_gas/test_emission_sources.py +++ b/tests/integration/test_sector_oil_gas/test_emission_sources.py @@ -27,8 +27,8 @@ def test_nsw_emission_sources(input_files, data_manager): ) # original drillholes dataset has been filtered down - assert len(drillholes_da.data) == 775 - assert len(df) == 92 + assert len(drillholes_da.data) > 0 + assert len(df) <= len(drillholes_da.data) # no sources where activity period doesn't intersect date period assert len(df[(df["activity_end"] < start_date) & (df["activity_start"] > start_date_end)]) == 0 @@ -53,8 +53,8 @@ def test_qld_emission_sources(input_files, data_manager): ) # original boreholes dataset has been filtered down - assert len(boreholes_da.data) == 23908 - assert len(df) == 8670 + assert len(boreholes_da.data) > 0 + assert len(df) <= len(boreholes_da.data) # no sources where activity period doesn't intersect date period assert len(df[(df["activity_end"] < start_date) & (df["activity_start"] > start_date_end)]) == 0 @@ -97,8 +97,8 @@ def test_wa_emission_sources(input_files, data_manager): ) # original wells dataset has been filtered down - assert len(wells_da.data) == 4363 - assert len(df) == 413 + assert len(wells_da.data) > 0 + assert len(df) <= len(wells_da.data) # no sources where activity period doesn't intersect date period assert len(df[(df["activity_end"] < start_date) & (df["activity_start"] > start_date_end)]) == 0 @@ -121,8 +121,8 @@ def test_offshore_emission_sources(input_files, data_manager): ) # original wells dataset has been filtered down - assert len(wells_da.data) == 3024 - assert len(df) == 900 + assert len(wells_da.data) > 0 + assert len(df) <= len(wells_da.data) # no sources where activity period doesn't intersect date period assert len(df[(df["activity_end"] < start_date) & (df["activity_start"] > start_date_end)]) == 0 @@ -141,7 +141,7 @@ def test_all_emission_sources(input_files, data_manager, config): prior_config=config, ) - assert len(df) == 9882 + assert len(df) > 0 # no sources where activity period doesn't intersect config period assert len(df[(df["activity_end"] < config.start_date) & (df["activity_start"] > end_date_end)]) == 0 From e65eceb1dfeeed70ac582e48d69fe2e896869f96 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Mon, 23 Mar 2026 16:11:07 +1100 Subject: [PATCH 16/29] Update data sources docs to reflect new oil and gas sector implementation --- docs/data-sources.md | 75 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 10 deletions(-) diff --git a/docs/data-sources.md b/docs/data-sources.md index a5654bb3..58827b35 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -193,7 +193,7 @@ Global dataset of CH4 flux estimates from termites. Termite emissions used in [Saunois et al. 2020](https://essd.copernicus.org/articles/12/1561/2020/) supplied by Simona Castaldi and Sergio Noce. -- Dataset: [termite_emissions_2010-2016.nc][6] +- Dataset: [termite_emissions_2010-2016.nc][5] - Resolution: 0.5 degree - Period: mean of 2010 – 2016 - Updates: never @@ -313,17 +313,73 @@ spatialised according to the [Land Use of Australia](#Land-Use-of-Australia) dat Oil and gas emissions reported in the [Australian UNFCCC Inventory](#Australian-UNFCCC-Inventory) -are spatialised according to facility-level estimates. +are spatialised according to locations of oil and gas boreholes/wells which lie +within petroleum titles/leases that were active during the period of interest. ### Sources -- Dataset: [Oil and gas production sources][5] - - Original source: [ClimateTrace](https://climatetrace.org/) +- New South Wales datasets: + - [Data.NSW - Coal Seam Gas Boreholes](https://data.nsw.gov.au/data/dataset/coal-seam-gas-borehole) + - [Data.NSW - NSW Drillholes Petroleum](https://data.nsw.gov.au/data/dataset/nsw-drillholes-petroleum) + - [Data.NSW - NSW Exploration and Mining Titles](https://data.nsw.gov.au/data/dataset/nsw-mining-titles) +- Queensland datasets: + - [Queensland borehole series](https://www.data.qld.gov.au/dataset/queensland-borehole-series) + - [Queensland mining and exploration tenure series](https://www.data.qld.gov.au/dataset/queensland-mining-and-exploration-tenure-series) +- Western Australia datasets: + - [WA Petroleum Wells (DMIRS-025)](https://catalogue.data.wa.gov.au/dataset/wa-onshore-petroleum-wells-dmirs-025) + - [WA Petroleum Titles (DMIRS-011)](https://catalogue.data.wa.gov.au/dataset/wa-petroleum-titles-dmirs-011) +- Offshore datasets: + - [National Offshore Petroleum Information Management System (NOPIMS)](https://www.nopta.gov.au/maps-and-public-data/nopims-info.html) + - [National Electronic Approvals Tracking System](https://public.neats.nopta.gov.au/) + +Locations of every borehole/drillhole/well in the public datasets from NSW, +QLD, WA and NOPTA are correlated with petroleum production titles and filtered +to only bores involved in petroleum production where the title period overlaps +with the prior period of interest. + +The national inventory total for oil and gas emissions is divided evenly between +these sites. The listed point location for each site is mapped to the relevant +domain grid cell, where emissions are allocated. -The national inventory total for oil and gas emissions is pro-rated to the -location of every facility noted in the ClimateTrace data which is -listed for the chosen period. The listed point location for each -climate trace emission is mapped to the relevant domain grid cell. +### Considerations + +The approach used to spatialise the oil and gas sector has several known flaws. + +1. Capped wells + +First and foremost, although some datasets list many bores/wells as "capped" or +"abandoned", they don't include the date when the capping occurred. For this +reason we consider every production well to be an emission source until the +date of expiry of the title. This could be incorrect in both ways: wells very +likely stop emitting methane when they are capped, and in that case this leads +to many false positives within the datasets. Alternatively, capped wells may +continue to emit methane after production (and the title) has ended, leading to +missing emissions on abandoned fields. + +2. Attribution of inventory emissions + +Attribution of equal emissions to every "active" bore/well is also +very naive. Wells for different resources (i.e. oil vs coal seam gas) or in +different regions (WA vs NSW) or in different infrastructure (onshore vs +offshore) are likely to have very different emission profiles. Until we have +solid evidence of what these profiles might be, we cannot model them. + +3. Refineries and pipelines + +Several types of major infrastructure are currently missing from this approach: +refineries and pipelines. This is a major omission, as there is reasonable +suspicion that the majority of emissions occur at processing facilities. We +hope to add these facilities at a later date. + +4. Missing regions + +Several Australian states are not included in the data sources: +- Northern Territory + - has onshore gas, will be added +- Victoria + - has offshore oil and gas, should be represented in NOPTA datasets +- ACT, SA + - no significant oil and gas production ## Sector: Stationary @@ -350,5 +406,4 @@ spatialised according to the [Land Use of Australia](#Land-Use-of-Australia) dat [2]: https://openmethane.s3.amazonaws.com/prior/inputs/EntericFermentation.nc [3]: https://openmethane.s3.amazonaws.com/prior/inputs/ch4-electricity.csv [4]: https://openmethane.s3.amazonaws.com/prior/inputs/landuse-sector-map.csv -[5]: https://openmethane.s3.amazonaws.com/prior/inputs/oil-and-gas-production-and-transport_emissions-sources.csv -[6]: https://openmethane.s3.amazonaws.com/prior/inputs/termite_emissions_2010-2016.nc +[5]: https://openmethane.s3.amazonaws.com/prior/inputs/termite_emissions_2010-2016.nc From ed7c20da30553597cfe5a24e3aeb945c489d4d9e Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Wed, 25 Mar 2026 10:11:49 +1100 Subject: [PATCH 17/29] Add NT oil and gas sources Combine Northern Territories petroleum wells dataset with petroleum titles dataset to generate a list of production wells within petroleum titles that are active within the period of interest. --- docs/data-sources.md | 17 ++-- .../sectors/oil_gas/data/nt_titles.py | 80 ++++++++++++++++++ .../sectors/oil_gas/data/nt_wells.py | 79 ++++++++++++++++++ .../oil_gas/emission_sources/all_sources.py | 15 ++++ .../oil_gas/emission_sources/nt_sources.py | 82 +++++++++++++++++++ .../test_emission_sources.py | 28 +++++++ 6 files changed, 294 insertions(+), 7 deletions(-) create mode 100644 src/openmethane_prior/sectors/oil_gas/data/nt_titles.py create mode 100644 src/openmethane_prior/sectors/oil_gas/data/nt_wells.py create mode 100644 src/openmethane_prior/sectors/oil_gas/emission_sources/nt_sources.py diff --git a/docs/data-sources.md b/docs/data-sources.md index 58827b35..65d5d70d 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -322,6 +322,10 @@ within petroleum titles/leases that were active during the period of interest. - [Data.NSW - Coal Seam Gas Boreholes](https://data.nsw.gov.au/data/dataset/coal-seam-gas-borehole) - [Data.NSW - NSW Drillholes Petroleum](https://data.nsw.gov.au/data/dataset/nsw-drillholes-petroleum) - [Data.NSW - NSW Exploration and Mining Titles](https://data.nsw.gov.au/data/dataset/nsw-mining-titles) +- Northern Territory datasets: + - [STRIKE](http://strike.nt.gov.au/wss.html) + - Downloads -> Petroleum Titles and Pipeline Titles -> All Petroleum and Pipeline Titles Layers + - Downloads -> Drilling -> Petroleum Wells - Queensland datasets: - [Queensland borehole series](https://www.data.qld.gov.au/dataset/queensland-borehole-series) - [Queensland mining and exploration tenure series](https://www.data.qld.gov.au/dataset/queensland-mining-and-exploration-tenure-series) @@ -332,7 +336,7 @@ within petroleum titles/leases that were active during the period of interest. - [National Offshore Petroleum Information Management System (NOPIMS)](https://www.nopta.gov.au/maps-and-public-data/nopims-info.html) - [National Electronic Approvals Tracking System](https://public.neats.nopta.gov.au/) -Locations of every borehole/drillhole/well in the public datasets from NSW, +Locations of every borehole/drillhole/well in the public datasets from NSW, NT, QLD, WA and NOPTA are correlated with petroleum production titles and filtered to only bores involved in petroleum production where the title period overlaps with the prior period of interest. @@ -373,13 +377,12 @@ hope to add these facilities at a later date. 4. Missing regions -Several Australian states are not included in the data sources: -- Northern Territory - - has onshore gas, will be added +- South Australia + - not yet implemented - Victoria - - has offshore oil and gas, should be represented in NOPTA datasets -- ACT, SA - - no significant oil and gas production + - oil and gas extraction currently entirely offshore, present in NOPTA dataset +- ACT + - no oil or gas production to date ## Sector: Stationary diff --git a/src/openmethane_prior/sectors/oil_gas/data/nt_titles.py b/src/openmethane_prior/sectors/oil_gas/data/nt_titles.py new file mode 100644 index 00000000..e9e4efd5 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/nt_titles.py @@ -0,0 +1,80 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import geopandas as gpd +import numpy as np +import os +import pathlib +import shutil +import urllib +import zipfile + +from openmethane_prior.lib import DataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo +from openmethane_prior.lib.data_manager.source import ConfiguredDataSource + + +def fetch_nt_titles(data_source: ConfiguredDataSource) -> pathlib.Path: + """ + Download a zip file specified by DataSource.url, extract the contents and + convert a Shapefile (.shp) to GeoJSON using geopandas. + """ + # download zip file to a temporary location, it should be cleaned up afterwards + zip_path, response = urllib.request.urlretrieve( + url=data_source.url, + filename=data_source.data_path / os.path.basename(data_source.url), + ) + + tmp_path = data_source.data_path / "zip_contents" + with zipfile.ZipFile(zip_path, 'r') as zip_ref: + zip_ref.extractall(tmp_path) + + # read the extracted Shapefile with geopandas + df = gpd.GeoDataFrame.from_file(tmp_path / "PETRO_TITLE_PROD_GRNT.shp") + # remove entries with no granted date, these are unusable + df = df[~np.isnat(df["DT_GRNT"])] + + # datetime64 objects aren't JSON serialisable, convert them to string + for field_name in df.columns: + if df.dtypes[field_name] == "datetime64[ms]": + df[field_name] = [ + str(dt) if not np.isnat(dt) else None + for dt in df[field_name].values + ] + + # write the data as geojson + with open(data_source.asset_path, "w") as asset_file: + asset_file.write(df.to_json()) + + # clean up the zip and extracted contents, leaving only the GeoJSON + os.remove(zip_path) + shutil.rmtree(tmp_path) + + return data_source.asset_path + + +# Locations of petroleum production titles in Australia's Northern Territories, +# via the Spatial Territory Resource Information Kit for Exploration (STRIKE), +# part of the NT Department of Mining and Energy. +# Source: http://strike.nt.gov.au/wss.html -> Downloads -> Petroleum Titles and Pipeline Titles +nt_titles_data_source = DataSource( + name="NT-petroleum-titles", + url="https://geoscience.nt.gov.au/contents/prod/Downloads/NT_PetroleumPipelineTitles_shp.zip", + file_path='NT-petroleum-titles.geojson', + fetch=fetch_nt_titles, + parse=parse_geo, +) diff --git a/src/openmethane_prior/sectors/oil_gas/data/nt_wells.py b/src/openmethane_prior/sectors/oil_gas/data/nt_wells.py new file mode 100644 index 00000000..0d8b8e5a --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/nt_wells.py @@ -0,0 +1,79 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import geopandas as gpd +import os +import pathlib +import shutil +import urllib +import zipfile + +from openmethane_prior.lib import DataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo +from openmethane_prior.lib.data_manager.source import ConfiguredDataSource + + +def fetch_nt_wells(data_source: ConfiguredDataSource) -> pathlib.Path: + """ + Download a zip file specified by DataSource.url, extract the contents and + convert a Shapefile (.shp) to GeoJSON using geopandas. + """ + # download zip file to a temporary location, it should be cleaned up afterwards + zip_path, response = urllib.request.urlretrieve( + url=data_source.url, + filename=data_source.data_path / os.path.basename(data_source.url), + ) + + tmp_path = data_source.data_path / "zip_contents" + with zipfile.ZipFile(zip_path, 'r') as zip_ref: + zip_ref.extractall(tmp_path) + + # read the extracted Shapefile with geopandas + df = gpd.GeoDataFrame.from_file(tmp_path / "PETROLEUM_WELLS.shp") + + # convert datetime fields with strings like "19891229000000" into RFC3339 + for field_name in df.columns: + if field_name.startswith("DT_"): + df[field_name] = [ + str(datetime.datetime.strptime(dt_str, "%Y%m%d%H%M%S")) if dt_str is not None else None + for dt_str in df[field_name].values + ] + + # write the data as geojson + with open(data_source.asset_path, "w") as asset_file: + asset_file.write(df.to_json()) + + # clean up the zip and extracted contents, leaving only the GeoJSON + os.remove(zip_path) + shutil.rmtree(tmp_path) + + return data_source.asset_path + + +# Locations of petroleum wells in Australia's Northern Territories, via the +# Spatial Territory Resource Information Kit for Exploration (STRIKE), part of +# the NT Department of Mining and Energy. +# Source: http://strike.nt.gov.au/wss.html -> Downloads -> Drilling -> Petroleum Wells +# Metadata: https://www.ntlis.nt.gov.au/metadata/export_data?type=html&metadata_id=2DBCB771210D06B6E040CD9B0F274EFE +nt_wells_data_source = DataSource( + name="NT-petroleum-wells", + url="https://geoscience.nt.gov.au/contents/prod/Downloads/Drilling/PETROLEUM_WELLS_shp.zip", + file_path='NT-petroleum-wells.geojson', + fetch=fetch_nt_wells, + parse=parse_geo, +) diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py index 92884da4..b5db2c82 100644 --- a/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py @@ -26,6 +26,7 @@ from .emission_source import normalise_emission_source_df from .nsw_sources import nsw_emission_sources +from .nt_sources import nt_emission_sources from .offshore_sources import offshore_emission_sources from .qld_sources import qld_emission_sources from .wa_sources import wa_emission_sources @@ -33,6 +34,8 @@ from ..data.nopta_wells import nopta_wells_data_source from ..data.nsw_drillholes import nsw_drillholes_data_source from ..data.nsw_titles import nsw_titles_data_source +from ..data.nt_titles import nt_titles_data_source +from ..data.nt_wells import nt_wells_data_source from ..data.qld_boreholes import qld_boreholes_data_source from ..data.qld_leases import qld_leases_data_source from ..data.wa_titles import wa_titles_data_source @@ -60,6 +63,17 @@ def all_emission_sources( nsw_df = normalise_emission_source_df(nsw_df, prior_config.crs) logger.debug(f"found {len(nsw_df)} NSW sources in {len(nsw_df['group_id'].unique())} titles") + nt_wells_da = data_manager.get_asset(nt_wells_data_source) + nt_titles_da = data_manager.get_asset(nt_titles_data_source) + nt_df = nt_emission_sources( + start_date=start_date, + end_date=end_date, + nt_wells_da=nt_wells_da, + nt_titles_da=nt_titles_da, + ) + nt_df = normalise_emission_source_df(nt_df, prior_config.crs) + logger.debug(f"found {len(nt_df)} NT sources in {len(nt_df['group_id'].unique())} titles") + qld_boreholes_da = data_manager.get_asset(qld_boreholes_data_source) qld_leases_da = data_manager.get_asset(qld_leases_data_source) qld_df = qld_emission_sources( @@ -84,6 +98,7 @@ def all_emission_sources( states_df: gpd.GeoDataFrame = pd.concat([ nsw_df, + nt_df, qld_df, wa_df, ]) diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/nt_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/nt_sources.py new file mode 100644 index 00000000..806c10b5 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/nt_sources.py @@ -0,0 +1,82 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import geopandas as gpd +import numpy as np + +from openmethane_prior.lib.data_manager.asset import DataAsset +from openmethane_prior.lib.utils import rows_in_period + + +def nt_emission_sources( + start_date: datetime.date, + end_date: datetime.date, + nt_wells_da: DataAsset, + nt_titles_da: DataAsset, +) -> gpd.GeoDataFrame: + """Create normalised emission source DataFrame by combining NT petroleum + wells dataset for locations, with petroleum title dataset for production + start/end dates.""" + nt_wells_df: gpd.GeoDataFrame = nt_wells_da.data + nt_titles_df: gpd.GeoDataFrame = nt_titles_da.data + + # filter out non-production wells + emitting_well_purposes = ["Development", "Production"] + nt_wells_df = nt_wells_df[nt_wells_df["PURPOSE"].isin(emitting_well_purposes)] + + # well datasets may have duplicate rows for a single location due to + # further drilling at a site to deepen/extend an existing hole + nt_wells_df = nt_wells_df.sort_values(by="DT_RELEASE") + nt_wells_df.drop_duplicates(subset="geometry", keep="first", inplace=True) + + # dataset includes multiple rows for each title when the controlling entity + # ("PTY_NAME") has changed, we only want a single row per title + nt_titles_df.drop_duplicates(subset="TITLEID", keep="last", inplace=True) + + # join wells with titles to use the title dates as start/end dates + sources_df = gpd.sjoin(nt_wells_df, nt_titles_df, how="inner", predicate="within") + + # start date of emissions must be after hole is drilled (DT_RELEASE, drill + # release date) and after the title is granted (DT_GRNT), so use the latter + # of the two dates + sources_df["start_date"] = [ + issued if not np.isnat(issued) and (np.isnat(drilled) or issued > drilled) else drilled + for issued, drilled in sources_df[["DT_GRNT", "DT_RELEASE"]].values + ] + del sources_df["DT_GRNT"] + del sources_df["DT_RELEASE"] + + # exclude any emission sources that would not have been emitting during + # the period between start_date and end_date + sources_df = rows_in_period(sources_df, start_date=start_date, end_date=end_date, end_field="DT_EXPIRY") + + # after the join with titles, there may be multiple rows for each well. + # drop duplicates so there is only one entry for each location. + sources_df = sources_df.drop_duplicates(["WELLNAME"]) + + # normalise output to match emission sources format + sources_df = sources_df.rename(columns={ + "WELLNAME": "data_source_id", + "TITLEID": "group_id", + "start_date": "activity_start", + "DT_EXPIRY": "activity_end", + }) + sources_df["data_source"] = nt_wells_da.name + sources_df["site_type"] = "drillhole-unknown" + + return sources_df diff --git a/tests/integration/test_sector_oil_gas/test_emission_sources.py b/tests/integration/test_sector_oil_gas/test_emission_sources.py index 7c035196..fadd0abe 100644 --- a/tests/integration/test_sector_oil_gas/test_emission_sources.py +++ b/tests/integration/test_sector_oil_gas/test_emission_sources.py @@ -4,12 +4,15 @@ from openmethane_prior.sectors.oil_gas.data.nopta_wells import nopta_wells_data_source from openmethane_prior.sectors.oil_gas.data.nsw_drillholes import nsw_drillholes_data_source from openmethane_prior.sectors.oil_gas.data.nsw_titles import nsw_titles_data_source +from openmethane_prior.sectors.oil_gas.data.nt_titles import nt_titles_data_source +from openmethane_prior.sectors.oil_gas.data.nt_wells import nt_wells_data_source from openmethane_prior.sectors.oil_gas.data.qld_boreholes import qld_boreholes_data_source from openmethane_prior.sectors.oil_gas.data.qld_leases import qld_leases_data_source from openmethane_prior.sectors.oil_gas.emission_sources.all_sources import all_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.nsw_sources import nsw_emission_sources from openmethane_prior.sectors.oil_gas.data.wa_titles import wa_titles_data_source from openmethane_prior.sectors.oil_gas.data.wa_wells import wa_wells_data_source +from openmethane_prior.sectors.oil_gas.emission_sources.nt_sources import nt_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.offshore_sources import offshore_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.qld_sources import qld_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.wa_sources import wa_emission_sources @@ -41,6 +44,31 @@ def test_nsw_emission_sources(input_files, data_manager): assert len(df["geometry"].unique()) == len(df) +def test_nt_emission_sources(input_files, data_manager): + start_date = datetime.datetime(2023, 1, 1, 0, 0) + start_date_end = datetime.datetime(2023, 1, 2, 0, 0) + wells_da = data_manager.get_asset(nt_wells_data_source) + df = nt_emission_sources( + start_date=start_date.date(), + end_date=start_date.date(), + nt_wells_da=wells_da, + nt_titles_da=data_manager.get_asset(nt_titles_data_source), + ) + + # original drillholes dataset has been filtered down + assert len(wells_da.data) > 0 + assert len(df) <= len(wells_da.data) + + # no sources where activity period doesn't intersect date period + assert len(df[(df["activity_end"] < start_date) & (df["activity_start"] > start_date_end)]) == 0 + + # no sources which aren't related to production + assert set(df["PURPOSE"].unique()) == {"Production", "Development"} + + # no duplicate entries for the same location + assert len(df["geometry"].unique()) == len(df) + + def test_qld_emission_sources(input_files, data_manager): start_date = datetime.datetime(2023, 1, 1, 0, 0) start_date_end = datetime.datetime(2023, 1, 2, 0, 0) From 504ea9c5136475553dff2406047b70c999f615e5 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Wed, 22 Apr 2026 16:29:27 +1000 Subject: [PATCH 18/29] Add SA oil and gas sources Combine South Australia petroleum wells dataset with well production dataset to generate a list of production wells which recorded oil or gas production within the period of interest. --- docs/data-sources.md | 12 ++- pyproject.toml | 1 + .../lib/data_manager/parsers.py | 39 +++++++ .../sectors/oil_gas/README.md | 8 ++ .../sectors/oil_gas/data/sa_wells.py | 39 +++++++ .../oil_gas/emission_sources/all_sources.py | 14 +++ .../oil_gas/emission_sources/sa_sources.py | 100 ++++++++++++++++++ .../test_emission_sources.py | 25 +++++ uv.lock | 23 ++++ 9 files changed, 256 insertions(+), 5 deletions(-) create mode 100644 src/openmethane_prior/sectors/oil_gas/data/sa_wells.py create mode 100644 src/openmethane_prior/sectors/oil_gas/emission_sources/sa_sources.py diff --git a/docs/data-sources.md b/docs/data-sources.md index 65d5d70d..cf3c40d0 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -329,6 +329,10 @@ within petroleum titles/leases that were active during the period of interest. - Queensland datasets: - [Queensland borehole series](https://www.data.qld.gov.au/dataset/queensland-borehole-series) - [Queensland mining and exploration tenure series](https://www.data.qld.gov.au/dataset/queensland-mining-and-exploration-tenure-series) +- South Australia datasets: + - [PEPS SA Downloads](https://peps.sa.gov.au/more/excels/) + - Well Details and Locations + - Monthly Production by Completion - Western Australia datasets: - [WA Petroleum Wells (DMIRS-025)](https://catalogue.data.wa.gov.au/dataset/wa-onshore-petroleum-wells-dmirs-025) - [WA Petroleum Titles (DMIRS-011)](https://catalogue.data.wa.gov.au/dataset/wa-petroleum-titles-dmirs-011) @@ -337,9 +341,9 @@ within petroleum titles/leases that were active during the period of interest. - [National Electronic Approvals Tracking System](https://public.neats.nopta.gov.au/) Locations of every borehole/drillhole/well in the public datasets from NSW, NT, -QLD, WA and NOPTA are correlated with petroleum production titles and filtered -to only bores involved in petroleum production where the title period overlaps -with the prior period of interest. +QLD, SA, WA and NOPTA are correlated with petroleum production titles and +filtered to only bores involved in petroleum production where the title period +overlaps with the prior period of interest. The national inventory total for oil and gas emissions is divided evenly between these sites. The listed point location for each site is mapped to the relevant @@ -377,8 +381,6 @@ hope to add these facilities at a later date. 4. Missing regions -- South Australia - - not yet implemented - Victoria - oil and gas extraction currently entirely offshore, present in NOPTA dataset - ACT diff --git a/pyproject.toml b/pyproject.toml index edb9cf5c..f7cdf061 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "pytest-mock>=3.15.1,<4", "bmi-arcgis-restapi>=2.4.15", "owslib>=0.35.0", + "openpyxl>=3.1.5", ] [dependency-groups] diff --git a/src/openmethane_prior/lib/data_manager/parsers.py b/src/openmethane_prior/lib/data_manager/parsers.py index 1b9b9e09..53e37515 100644 --- a/src/openmethane_prior/lib/data_manager/parsers.py +++ b/src/openmethane_prior/lib/data_manager/parsers.py @@ -44,3 +44,42 @@ def parse_geo(data_source: ConfiguredDataSource, source_crs: pyproj.CRS = None): geo_df = geo_df.to_crs(data_source.prior_config.crs) return geo_df + + +def parse_xlsx(data_source: ConfiguredDataSource) -> pd.DataFrame: + """Read and parse a ConfiguredDataSource XLSX asset as a pandas DataFrame.""" + return pd.read_excel(data_source.asset_path) + + +def parse_geo_xlsx( + x_column: str, + y_column: str, + source_crs: pyproj.CRS | str, +): + """Create a parser which will read an Excel-based ConfiguredDataSource, + extracting coordinates from columns in x_column and y_column, and project + coordinates from the source_crs to the PriorConfig domain CRS. + + This can be used to configure a DataSource like: + DataSource( + file_path="example.xlsx", + parse=parse_geo_xlsx("x_col", "y_col", "EPSG:7844") + ) + """ + def _parser(data_source: ConfiguredDataSource): + df = parse_xlsx(data_source) + + df["geometry"] = gpd.points_from_xy( + x=df[x_column], + y=df[y_column], + crs=source_crs, + ) + geo_df = gpd.GeoDataFrame(df) + + # convert the geometries into the prior projection to ensure downstream + # comparisons are done using the same coordinate system + geo_df = geo_df.to_crs(data_source.prior_config.crs) + + return geo_df + + return _parser \ No newline at end of file diff --git a/src/openmethane_prior/sectors/oil_gas/README.md b/src/openmethane_prior/sectors/oil_gas/README.md index 46f64fd4..24f3f79b 100644 --- a/src/openmethane_prior/sectors/oil_gas/README.md +++ b/src/openmethane_prior/sectors/oil_gas/README.md @@ -69,6 +69,14 @@ for years after production has ceased. Assuming that capped wells don't continue to emit methane, this will misallocate potentially significant emissions. Unfortunately we currently don't have a better solution. +#### South Australia + +Unlike other states, South Australia also provides monthly per-well production +amounts for both oil and gas. For South Australia, only wells which produced +either oil or gas within the period of interest will be considered an emission +source. + + ### Processing facilities Not yet implemented. diff --git a/src/openmethane_prior/sectors/oil_gas/data/sa_wells.py b/src/openmethane_prior/sectors/oil_gas/data/sa_wells.py new file mode 100644 index 00000000..38853f3b --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/sa_wells.py @@ -0,0 +1,39 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from openmethane_prior.lib import DataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo_xlsx, parse_xlsx + +# Locations of all petroleum wells in the Australian state of South Australia +# via the PEPS SA Portal. +# Source: https://peps.sa.gov.au/more/excels/ -> Well Details and Locations +sa_wells_data_source = DataSource( + name="SA-petroleum-wells", + file_path="SAPetroleumWells.xlsx", + url="https://onepeps-api.azurewebsites.net/api/excel/file/Wells.xlsx", + parse=parse_geo_xlsx("GDA20 X", "GDA20 Y", "EPSG:7844"), +) + + +# Production amounts by month for each well in the SA PEPS wells dataset. +# Source: https://peps.sa.gov.au/more/excels/ -> Monthly Production by Completion +sa_wells_production_data_source = DataSource( + name="SA-wells-production", + file_path="MonthlyProductionByCompletion.xlsx", + url="https://onepeps-api.azurewebsites.net/api/excel/file/MonthlyProductionByCompletion.xlsx", + parse=parse_xlsx, +) diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py index b5db2c82..109a8864 100644 --- a/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py @@ -29,6 +29,7 @@ from .nt_sources import nt_emission_sources from .offshore_sources import offshore_emission_sources from .qld_sources import qld_emission_sources +from .sa_sources import sa_emission_sources from .wa_sources import wa_emission_sources from ..data.nopta_titles import nopta_titles_data_source from ..data.nopta_wells import nopta_wells_data_source @@ -38,6 +39,7 @@ from ..data.nt_wells import nt_wells_data_source from ..data.qld_boreholes import qld_boreholes_data_source from ..data.qld_leases import qld_leases_data_source +from ..data.sa_wells import sa_wells_data_source, sa_wells_production_data_source from ..data.wa_titles import wa_titles_data_source from ..data.wa_wells import wa_wells_data_source @@ -85,6 +87,17 @@ def all_emission_sources( qld_df = normalise_emission_source_df(qld_df, prior_config.crs) logger.debug(f"found {len(qld_df)} QLD sources in {len(qld_df['group_id'].unique())} titles") + sa_wells_da = data_manager.get_asset(sa_wells_data_source) + sa_production_da = data_manager.get_asset(sa_wells_production_data_source) + sa_df = sa_emission_sources( + start_date=start_date, + end_date=end_date, + sa_wells_da=sa_wells_da, + sa_production_da=sa_production_da, + ) + sa_df = normalise_emission_source_df(sa_df, prior_config.crs) + logger.debug(f"found {len(sa_df)} SA sources in {len(sa_df['group_id'].unique())} titles") + wa_wells_da = data_manager.get_asset(wa_wells_data_source) wa_titles_da = data_manager.get_asset(wa_titles_data_source) wa_df = wa_emission_sources( @@ -100,6 +113,7 @@ def all_emission_sources( nsw_df, nt_df, qld_df, + sa_df, wa_df, ]) diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/sa_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/sa_sources.py new file mode 100644 index 00000000..71595086 --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/sa_sources.py @@ -0,0 +1,100 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import geopandas as gpd +import numpy as np +import pandas as pd + +from openmethane_prior.lib.data_manager.asset import DataAsset +from openmethane_prior.lib.utils import rows_in_period + + +def start_of_month(date: pd.Timestamp) -> np.datetime64: + return np.datetime64(pd.Timestamp(year=date.year, month=date.month, day=1)) + +def sa_emission_sources( + start_date: datetime.date, + end_date: datetime.date, + sa_wells_da: DataAsset, + sa_production_da: DataAsset, +) -> gpd.GeoDataFrame: + """Create normalised emission source DataFrame by combining SA petroleum + wells dataset for locations, with petroleum title dataset for production + start/end dates.""" + sa_wells_df: gpd.GeoDataFrame = sa_wells_da.data + sa_production_df: gpd.GeoDataFrame = sa_production_da.data + + # filter out non-production wells + sa_emitting_types = [ + "Oil Shows", "Oil", + "Gas Shows", "Gas", + "Oil and Gas", "Oil and Gas Shows", "Oil with Gas Shows", + "Gas with Oil Shows", "CO2 with Oil Shows", + ] + wells_df = sa_wells_df[sa_wells_df["Type"].isin(sa_emitting_types)] + + # "Month End" is read from a date on the last day of the month, construct + # a start/end date reflecting the full period ending on midnight of "Month End", + # then filter to only rows that intersect with the prior period. + sa_production_df["activity_start"] = sa_production_df["Month End"].map(start_of_month) + sa_production_df["activity_end"] = sa_production_df["Month End"] + np.timedelta64(1, "D") + production_df = rows_in_period( + sa_production_df, + start_date=start_date, + end_date=end_date, + start_field="activity_start", + end_field="activity_end", + ) + + # Production values are split by Co-Formation, but we just want the sum + # of production for each well + production_df = production_df.groupby("WellID", as_index=False) + production_df = production_df.agg({ + "Oil (m3)": "sum", + "Gas (m3E6)": "sum", + "Type": "first", + "Field": "first", + "activity_start": "first", + "activity_end": "first", + "Month Days": "first", + }) + production_df = production_df.rename({"WellID": "Well ID"}) # join column + + # join well details with production figures + sources_df = wells_df.join(production_df, on="Well ID", how="inner", rsuffix=" Production") + + # remove wells which didn't produce during the period of interest + sources_df = sources_df[(sources_df["Oil (m3)"] > 0) | (sources_df["Gas (m3E6)"] > 0)] + + sources_df["License"] = [ + None if current_ppl is None else f"PPL {current_ppl:0.0f}" + for current_ppl in sources_df["Current PPL"].values + ] + + # normalise output to match emission sources format + sources_df = sources_df.rename(columns={ + "Well ID": "data_source_id", + "License": "group_id", + }) + sources_df["data_source"] = sa_wells_da.name + sources_df["site_type"] = [ + "drillhole-oil" if production_type == "Oil" else "drillhole-gas" + for production_type in sources_df["Type Production"].values + ] + + return sources_df diff --git a/tests/integration/test_sector_oil_gas/test_emission_sources.py b/tests/integration/test_sector_oil_gas/test_emission_sources.py index fadd0abe..a62c2595 100644 --- a/tests/integration/test_sector_oil_gas/test_emission_sources.py +++ b/tests/integration/test_sector_oil_gas/test_emission_sources.py @@ -8,6 +8,7 @@ from openmethane_prior.sectors.oil_gas.data.nt_wells import nt_wells_data_source from openmethane_prior.sectors.oil_gas.data.qld_boreholes import qld_boreholes_data_source from openmethane_prior.sectors.oil_gas.data.qld_leases import qld_leases_data_source +from openmethane_prior.sectors.oil_gas.data.sa_wells import sa_wells_data_source, sa_wells_production_data_source from openmethane_prior.sectors.oil_gas.emission_sources.all_sources import all_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.nsw_sources import nsw_emission_sources from openmethane_prior.sectors.oil_gas.data.wa_titles import wa_titles_data_source @@ -15,6 +16,7 @@ from openmethane_prior.sectors.oil_gas.emission_sources.nt_sources import nt_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.offshore_sources import offshore_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.qld_sources import qld_emission_sources +from openmethane_prior.sectors.oil_gas.emission_sources.sa_sources import sa_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.wa_sources import wa_emission_sources @@ -113,6 +115,29 @@ def test_qld_emission_sources(input_files, data_manager): assert len(df["geometry"].unique()) == len(df) +def test_sa_emission_sources(input_files, data_manager): + start_date = datetime.datetime(2023, 1, 1, 0, 0) + start_date_end = datetime.datetime(2023, 1, 2, 0, 0) + wells_da = data_manager.get_asset(sa_wells_data_source) + production_da = data_manager.get_asset(sa_wells_production_data_source) + df = sa_emission_sources( + start_date=start_date.date(), + end_date=start_date.date(), + sa_wells_da=wells_da, + sa_production_da=production_da, + ) + + # original wells dataset has been filtered down + assert len(wells_da.data) > 0 + assert len(df) <= len(wells_da.data) + + # no sources where activity period doesn't intersect date period + assert len(df[(df["activity_end"] < start_date) & (df["activity_start"] > start_date_end)]) == 0 + + # # no sources which didn't have recorded production within the period + assert len(df[(df["Oil (m3)"] <= 0) & (df["Gas (m3E6)"] <= 0)]) == 0 + + def test_wa_emission_sources(input_files, data_manager): start_date = datetime.datetime(2023, 1, 1, 0, 0) start_date_end = datetime.datetime(2023, 1, 2, 0, 0) diff --git a/uv.lock b/uv.lock index 05e354b5..9e22ff2e 100644 --- a/uv.lock +++ b/uv.lock @@ -427,6 +427,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/21/1e0d8de234e9d0c675ea8fd50f9e7ad66fae32c207bc982f1d14f7c0835b/environs-11.2.1-py3-none-any.whl", hash = "sha256:9d2080cf25807a26fc0d4301e2d7b62c64fbf547540f21e3a30cc02bc5fbe948", size = 12923, upload-time = "2024-11-20T17:38:39.013Z" }, ] +[[package]] +name = "et-xmlfile" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234, upload-time = "2024-10-25T17:25:40.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, +] + [[package]] name = "executing" version = "2.2.1" @@ -1225,6 +1234,7 @@ dependencies = [ { name = "geopandas" }, { name = "netcdf4" }, { name = "numpy" }, + { name = "openpyxl" }, { name = "owslib" }, { name = "pandas" }, { name = "prettyprinter" }, @@ -1261,6 +1271,7 @@ requires-dist = [ { name = "geopandas", specifier = ">=1.1.3,<2" }, { name = "netcdf4", specifier = ">=1.6.5,<2" }, { name = "numpy", specifier = ">=2.4.3,<3" }, + { name = "openpyxl", specifier = ">=3.1.5" }, { name = "owslib", specifier = ">=0.35.0" }, { name = "pandas", specifier = ">=2.2.2,<3" }, { name = "prettyprinter", specifier = ">=0.18.0,<0.19" }, @@ -1286,6 +1297,18 @@ notebooks = [ ] tests = [{ name = "pytest", specifier = ">=8.2.1,<9" }] +[[package]] +name = "openpyxl" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "et-xmlfile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464, upload-time = "2024-06-28T14:03:44.161Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" }, +] + [[package]] name = "overrides" version = "7.7.0" From f04103813d8266c31e3516c664a2dfbd07809a14 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Wed, 25 Mar 2026 16:47:11 +1100 Subject: [PATCH 19/29] Convert SA well production data source to CSV The original XLSX source for this file is 50MB+ and takes more than 60 seconds to parse. Since we only care about the data, and not the Excel functionality, saving as a CSV reduces parsing time to less than 5s. Although this won't make a difference in production where the source file is fetched and parsed either way, it significantly speeds up development and testing. --- .../sectors/oil_gas/data/sa_wells.py | 28 +++++++++++++++++-- .../oil_gas/emission_sources/sa_sources.py | 1 + 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/openmethane_prior/sectors/oil_gas/data/sa_wells.py b/src/openmethane_prior/sectors/oil_gas/data/sa_wells.py index 38853f3b..4a261367 100644 --- a/src/openmethane_prior/sectors/oil_gas/data/sa_wells.py +++ b/src/openmethane_prior/sectors/oil_gas/data/sa_wells.py @@ -15,8 +15,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import os.path +import pandas as pd +import urllib + from openmethane_prior.lib import DataSource -from openmethane_prior.lib.data_manager.parsers import parse_geo_xlsx, parse_xlsx +from openmethane_prior.lib.data_manager.parsers import parse_geo_xlsx, parse_csv +from openmethane_prior.lib.data_manager.source import ConfiguredDataSource # Locations of all petroleum wells in the Australian state of South Australia # via the PEPS SA Portal. @@ -29,11 +34,28 @@ ) +def fetch_xlsx_as_csv(data_source: ConfiguredDataSource): + """Fetch a source file in xlsx format, but save it to disk as csv.""" + if data_source.url is None: + raise ValueError("DataSource must have url set to use default fetch") + + # urlretrieve will throw on non-successful fetches + save_path, response = urllib.request.urlretrieve(url=data_source.url) + + df = pd.read_excel(save_path) + df.to_csv(data_source.asset_path) + + # clean up the xlsx file + os.remove(save_path) + + return data_source.asset_path + # Production amounts by month for each well in the SA PEPS wells dataset. # Source: https://peps.sa.gov.au/more/excels/ -> Monthly Production by Completion sa_wells_production_data_source = DataSource( name="SA-wells-production", - file_path="MonthlyProductionByCompletion.xlsx", + file_path="SA-wells-production.csv", url="https://onepeps-api.azurewebsites.net/api/excel/file/MonthlyProductionByCompletion.xlsx", - parse=parse_xlsx, + fetch=fetch_xlsx_as_csv, + parse=parse_csv, ) diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/sa_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/sa_sources.py index 71595086..0f9c464f 100644 --- a/src/openmethane_prior/sectors/oil_gas/emission_sources/sa_sources.py +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/sa_sources.py @@ -51,6 +51,7 @@ def sa_emission_sources( # "Month End" is read from a date on the last day of the month, construct # a start/end date reflecting the full period ending on midnight of "Month End", # then filter to only rows that intersect with the prior period. + sa_production_df["Month End"] = pd.to_datetime(sa_production_df["Month End"]) sa_production_df["activity_start"] = sa_production_df["Month End"].map(start_of_month) sa_production_df["activity_end"] = sa_production_df["Month End"] + np.timedelta64(1, "D") production_df = rows_in_period( From ad986c849540564f7993b7ab08f2d936f64a6206 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Wed, 25 Mar 2026 16:56:39 +1100 Subject: [PATCH 20/29] Update oil and gas notebook to use all_emission_sources --- notebooks/oil_gas_locations.py | 81 +++++++--------------------------- 1 file changed, 16 insertions(+), 65 deletions(-) diff --git a/notebooks/oil_gas_locations.py b/notebooks/oil_gas_locations.py index f146cbc1..677bfae6 100644 --- a/notebooks/oil_gas_locations.py +++ b/notebooks/oil_gas_locations.py @@ -1,25 +1,12 @@ -import datetime import geopandas as gpd import os -import pandas as pd import pathlib from openmethane_prior.lib import PriorConfig from openmethane_prior.lib.data_manager.manager import DataManager -from openmethane_prior.sectors.oil_gas.data.nopta_titles import nopta_titles_data_source -from openmethane_prior.sectors.oil_gas.data.nopta_wells import nopta_wells_data_source -from openmethane_prior.sectors.oil_gas.data.nsw_drillholes import nsw_drillholes_data_source -from openmethane_prior.sectors.oil_gas.data.nsw_titles import nsw_titles_data_source -from openmethane_prior.sectors.oil_gas.data.qld_boreholes import qld_boreholes_data_source -from openmethane_prior.sectors.oil_gas.data.qld_leases import qld_leases_data_source -from openmethane_prior.sectors.oil_gas.data.wa_titles import wa_titles_data_source -from openmethane_prior.sectors.oil_gas.data.wa_wells import wa_wells_data_source -from openmethane_prior.sectors.oil_gas.emission_sources.nsw_sources import nsw_emission_sources -from openmethane_prior.sectors.oil_gas.emission_sources.offshore_sources import offshore_emission_sources -from openmethane_prior.sectors.oil_gas.emission_sources.qld_sources import qld_emission_sources -from openmethane_prior.sectors.oil_gas.emission_sources.wa_sources import wa_emission_sources +from openmethane_prior.sectors.oil_gas.emission_sources.all_sources import all_emission_sources os.environ["DOMAIN_FILE"] = "https://openmethane.s3.amazonaws.com/domains/aust10km/v1/domain.aust10km.nc" -os.environ["START_DATE"] = "2023-07-01" -os.environ["END_DATE"] = "2023-07-01" +os.environ["START_DATE"] = "2022-12-01" +os.environ["END_DATE"] = "2022-12-01" os.environ["INPUTS"] = "../data/inputs" os.environ["INPUT_CACHE"] = "../data/.cache" prior_config = PriorConfig.from_env() @@ -30,59 +17,23 @@ prior_config=prior_config, ) -nsw_drillholes_da = data_manager.get_asset(nsw_drillholes_data_source) -nsw_titles_da = data_manager.get_asset(nsw_titles_data_source) -nsw_df = nsw_emission_sources( - start_date=datetime.date(2021, 1, 1), - end_date=datetime.date(2023, 1, 2), - nsw_drillholes_da=nsw_drillholes_da, - nsw_titles_da=nsw_titles_da, -) -qld_boreholes_da = data_manager.get_asset(qld_boreholes_data_source) -qld_leases_da = data_manager.get_asset(qld_leases_data_source) -qld_df = qld_emission_sources( - start_date=datetime.date(2021, 1, 1), - end_date=datetime.date(2023, 1, 2), - qld_boreholes_da=qld_boreholes_da, - qld_leases_da=qld_leases_da, -) -wa_wells_da = data_manager.get_asset(wa_wells_data_source) -wa_titles_da = data_manager.get_asset(wa_titles_data_source) -wa_df = wa_emission_sources( - start_date=datetime.date(2021, 1, 1), - end_date=datetime.date(2023, 1, 2), - wa_wells_da=wa_wells_da, - wa_titles_da=wa_titles_da, -) -offshore_wells_da = data_manager.get_asset(nopta_wells_data_source) -offshore_titles_da = data_manager.get_asset(nopta_titles_data_source) -offshore_df = offshore_emission_sources( - start_date=datetime.date(2021, 1, 1), - end_date=datetime.date(2023, 1, 2), - offshore_wells_da=offshore_wells_da, - offshore_titles_da=offshore_titles_da, +emission_sources_df = all_emission_sources( + data_manager=data_manager, + prior_config=prior_config, ) -print(f"{nsw_df.crs=}") -print(f"{qld_df.crs=}") -print(f"{wa_df.crs=}") -print(f"{offshore_df.crs=}") -all_df = pd.concat([ - nsw_df, - qld_df, - wa_df, - offshore_df, -]).to_crs("EPSG:4326") # for plotting au_shp = gpd.GeoDataFrame.from_file("~/Downloads/STE_2021_AUST_SHP_GDA2020/STE_2021_AUST_GDA2020.shx").to_crs("EPSG:4326") au_map = au_shp.plot(color='white', edgecolor='#CCCCCC', figsize=(32,24)) -# uncomment to show a smaller extent around QLD -# au_map.set_xlim(110, 130) -# au_map.set_ylim(-35, -10) -nsw_titles_da.data.to_crs("EPSG:4326").plot(ax=au_map, alpha=0.5) -qld_leases_da.data.to_crs("EPSG:4326").plot(ax=au_map, alpha=0.5) -# # wa_wells_da.data.to_crs("EPSG:4326").plot(ax=au_map, markersize=3, column="class", legend=True) -wa_titles_da.data.to_crs("EPSG:4326").plot(ax=au_map, alpha=0.5) -all_df.plot(ax=au_map, markersize=3, column="site_type", legend=True) +region_extents = { + "QLD": (137, 155, -30, -20), + "VIC": (146, 150, -41, -36), + "WA": (110, 130, -35, -10), +} +region = "QLD" # set to "QLD", "VIC" or "WA" to render a sub-region +if region is not None: + au_map.set_xlim(region_extents[region][0], region_extents[region][1]) + au_map.set_ylim(region_extents[region][2], region_extents[region][3]) +emission_sources_df.to_crs("EPSG:4326").plot(ax=au_map, markersize=3, column="site_type", legend=True) au_map From b8a7c8a6a2050203c112cd19562d582b8d4c82df Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 26 Mar 2026 12:45:07 +1100 Subject: [PATCH 21/29] Fix test data in test_utils.py causing a warning --- tests/unit/test_utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 6272eaca..5ddfe3d2 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -81,11 +81,11 @@ def test_rows_in_period(): dt = lambda st: np.datetime64(datetime.datetime.fromisoformat(st)) test_df = pd.DataFrame.from_records([ - (dt("2022-12-31T00:00:00Z"), dt("2022-12-31T23:59:59Z"), "a"), - (dt("2022-12-31T00:00:00Z"), dt("2023-01-01T00:00:00Z"), "b"), - (dt("2023-01-01T00:00:01Z"), dt("2023-01-01T23:59:59Z"), "c"), - (dt("2023-01-02T00:00:00Z"), dt("2023-02-02T23:59:59Z"), "d"), - (dt("2023-01-02T00:00:01Z"), dt("2023-02-02T23:59:59Z"), "e"), + (dt("2022-12-31T00:00:00"), dt("2022-12-31T23:59:59"), "a"), + (dt("2022-12-31T00:00:00"), dt("2023-01-01T00:00:00"), "b"), + (dt("2023-01-01T00:00:01"), dt("2023-01-01T23:59:59"), "c"), + (dt("2023-01-02T00:00:00"), dt("2023-02-02T23:59:59"), "d"), + (dt("2023-01-02T00:00:01"), dt("2023-02-02T23:59:59"), "e"), ], columns=["start_date", "end_date", "test"]) result_df = rows_in_period( @@ -98,9 +98,9 @@ def test_rows_in_period(): # works with arbitrary start/end field names "start_test" and "end_test" test_df = pd.DataFrame.from_records([ - (dt("2022-12-31T00:00:00Z"), dt("2022-12-31T23:59:59Z"), "f"), - (dt("2022-12-31T00:00:00Z"), dt("2023-01-01T00:00:00Z"), "g"), - (dt("2023-01-02T00:00:01Z"), dt("2023-02-02T23:59:59Z"), "h"), + (dt("2022-12-31T00:00:00"), dt("2022-12-31T23:59:59"), "f"), + (dt("2022-12-31T00:00:00"), dt("2023-01-01T00:00:00"), "g"), + (dt("2023-01-02T00:00:01"), dt("2023-02-02T23:59:59"), "h"), ], columns=["start_test", "end_test", "test"]) result_df = rows_in_period( From dca3ccd18450f87266cefe22f75ff3c1e26d8b86 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 26 Mar 2026 15:21:32 +1100 Subject: [PATCH 22/29] Add changelog for #161 --- changelog/161.feature.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/161.feature.md diff --git a/changelog/161.feature.md b/changelog/161.feature.md new file mode 100644 index 00000000..42054676 --- /dev/null +++ b/changelog/161.feature.md @@ -0,0 +1 @@ +Utilise state-based petroleum well and borehole datasets to locate oil and gas emission sources From 7b4bf6f93b82589e4d413fbfcecaeac850cb67c7 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 26 Mar 2026 15:50:17 +1100 Subject: [PATCH 23/29] Instantiate PriorConfig directly in oil_gas_locations notebook --- notebooks/oil_gas_locations.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/notebooks/oil_gas_locations.py b/notebooks/oil_gas_locations.py index 677bfae6..6be077bd 100644 --- a/notebooks/oil_gas_locations.py +++ b/notebooks/oil_gas_locations.py @@ -1,19 +1,20 @@ +import datetime import geopandas as gpd -import os import pathlib from openmethane_prior.lib import PriorConfig from openmethane_prior.lib.data_manager.manager import DataManager from openmethane_prior.sectors.oil_gas.emission_sources.all_sources import all_emission_sources -os.environ["DOMAIN_FILE"] = "https://openmethane.s3.amazonaws.com/domains/aust10km/v1/domain.aust10km.nc" -os.environ["START_DATE"] = "2022-12-01" -os.environ["END_DATE"] = "2022-12-01" -os.environ["INPUTS"] = "../data/inputs" -os.environ["INPUT_CACHE"] = "../data/.cache" -prior_config = PriorConfig.from_env() +prior_config = PriorConfig( + domain_path="https://openmethane.s3.amazonaws.com/domains/aust10km/v1/domain.aust10km.nc", + inventory_domain_path="https://openmethane.s3.amazonaws.com/domains/aust10km/v1/domain.aust10km.nc", + start_date=datetime.datetime.fromisoformat("2022-12-01"), + input_path=pathlib.Path("../data/inputs"), + input_cache=pathlib.Path("../data/.cache"), +) prior_config.prepare_paths() prior_config.load_cached_inputs() data_manager = DataManager( - data_path=pathlib.Path("../data/inputs"), + data_path=prior_config.input_path, prior_config=prior_config, ) From 0e8bcc384e893c94dbf621f61a3b026b0690401a Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Wed, 1 Apr 2026 11:31:40 +1100 Subject: [PATCH 24/29] Add state to emission_source and populate it for each well/drillhole row --- .../sectors/oil_gas/emission_sources/all_sources.py | 5 +++++ .../sectors/oil_gas/emission_sources/emission_source.py | 4 ++++ .../sectors/oil_gas/emission_sources/offshore_sources.py | 1 + tests/unit/test_oil_gas/test_emission_source.py | 2 ++ 4 files changed, 12 insertions(+) diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py index 109a8864..8b2f2f0f 100644 --- a/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py @@ -62,6 +62,7 @@ def all_emission_sources( nsw_drillholes_da=nsw_drillholes_da, nsw_titles_da=nsw_titles_da, ) + nsw_df["state"] = "NSW" nsw_df = normalise_emission_source_df(nsw_df, prior_config.crs) logger.debug(f"found {len(nsw_df)} NSW sources in {len(nsw_df['group_id'].unique())} titles") @@ -73,6 +74,7 @@ def all_emission_sources( nt_wells_da=nt_wells_da, nt_titles_da=nt_titles_da, ) + nt_df["state"] = "NT" nt_df = normalise_emission_source_df(nt_df, prior_config.crs) logger.debug(f"found {len(nt_df)} NT sources in {len(nt_df['group_id'].unique())} titles") @@ -84,6 +86,7 @@ def all_emission_sources( qld_boreholes_da=qld_boreholes_da, qld_leases_da=qld_leases_da, ) + qld_df["state"] = "QLD" qld_df = normalise_emission_source_df(qld_df, prior_config.crs) logger.debug(f"found {len(qld_df)} QLD sources in {len(qld_df['group_id'].unique())} titles") @@ -95,6 +98,7 @@ def all_emission_sources( sa_wells_da=sa_wells_da, sa_production_da=sa_production_da, ) + sa_df["state"] = "SA" sa_df = normalise_emission_source_df(sa_df, prior_config.crs) logger.debug(f"found {len(sa_df)} SA sources in {len(sa_df['group_id'].unique())} titles") @@ -106,6 +110,7 @@ def all_emission_sources( wa_wells_da=wa_wells_da, wa_titles_da=wa_titles_da, ) + wa_df["state"] = "WA" wa_df = normalise_emission_source_df(wa_df, prior_config.crs) logger.debug(f"found {len(wa_df)} WA sources in {len(wa_df['group_id'].unique())} titles") diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/emission_source.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/emission_source.py index 9bc05127..1a72a1c7 100644 --- a/src/openmethane_prior/sectors/oil_gas/emission_sources/emission_source.py +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/emission_source.py @@ -51,6 +51,10 @@ # (Optional) a string identifying a group of locations this site is part # of. A group could be a title/license area, a field, etc. "group_id": str, + + # (Optional) the state/region the emission source is in the jurisdiction + # of. Can be useful for attributing inventory emissions, for example. + "state": str, } emission_source_site_types = [ diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/offshore_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/offshore_sources.py index e1ee274a..2dbaf7a8 100644 --- a/src/openmethane_prior/sectors/oil_gas/emission_sources/offshore_sources.py +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/offshore_sources.py @@ -40,6 +40,7 @@ def offshore_emission_sources( # emissions are expected where production is ongoing, unlike exploration or retention titles_df = offshore_titles_df[offshore_titles_df["TitleType"] == "Production Licence"] + titles_df = titles_df.rename(columns={"state": "title_state"}) # NOPTA wells dataset appears to re-use lat/lon coords for multiple wells # within the same field. Since these contain unique values for WellName and diff --git a/tests/unit/test_oil_gas/test_emission_source.py b/tests/unit/test_oil_gas/test_emission_source.py index 30e30950..b811b23e 100644 --- a/tests/unit/test_oil_gas/test_emission_source.py +++ b/tests/unit/test_oil_gas/test_emission_source.py @@ -15,6 +15,7 @@ def test_normalise_emission_source(): "data_source": ["nsw-drillholes"], "data_source_id": ["012345"], "group_id": ["xyz"], + "state": ["VIC"], "extra_column": [0], }, crs="EPSG:7844") @@ -28,6 +29,7 @@ def test_normalise_emission_source(): "data_source", "data_source_id", "group_id", + "state", ] assert result_df.crs == "EPSG:4326" From b55303549a0126a59b23d633b980b6b542f9bd38 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Fri, 10 Apr 2026 14:39:12 +1000 Subject: [PATCH 25/29] Fix code comment which still refers to Climate TRACE oil and gas data --- src/openmethane_prior/sectors/oil_gas/sector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openmethane_prior/sectors/oil_gas/sector.py b/src/openmethane_prior/sectors/oil_gas/sector.py index 487798de..fe7402ae 100644 --- a/src/openmethane_prior/sectors/oil_gas/sector.py +++ b/src/openmethane_prior/sectors/oil_gas/sector.py @@ -42,7 +42,7 @@ def process_emissions(sector: AustraliaPriorSector, sector_config: PriorSectorCo category_codes=sector.unfccc_categories, ) - # now read climate_trace facilities emissions for oil and gas + # create a DataFrame with all potential methane emission sources in the sector emission_sources_df = all_emission_sources( data_manager=sector_config.data_manager, prior_config=config, From 7e19e85d11f4e9192d2e1af2ce636d6e9f1aa1c9 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 23 Apr 2026 10:20:50 +1000 Subject: [PATCH 26/29] Consolidate region well and titles into a single file for each source Every well and titles dataset for each state comes from the same source, and sometimes share other fetching or parsing functionality. This moves the two DataSource definitions for each regional source into a single file. --- .../oil_gas/data/{nopta_wells.py => nopta.py} | 113 ++++++++++++---- .../sectors/oil_gas/data/nopta_titles.py | 118 ----------------- .../data/{nsw_drillholes.py => nsw_geo.py} | 51 ++++++- .../sectors/oil_gas/data/nsw_titles.py | 69 ---------- .../sectors/oil_gas/data/nt_geo.py | 124 ++++++++++++++++++ .../sectors/oil_gas/data/nt_titles.py | 80 ----------- .../sectors/oil_gas/data/nt_wells.py | 79 ----------- .../data/{qld_boreholes.py => qld_gis.py} | 66 +++++++++- .../sectors/oil_gas/data/qld_leases.py | 94 ------------- .../oil_gas/data/{wa_titles.py => wa_gis.py} | 47 ++++++- .../sectors/oil_gas/data/wa_wells.py | 65 --------- .../oil_gas/emission_sources/all_sources.py | 15 +-- .../test_emission_sources.py | 29 ++-- 13 files changed, 388 insertions(+), 562 deletions(-) rename src/openmethane_prior/sectors/oil_gas/data/{nopta_wells.py => nopta.py} (61%) delete mode 100644 src/openmethane_prior/sectors/oil_gas/data/nopta_titles.py rename src/openmethane_prior/sectors/oil_gas/data/{nsw_drillholes.py => nsw_geo.py} (63%) delete mode 100644 src/openmethane_prior/sectors/oil_gas/data/nsw_titles.py create mode 100644 src/openmethane_prior/sectors/oil_gas/data/nt_geo.py delete mode 100644 src/openmethane_prior/sectors/oil_gas/data/nt_titles.py delete mode 100644 src/openmethane_prior/sectors/oil_gas/data/nt_wells.py rename src/openmethane_prior/sectors/oil_gas/data/{qld_boreholes.py => qld_gis.py} (63%) delete mode 100644 src/openmethane_prior/sectors/oil_gas/data/qld_leases.py rename src/openmethane_prior/sectors/oil_gas/data/{wa_titles.py => wa_gis.py} (69%) delete mode 100644 src/openmethane_prior/sectors/oil_gas/data/wa_wells.py diff --git a/src/openmethane_prior/sectors/oil_gas/data/nopta_wells.py b/src/openmethane_prior/sectors/oil_gas/data/nopta.py similarity index 61% rename from src/openmethane_prior/sectors/oil_gas/data/nopta_wells.py rename to src/openmethane_prior/sectors/oil_gas/data/nopta.py index 6de3ca1b..a598a806 100644 --- a/src/openmethane_prior/sectors/oil_gas/data/nopta_wells.py +++ b/src/openmethane_prior/sectors/oil_gas/data/nopta.py @@ -23,16 +23,97 @@ from .esri_types import map_esri_date_to_str +NOPTA_ARCGIS_URL="https://arcgis.nopta.gov.au/arcgis/rest/services" -def fetch_nopta_wells(data_source: ConfiguredDataSource): - # nopta_wells_arcgis = restapi.ArcServer(url="https://arcgis.nopta.gov.au/arcgis/rest/services") - nopta_wells = restapi.MapService( - url="https://arcgis.nopta.gov.au/arcgis/rest/services/Public/Petroleum_Wells/MapServer" +offshore_area_state_mapping = { + "Western Australia": "WA", + "Victoria": "VIC", + "Northern Territory": "NT", + "Queensland": "QLD", + "South Australia": "SA", + "Tasmania": "TAS", + "New South Wales": "NSW", + "ACT": "ACT", + # https://www.infrastructure.gov.au/territories-regions/territories/ashmore-and-cartier-islands + "Territory of Ashmore and Cartier Islands": "NT", + "Outside of Australia": None, + "UNK": None, +} +def map_offshore_area_to_state(offshore_area: str) -> str | None: + if offshore_area in offshore_area_state_mapping: + return offshore_area_state_mapping[offshore_area] + return None + + +def fetch_nopta_titles(data_source: ConfiguredDataSource): + titles_url = NOPTA_ARCGIS_URL + "/Public/TitlesCompany_NOPTA/MapServer" + titles_layer = restapi.MapService(url=titles_url).layer("Titles and Permits Current") + + layer_features = titles_layer.query( + # where="Type in ('Petroleum','Mineral or Coal') AND Purpose in ('Development','Appraisal', 'Exploration')", + # descriptions of available fields + # https://www.nopta.gov.au/maps-and-public-data/documents/DataDescription_OffshorePetroleumWells.docx + fields=[ + "OBJECTID", + "Title", + "RelTitle", + "TitleType", + "ExpiryDate", + "GrantDate", + "LastReDate", + "NoOfRenews", + "EndDate", + "Status", + "FieldName", + "BasinName", + "SubBasin", + "OffShoreAr", + "TitleOprat", + "TitleHold", + "NoOfBlocks", + "AreaKM2", + "NEATS_Links", + "TITLE_NUMBER_NEATS", + ], + exceed_limit=True, ) - petroleum_wells_layer = nopta_wells.layer("Petroleum Wells") + for feature in layer_features["features"]: + # convert esriFieldTypeDate to RFC3339 date + for feature_key in feature["properties"]: + if feature_key.endswith("Date"): + feature["properties"][feature_key] = map_esri_date_to_str(feature["properties"][feature_key]) - layer_features = petroleum_wells_layer.query( + with open(data_source.asset_path, "w") as asset_file: + json.dump(layer_features.json, asset_file) + return data_source.asset_path + + +def parse_nopta_titles(data_source: ConfiguredDataSource): + df = parse_geo(data_source=data_source) + + df["state"] = df["OffShoreAr"].map(map_offshore_area_to_state) + + return df + + +# Locations of all offshore petroleum and gas titles (licenses) administered +# by the National Offshore Petroleum Titles Administrator (NOPTA), via the +# National Electronic Approvals Tracking System (NEATS). +# Source: https://www.nopta.gov.au/maps-and-public-data/neats-info.html +nopta_titles_data_source = DataSource( + name="NOPTA-titles", + file_path="NOPTA-titles.geojson", + fetch=fetch_nopta_titles, + parse=parse_nopta_titles, +) + + +def fetch_nopta_wells(data_source: ConfiguredDataSource): + wells_url = NOPTA_ARCGIS_URL + "/Public/Petroleum_Wells/MapServer" + wells_layer = restapi.MapService(url=wells_url).layer("Petroleum Wells") + + layer_features = wells_layer.query( where="Type in ('Petroleum','Mineral or Coal') AND Purpose in ('Development','Appraisal', 'Exploration')", # descriptions of available fields # https://www.nopta.gov.au/maps-and-public-data/documents/DataDescription_OffshorePetroleumWells.docx @@ -60,26 +141,6 @@ def fetch_nopta_wells(data_source: ConfiguredDataSource): return data_source.asset_path -offshore_area_state_mapping = { - "Western Australia": "WA", - "Victoria": "VIC", - "Northern Territory": "NT", - "Queensland": "QLD", - "South Australia": "SA", - "Tasmania": "TAS", - "New South Wales": "NSW", - "ACT": "ACT", - # https://www.infrastructure.gov.au/territories-regions/territories/ashmore-and-cartier-islands - "Territory of Ashmore and Cartier Islands": "NT", - "Outside of Australia": None, - "UNK": None, -} -def map_offshore_area_to_state(offshore_area: str) -> str | None: - if offshore_area in offshore_area_state_mapping: - return offshore_area_state_mapping[offshore_area] - return None - - def parse_nopta_wells(data_source: ConfiguredDataSource): wells_df = parse_geo(data_source=data_source) diff --git a/src/openmethane_prior/sectors/oil_gas/data/nopta_titles.py b/src/openmethane_prior/sectors/oil_gas/data/nopta_titles.py deleted file mode 100644 index 98ee6281..00000000 --- a/src/openmethane_prior/sectors/oil_gas/data/nopta_titles.py +++ /dev/null @@ -1,118 +0,0 @@ -# -# Copyright 2026 The Superpower Institute Ltd. -# -# This file is part of OpenMethane. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import json -import restapi # https://github.com/Bolton-and-Menk-GIS/restapi - -from openmethane_prior.lib import DataSource, ConfiguredDataSource -from openmethane_prior.lib.data_manager.parsers import parse_geo - -from .esri_types import map_esri_date_to_str - - -def fetch_nopta_titles(data_source: ConfiguredDataSource): - nopta_titles = restapi.MapService( - url="https://arcgis.nopta.gov.au/arcgis/rest/services/Public/TitlesCompany_NOPTA/MapServer" - ) - - titles_layer = nopta_titles.layer("Titles and Permits Current") - - layer_features = titles_layer.query( - # where="Type in ('Petroleum','Mineral or Coal') AND Purpose in ('Development','Appraisal', 'Exploration')", - # descriptions of available fields - # https://www.nopta.gov.au/maps-and-public-data/documents/DataDescription_OffshorePetroleumWells.docx - fields=[ - "OBJECTID", - "Title", - "RelTitle", - "TitleType", - "ExpiryDate", - "GrantDate", - "LastReDate", - "NoOfRenews", - "EndDate", - "Status", - "FieldName", - "BasinName", - "SubBasin", - "OffShoreAr", - "TitleOprat", - "TitleHold", - "NoOfBlocks", - "AreaKM2", - "NEATS_Links", - "TITLE_NUMBER_NEATS", - ], - exceed_limit=True, - ) - - for feature in layer_features["features"]: - # convert esriFieldTypeDate to RFC3339 date - for feature_key in feature["properties"]: - if feature_key.endswith("Date"): - feature["properties"][feature_key] = map_esri_date_to_str(feature["properties"][feature_key]) - - with open(data_source.asset_path, "w") as asset_file: - json.dump(layer_features.json, asset_file) - return data_source.asset_path - - -offshore_area_state_mapping = { - "Western Australia": "WA", - "Victoria": "VIC", - "Northern Territory": "NT", - "Queensland": "QLD", - "South Australia": "SA", - "Tasmania": "TAS", - "New South Wales": "NSW", - "ACT": "ACT", - # https://www.infrastructure.gov.au/territories-regions/territories/ashmore-and-cartier-islands - "Territory of Ashmore and Cartier Islands": "NT", - "Outside of Australia": None, - "UNK": None, -} -def map_offshore_area_to_state(offshore_area: str) -> str | None: - if offshore_area in offshore_area_state_mapping: - return offshore_area_state_mapping[offshore_area] - return None - - -def parse_nopta_titles(data_source: ConfiguredDataSource): - df = parse_geo(data_source=data_source) - - # # NOPIMS dataset may record multiple boreholes for a single well, all - # # with identical WellName and location. This will remove duplicate rows - # # so that only a single location is present for each well, keeping the - # # earliest "RigReleaseDate" when the first bore was drilled at the well. - # wells_df = wells_df.sort_values(by="RigReleaseDate") - # wells_df.drop_duplicates(subset="WellName", keep="first", inplace=True) - # - df["state"] = df["OffShoreAr"].map(map_offshore_area_to_state) - - return df - - -# Locations of all offshore petroleum and gas titles (licenses) administered -# by the National Offshore Petroleum Titles Administrator (NOPTA), via the -# National Electronic Approvals Tracking System (NEATS). -# Source: https://www.nopta.gov.au/maps-and-public-data/neats-info.html -nopta_titles_data_source = DataSource( - name="NOPTA-titles", - file_path="NOPTA-titles.geojson", - fetch=fetch_nopta_titles, - parse=parse_nopta_titles, -) diff --git a/src/openmethane_prior/sectors/oil_gas/data/nsw_drillholes.py b/src/openmethane_prior/sectors/oil_gas/data/nsw_geo.py similarity index 63% rename from src/openmethane_prior/sectors/oil_gas/data/nsw_drillholes.py rename to src/openmethane_prior/sectors/oil_gas/data/nsw_geo.py index 8103eff4..dfd21ac2 100644 --- a/src/openmethane_prior/sectors/oil_gas/data/nsw_drillholes.py +++ b/src/openmethane_prior/sectors/oil_gas/data/nsw_geo.py @@ -25,8 +25,10 @@ from openmethane_prior.lib.data_manager.source import ConfiguredDataSource +NSW_GEOSCIENCE_WFS_URL="https://public-gs.geoscience.nsw.gov.au/geoserver/wfs" + def fetch_nsw_drillholes(data_source: ConfiguredDataSource): - geoserver_wfs = WebFeatureService("https://public-gs.geoscience.nsw.gov.au/geoserver/wfs", version="2.0.0") + geoserver_wfs = WebFeatureService(NSW_GEOSCIENCE_WFS_URL, version="2.0.0") desired_properties = [ "program", @@ -77,8 +79,53 @@ def fetch_nsw_drillholes(data_source: ConfiguredDataSource): # Source: https://data.nsw.gov.au/data/dataset/coal-seam-gas-borehole # Source: https://data.nsw.gov.au/data/dataset/nsw-drillholes-petroleum nsw_drillholes_data_source = DataSource( - name="nsw-drillholes-csg-petroleum", + name="NSW-drillholes-csg-petroleum", file_path="NSW-drillholes-csg-petroleum.geojson", fetch=fetch_nsw_drillholes, parse=parse_geo, ) + + +def fetch_nsw_titles(data_source: ConfiguredDataSource): + geoserver_wfs = WebFeatureService(NSW_GEOSCIENCE_WFS_URL, version="2.0.0") + + desired_properties = [ + "tas_id", + "title", + "holder", + "company", + "grant_date", + "expiry_date", + "minerals", + "resource", + "operation", + "geom", + ] + + nsw_titles_feature = geoserver_wfs.getfeature( + typename="mining-and-exploration:titles_title_granted", + srsname="urn:ogc:def:crs:EPSG::4326", + propertyname=desired_properties, + outputFormat="application/json", + ) + + features_df = gpd.GeoDataFrame.from_features(json.loads(nsw_titles_feature.read())) + + # only include titles relevant to petroleum production + features_df = features_df[features_df["resource"] == "PETROLEUM"] + features_df = features_df[features_df["operation"] == "MINING"] + + with open(data_source.asset_path, "w") as asset_file: + asset_file.write(features_df.to_json()) + return data_source.asset_path + + +# Locations of coal seam gas and petroleum production titles in the +# Australian state of New South Wales, via Geosciences NSW. +# Source: https://data.nsw.gov.au/data/dataset/nsw-mining-titles +nsw_titles_data_source = DataSource( + name="NSW-titles-csg-petroleum", + file_path="NSW-titles-csg-petroleum.geojson", + fetch=fetch_nsw_titles, + parse=parse_geo, +) diff --git a/src/openmethane_prior/sectors/oil_gas/data/nsw_titles.py b/src/openmethane_prior/sectors/oil_gas/data/nsw_titles.py deleted file mode 100644 index 0e3f959f..00000000 --- a/src/openmethane_prior/sectors/oil_gas/data/nsw_titles.py +++ /dev/null @@ -1,69 +0,0 @@ -# -# Copyright 2026 The Superpower Institute Ltd. -# -# This file is part of OpenMethane. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import geopandas as gpd -import json -from owslib.wfs import WebFeatureService - -from openmethane_prior.lib import DataSource -from openmethane_prior.lib.data_manager.parsers import parse_geo -from openmethane_prior.lib.data_manager.source import ConfiguredDataSource - - -def fetch_nsw_titles(data_source: ConfiguredDataSource): - geoserver_wfs = WebFeatureService("https://public-gs.geoscience.nsw.gov.au/geoserver/wfs", version="2.0.0") - - desired_properties = [ - "tas_id", - "title", - "holder", - "company", - "grant_date", - "expiry_date", - "minerals", - "resource", - "operation", - "geom", - ] - - nsw_titles_feature = geoserver_wfs.getfeature( - typename="mining-and-exploration:titles_title_granted", - srsname="urn:ogc:def:crs:EPSG::4326", - propertyname=desired_properties, - outputFormat="application/json", - ) - - features_df = gpd.GeoDataFrame.from_features(json.loads(nsw_titles_feature.read())) - - # only include titles relevant to petroleum production - features_df = features_df[features_df["resource"] == "PETROLEUM"] - features_df = features_df[features_df["operation"] == "MINING"] - - with open(data_source.asset_path, "w") as asset_file: - asset_file.write(features_df.to_json()) - return data_source.asset_path - - -# Locations of coal seam gas and petroleum production titles in the -# Australian state of New South Wales, via Geosciences NSW. -# Source: https://data.nsw.gov.au/data/dataset/nsw-mining-titles -nsw_titles_data_source = DataSource( - name="nsw-titles-csg-petroleum", - file_path="NSW-titles-csg-petroleum.geojson", - fetch=fetch_nsw_titles, - parse=parse_geo, -) diff --git a/src/openmethane_prior/sectors/oil_gas/data/nt_geo.py b/src/openmethane_prior/sectors/oil_gas/data/nt_geo.py new file mode 100644 index 00000000..5e28d1bd --- /dev/null +++ b/src/openmethane_prior/sectors/oil_gas/data/nt_geo.py @@ -0,0 +1,124 @@ +# +# Copyright 2026 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import geopandas as gpd +import numpy as np +import os +import pathlib +import shutil +import tempfile +import urllib +import zipfile + +from openmethane_prior.lib import DataSource +from openmethane_prior.lib.data_manager.parsers import parse_geo +from openmethane_prior.lib.data_manager.source import ConfiguredDataSource + +def fetch_zipped_shp_to_gdf( + url: str, + shp_file: str, +) -> gpd.GeoDataFrame: + """Fetch a zip file specified by DataSource.url, extract the contents and + read in a Shapefile (.shp) from the archive as a GeoDataFrame.""" + + # use a temporary path to store and extract the zip contents, which will + # be cleaned up automatically afterwards + with tempfile.TemporaryDirectory(prefix="openmethane-prior") as tmp_dir: + tmp_path = pathlib.Path(tmp_dir) + + # download and extract the zip file to a known location + zip_path, response = urllib.request.urlretrieve( + url=url, + filename=tmp_path / os.path.basename(url), + ) + # entire zip must be extracted, as the .shp file depends on sibling files + with zipfile.ZipFile(zip_path, 'r') as zip_ref: + zip_ref.extractall(tmp_path) + + # read the extracted Shapefile with geopandas + return gpd.read_file(tmp_path / shp_file) + + +def fetch_nt_titles(data_source: ConfiguredDataSource) -> pathlib.Path: + """Fetch a zipped Shapefile (.shp) and convert to GeoJSON using geopandas.""" + # fetch the zipfile in data_source.url and read the included Shapefile + df = fetch_zipped_shp_to_gdf(data_source.url, "PETRO_TITLE_PROD_GRNT.shp") + + # remove entries with no granted date, these are unusable + df = df[~np.isnat(df["DT_GRNT"])] + + # datetime64 objects aren't JSON serialisable, convert them to string + for field_name in df.columns: + if df.dtypes[field_name] == "datetime64[ms]": + df[field_name] = [ + str(dt) if not np.isnat(dt) else None + for dt in df[field_name].values + ] + + # write the data as geojson + with open(data_source.asset_path, "w") as asset_file: + asset_file.write(df.to_json()) + + return data_source.asset_path + + +# Locations of petroleum production titles in Australia's Northern Territories, +# via the Spatial Territory Resource Information Kit for Exploration (STRIKE), +# part of the NT Department of Mining and Energy. +# Source: http://strike.nt.gov.au/wss.html -> Downloads -> Petroleum Titles and Pipeline Titles +nt_titles_data_source = DataSource( + name="NT-petroleum-titles", + url="https://geoscience.nt.gov.au/contents/prod/Downloads/NT_PetroleumPipelineTitles_shp.zip", + file_path='NT-petroleum-titles.geojson', + fetch=fetch_nt_titles, + parse=parse_geo, +) + + +def fetch_nt_wells(data_source: ConfiguredDataSource) -> pathlib.Path: + """Fetch a zipped Shapefile (.shp) and convert to GeoJSON using geopandas.""" + # fetch the zipfile in data_source.url and read the included Shapefile + df = fetch_zipped_shp_to_gdf(data_source.url, "PETROLEUM_WELLS.shp") + + # convert datetime fields with strings like "19891229000000" into RFC3339 + for field_name in df.columns: + if field_name.startswith("DT_"): + df[field_name] = [ + str(datetime.datetime.strptime(dt_str, "%Y%m%d%H%M%S")) if dt_str is not None else None + for dt_str in df[field_name].values + ] + + # write the data as geojson + with open(data_source.asset_path, "w") as asset_file: + asset_file.write(df.to_json()) + + return data_source.asset_path + + +# Locations of petroleum wells in Australia's Northern Territories, via the +# Spatial Territory Resource Information Kit for Exploration (STRIKE), part of +# the NT Department of Mining and Energy. +# Source: http://strike.nt.gov.au/wss.html -> Downloads -> Drilling -> Petroleum Wells +# Metadata: https://www.ntlis.nt.gov.au/metadata/export_data?type=html&metadata_id=2DBCB771210D06B6E040CD9B0F274EFE +nt_wells_data_source = DataSource( + name="NT-petroleum-wells", + url="https://geoscience.nt.gov.au/contents/prod/Downloads/Drilling/PETROLEUM_WELLS_shp.zip", + file_path='NT-petroleum-wells.geojson', + fetch=fetch_nt_wells, + parse=parse_geo, +) diff --git a/src/openmethane_prior/sectors/oil_gas/data/nt_titles.py b/src/openmethane_prior/sectors/oil_gas/data/nt_titles.py deleted file mode 100644 index e9e4efd5..00000000 --- a/src/openmethane_prior/sectors/oil_gas/data/nt_titles.py +++ /dev/null @@ -1,80 +0,0 @@ -# -# Copyright 2026 The Superpower Institute Ltd. -# -# This file is part of OpenMethane. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import geopandas as gpd -import numpy as np -import os -import pathlib -import shutil -import urllib -import zipfile - -from openmethane_prior.lib import DataSource -from openmethane_prior.lib.data_manager.parsers import parse_geo -from openmethane_prior.lib.data_manager.source import ConfiguredDataSource - - -def fetch_nt_titles(data_source: ConfiguredDataSource) -> pathlib.Path: - """ - Download a zip file specified by DataSource.url, extract the contents and - convert a Shapefile (.shp) to GeoJSON using geopandas. - """ - # download zip file to a temporary location, it should be cleaned up afterwards - zip_path, response = urllib.request.urlretrieve( - url=data_source.url, - filename=data_source.data_path / os.path.basename(data_source.url), - ) - - tmp_path = data_source.data_path / "zip_contents" - with zipfile.ZipFile(zip_path, 'r') as zip_ref: - zip_ref.extractall(tmp_path) - - # read the extracted Shapefile with geopandas - df = gpd.GeoDataFrame.from_file(tmp_path / "PETRO_TITLE_PROD_GRNT.shp") - # remove entries with no granted date, these are unusable - df = df[~np.isnat(df["DT_GRNT"])] - - # datetime64 objects aren't JSON serialisable, convert them to string - for field_name in df.columns: - if df.dtypes[field_name] == "datetime64[ms]": - df[field_name] = [ - str(dt) if not np.isnat(dt) else None - for dt in df[field_name].values - ] - - # write the data as geojson - with open(data_source.asset_path, "w") as asset_file: - asset_file.write(df.to_json()) - - # clean up the zip and extracted contents, leaving only the GeoJSON - os.remove(zip_path) - shutil.rmtree(tmp_path) - - return data_source.asset_path - - -# Locations of petroleum production titles in Australia's Northern Territories, -# via the Spatial Territory Resource Information Kit for Exploration (STRIKE), -# part of the NT Department of Mining and Energy. -# Source: http://strike.nt.gov.au/wss.html -> Downloads -> Petroleum Titles and Pipeline Titles -nt_titles_data_source = DataSource( - name="NT-petroleum-titles", - url="https://geoscience.nt.gov.au/contents/prod/Downloads/NT_PetroleumPipelineTitles_shp.zip", - file_path='NT-petroleum-titles.geojson', - fetch=fetch_nt_titles, - parse=parse_geo, -) diff --git a/src/openmethane_prior/sectors/oil_gas/data/nt_wells.py b/src/openmethane_prior/sectors/oil_gas/data/nt_wells.py deleted file mode 100644 index 0d8b8e5a..00000000 --- a/src/openmethane_prior/sectors/oil_gas/data/nt_wells.py +++ /dev/null @@ -1,79 +0,0 @@ -# -# Copyright 2026 The Superpower Institute Ltd. -# -# This file is part of OpenMethane. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import datetime -import geopandas as gpd -import os -import pathlib -import shutil -import urllib -import zipfile - -from openmethane_prior.lib import DataSource -from openmethane_prior.lib.data_manager.parsers import parse_geo -from openmethane_prior.lib.data_manager.source import ConfiguredDataSource - - -def fetch_nt_wells(data_source: ConfiguredDataSource) -> pathlib.Path: - """ - Download a zip file specified by DataSource.url, extract the contents and - convert a Shapefile (.shp) to GeoJSON using geopandas. - """ - # download zip file to a temporary location, it should be cleaned up afterwards - zip_path, response = urllib.request.urlretrieve( - url=data_source.url, - filename=data_source.data_path / os.path.basename(data_source.url), - ) - - tmp_path = data_source.data_path / "zip_contents" - with zipfile.ZipFile(zip_path, 'r') as zip_ref: - zip_ref.extractall(tmp_path) - - # read the extracted Shapefile with geopandas - df = gpd.GeoDataFrame.from_file(tmp_path / "PETROLEUM_WELLS.shp") - - # convert datetime fields with strings like "19891229000000" into RFC3339 - for field_name in df.columns: - if field_name.startswith("DT_"): - df[field_name] = [ - str(datetime.datetime.strptime(dt_str, "%Y%m%d%H%M%S")) if dt_str is not None else None - for dt_str in df[field_name].values - ] - - # write the data as geojson - with open(data_source.asset_path, "w") as asset_file: - asset_file.write(df.to_json()) - - # clean up the zip and extracted contents, leaving only the GeoJSON - os.remove(zip_path) - shutil.rmtree(tmp_path) - - return data_source.asset_path - - -# Locations of petroleum wells in Australia's Northern Territories, via the -# Spatial Territory Resource Information Kit for Exploration (STRIKE), part of -# the NT Department of Mining and Energy. -# Source: http://strike.nt.gov.au/wss.html -> Downloads -> Drilling -> Petroleum Wells -# Metadata: https://www.ntlis.nt.gov.au/metadata/export_data?type=html&metadata_id=2DBCB771210D06B6E040CD9B0F274EFE -nt_wells_data_source = DataSource( - name="NT-petroleum-wells", - url="https://geoscience.nt.gov.au/contents/prod/Downloads/Drilling/PETROLEUM_WELLS_shp.zip", - file_path='NT-petroleum-wells.geojson', - fetch=fetch_nt_wells, - parse=parse_geo, -) diff --git a/src/openmethane_prior/sectors/oil_gas/data/qld_boreholes.py b/src/openmethane_prior/sectors/oil_gas/data/qld_gis.py similarity index 63% rename from src/openmethane_prior/sectors/oil_gas/data/qld_boreholes.py rename to src/openmethane_prior/sectors/oil_gas/data/qld_gis.py index c5b32c9f..7b23695f 100644 --- a/src/openmethane_prior/sectors/oil_gas/data/qld_boreholes.py +++ b/src/openmethane_prior/sectors/oil_gas/data/qld_gis.py @@ -17,6 +17,7 @@ # import geopandas as gpd import numpy as np +import pandas as pd import restapi # https://github.com/Bolton-and-Menk-GIS/restapi from openmethane_prior.lib import DataSource @@ -24,13 +25,13 @@ from openmethane_prior.lib.data_manager.source import ConfiguredDataSource from openmethane_prior.sectors.oil_gas.data.esri_types import map_esri_date_to_str +QLD_SPATIAL_ARCGIS_URL="https://spatial-gis.information.qld.gov.au/arcgis/rest/services" # Queensland borehole series - REST Service (ArcGIS) # https://www.data.qld.gov.au/dataset/queensland-borehole-series/resource/c206a53a-59de-48c2-9544-b7b7323ed5dd def fetch_qld_boreholes(data_source: ConfiguredDataSource): - # qld_spatial_arcgis = restapi.ArcServer(url="https://spatial-gis.information.qld.gov.au/arcgis/rest/services") qld_spatial_boreholes = restapi.MapService( - url="https://spatial-gis.information.qld.gov.au/arcgis/rest/services/GeoscientificInformation/Boreholes/MapServer" + url=f"{QLD_SPATIAL_ARCGIS_URL}/GeoscientificInformation/Boreholes/MapServer" ) boreholes_features = None @@ -74,8 +75,6 @@ def fetch_qld_boreholes(data_source: ConfiguredDataSource): def parse_qld_boreholes(data_source: ConfiguredDataSource): boreholes_df = parse_geo(data_source=data_source) - # - # lease/tenement name, i.e. "PL 100", is more useful than "PL" and 100.0, # so combine tenure_no and tenure_type into a single string boreholes_df["tenure"] = [ @@ -85,12 +84,69 @@ def parse_qld_boreholes(data_source: ConfiguredDataSource): return boreholes_df + # Locations of all boreholes in the Australian state of Queensland, via the # Queensland Open Data Portal. # Source: https://www.data.qld.gov.au/dataset/queensland-borehole-series qld_boreholes_data_source = DataSource( - name="qld-boreholes", + name="QLD-boreholes", file_path="QLD-boreholes.geojson", fetch=fetch_qld_boreholes, parse=parse_qld_boreholes, ) + + +# Petroleum leases - Queensland - REST Service (ArcGIS) +# https://www.data.qld.gov.au/dataset/queensland-mining-and-exploration-tenure-series/resource/544ecb39-9c25-42f9-9928-b44c7ff05b30 +def fetch_qld_leases(data_source: ConfiguredDataSource): + # qld_spatial_arcgis = restapi.ArcServer(url="https://spatial-gis.information.qld.gov.au/arcgis/rest/services") + qld_current_leases = restapi.MapService( + url=f"{QLD_SPATIAL_ARCGIS_URL}/Economy/MinesPermitsCurrent/MapServer" + ) + qld_historic_leases = restapi.MapService( + url=f"{QLD_SPATIAL_ARCGIS_URL}/Economy/MinesPermitsHistoric/MapServer" + ) + + fields = [ + "displayname", + "permittype", + "permitstatus", + "approvedate", + "expirydate", + "permitminerals", + "permitpurpose", + ] + + leases_layer = qld_current_leases.layer("PL granted") + leases_features = leases_layer.query( + fields=fields, + exceed_limit=True, + ) + historic_layer = qld_historic_leases.layer("Historical petroleum lease") + historic_features = historic_layer.query( + fields=fields, + exceed_limit=True, + ) + + df = pd.concat([ + gpd.GeoDataFrame.from_features(leases_features.features), + gpd.GeoDataFrame.from_features(historic_features.features), + ]) + + df["approvedate"] = df["approvedate"].map(map_esri_date_to_str) + df["expirydate"] = df["expirydate"].map(map_esri_date_to_str) + + with open(data_source.asset_path, "w") as asset_file: + asset_file.write(df.to_json()) + return data_source.asset_path + + +# Locations of all mining leases in the Australian state of Queensland, via the +# Queensland Open Data Portal. +# Source: https://www.data.qld.gov.au/dataset/queensland-mining-and-exploration-tenure-series +qld_leases_data_source = DataSource( + name="QLD-leases", + file_path="QLD-leases.geojson", + fetch=fetch_qld_leases, + parse=parse_geo, +) diff --git a/src/openmethane_prior/sectors/oil_gas/data/qld_leases.py b/src/openmethane_prior/sectors/oil_gas/data/qld_leases.py deleted file mode 100644 index c3236685..00000000 --- a/src/openmethane_prior/sectors/oil_gas/data/qld_leases.py +++ /dev/null @@ -1,94 +0,0 @@ -# -# Copyright 2026 The Superpower Institute Ltd. -# -# This file is part of OpenMethane. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import geopandas as gpd -import pandas as pd -import restapi # https://github.com/Bolton-and-Menk-GIS/restapi - -from openmethane_prior.lib import DataSource -from openmethane_prior.lib.data_manager.parsers import parse_geo -from openmethane_prior.lib.data_manager.source import ConfiguredDataSource -from openmethane_prior.sectors.oil_gas.data.esri_types import map_esri_date_to_str - - -# Petroleum leases - Queensland - REST Service (ArcGIS) -# https://www.data.qld.gov.au/dataset/queensland-mining-and-exploration-tenure-series/resource/544ecb39-9c25-42f9-9928-b44c7ff05b30 -def fetch_qld_leases(data_source: ConfiguredDataSource): - # qld_spatial_arcgis = restapi.ArcServer(url="https://spatial-gis.information.qld.gov.au/arcgis/rest/services") - qld_current_leases = restapi.MapService( - url="https://spatial-gis.information.qld.gov.au/arcgis/rest/services/Economy/MinesPermitsCurrent/MapServer" - ) - qld_historic_leases = restapi.MapService( - url="https://spatial-gis.information.qld.gov.au/arcgis/rest/services/Economy/MinesPermitsHistoric/MapServer" - ) - - fields = [ - "displayname", - "permittype", - "permitstatus", - "approvedate", - "expirydate", - "permitminerals", - "permitpurpose", - ] - - leases_layer = qld_current_leases.layer("PL granted") - leases_features = leases_layer.query( - fields=fields, - exceed_limit=True, - ) - historic_layer = qld_historic_leases.layer("Historical petroleum lease") - historic_features = historic_layer.query( - fields=fields, - exceed_limit=True, - ) - - df = pd.concat([ - gpd.GeoDataFrame.from_features(leases_features.features), - gpd.GeoDataFrame.from_features(historic_features.features), - ]) - - df["approvedate"] = df["approvedate"].map(map_esri_date_to_str) - df["expirydate"] = df["expirydate"].map(map_esri_date_to_str) - - with open(data_source.asset_path, "w") as asset_file: - asset_file.write(df.to_json()) - return data_source.asset_path - - - -def parse_qld_leases(data_source: ConfiguredDataSource): - leases_df = parse_geo(data_source=data_source) - - # lease/tenement name, i.e. "PL 100", is more useful than "PL" and 100.0, - # so combine tenure_no and tenure_type into a single string - # leases_df["tenure"] = [ - # None if t_no is np.nan else f"{t_type} {t_no:0.0f}" - # for t_no, t_type in leases_df[["tenure_no", "tenure_type"]].values - # ] - - return leases_df - -# Locations of all mining leases in the Australian state of Queensland, via the -# Queensland Open Data Portal. -# Source: https://www.data.qld.gov.au/dataset/queensland-mining-and-exploration-tenure-series -qld_leases_data_source = DataSource( - name="qld-leases", - file_path="QLD-leases.geojson", - fetch=fetch_qld_leases, - parse=parse_qld_leases, -) diff --git a/src/openmethane_prior/sectors/oil_gas/data/wa_titles.py b/src/openmethane_prior/sectors/oil_gas/data/wa_gis.py similarity index 69% rename from src/openmethane_prior/sectors/oil_gas/data/wa_titles.py rename to src/openmethane_prior/sectors/oil_gas/data/wa_gis.py index efe60ce1..880d38ef 100644 --- a/src/openmethane_prior/sectors/oil_gas/data/wa_titles.py +++ b/src/openmethane_prior/sectors/oil_gas/data/wa_gis.py @@ -25,12 +25,13 @@ from .esri_types import map_esri_date_to_str +WA_ARCGIS_URL="https://public-services.slip.wa.gov.au/public/rest/services" + # WA Petroleum Titles (DMIRS-011) - REST Service (ArcGIS) # https://catalogue.data.wa.gov.au/dataset/wa-petroleum-titles-dmirs-011/resource/f5bb4e83-241e-4dcf-8efe-41707866af4e def fetch_wa_titles(data_source: ConfiguredDataSource): - # wa_arcgis = restapi.ArcServer(url="https://public-services.slip.wa.gov.au/public/rest/services") wa_arcgis_mining = restapi.MapService( - url="https://public-services.slip.wa.gov.au/public/rest/services/SLIP_Public_Services/Industry_and_Mining/MapServer" + url=f"{WA_ARCGIS_URL}/SLIP_Public_Services/Industry_and_Mining/MapServer" ) # DMIRS-011 contains current active petroleum titles, but has expired @@ -89,8 +90,48 @@ def fetch_wa_titles(data_source: ConfiguredDataSource): # Australia via the Data WA Portal. # Source: https://catalogue.data.wa.gov.au/dataset/wa-petroleum-titles-dmirs-011 wa_titles_data_source = DataSource( - name="wa-petroleum-titles", + name="WA-petroleum-titles", file_path="WA-petroleum-titles.geojson", fetch=fetch_wa_titles, parse=parse_geo, ) + + +# WA Petroleum Wells (DMIRS-025) - REST Service (ArcGIS) +# https://catalogue.data.wa.gov.au/dataset/mineral-exploration-drillholes-open-file/resource/1c61171a-3b23-4f2b-baae-04615c5bb39e +def fetch_wa_wells(data_source: ConfiguredDataSource): + wa_arcgis_mining = restapi.MapService( + url=f"{WA_ARCGIS_URL}/SLIP_Public_Services/Industry_and_Mining/MapServer" + ) + + wells_layer = wa_arcgis_mining.layer("WA Onshore Petroleum Wells (DMIRS-025)") + wells_features = wells_layer.query( + fields=[ + "well_name", + "uwi", + "lease_no", + "operator", + "class", + "status", + "rig_release_date", + ], + exceed_limit=True, + ) + + df = gpd.GeoDataFrame.from_features(wells_features.features) + df["rig_release_date"] = df["rig_release_date"].map(map_esri_date_to_str) + + with open(data_source.asset_path, "w") as asset_file: + asset_file.write(df.to_json()) + return data_source.asset_path + + +# Locations of all petroleum wells in the Australian state of Western Australia +# via the Data WA Portal. +# Source: https://catalogue.data.wa.gov.au/dataset/wa-onshore-petroleum-wells-dmirs-025 +wa_wells_data_source = DataSource( + name="WA-petroleum-wells", + file_path="WA-petroleum-wells.geojson", + fetch=fetch_wa_wells, + parse=parse_geo, +) diff --git a/src/openmethane_prior/sectors/oil_gas/data/wa_wells.py b/src/openmethane_prior/sectors/oil_gas/data/wa_wells.py deleted file mode 100644 index c0b4b135..00000000 --- a/src/openmethane_prior/sectors/oil_gas/data/wa_wells.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright 2026 The Superpower Institute Ltd. -# -# This file is part of OpenMethane. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import geopandas as gpd -import restapi # https://github.com/Bolton-and-Menk-GIS/restapi - -from openmethane_prior.lib import DataSource -from openmethane_prior.lib.data_manager.parsers import parse_geo -from openmethane_prior.lib.data_manager.source import ConfiguredDataSource -from .esri_types import map_esri_date_to_str - - -# WA Petroleum Wells (DMIRS-025) - REST Service (ArcGIS) -# https://catalogue.data.wa.gov.au/dataset/mineral-exploration-drillholes-open-file/resource/1c61171a-3b23-4f2b-baae-04615c5bb39e -def fetch_wa_wells(data_source: ConfiguredDataSource): - # wa_arcgis = restapi.ArcServer(url="https://public-services.slip.wa.gov.au/public/rest/services") - wa_arcgis_mining = restapi.MapService( - url="https://public-services.slip.wa.gov.au/public/rest/services/SLIP_Public_Services/Industry_and_Mining/MapServer" - ) - - wells_layer = wa_arcgis_mining.layer("WA Onshore Petroleum Wells (DMIRS-025)") - wells_features = wells_layer.query( - fields=[ - "well_name", - "uwi", - "lease_no", - "operator", - "class", - "status", - "rig_release_date", - ], - exceed_limit=True, - ) - - df = gpd.GeoDataFrame.from_features(wells_features.features) - df["rig_release_date"] = df["rig_release_date"].map(map_esri_date_to_str) - - with open(data_source.asset_path, "w") as asset_file: - asset_file.write(df.to_json()) - return data_source.asset_path - - -# Locations of all petroleum wells in the Australian state of Western Australia -# via the Data WA Portal. -# Source: https://catalogue.data.wa.gov.au/dataset/wa-onshore-petroleum-wells-dmirs-025 -wa_wells_data_source = DataSource( - name="wa-petroleum-wells", - file_path="WA-petroleum-wells.geojson", - fetch=fetch_wa_wells, - parse=parse_geo, -) diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py index 8b2f2f0f..2865fa06 100644 --- a/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py @@ -31,17 +31,12 @@ from .qld_sources import qld_emission_sources from .sa_sources import sa_emission_sources from .wa_sources import wa_emission_sources -from ..data.nopta_titles import nopta_titles_data_source -from ..data.nopta_wells import nopta_wells_data_source -from ..data.nsw_drillholes import nsw_drillholes_data_source -from ..data.nsw_titles import nsw_titles_data_source -from ..data.nt_titles import nt_titles_data_source -from ..data.nt_wells import nt_wells_data_source -from ..data.qld_boreholes import qld_boreholes_data_source -from ..data.qld_leases import qld_leases_data_source +from ..data.nopta import nopta_titles_data_source, nopta_wells_data_source +from ..data.nsw_geo import nsw_drillholes_data_source, nsw_titles_data_source +from ..data.nt_geo import nt_titles_data_source, nt_wells_data_source +from ..data.qld_gis import qld_boreholes_data_source, qld_leases_data_source from ..data.sa_wells import sa_wells_data_source, sa_wells_production_data_source -from ..data.wa_titles import wa_titles_data_source -from ..data.wa_wells import wa_wells_data_source +from ..data.wa_gis import wa_titles_data_source, wa_wells_data_source logger = logger.get_logger(__name__) diff --git a/tests/integration/test_sector_oil_gas/test_emission_sources.py b/tests/integration/test_sector_oil_gas/test_emission_sources.py index a62c2595..90363969 100644 --- a/tests/integration/test_sector_oil_gas/test_emission_sources.py +++ b/tests/integration/test_sector_oil_gas/test_emission_sources.py @@ -1,18 +1,25 @@ import datetime -from openmethane_prior.sectors.oil_gas.data.nopta_titles import nopta_titles_data_source -from openmethane_prior.sectors.oil_gas.data.nopta_wells import nopta_wells_data_source -from openmethane_prior.sectors.oil_gas.data.nsw_drillholes import nsw_drillholes_data_source -from openmethane_prior.sectors.oil_gas.data.nsw_titles import nsw_titles_data_source -from openmethane_prior.sectors.oil_gas.data.nt_titles import nt_titles_data_source -from openmethane_prior.sectors.oil_gas.data.nt_wells import nt_wells_data_source -from openmethane_prior.sectors.oil_gas.data.qld_boreholes import qld_boreholes_data_source -from openmethane_prior.sectors.oil_gas.data.qld_leases import qld_leases_data_source -from openmethane_prior.sectors.oil_gas.data.sa_wells import sa_wells_data_source, sa_wells_production_data_source +from openmethane_prior.sectors.oil_gas.data.nopta import ( + nopta_titles_data_source, nopta_wells_data_source, +) +from openmethane_prior.sectors.oil_gas.data.nsw_geo import ( + nsw_drillholes_data_source, nsw_titles_data_source, +) +from openmethane_prior.sectors.oil_gas.data.nt_geo import ( + nt_titles_data_source, nt_wells_data_source, +) +from openmethane_prior.sectors.oil_gas.data.qld_gis import ( + qld_boreholes_data_source, qld_leases_data_source, +) +from openmethane_prior.sectors.oil_gas.data.sa_wells import ( + sa_wells_data_source, sa_wells_production_data_source, +) +from openmethane_prior.sectors.oil_gas.data.wa_gis import ( + wa_titles_data_source, wa_wells_data_source, +) from openmethane_prior.sectors.oil_gas.emission_sources.all_sources import all_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.nsw_sources import nsw_emission_sources -from openmethane_prior.sectors.oil_gas.data.wa_titles import wa_titles_data_source -from openmethane_prior.sectors.oil_gas.data.wa_wells import wa_wells_data_source from openmethane_prior.sectors.oil_gas.emission_sources.nt_sources import nt_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.offshore_sources import offshore_emission_sources from openmethane_prior.sectors.oil_gas.emission_sources.qld_sources import qld_emission_sources From 55ae5f6509d39cc9016e8c326fb1cf74889798a0 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 23 Apr 2026 10:40:56 +1000 Subject: [PATCH 27/29] Refactor DataSources with custom fetch methods to use `url` param This doesn't result in a functional change, but has been requested in PR review as a more self-documenting way to keep the source URL closer to the DataSource which includes the documentation about where the URL was discovered. --- src/openmethane_prior/sectors/oil_gas/data/nopta.py | 4 ++-- src/openmethane_prior/sectors/oil_gas/data/wa_gis.py | 11 ++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/openmethane_prior/sectors/oil_gas/data/nopta.py b/src/openmethane_prior/sectors/oil_gas/data/nopta.py index a598a806..2f9ccfd4 100644 --- a/src/openmethane_prior/sectors/oil_gas/data/nopta.py +++ b/src/openmethane_prior/sectors/oil_gas/data/nopta.py @@ -46,8 +46,7 @@ def map_offshore_area_to_state(offshore_area: str) -> str | None: def fetch_nopta_titles(data_source: ConfiguredDataSource): - titles_url = NOPTA_ARCGIS_URL + "/Public/TitlesCompany_NOPTA/MapServer" - titles_layer = restapi.MapService(url=titles_url).layer("Titles and Permits Current") + titles_layer = restapi.MapService(url=data_source.url).layer("Titles and Permits Current") layer_features = titles_layer.query( # where="Type in ('Petroleum','Mineral or Coal') AND Purpose in ('Development','Appraisal', 'Exploration')", @@ -104,6 +103,7 @@ def parse_nopta_titles(data_source: ConfiguredDataSource): nopta_titles_data_source = DataSource( name="NOPTA-titles", file_path="NOPTA-titles.geojson", + url=f"{NOPTA_ARCGIS_URL}/Public/TitlesCompany_NOPTA/MapServer", fetch=fetch_nopta_titles, parse=parse_nopta_titles, ) diff --git a/src/openmethane_prior/sectors/oil_gas/data/wa_gis.py b/src/openmethane_prior/sectors/oil_gas/data/wa_gis.py index 880d38ef..7e581e91 100644 --- a/src/openmethane_prior/sectors/oil_gas/data/wa_gis.py +++ b/src/openmethane_prior/sectors/oil_gas/data/wa_gis.py @@ -30,9 +30,7 @@ # WA Petroleum Titles (DMIRS-011) - REST Service (ArcGIS) # https://catalogue.data.wa.gov.au/dataset/wa-petroleum-titles-dmirs-011/resource/f5bb4e83-241e-4dcf-8efe-41707866af4e def fetch_wa_titles(data_source: ConfiguredDataSource): - wa_arcgis_mining = restapi.MapService( - url=f"{WA_ARCGIS_URL}/SLIP_Public_Services/Industry_and_Mining/MapServer" - ) + wa_arcgis_mining = restapi.MapService(url=data_source.url) # DMIRS-011 contains current active petroleum titles, but has expired # titles periodically removed. Our interest is only in "Production License" @@ -92,6 +90,7 @@ def fetch_wa_titles(data_source: ConfiguredDataSource): wa_titles_data_source = DataSource( name="WA-petroleum-titles", file_path="WA-petroleum-titles.geojson", + url=f"{WA_ARCGIS_URL}/SLIP_Public_Services/Industry_and_Mining/MapServer", fetch=fetch_wa_titles, parse=parse_geo, ) @@ -100,10 +99,7 @@ def fetch_wa_titles(data_source: ConfiguredDataSource): # WA Petroleum Wells (DMIRS-025) - REST Service (ArcGIS) # https://catalogue.data.wa.gov.au/dataset/mineral-exploration-drillholes-open-file/resource/1c61171a-3b23-4f2b-baae-04615c5bb39e def fetch_wa_wells(data_source: ConfiguredDataSource): - wa_arcgis_mining = restapi.MapService( - url=f"{WA_ARCGIS_URL}/SLIP_Public_Services/Industry_and_Mining/MapServer" - ) - + wa_arcgis_mining = restapi.MapService(data_source.url) wells_layer = wa_arcgis_mining.layer("WA Onshore Petroleum Wells (DMIRS-025)") wells_features = wells_layer.query( fields=[ @@ -132,6 +128,7 @@ def fetch_wa_wells(data_source: ConfiguredDataSource): wa_wells_data_source = DataSource( name="WA-petroleum-wells", file_path="WA-petroleum-wells.geojson", + url=f"{WA_ARCGIS_URL}/SLIP_Public_Services/Industry_and_Mining/MapServer", fetch=fetch_wa_wells, parse=parse_geo, ) From 3a83e5966cfadb2bcaa6b38b446a95f29afd9d91 Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Thu, 16 Apr 2026 15:59:47 +1000 Subject: [PATCH 28/29] Fetch additional QLD lease fields to help with manual SGM facility attribution --- src/openmethane_prior/sectors/oil_gas/data/qld_gis.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/openmethane_prior/sectors/oil_gas/data/qld_gis.py b/src/openmethane_prior/sectors/oil_gas/data/qld_gis.py index 7b23695f..20c51b38 100644 --- a/src/openmethane_prior/sectors/oil_gas/data/qld_gis.py +++ b/src/openmethane_prior/sectors/oil_gas/data/qld_gis.py @@ -115,6 +115,8 @@ def fetch_qld_leases(data_source: ConfiguredDataSource): "expirydate", "permitminerals", "permitpurpose", + "permitname", + "authorisedholdername", ] leases_layer = qld_current_leases.layer("PL granted") From cd1d00d98f285082a8c7291754dba2982427a22c Mon Sep 17 00:00:00 2001 From: Lindsay Gaines Date: Mon, 27 Apr 2026 07:47:44 +1000 Subject: [PATCH 29/29] Update QLD bore filters after dataset update The reference dataset was updated 22/04/2026, and appears to have cleaned up some of the `status` and other field values. Filters have been updated and test cases have been adjusted to remove field values that no longer appear in the data. --- .../oil_gas/emission_sources/qld_sources.py | 2 +- .../test_emission_sources.py | 17 +++++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/openmethane_prior/sectors/oil_gas/emission_sources/qld_sources.py b/src/openmethane_prior/sectors/oil_gas/emission_sources/qld_sources.py index ca9186c0..c3f8b0b5 100644 --- a/src/openmethane_prior/sectors/oil_gas/emission_sources/qld_sources.py +++ b/src/openmethane_prior/sectors/oil_gas/emission_sources/qld_sources.py @@ -50,7 +50,7 @@ def qld_emission_sources( # filter bores without hydrocarbons non_emitting_bore_result = ['NO HYDROCARBONS', 'UNKNOWN', 'COAL', 'WATER', 'NO COAL INTERSECTED'] qld_boreholes_df = qld_boreholes_df[~qld_boreholes_df["result"].isin(non_emitting_bore_result)] - non_emitting_bore_status = ['WATER BORE', 'UNKNOWN'] + non_emitting_bore_status = ['WATER SUPPLY', 'PROPOSED', 'UNKNOWN', 'NEVER USED'] qld_boreholes_df = qld_boreholes_df[~qld_boreholes_df["status"].isin(non_emitting_bore_status)] # bore datasets may have duplicate rows for a single location due to diff --git a/tests/integration/test_sector_oil_gas/test_emission_sources.py b/tests/integration/test_sector_oil_gas/test_emission_sources.py index 90363969..df9f5fad 100644 --- a/tests/integration/test_sector_oil_gas/test_emission_sources.py +++ b/tests/integration/test_sector_oil_gas/test_emission_sources.py @@ -97,25 +97,22 @@ def test_qld_emission_sources(input_files, data_manager): assert len(df[(df["activity_end"] < start_date) & (df["activity_start"] > start_date_end)]) == 0 # no sources which aren't related to hydrocarbon production - allowed_bore_types = { - "COAL SEAM GAS", "PETROLEUM", "UNCONVENTIONAL PETROLEUM", "GREENHOUSE GAS STORAGE", - } + allowed_bore_types = {"COAL SEAM GAS", "PETROLEUM"} assert set(df["bore_type"].unique()) - allowed_bore_types == set() - allowed_bore_subtypes = { - "DEVELOPMENT WELL", "COAL SEAM GAS INJECTION WELL", "PETROLEUM INJECTION WELL", - } + allowed_bore_subtypes = {"DEVELOPMENT WELL", "COAL SEAM GAS INJECTION WELL"} assert set(df["bore_subtype"].unique()) - allowed_bore_subtypes == set() allowed_results = { - "GAS", "OIL AND GAS", "GAS PLUS CONDENSATE SHOW", "OIL", "DRY PLUS GAS SHOW", - "DRY PLUS OIL SHOW", "OIL PLUS GAS SHOW", "GAS AND CONDENSATE", - "GAS PLUS OIL SHOW", "COAL SEAM GAS", "DRY PLUS OIL AND GAS SHOW", + 'COAL SEAM GAS', + 'DRY PLUS GAS SHOW', 'DRY PLUS OIL AND GAS SHOW', 'DRY PLUS OIL SHOW', + 'GAS', 'GAS AND CONDENSATE', 'GAS PLUS CONDENSATE SHOW', 'GAS PLUS OIL SHOW', + 'OIL', 'OIL AND GAS', 'OIL PLUS GAS SHOW', } assert set(df["result"].unique()) - allowed_results == set() allowed_status = { - "PLUGGED AND ABANDONED", "SUSPENDED/CAPPED/SHUT-IN", "PRODUCING HYDROCARBONS", + "PLUGGED AND ABANDONED", 'PRODUCING', 'COMPLETED', 'SUSPENDED', } assert set(df["status"].unique()) - allowed_status == set()