Add Safeguard Mechanism facilities to oil & gas sector

changelog/170.feature.md

@@ -0,0 +1 @@
+Add Safeguard Mechanism facilities to oil & gas sector

docs/data-sources.md

@@ -110,6 +110,41 @@ supporting dataset allows us to place those emissions in the correct grid cell
 or grid cells.
 
 
+## Safeguard Mechanism Facility Locations
+
+The Safeguard Mechanism Baselines and Emissions dataset includes facility-level
+methane emissions reported to the CER, but the locations of facilities are not
+made public. The
+[Safeguard Mechanism Facility Locations](https://docs.google.com/spreadsheets/d/1vET6DVXo3K9MeMYJj9sksSTmQjV3v9JmIPSlR6HS4NA)
+dataset was created by
+[The Superpower Institute](https://www.superpowerinstitute.com.au/) to detail
+the locations of sites and infrastructure related to each Safeguard facility so
+that their emissions can be allocated geographically.
+
+The dataset includes two parts:
+- external dataset references
+- direct locations
+
+Where possible, Safeguard facility locations have been identified in other
+public datasets, such as the National Pollutant Inventory or Climate TRACE.
+In these cases an external dataset reference is created by recording the
+Safeguard facility name alongside the `DataSource.name` of the external dataset
+and an identifier in the external dataset.
+
+For facility locations that can't be found in existing datasets, research by
+The Superpower Institute has used company reports, public datasets and
+satellite imagery to find and document sites and infrastructure related to
+each facility. These direct locations include a geographical coordinate, and
+operation dates for locations where the information is available. Each direct
+location record also includes references to public documents where the
+information was sourced.
+
+Both of these datasets have been made available via a public, read-only Google
+Sheet. The public sheet is backed by a private dataset maintained by The
+Superpower Institute, with data synced using
+[`IMPORTRANGE`](https://support.google.com/docs/answer/3093340).
+
+
 ## Land Use of Australia
 
 Some sectoral inventories are spatialised by identifying which Australian Land
@@ -338,6 +373,16 @@ within petroleum titles/leases that were active during the period of interest,
 and sites and facilities in ANZSIC sectors associated with the oil and gas
 industries.
 
+Facility estimates are taken from
+[Safeguard Mechanism estimates](#Safeguard-Mechanism-Baselines-and-Emissions)
+if available, using facility locations researched by The Superpower Institute.
+
+For wells and facilities not covered by the Safeguard Mechanism, the National
+Inventory total for "Fugitive emissions from fuels, Oil and Natural Gas", minus
+emissions already allocated to Safeguard facilities, are distributed equally
+across all potential emission sources not associated with any SGM facility.
+Listed point location for each source is mapped to the relevant grid cell.
+
 ### Sources
 
 - New South Wales datasets:

pyproject.toml

@@ -35,6 +35,7 @@ dependencies = [
 tests = ["pytest>=8.2.1,<9"]
 dev = [
     "licenseheaders>=0.8.8,<0.9",
+    "pandas-stubs~=2.2.3",
     "ruff>=0.5.1,<0.6",
     "towncrier>=24.7.1,<25",
 ]

src/openmethane_prior/data_sources/inventory/__init__.py

@@ -16,5 +16,9 @@
 # limitations under the License.
 #
 
-from .data import inventory_data_source, unfccc_codes_data_source
+from .data import (
+    inventory_data_source,
+    qld_inventory_data_source,
+    unfccc_codes_data_source,
+)
 from .inventory import get_sector_emissions_by_code

src/openmethane_prior/data_sources/inventory/data.py

@@ -52,3 +52,10 @@ def parse_inventory(data_source: ConfiguredDataSource) -> pd.DataFrame:
     parse=parse_inventory,
 )
 
+
+qld_inventory_data_source = DataSource(
+    name="ANGA-UNFCCC-inventory-QLD",
+    url="https://greenhouseaccounts.climatechange.gov.au/OData/AR5_ParisInventory_QLD",
+    data_sources=[unfccc_codes_data_source],
+    parse=parse_inventory,
+)

src/openmethane_prior/data_sources/safeguard/data.py

@@ -15,10 +15,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
 import pandas as pd
+import pathlib
 
 from openmethane_prior.lib import ConfiguredDataSource, DataSource
+from openmethane_prior.lib.data_manager.fetchers import fetch_google_sheet_by_name_csv
+
 from .facility import create_facilities_from_safeguard_rows
 
 # NGER emissions numbers are reported in CO2 equivalent, so to calculate raw
@@ -59,6 +61,7 @@
     "safeguard_facility_name", # exact facility_name from Safeguard Mechanism
     "data_source_name", # DataSource name with facility details
     "data_source_id", # identifier in reference DataSource
+    "notes", # notes about how this location is related to the facility
 ]
 
 
@@ -71,17 +74,6 @@ def parse_csv_numeric(csv_value: str) -> float | None:
     return float(raw.replace(",", ""))
 
 
-def parse_location_csv(data_source: ConfiguredDataSource):
-    locations_rows_df = pd.read_csv(
-        filepath_or_buffer=data_source.asset_path,
-        header=0,
-        names=safeguard_locations_csv_columns,
-    )
-
-    # filter out any rows with incomplete information
-    return locations_rows_df.dropna()
-
-
 def parse_safeguard_csv(data_source: ConfiguredDataSource):
     """Read the Safeguard Mechanism Baselines and Emissions Table CSV,
     returning only the data columns useful for methane estimation."""
@@ -99,17 +91,68 @@ def parse_safeguard_csv(data_source: ConfiguredDataSource):
         ch4_gwp=ar5_ch4_gwp, co2_gwp=ar5_co2_gwp,
     )
 
-
-safeguard_locations_data_source = DataSource(
-    name="safeguard-locations",
-    url="https://openmethane.s3.amazonaws.com/prior/inputs/facility-locations-v1.csv",
-    parse=parse_location_csv,
-)
-
-
+# Facility-level emissions reported under the Safeguard Mechanism.
+# Source: https://cer.gov.au/markets/reports-and-data/safeguard-data/2023-24-baselines-and-emissions-data
 safeguard_mechanism_data_source = DataSource(
     name="safeguard-mechanism",
     url="https://cer.gov.au/document/2023-24-baselines-and-emissions-table",
     file_path="2023-24-baselines-and-emissions-table.csv",
     parse=parse_safeguard_csv,
 )
+
+
+def fetch_location_csv(data_source: ConfiguredDataSource) -> pathlib.Path:
+    # Primary source of external locations
+    external_locations_df = fetch_google_sheet_by_name_csv(data_source.url, "External facility locations")
+
+    # Petroleum titles sheet has some extra details, but contains columns for
+    # Facility name, Data source, and Production license which can be converted
+    # into external location references.
+    petroleum_titles_df = fetch_google_sheet_by_name_csv(data_source.url, "Petroleum titles")
+    petroleum_locations_df = petroleum_titles_df \
+        .drop(columns=["Responsible emitter", "State"]) \
+        .rename(columns={
+            "Data source": "Supporting data source",
+            "Production license": "Supporting data id",
+        })
+
+    # Oil and gas sites, which are used as an independent DataSource in the
+    # oil_gas sector, already include the Facility name. Create a row for each
+    # facility in this with the facility name as both the name and the id.
+    oil_gas_sites = fetch_google_sheet_by_name_csv(data_source.url, "Oil and gas sites")
+    oil_gas_facilities = oil_gas_sites["Facility name"].unique()
+    oil_gas_locations_df = pd.DataFrame({
+        "Facility name": oil_gas_facilities,
+        "Supporting data id": oil_gas_facilities,
+        "Supporting data source": "oil-gas-sites",
+    })
+
+    # Combine the sources of facility locations and write them to a CSV file
+    df = pd.concat([external_locations_df, oil_gas_locations_df, petroleum_locations_df])
+    df.to_csv(data_source.asset_path, index=False)
+    return data_source.asset_path
+
+
+def parse_location_csv(data_source: ConfiguredDataSource):
+    locations_rows_df = pd.read_csv(
+        filepath_or_buffer=data_source.asset_path,
+        header=0,
+        names=safeguard_locations_csv_columns,
+        usecols=["safeguard_facility_name", "data_source_name", "data_source_id"],
+    )
+
+    # filter out any rows with incomplete information
+    return locations_rows_df.dropna()
+
+
+# Locations of Safeguard Mechanism facilities which are found in other data
+# sources are recorded in The Superpower Institute's Safeguard Mechanism
+# Facility Locations dataset.
+# Source: https://docs.google.com/spreadsheets/d/1vET6DVXo3K9MeMYJj9sksSTmQjV3v9JmIPSlR6HS4NA
+safeguard_locations_data_source = DataSource(
+    name="safeguard-locations",
+    file_path="safeguard-facility-locations.csv",
+    url="https://docs.google.com/spreadsheets/d/1vET6DVXo3K9MeMYJj9sksSTmQjV3v9JmIPSlR6HS4NA",
+    fetch=fetch_location_csv,
+    parse=parse_location_csv,
+)

src/openmethane_prior/lib/data_manager/fetchers.py

@@ -0,0 +1,70 @@
+#
+# Copyright 2025 The Superpower Institute Ltd.
+#
+# This file is part of Open Methane.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import pandas as pd
+import pathlib
+import urllib.parse
+
+from .source import ConfiguredDataSource
+
+
+def fetch_google_sheet_by_name_csv(
+    sheet_id_or_url: str,
+    sheet_name: str,
+) -> pd.DataFrame:
+    """Fetch a single sheet from a publicly viewable Google Sheet and return
+    the contents as a pandas DataFrame."""
+    if "://docs.google.com" in sheet_id_or_url:
+        # if a url is provided, extract the sheet_id
+        sheet_url = urllib.parse.urlparse(sheet_id_or_url)
+        sheet_id = sheet_url.path.rpartition('/')[-1]
+    else:
+        sheet_id = sheet_id_or_url
+
+    # make the sheet name url-safe
+    sheet_name_param = urllib.parse.quote(sheet_name)
+
+    # Google Sheets provides a CSV endpoint for downloading
+    url = f"https://docs.google.com/spreadsheets/d/{sheet_id}/gviz/tq?tqx=out:csv&sheet={sheet_name_param}"
+    df = pd.read_csv(url)
+
+    # remove empty columns with no name or data
+    for column_name in df.columns:
+        if "Unnamed: " in column_name and df[column_name].isnull().all():
+            del df[column_name]
+
+    return df
+
+
+def fetch_google_sheet_csv(sheet_name: str):
+    """Return a DataSource fetch-compatible method for a DataSource where a
+    Google Sheet is specified by the url parameter and only a single sheet
+    should be fetched.
+
+    Usage:
+        DataSource(
+            url="https://docs.google.com/spreadsheets/d/EXAMPLE_SHEET_ID",
+            fetch=fetch_google_sheet_csv("Sheet name"),
+            ...
+        )
+    """
+    def _data_source_fetch(data_source: ConfiguredDataSource) -> pathlib.Path:
+        df = fetch_google_sheet_by_name_csv(data_source.url, sheet_name)
+        df.to_csv(data_source.asset_path, index=False)
+        return data_source.asset_path
+
+    return _data_source_fetch

src/openmethane_prior/sectors/oil_gas/data/oil_gas_sites.py

@@ -20,6 +20,7 @@
 import pandas as pd
 
 from openmethane_prior.lib import DataSource
+from openmethane_prior.lib.data_manager.fetchers import fetch_google_sheet_csv
 from openmethane_prior.lib.data_manager.source import ConfiguredDataSource
 
 
@@ -35,7 +36,7 @@ def parse_oil_gas_sites_csv(data_source: ConfiguredDataSource) -> gpd.GeoDataFra
     )
 
     # remove entries without lat/lon, these aren't useful to us
-    df = df[(~np.isnan(df["Longitude"])) & (~np.isnan(df["Latitude"]))]
+    df = df[(~pd.isna(df["Longitude"])) & (~pd.isna(df["Latitude"]))]
 
     gdf = gpd.GeoDataFrame(
         data=df,
@@ -50,11 +51,14 @@ def parse_oil_gas_sites_csv(data_source: ConfiguredDataSource) -> gpd.GeoDataFra
     return gdf
 
 
-# Processing facilities in the oil and gas industry in Australia. This dataset
-# was manually created from research conducted by The Superpower Institute to
-# identify locations of processing sites linked to Safeguard Mechanism facilities.
+# Processing facilities in the oil and gas industry in Australia. This dataset,
+# created by The Superpower Institute, identifies locations of oil and gas
+# sites linked to Safeguard Mechanism facilities.
+# Source: https://docs.google.com/spreadsheets/d/1vET6DVXo3K9MeMYJj9sksSTmQjV3v9JmIPSlR6HS4NA
 oil_gas_sites_data_source = DataSource(
     name="oil-gas-sites",
-    url="https://openmethane.s3.amazonaws.com/prior/inputs/oil-gas-sites-v0.3.csv",
+    file_path="oil-gas-sites.csv",
+    url="https://docs.google.com/spreadsheets/d/1vET6DVXo3K9MeMYJj9sksSTmQjV3v9JmIPSlR6HS4NA",
+    fetch=fetch_google_sheet_csv("Oil and gas sites"),
     parse=parse_oil_gas_sites_csv,
 )

src/openmethane_prior/sectors/oil_gas/emission_sources/emission_source.py → src/openmethane_prior/sectors/oil_gas/emission_source.py

@@ -17,6 +17,7 @@
 #
 import numpy as np
 import geopandas as gpd
+import pandas as pd
 from typing import Any
 
 """
@@ -74,4 +75,21 @@ def normalise_emission_source_df(
 
     normalised_df = normalised_df.to_crs(crs)
 
-    return normalised_df
+    return normalised_df
+
+
+def allocate_emissions_to_sources(
+    sources_df: pd.DataFrame,
+    sources_mask: "pd.Series[bool] | np.typing.NDArray[np.bool_]",
+    emission_mass: float,
+):
+    """Distribute a single total emission across all emission sources in
+    sources_df which match sources_mask."""
+    # since we will use addition to allocate emission to each source, ensure
+    # nans are replaced with zeros prior to addition
+    is_nan = pd.isna(sources_df["emissions_quantity"])
+    sources_df.loc[sources_mask & is_nan, "emissions_quantity"] = 0
+
+    # naively allocate the emissions across each emission source equally
+    # TODO: apply weighting to emissions distribution based on site_type
+    sources_df.loc[sources_mask, "emissions_quantity"] += emission_mass / sources_mask.sum()

src/openmethane_prior/sectors/oil_gas/emission_sources/all_sources.py

@@ -25,7 +25,7 @@
     PriorConfig,
 )
 
-from .emission_source import normalise_emission_source_df
+from ..emission_source import normalise_emission_source_df
 from .nsw_sources import nsw_emission_sources
 from .nt_sources import nt_emission_sources
 from .offshore_sources import offshore_emission_sources
@@ -46,6 +46,7 @@
 def all_emission_sources(
     data_manager: DataManager,
     prior_config: PriorConfig,
+    anzsic_codes: list[str],
 ) -> gpd.GeoDataFrame:
     """Assemble a single DataFrame from multiple data sources, where each row
     represents a possible location of emissions in the oil and gas sector."""
@@ -145,6 +146,7 @@ def all_emission_sources(
         end_date=end_date,
         oil_gas_sites_da=oil_gas_sites_da,
         npi_da=npi_da,
+        anzsic_codes=anzsic_codes,
     )
     logger.debug(f"found {len(sites_df[sites_df['data_source'] == oil_gas_sites_da.name])} oil and gas facilities")
     logger.debug(f"found {len(sites_df[sites_df['data_source'] == npi_da.name])} NPI facilities")

src/openmethane_prior/sectors/oil_gas/emission_sources/qld_sources.py

@@ -48,9 +48,9 @@ def qld_emission_sources(
     qld_boreholes_df = qld_boreholes_df[qld_boreholes_df["bore_subtype"].isin(emitting_bore_subtypes)]
 
     # filter bores without hydrocarbons
-    non_emitting_bore_result = ['NO HYDROCARBONS', 'UNKNOWN', 'COAL', 'WATER', 'NO COAL INTERSECTED']
+    non_emitting_bore_result = ["NO HYDROCARBONS", "WATER", "NO COAL INTERSECTED"]
     qld_boreholes_df = qld_boreholes_df[~qld_boreholes_df["result"].isin(non_emitting_bore_result)]
-    non_emitting_bore_status = ['WATER SUPPLY', 'PROPOSED', 'UNKNOWN', 'NEVER USED']
+    non_emitting_bore_status = ["WATER SUPPLY", "PROPOSED", "UNKNOWN", "NEVER USED", "PLUGGED AND ABANDONED"]
     qld_boreholes_df = qld_boreholes_df[~qld_boreholes_df["status"].isin(non_emitting_bore_status)]
 
     # bore datasets may have duplicate rows for a single location due to