diff --git a/metloom/pointdata/__init__.py b/metloom/pointdata/__init__.py index 7ad0419..d152c2e 100644 --- a/metloom/pointdata/__init__.py +++ b/metloom/pointdata/__init__.py @@ -1,6 +1,6 @@ from .base import PointData, PointDataCollection from .cdec import CDECPointData -from .snotel import SnotelPointData +from metloom.pointdata.snotel.snotel import SnotelPointData from .mesowest import MesowestPointData from .usgs import USGSPointData from .geosphere_austria import GeoSphereHistPointData, GeoSphereCurrentPointData diff --git a/metloom/pointdata/base.py b/metloom/pointdata/base.py index 47f0591..4d4cd5d 100644 --- a/metloom/pointdata/base.py +++ b/metloom/pointdata/base.py @@ -7,7 +7,7 @@ import geopandas as gpd -from ..variables import SensorDescription, VariableBase +from ..sensors import SensorDescription, VariableBase LOG = logging.getLogger("metloom.pointdata.base") diff --git a/metloom/pointdata/snotel/__init__.py b/metloom/pointdata/snotel/__init__.py new file mode 100644 index 0000000..f807784 --- /dev/null +++ b/metloom/pointdata/snotel/__init__.py @@ -0,0 +1,4 @@ +from .snotel import SnotelPointData +from .variables import SnotelVariables + +__all__ = ["SnotelPointData", "SnotelVariables"] diff --git a/metloom/pointdata/snotel.py b/metloom/pointdata/snotel/snotel.py similarity index 65% rename from metloom/pointdata/snotel.py rename to metloom/pointdata/snotel/snotel.py index f978d92..c3b8985 100644 --- a/metloom/pointdata/snotel.py +++ b/metloom/pointdata/snotel/snotel.py @@ -3,16 +3,16 @@ import logging import geopandas as gpd import pandas as pd -from functools import reduce +from functools import cached_property +import requests +import numpy as np -from .base import PointData -from ..variables import SnotelVariables, SensorDescription -from ..dataframe_utils import append_df, merge_df +from metloom.pointdata.base import PointData +from .variables import SnotelVariables, SensorDescription +from metloom.dataframe_utils import merge_df from .snotel_client import ( - DailySnotelDataClient, MetaDataSnotelClient, HourlySnotelDataClient, - SemiMonthlySnotelClient, PointSearchSnotelClient, SeriesSnotelClient, - ElementSnotelClient + PointSearchSnotelClient ) LOG = logging.getLogger("metloom.pointdata.snotel") @@ -30,6 +30,7 @@ class SnotelPointData(PointData): ALLOWED_VARIABLES = SnotelVariables DATASOURCE = "NRCS" + API_URL = "https://wcc.sc.egov.usda.gov/awdbRestApi/" def __init__(self, station_id, name, metadata=None): """ @@ -42,8 +43,10 @@ def __init__(self, station_id, name, metadata=None): self._raw_elements = None self._tzinfo = None - def _snotel_response_to_df(self, result_map: Dict[SensorDescription, List[dict]], - duration: str, include_measurement_date=False): + def _snotel_response_to_df( + self, result_map: Dict[SensorDescription, Dict[str, List[dict]]], + duration: str, include_measurement_date=False + ): """ Convert the response from climata.snotel classes into Args: @@ -61,17 +64,21 @@ def _snotel_response_to_df(self, result_map: Dict[SensorDescription, List[dict]] if include_measurement_date: final_columns += ["measurementDate"] - for variable, data in result_map.items(): + date_key = "collectionDate" if duration == "SEMIMONTHLY" else "date" + for variable, info in result_map.items(): + data = info["values"] + element = info["stationElement"] + unit_name = element["storedUnitCode"] transformed = [] for row in data: row_obj = { - "datetime": row["datetime"], + "datetime": row[date_key], "site": self.id, - variable.name: row["value"], - f"{variable.name}_units": self._get_units(variable, duration), + variable.name: row.get("value", np.nan), + f"{variable.name}_units": unit_name, } if include_measurement_date: - row_obj["measurementDate"] = row["datetime"] + row_obj["measurementDate"] = row[date_key] transformed.append(row_obj) final_columns += [variable.name, f"{variable.name}_units"] @@ -108,21 +115,62 @@ def _snotel_response_to_df(self, result_map: Dict[SensorDescription, List[dict]] self.validate_sensor_df(df) return df - def _fetch_data_for_variables(self, client: SeriesSnotelClient, - variables: List[SensorDescription], - duration: str, - include_measurement_date=False, - ): + def _fetch_data_for_variables( + self, start_date: datetime, end_date: datetime, + variables: List[SensorDescription], + duration: str, + include_measurement_date=False, + ): + """ + Fetch data for the given variables using the Snotel API. + Args: + start_date: start date for the data + end_date: end date for the data + variables: list of SensorDescription objects for the variables + duration: string representation of the duration tag for the + API (i.e. HOURLY) + include_measurement_date: boolean for including the + 'measurementDate' column in the resulting dataframe. + + """ + endpoint_url = self.API_URL + "services/v1/data" result_map = {} + params = dict( + beginDate=start_date.strftime("%Y-%m-%d %H:%M"), + endDate=end_date.strftime("%Y-%m-%d %H:%M"), + stationTriplets=self.id, + duration=duration, + ) for variable in variables: - params = variable.extra or {} - data = client.get_data(element_cd=variable.code, **params) - if len(data) > 0: - result_map[variable] = data + extra = variable.extra or {} + height_depth = extra.get("height_depth", {}) + # Add the height depth for sensors with a heigh component + if height_depth: + code = variable.code + f":{height_depth['value']}" + else: + code = variable.code + + # TODO: we could request multiple variables at once + result = requests.get( + endpoint_url, params={**params, "elements": code} + ) + result.raise_for_status() + data = result.json() + # Get the first station return, since we only requested one station + if len(data) == 0: + return None + data = data[0]["data"] + # TODO: this is where we could iterate through multiple variables + # if we wanted to. We would need to be careful of the meas height + if len(data) == 1: + result_map[variable] = data[0] + elif len(data) > 1: + raise RuntimeError("We received too many results") else: LOG.warning(f"No {variable.name} found for {self.name}") return self._snotel_response_to_df( - result_map, duration, include_measurement_date=include_measurement_date + result_map, duration, + include_measurement_date=include_measurement_date ) def get_daily_data( @@ -134,13 +182,9 @@ def get_daily_data( """ See docstring for PointData.get_daily_data """ - client = DailySnotelDataClient( - station_triplet=self.id, - begin_date=start_date, - end_date=end_date, + return self._fetch_data_for_variables( + start_date, end_date, variables, "DAILY" ) - return self._fetch_data_for_variables(client, variables, - client.DURATION) def get_hourly_data( self, @@ -151,13 +195,9 @@ def get_hourly_data( """ See docstring for PointData.get_hourly_data """ - - client = HourlySnotelDataClient( - station_triplet=self.id, - begin_date=start_date, - end_date=end_date, + return self._fetch_data_for_variables( + start_date, end_date, variables, "HOURLY" ) - return self._fetch_data_for_variables(client, variables, "HOURLY") def get_snow_course_data( self, @@ -168,48 +208,43 @@ def get_snow_course_data( """ See docstring for PointData.get_snow_course_data """ - client = SemiMonthlySnotelClient( - station_triplet=self.id, - begin_date=start_date, - end_date=end_date, - ) + return self._fetch_data_for_variables( - client, variables, client.DURATION, include_measurement_date=True + start_date, end_date, variables, "SEMIMONTHLY", include_measurement_date=True ) - def _get_all_metadata(self): + @classmethod + def _metadata_call(cls, point_ids): """ - Set _raw_metadata once using Snotel API + Call the Snotel API to get metadata for a point_id + Args: + point_ids: string of comma separated station triplets + Returns: + dict: metadata for the point """ - if self._raw_metadata is None: - client = MetaDataSnotelClient(station_triplet=self.id) - self._raw_metadata = client.get_data() - return self._raw_metadata + endpoint_url = cls.API_URL + "services/v1/stations" + params = dict( + stationTriplets=point_ids, + ) + result = requests.get( + endpoint_url, params=params + ) + result.raise_for_status() + data = result.json() + return data - def _get_all_elements(self): + @cached_property + def _all_metadata(self): """ Set _raw_metadata once using Snotel API """ - if self._raw_elements is None: - client = ElementSnotelClient(station_triplet=self.id) - self._raw_elements = client.get_data() - return self._raw_elements - - def _get_units(self, variable: SensorDescription, duration: str): - units = None - for meta in self._get_all_elements(): - if meta["elementCd"] == variable.code and meta["duration"] == duration: - units = meta["storedUnitCd"] - break - if units is None: - raise ValueError(f"Could not find units for {variable}") - return units + return self._metadata_call(self.id) def _get_metadata(self): """ See docstring for PointData._get_metadata """ - all_metadata = self._get_all_metadata() + all_metadata = self._all_metadata if isinstance(all_metadata, list): data = all_metadata[0] else: @@ -222,9 +257,9 @@ def _get_tzinfo(self): """ Return timezone info that pandas can use from the raw_metadata """ - metadata = self._get_all_metadata() + metadata = self._all_metadata # Snow courses might not have a timezone attached - tz_hours = metadata.get("stationDataTimeZone") + tz_hours = metadata[0].get("dataTimeZone") if tz_hours is None: LOG.error(f"Could not find timezone info for {self.id} ({self.name})") tz_hours = 0 @@ -294,17 +329,17 @@ def points_from_geometry( # no duplicate codes point_codes = list(set(point_codes)) - dfs = [ - pd.DataFrame.from_records( - [MetaDataSnotelClient(station_triplet=code).get_data()] - ).set_index("stationTriplet") for code in point_codes - ] + codes_string = ",".join(point_codes) + df = pd.DataFrame.from_records( + cls._metadata_call(codes_string) + ) - if len(dfs) > 0: - df = reduce(lambda a, b: append_df(a, b), dfs) - else: + if len(df) == 0: + # Short circuit, return empty class return cls.ITERATOR_CLASS([]) + df = df.set_index("stationTriplet") + df.reset_index(inplace=True) gdf = gpd.GeoDataFrame( df, diff --git a/metloom/pointdata/snotel/snotel_client.py b/metloom/pointdata/snotel/snotel_client.py new file mode 100644 index 0000000..b2131c8 --- /dev/null +++ b/metloom/pointdata/snotel/snotel_client.py @@ -0,0 +1,106 @@ +from datetime import datetime +import zeep + +from metloom.request_utils import no_ssl_verification + + +class BaseSnotelClient: + """ + Base snotel client class. Used for interacting with SNOTEL SOAP client. + This is just a base class and not meant for direct use. + + Example use with extended class:: + + MetaDataSnotelClient('TNY:CA:SNOW').get_data() + + """ + URL = "https://wcc.sc.egov.usda.gov/awdbWebService/services?WSDL" + # map allowed params to API filter params + PARAMS_MAP = { + 'station_triplet': 'stationTriplet', + 'station_triplets': 'stationTriplets', + 'begin_date': 'beginDate', + 'end_date': 'endDate', + 'max_longitude': 'maxLongitude', + 'min_longitude': 'minLongitude', + 'max_latitude': 'maxLatitude', + 'min_latitude': 'minLatitude', + 'network_cds': 'networkCds', + "duration": "duration", + 'ordinal': 'ordinal', + 'element_cd': 'elementCd', + 'element_cds': 'elementCds', + 'parameter': 'parameter', + 'height_depth': 'heightDepth', + } + SERVICE_NAME = None + DEFAULT_PARAMS = {} + + def __init__(self, **kwargs): + self.params = self._get_params(**kwargs) + if getattr(self, "DURATION", None) is not None: + extra_params = {"duration": self.DURATION, **self.DEFAULT_PARAMS} + else: + extra_params = self.DEFAULT_PARAMS + # add in default params to self.params, don't replace user entries + for key, value in extra_params.items(): + if key not in self.params: + self.params[key] = value + + def _get_params(self, **kwargs): + """ + map input parameter keys to keys expected by the SOAP client + """ + params = {} + for key, value in kwargs.items(): + mapped_key = self.PARAMS_MAP.get(key) + if mapped_key is None: + raise ValueError( + f"Could not find valid mapped key for {key}" + ) + if isinstance(value, datetime): + value = value.date().isoformat() + params.update({mapped_key: value}) + return params + + @classmethod + def _make_request(cls, **params): + """ + Make the request to the SOAP client for the implemented service. + """ + with no_ssl_verification(): + client = zeep.Client( + cls.URL, + ) + service = getattr(client.service, cls.SERVICE_NAME) + response = service(**params) + return response + + def get_data(self): + """ + Make the actual request and return data. + """ + data = self._make_request(**self.params) + return data + + +class PointSearchSnotelClient(BaseSnotelClient): + """ + Search for stations based on criteria. This search is default logical + ``AND`` meaning all criteria need to be true. + get_data returns a list of string station triplets + """ + SERVICE_NAME = 'getStations' + DEFAULT_PARAMS = { + 'logicalAnd': 'true', + } + + def __init__(self, max_latitude: float, min_latitude: float, + max_longitude: float, min_longitude: float, + network_cds: str, element_cds: str, **kwargs): + super(PointSearchSnotelClient, self).__init__( + max_latitude=max_latitude, min_latitude=min_latitude, + max_longitude=max_longitude, min_longitude=min_longitude, + network_cds=network_cds, element_cds=element_cds, + **kwargs + ) diff --git a/metloom/pointdata/snotel/variables.py b/metloom/pointdata/snotel/variables.py new file mode 100644 index 0000000..c86f5e2 --- /dev/null +++ b/metloom/pointdata/snotel/variables.py @@ -0,0 +1,108 @@ +from dataclasses import field, make_dataclass + +from metloom.sensors import SensorDescription, VariableBase + +# Available sensors from Snotel +SnotelVariables = make_dataclass( + "SnotelVariables", + [ + ( + "SNOWDEPTH", + SensorDescription, + field(default=SensorDescription("SNWD", "SNOWDEPTH")), + ), + ("SWE", SensorDescription, field(default=SensorDescription("WTEQ", "SWE"))), + ( + "TEMP", + SensorDescription, + field(default=SensorDescription("TOBS", "AIR TEMP")), + ), + ( + "TEMPAVG", + SensorDescription, + field(default=SensorDescription("TAVG", "AVG AIR TEMP", "AIR TEMPERATURE AVERAGE")), + ), + ( + "TEMPMIN", + SensorDescription, + field(default=SensorDescription("TMIN", "MIN AIR TEMP", "AIR TEMPERATURE MINIMUM")), + ), + ( + "TEMPMAX", + SensorDescription, + field(default=SensorDescription("TMAX", "MAX AIR TEMP", "AIR TEMPERATURE MAXIMUM")), + ), + ( + "PRECIPITATION", + SensorDescription, + field(default=SensorDescription("PRCPSA", "PRECIPITATION", "PRECIPITATION INCREMENT SNOW-ADJUSTED")), + ), + ( + "PRECIPITATIONACCUM", + SensorDescription, + field(default=SensorDescription("PREC", "ACCUMULATED PRECIPITATION", "PRECIPITATION ACCUMULATION")), + ), + # TODO for the SCAN network this appears to be "RHUM", we may need a new class + ( + "RH", + SensorDescription, + field(default=SensorDescription("RHUMV", "Relative Humidity", "RELATIVE HUMIDITY")), + ), + ( + "STREAMVOLUMEOBS", + SensorDescription, + field(default=SensorDescription("SRVO", "STREAM VOLUME OBS", "STREAM VOLUME OBS")), + ), + ( + "STREAMVOLUMEADJ", + SensorDescription, + field(default=SensorDescription("SRVOX", "STREAM VOLUME ADJ", "STREAM VOLUME ADJ")), + ), + ] + + [ + ( + f"TEMPGROUND{abs(d)}IN", + SensorDescription, + field( + default=SensorDescription( + "STO", + f"GROUND TEMPERATURE -{d}IN", + f"GROUND TEMPERATURE OBS -{d}IN", + extra={"height_depth": {"value": -d, "unitCd": "in"}}, + ) + ), + ) + for d in [2, 4, 8, 20] + ] + + [ + ( + f"SOILMOISTURE{abs(d)}IN", + SensorDescription, + field( + default=SensorDescription( + "SMS", + f"SOIL MOISTURE -{d}IN", + f"SOIL MOISTURE PERCENT -{d}IN", + extra={"height_depth": {"value": -d, "unitCd": "in"}}, + ) + ), + ) + for d in [2, 4, 8, 20] + ] + + [ + ( + f"TEMPPROFILENEG{abs(d)}IN" if d < 0 else f"TEMPPROFILE{d}IN", + SensorDescription, + field( + default=SensorDescription( + "PTEMP", + f"PROFILE TEMPERATURE {d}IN", + f"PROFILE TEMPERATURE OBS{d}IN", + extra={"height_depth": {"value": d, "unitCd": "in"}}, + ) + ), + ) + for d in [-8, 0, 8, 16, 24, 31, 39, 47, 55, 63, 71, 79, 87, 94, 102, 110, 118, 126] # noqa: E501 + ], + bases=(VariableBase,), +) diff --git a/metloom/pointdata/snotel_client.py b/metloom/pointdata/snotel_client.py deleted file mode 100644 index ff4451c..0000000 --- a/metloom/pointdata/snotel_client.py +++ /dev/null @@ -1,244 +0,0 @@ -from datetime import datetime -import pandas as pd -import zeep - -from metloom.request_utils import no_ssl_verification - - -class BaseSnotelClient: - """ - Base snotel client class. Used for interacting with SNOTEL SOAP client. - This is just a base class and not meant for direct use. - - Example use with extended class:: - - MetaDataSnotelClient('TNY:CA:SNOW').get_data() - - """ - URL = "https://wcc.sc.egov.usda.gov/awdbWebService/services?WSDL" - # map allowed params to API filter params - PARAMS_MAP = { - 'station_triplet': 'stationTriplet', - 'station_triplets': 'stationTriplets', - 'begin_date': 'beginDate', - 'end_date': 'endDate', - 'max_longitude': 'maxLongitude', - 'min_longitude': 'minLongitude', - 'max_latitude': 'maxLatitude', - 'min_latitude': 'minLatitude', - 'network_cds': 'networkCds', - "duration": "duration", - 'ordinal': 'ordinal', - 'element_cd': 'elementCd', - 'element_cds': 'elementCds', - 'parameter': 'parameter', - 'height_depth': 'heightDepth', - } - SERVICE_NAME = None - DEFAULT_PARAMS = {} - - def __init__(self, **kwargs): - self.params = self._get_params(**kwargs) - if getattr(self, "DURATION", None) is not None: - extra_params = {"duration": self.DURATION, **self.DEFAULT_PARAMS} - else: - extra_params = self.DEFAULT_PARAMS - # add in default params to self.params, don't replace user entries - for key, value in extra_params.items(): - if key not in self.params: - self.params[key] = value - - def _get_params(self, **kwargs): - """ - map input parameter keys to keys expected by the SOAP client - """ - params = {} - for key, value in kwargs.items(): - mapped_key = self.PARAMS_MAP.get(key) - if mapped_key is None: - raise ValueError( - f"Could not find valid mapped key for {key}" - ) - if isinstance(value, datetime): - value = value.date().isoformat() - params.update({mapped_key: value}) - return params - - @classmethod - def _make_request(cls, **params): - """ - Make the request to the SOAP client for the implemented service. - """ - with no_ssl_verification(): - client = zeep.Client( - cls.URL, - ) - service = getattr(client.service, cls.SERVICE_NAME) - response = service(**params) - return response - - def get_data(self): - """ - Make the actual request and return data. - """ - data = self._make_request(**self.params) - return data - - -class MetaDataSnotelClient(BaseSnotelClient): - """ - Read metadata from the metadata service for a particular station triplet - """ - SERVICE_NAME = "getStationMetadata" - - def __init__(self, station_triplet: str, **kwargs): - super(MetaDataSnotelClient, self).__init__( - station_triplet=station_triplet, **kwargs - ) - - def get_data(self): - """ - Returns a dictionary of metadata values - """ - data = self._make_request(**self.params) - # change ordered dict of values to regular dict - return dict(data.__values__) - - -class ElementSnotelClient(BaseSnotelClient): - """ - Get all station elements for a station triplet. Station triplets - are descriptions of each sensor on the station - - get_data returns a list of zeep objects. Zeep objects are indexible - or attributes can be accessed with getattr or ``.`` - """ - SERVICE_NAME = "getStationElements" - - def __init__(self, station_triplet: str, **kwargs): - super(ElementSnotelClient, self).__init__( - station_triplet=station_triplet, **kwargs - ) - - -class PointSearchSnotelClient(BaseSnotelClient): - """ - Search for stations based on criteria. This search is default logical - ``AND`` meaning all criteria need to be true. - get_data returns a list of string station triplets - """ - SERVICE_NAME = 'getStations' - DEFAULT_PARAMS = { - 'logicalAnd': 'true', - } - - def __init__(self, max_latitude: float, min_latitude: float, - max_longitude: float, min_longitude: float, - network_cds: str, element_cds: str, **kwargs): - super(PointSearchSnotelClient, self).__init__( - max_latitude=max_latitude, min_latitude=min_latitude, - max_longitude=max_longitude, min_longitude=min_longitude, - network_cds=network_cds, element_cds=element_cds, - **kwargs - ) - - -class SeriesSnotelClient(BaseSnotelClient): - """ - Base extension for services that return timseries data. - """ - SERVICE_NAME = "getData" - DURATION = "DAILY" - DEFAULT_PARAMS = { - "ordinal": 1, - "getFlags": "true", - "alwaysReturnDailyFeb29": "false", - } - - def __init__(self, begin_date: datetime, end_date: datetime, - station_triplet: str, **kwargs): - super(SeriesSnotelClient, self).__init__( - begin_date=begin_date, end_date=end_date, - station_triplets=[station_triplet], **kwargs) - - @staticmethod - def _parse_data(raw_data): - """ - Parse the return data to return a consistent format for timeseries - data - """ - data = raw_data[0] - mapped_data = [] - collection_dates = getattr(data, "collectionDates", None) - if len(data["values"]) == 0: - return mapped_data - if collection_dates: - date_list = [pd.to_datetime(d) for d in collection_dates] - else: - date_list = pd.date_range(data["beginDate"], data["endDate"]) - - for date_obj, flag, value in zip(date_list, data["flags"], data["values"]): - if date_obj is not None: - mapped_data.append({ - "datetime": date_obj, - "flag": flag, - "value": float(value) if value is not None else None - }) - return mapped_data - - def get_data(self, element_cd: str, **extra_params): - """ - get the timeseires data - - Args: - element_cd: the variable code from the allowed variable codes - in the API - extra_params: kwargs for any extra parameters. These override - the default parameters - """ - extra_params.update(element_cd=element_cd) - mapped_params = self._get_params(**extra_params) - params = {**self.params, **mapped_params} - data = self._make_request(**params) - return self._parse_data(data) - - -class DailySnotelDataClient(SeriesSnotelClient): - """ - Class for getting daily data - """ - pass - - -class SemiMonthlySnotelClient(SeriesSnotelClient): - """ - Class for getting semi monthly (snow course) data - """ - DURATION = "SEMIMONTHLY" - - -class HourlySnotelDataClient(SeriesSnotelClient): - """ - Class for getting hourly data - """ - DURATION = None - SERVICE_NAME = "getHourlyData" - DEFAULT_PARAMS = { - "ordinal": 1, - } - - @staticmethod - def _parse_data(raw_data): - """ - Clean the hourly data to be consistent with timeseries results - """ - data = raw_data[0] - mapped_data = [] - for row in data["values"]: - value = row["value"] - mapped_data.append({ - "datetime": pd.to_datetime(row["dateTime"]), - "flag": row["flag"], - "value": float(value) if value is not None else None - }) - return mapped_data diff --git a/metloom/sensors.py b/metloom/sensors.py new file mode 100644 index 0000000..fb0fe5d --- /dev/null +++ b/metloom/sensors.py @@ -0,0 +1,67 @@ +from dataclasses import field, dataclass +import typing + + +@dataclass(eq=True, frozen=True) +class SensorDescription: + """ + data class for describing a snow sensor + """ + + code: str = "-1" # code used within the applicable API + name: str = "basename" # desired name for the sensor + description: str = None # description of the sensor + accumulated: bool = False # whether the data is accumulated + units: str = None # Optional units kwarg + extra: typing.Any = field(default=None, hash=False) # Optional extra data for sub-class specific information + + +@dataclass(eq=True, frozen=True) +class InstrumentDescription(SensorDescription): + """ + Extend the Sensor Description to include instrument + """ + + # description of the specific instrument for the variable + instrument: str = None + + +class VariableBase: + """ + Base class to store all variables for a specific datasource. Each + datasource should implement the class. The goal is that the variables + are synonymous across implementations.(i.e. PRECIPITATION should have the + same meaning in each implementation). + Additionally, variables with the same meaning should have the same + `name` attribute of the SensorDescription. This way, if multiple datsources + are used to sample the same variable, they can be written to the same + column in a csv. + + Variables in this base class should ideally be implemented by all classes + and cannot be directly used from the base class. + """ + + PRECIPITATION = SensorDescription() + SWE = SensorDescription() + SNOWDEPTH = SensorDescription() + + @staticmethod + def _validate_sensor(sensor: SensorDescription): + """ + Validate that a sensor is not using the default values since they + are meaningless + """ + default = SensorDescription() + if sensor.name == default.name and sensor.code == default.code: + raise ValueError(f"{sensor.name} is the default implementation") + + @classmethod + def from_code(cls, code): + """ + Get the correct sensor description from the code + """ + for k, v in cls.__dict__.items(): + if isinstance(v, SensorDescription) and v.code == str(code): + cls._validate_sensor(v) + return v + raise ValueError(f"Could not find sensor for code {code}") diff --git a/metloom/variables.py b/metloom/variables.py index ba4bc97..d7600f1 100644 --- a/metloom/variables.py +++ b/metloom/variables.py @@ -1,70 +1,7 @@ -from dataclasses import dataclass, field, make_dataclass -import typing +from metloom.sensors import VariableBase, SensorDescription, InstrumentDescription - -@dataclass(eq=True, frozen=True) -class SensorDescription: - """ - data class for describing a snow sensor - """ - - code: str = "-1" # code used within the applicable API - name: str = "basename" # desired name for the sensor - description: str = None # description of the sensor - accumulated: bool = False # whether the data is accumulated - units: str = None # Optional units kwarg - extra: typing.Any = field(default=None, hash=False) # Optional extra data for sub-class specific information - - -@dataclass(eq=True, frozen=True) -class InstrumentDescription(SensorDescription): - """ - Extend the Sensor Description to include instrument - """ - - # description of the specific instrument for the variable - instrument: str = None - - -class VariableBase: - """ - Base class to store all variables for a specific datasource. Each - datasource should implement the class. The goal is that the variables - are synonymous across implementations.(i.e. PRECIPITATION should have the - same meaning in each implementation). - Additionally, variables with the same meaning should have the same - `name` attribute of the SensorDescription. This way, if multiple datsources - are used to sample the same variable, they can be written to the same - column in a csv. - - Variables in this base class should ideally be implemented by all classes - and cannot be directly used from the base class. - """ - - PRECIPITATION = SensorDescription() - SWE = SensorDescription() - SNOWDEPTH = SensorDescription() - - @staticmethod - def _validate_sensor(sensor: SensorDescription): - """ - Validate that a sensor is not using the default values since they - are meaningless - """ - default = SensorDescription() - if sensor.name == default.name and sensor.code == default.code: - raise ValueError(f"{sensor.name} is the default implementation") - - @classmethod - def from_code(cls, code): - """ - Get the correct sensor description from the code - """ - for k, v in cls.__dict__.items(): - if isinstance(v, SensorDescription) and v.code == str(code): - cls._validate_sensor(v) - return v - raise ValueError(f"Could not find sensor for code {code}") +# Maintain the import path for compatibility +from metloom.pointdata.snotel import SnotelVariables # noqa: F401 class CdecStationVariables(VariableBase): @@ -93,112 +30,6 @@ class CdecStationVariables(VariableBase): WINDDIR = SensorDescription("10", "WIND DIRECTION", "WIND DIRECTION") -# Available sensors from Snotel -SnotelVariables = make_dataclass( - "SnotelVariables", - [ - ( - "SNOWDEPTH", - SensorDescription, - field(default=SensorDescription("SNWD", "SNOWDEPTH")), - ), - ("SWE", SensorDescription, field(default=SensorDescription("WTEQ", "SWE"))), - ( - "TEMP", - SensorDescription, - field(default=SensorDescription("TOBS", "AIR TEMP")), - ), - ( - "TEMPAVG", - SensorDescription, - field(default=SensorDescription("TAVG", "AVG AIR TEMP", "AIR TEMPERATURE AVERAGE")), - ), - ( - "TEMPMIN", - SensorDescription, - field(default=SensorDescription("TMIN", "MIN AIR TEMP", "AIR TEMPERATURE MINIMUM")), - ), - ( - "TEMPMAX", - SensorDescription, - field(default=SensorDescription("TMAX", "MAX AIR TEMP", "AIR TEMPERATURE MAXIMUM")), - ), - ( - "PRECIPITATION", - SensorDescription, - field(default=SensorDescription("PRCPSA", "PRECIPITATION", "PRECIPITATION INCREMENT SNOW-ADJUSTED")), - ), - ( - "PRECIPITATIONACCUM", - SensorDescription, - field(default=SensorDescription("PREC", "ACCUMULATED PRECIPITATION", "PRECIPITATION ACCUMULATION")), - ), - # TODO for the SCAN network this appears to be "RHUM", we may need a new class - ( - "RH", - SensorDescription, - field(default=SensorDescription("RHUMV", "Relative Humidity", "RELATIVE HUMIDITY")), - ), - ( - "STREAMVOLUMEOBS", - SensorDescription, - field(default=SensorDescription("SRVO", "STREAM VOLUME OBS", "STREAM VOLUME OBS")), - ), - ( - "STREAMVOLUMEADJ", - SensorDescription, - field(default=SensorDescription("SRVOX", "STREAM VOLUME ADJ", "STREAM VOLUME ADJ")), - ), - ] - + [ - ( - f"TEMPGROUND{abs(d)}IN", - SensorDescription, - field( - default=SensorDescription( - "STO", - f"GROUND TEMPERATURE -{d}IN", - f"GROUND TEMPERATURE OBS -{d}IN", - extra={"height_depth": {"value": -d, "unitCd": "in"}}, - ) - ), - ) - for d in [2, 4, 8, 20] - ] - + [ - ( - f"SOILMOISTURE{abs(d)}IN", - SensorDescription, - field( - default=SensorDescription( - "SMS", - f"SOIL MOISTURE -{d}IN", - f"SOIL MOISTURE PERCENT -{d}IN", - extra={"height_depth": {"value": -d, "unitCd": "in"}}, - ) - ), - ) - for d in [2, 4, 8, 20] - ] - + [ - ( - f"TEMPPROFILENEG{abs(d)}IN" if d < 0 else f"TEMPPROFILE{d}IN", - SensorDescription, - field( - default=SensorDescription( - "PTEMP", - f"PROFILE TEMPERATURE {d}IN", - f"PROFILE TEMPERATURE OBS{d}IN", - extra={"height_depth": {"value": d, "unitCd": "in"}}, - ) - ), - ) - for d in [-8, 0, 8, 16, 24, 31, 39, 47, 55, 63, 71, 79, 87, 94, 102, 110, 118, 126] # noqa: E501 - ], - bases=(VariableBase,), -) - - class MesowestVariables(VariableBase): """ Available sensors from Mesowest diff --git a/setup.py b/setup.py index 4e4c955..25114b3 100644 --- a/setup.py +++ b/setup.py @@ -14,9 +14,9 @@ 'geopandas>=1.0.0,<2.0.0', 'pandas>=1.0.0,<3.0.0', 'lxml>=5.4.0,<6.0.0', - 'requests>2.0.0,<3.0.0', - 'beautifulsoup4>4,<5', + 'requests>=2.32.4,<3.0.0', 'zeep>4.0.0', + 'beautifulsoup4>4,<5', 'pydash>=8.0.0,<9.0.0', ] diff --git a/tests/data/snotel_mocks/daily.json b/tests/data/snotel_mocks/daily.json new file mode 100644 index 0000000..419e798 --- /dev/null +++ b/tests/data/snotel_mocks/daily.json @@ -0,0 +1,34 @@ +[ + { + "stationTriplet": "538:CO:SNTL", + "data": [ + { + "stationElement": { + "elementCode": "WTEQ", + "ordinal": 1, + "durationName": "DAILY", + "dataPrecision": 1, + "storedUnitCode": "in", + "originalUnitCode": "in", + "beginDate": "1980-07-23 00:00", + "endDate": "2100-01-01 00:00", + "derivedData": false + }, + "values": [ + { + "date": "2020-03-20", + "value": 11.6 + }, + { + "date": "2020-03-21", + "value": 11.6 + }, + { + "date": "2020-03-22", + "value": 11.8 + } + ] + } + ] +} +] diff --git a/tests/data/snotel_mocks/hourly_precip.json b/tests/data/snotel_mocks/hourly_precip.json new file mode 100644 index 0000000..9f13479 --- /dev/null +++ b/tests/data/snotel_mocks/hourly_precip.json @@ -0,0 +1,38 @@ +[ + { + "stationTriplet": "538:CO:SNTL", + "data": [ + { + "stationElement": { + "elementCode": "PREC", + "ordinal": 1, + "durationName": "HOURLY", + "dataPrecision": 1, + "storedUnitCode": "in", + "originalUnitCode": "in", + "beginDate": "1985-10-01 00:00", + "endDate": "2100-01-01 00:00", + "derivedData": false + }, + "values": [ + { + "date": "2020-01-02 00:00", + "value": 6 + }, + { + "date": "2020-01-02 01:00", + "value": 6 + }, + { + "date": "2020-01-02 02:00", + "value": 6.1 + }, + { + "date": "2020-01-02 03:00", + "value": 6.5 + } + ] + } + ] + } +] diff --git a/tests/data/snotel_mocks/hourly_soil.json b/tests/data/snotel_mocks/hourly_soil.json new file mode 100644 index 0000000..2a767be --- /dev/null +++ b/tests/data/snotel_mocks/hourly_soil.json @@ -0,0 +1,34 @@ +[ + { + "stationTriplet": "538:CO:SNTL", + "data": [ + { + "stationElement": { + "elementCode": "STO", + "ordinal": 1, + "durationName": "HOURLY", + "dataPrecision": 1, + "storedUnitCode": "degF", + "originalUnitCode": "degF", + "beginDate": "1985-10-01 00:00", + "endDate": "2100-01-01 00:00", + "derivedData": false + }, + "values": [ + { + "date": "2020-01-02 00:00", + "value": -1.0 + }, + { + "date": "2020-01-02 01:00", + "value": -1.2 + }, + { + "date": "2020-01-02 02:00", + "value": -2.0 + } + ] + } + ] + } +] diff --git a/tests/data/snotel_mocks/hourly_swe.json b/tests/data/snotel_mocks/hourly_swe.json new file mode 100644 index 0000000..0a251bf --- /dev/null +++ b/tests/data/snotel_mocks/hourly_swe.json @@ -0,0 +1,34 @@ +[ + { + "stationTriplet": "538:CO:SNTL", + "data": [ + { + "stationElement": { + "elementCode": "WTEQ", + "ordinal": 1, + "durationName": "HOURLY", + "dataPrecision": 1, + "storedUnitCode": "in", + "originalUnitCode": "in", + "beginDate": "1979-10-01 00:00", + "endDate": "2100-01-01 00:00", + "derivedData": false + }, + "values": [ + { + "date": "2020-01-02 00:00", + "value": 6.9 + }, + { + "date": "2020-01-02 01:00", + "value": 6.9 + }, + { + "date": "2020-01-02 02:00", + "value": 6.8 + } + ] + } + ] + } +] diff --git a/tests/data/snotel_mocks/semimonthly_swe.json b/tests/data/snotel_mocks/semimonthly_swe.json new file mode 100644 index 0000000..dd76c6e --- /dev/null +++ b/tests/data/snotel_mocks/semimonthly_swe.json @@ -0,0 +1,50 @@ +[ + { + "stationTriplet": "658:CO:SNTL", + "data": [ + { + "stationElement": { + "elementCode": "WTEQ", + "ordinal": 1, + "durationName": "SEMIMONTHLY", + "dataPrecision": 1, + "storedUnitCode": "in", + "originalUnitCode": "in", + "beginDate": "1985-10-01 00:00", + "endDate": "2100-01-01 00:00", + "derivedData": false + }, + "values": [ + { + "month": 1, + "monthPart": "1", + "year": 2020, + "collectionDate": "2020-01-16 00:00", + "value": 6.4 + }, + { + "month": 1, + "monthPart": "2", + "year": 2020, + "collectionDate": "2020-02-01 00:00", + "value": 7.3 + }, + { + "month": 2, + "monthPart": "1", + "year": 2020, + "collectionDate": "2020-02-16 00:00", + "value": 8.8 + }, + { + "month": 2, + "monthPart": "2", + "year": 2020, + "collectionDate": "2020-03-01 00:00", + "value": 9.9 + } + ] + } + ] + } +] diff --git a/tests/test_snotel.py b/tests/test_snotel.py index 2de41a9..3f1da98 100644 --- a/tests/test_snotel.py +++ b/tests/test_snotel.py @@ -1,4 +1,5 @@ from datetime import timezone, timedelta, datetime +from pathlib import Path from unittest.mock import patch, MagicMock from collections import OrderedDict @@ -10,6 +11,7 @@ from metloom.pointdata import SnotelPointData from metloom.variables import SnotelVariables from tests.test_point_data import BasePointDataTest +from tests.utils import read_json class MockZeepObject: @@ -31,55 +33,54 @@ def __getitem__(self, item): class TestSnotelPointData(BasePointDataTest): + MOCKS_DIR = Path(__file__).parent.joinpath("data/snotel_mocks/").absolute() @pytest.fixture(scope="class") def points(self): - return gpd.points_from_xy([-107.67552], [37.9339], z=[9800.0])[0] + return gpd.points_from_xy([-107.6762], [37.93389], z=[9800.0])[0] + + @classmethod + def side_effect(cls, *args, **kwargs): + """ + All request side effects + """ + url = args[0] + if "services/v1/stations" in url: + result = cls.snotel_meta_sideeffect(*args, **kwargs) + elif "services/v1/data" in url: + result = cls.snotel_data_sideeffect(*args, **kwargs) + else: + raise ValueError("Unknown URL in mock: " + url) + obj = MagicMock() + obj.json.return_value = result + return obj @staticmethod def snotel_meta_sideeffect(*args, **kwargs): """ Mock out the metadata response """ - code = kwargs["stationTriplet"] + codes = kwargs["params"]["stationTriplets"].split(",") available_stations = { "538:CO:SNTL": { - "actonId": "07M27S", - "beginDate": "1979-10-01 00:00:00", - "countyName": "Ouray", - "elevation": 9800.0, - "endDate": "2100-01-01 00:00:00", - "fipsCountryCd": "US", - "fipsCountyCd": "091", - "fipsStateNumber": "08", - "huc": "140200060201", - "hud": "14020006", - "latitude": 37.9339, - "longitude": -107.67552, - "name": "Idarado", - "shefId": "IDRC2", "stationTriplet": "538:CO:SNTL", - "stationDataTimeZone": -8.0 - }, - "538:CO:SNOW": { - "actonId": "07M27S", - "beginDate": "1979-10-01 00:00:00", + "stationId": "538", + "stateCode": "CO", + "networkCode": "SNTL", + "name": "Idarado", + "dcoCode": "CO", "countyName": "Ouray", - "elevation": 9800.0, - "endDate": "2100-01-01 00:00:00", - "fipsCountryCd": "US", - "fipsCountyCd": "091", - "fipsStateNumber": "08", "huc": "140200060201", - "hud": "14020006", - "latitude": 37.9339, - "longitude": -107.67552, - "name": "Idarado", + "elevation": 9800, + "latitude": 37.93389, + "longitude": -107.6762, + "dataTimeZone": -8, "shefId": "IDRC2", - "stationTriplet": "538:CO:SNOW", + "operator": "NRCS", + "beginDate": "1979-10-01 00:00", + "endDate": "2100-01-01 00:00" }, "FFF:CA:SNOW": { - "actonId": None, "beginDate": "1930-02-01 00:00:00", "countyName": "Tuolumne", "elevation": 6500.0, @@ -92,10 +93,10 @@ def snotel_meta_sideeffect(*args, **kwargs): "longitude": -119.78, "name": "Fake1", "shefId": None, + "dataTimeZone": -8, "stationTriplet": "FFF:CA:SNOW", }, "BBB:CA:SNOW": { - "actonId": None, "beginDate": "1948-02-01 00:00:00", "countyName": "Tuolumne", "elevation": 9300.0, @@ -109,166 +110,63 @@ def snotel_meta_sideeffect(*args, **kwargs): "longitude": -119.61667, "name": "Fake2", "shefId": None, + "dataTimeZone": -8, "stationTriplet": "BBB:CA:SNOW", }, } - return MockZeepObject(available_stations[code]) + return [available_stations[code] for code in codes if code in available_stations] - @staticmethod - def snotel_data_sideeffect(*args, **kwargs): - duration = kwargs["duration"] + @classmethod + def snotel_data_sideeffect(cls, *args, **kwargs): + duration = kwargs["params"]["duration"] + fname = None if duration == "SEMIMONTHLY": - return [ - MockZeepObject({ - 'beginDate': '2020-01-20 00:00:00', - 'collectionDates': ['2020-01-28', '2020-02-27'], - 'duration': 'SEMIMONTHLY', - 'endDate': '2020-03-14 00:00:00', - 'flags': ['V', 'V'], 'stationTriplet': '538:CO:SNTL', - 'values': [13.19, 13.17]}) - ] - if duration == "DAILY": - return [MockZeepObject({ - 'beginDate': '2020-03-20 00:00:00', - 'collectionDates': [], 'duration': 'DAILY', - 'endDate': '2020-03-22 00:00:00', 'flags': ['V', 'V', 'V'], - 'stationTriplet': '538:CO:SNTL', - 'values': [13.19, 13.17, 13.14]})] + fname = cls.MOCKS_DIR.joinpath("semimonthly_swe.json") + elif duration == "DAILY": + fname = cls.MOCKS_DIR.joinpath("daily.json") + elif duration == "HOURLY": + element_cd = kwargs["params"]["elements"] + cd_file_map = { + "WTEQ": "hourly_swe.json", + "PRCPSA": "hourly_precip.json", + "STO:-2": "hourly_soil.json", + } + fname = cls.MOCKS_DIR.joinpath(cd_file_map.get(element_cd)) - @staticmethod - def snotel_hourly_sideeffect(*args, **kwargs): - element_cd = kwargs["elementCd"] - if element_cd == "WTEQ": - return [ - { - 'beginDate': '2020-01-02 00:00', 'endDate': '2020-01-20 00:00', - 'stationTriplet': '538:CO:SNTL', - 'values': [ - { - 'dateTime': '2020-03-20 00:00', - 'flag': 'V', - 'value': 13.19 - }, { - 'dateTime': '2020-03-20 01:00', - 'flag': 'V', - 'value': 13.17 - }, { - 'dateTime': '2020-03-20 02:00', - 'flag': 'V', - 'value': 13.14 - }]}] - elif element_cd == "PRCPSA": - return [ - { - 'beginDate': '2020-01-02 00:00', - 'endDate': '2020-01-20 00:00', - 'stationTriplet': '538:CO:SNTL', - 'values': [ - { - 'dateTime': '2020-03-20 00:00', - 'flag': 'V', - 'value': 4.1 - }, { - 'dateTime': '2020-03-20 02:00', - 'flag': 'V', - 'value': 4.3 - }, { - 'dateTime': '2020-03-20 03:00', - 'flag': 'V', - 'value': 4.4 - }]}] - elif element_cd == "STO": - return [ - { - 'beginDate': '2020-01-02 00:00', - 'endDate': '2020-01-20 00:00', - 'stationTriplet': '538:CO:SNTL', - 'values': [ - { - 'dateTime': '2020-03-20 00:00', - 'flag': 'V', - 'value': -0.3, - }, { - 'dateTime': '2020-03-20 01:00', - 'flag': 'V', - 'value': -0.4, - }, { - 'dateTime': '2020-03-20 02:00', - 'flag': 'V', - 'value': -0.5, - }]}] - else: - raise ValueError(f"{element_cd} not configured in this mock") + if fname is None: + raise ValueError("No mock file found for duration: " + duration) - @pytest.fixture(scope="class") - def mock_elements(self): - return [ - MockZeepObject( - { - 'beginDate': '1980-07-23 00:00:00', 'dataPrecision': 1, - 'duration': 'DAILY', 'elementCd': 'WTEQ', - 'endDate': '2100-01-01 00:00:00', 'heightDepth': None, - 'ordinal': 1, - 'originalUnitCd': 'in', 'stationTriplet': '538:CO:SNTL', - 'storedUnitCd': 'in'}), - MockZeepObject( - {'beginDate': '1980-07-23 00:00:00', 'dataPrecision': 1, - 'duration': 'SEMIMONTHLY', 'elementCd': 'WTEQ', - 'endDate': '2100-01-01 00:00:00', 'heightDepth': None, - 'ordinal': 1, - 'originalUnitCd': 'in', 'stationTriplet': '538:CO:SNTL', - 'storedUnitCd': 'in'}), - MockZeepObject( - {'beginDate': '1979-10-01 00:00:00', 'dataPrecision': 1, - 'duration': 'HOURLY', 'elementCd': 'WTEQ', - 'endDate': '2100-01-01 00:00:00', 'heightDepth': None, - 'ordinal': 1, - 'originalUnitCd': 'in', 'stationTriplet': '538:CO:SNTL', - 'storedUnitCd': 'in'}), - MockZeepObject( - {'beginDate': '1980-07-23 00:00:00', 'dataPrecision': 1, - 'duration': 'MONTHLY', 'elementCd': 'WTEQ', - 'endDate': '2100-01-01 00:00:00', 'heightDepth': None, - 'ordinal': 1, - 'originalUnitCd': 'in', 'stationTriplet': '538:CO:SNTL', - 'storedUnitCd': 'in'}), - MockZeepObject( - {'beginDate': '1979-10-01 00:00:00', 'dataPrecision': 1, - 'duration': 'HOURLY', 'elementCd': 'PRCPSA', - 'endDate': '2100-01-01 00:00:00', 'heightDepth': None, - 'ordinal': 1, - 'originalUnitCd': 'in', 'stationTriplet': '538:CO:SNTL', - 'storedUnitCd': 'in'}), - MockZeepObject( - {'beginDate': '1979-10-01 00:00:00', 'dataPrecision': 1, - 'duration': 'HOURLY', 'elementCd': 'STO', - 'endDate': '2100-01-01 00:00:00', - 'heightDepth': {'unitCd': 'in', 'value': '-2'}, - 'ordinal': 1, - 'originalUnitCd': 'degF', 'stationTriplet': '538:CO:SNTL', - 'storedUnitCd': 'degF'}) - ] + return read_json(fname) + + @pytest.fixture + def mock_requests(self): + with patch("requests.get") as mock_get: + # Mock our gets + mock_get.side_effect = self.side_effect + yield mock_get @pytest.fixture - def mock_zeep_client(self, mock_elements): - with patch("metloom.pointdata.snotel_client.zeep.Client") as mock_client: + def mock_zeep_find(self): + """ + Mock the zeep client to return a mock service with + getStations method returning a list of station triplets. + """ + with patch( + "metloom.pointdata.snotel.snotel_client.zeep.Client" + ) as mock_client: mock_service = MagicMock() # setup the individual services - mock_service.getStationMetadata.side_effect = self.snotel_meta_sideeffect - mock_service.getStationElements.return_value = mock_elements mock_service.getStations.return_value = ["FFF:CA:SNOW", "BBB:CA:SNOW"] - mock_service.getData.side_effect = self.snotel_data_sideeffect - mock_service.getHourlyData.side_effect = self.snotel_hourly_sideeffect # assign service to client mock_client.return_value.service = mock_service yield mock_client - def test_metadata(self, mock_zeep_client): + def test_metadata(self, mock_requests): obj = SnotelPointData("538:CO:SNTL", "eh") assert ( obj.metadata == gpd.points_from_xy( - [-107.67552], [37.9339], z=[9800.0])[0] + [-107.6762], [37.93389], z=[9800.0])[0] ) assert obj.tzinfo == timezone(timedelta(hours=-8.0)) @@ -277,27 +175,27 @@ def test_metadata(self, mock_zeep_client): [ ( "538:CO:SNTL", - ["2020-03-20 00:00", "2020-03-20 01:00", "2020-03-20 02:00"], - ["2020-03-20 08:00", "2020-03-20 09:00", "2020-03-20 10:00"], + ["2020-01-02 00:00", "2020-01-02 01:00", "2020-01-02 02:00"], + ["2020-01-02 08:00", "2020-01-02 09:00", "2020-01-02 10:00"], { - SnotelVariables.SWE.name: [13.19, 13.17, 13.14], + SnotelVariables.SWE.name: [6.9, 6.9, 6.8], f"{SnotelVariables.SWE.name}_units": ["in", "in", "in"] }, - datetime(2020, 3, 20, 0), - datetime(2020, 3, 20, 2), + datetime(2020, 1, 2, 0), + datetime(2020, 1, 2, 2), "get_hourly_data", ), ( "538:CO:SNTL", - ["2020-03-20 00:00", "2020-03-20 01:00", "2020-03-20 02:00"], - ["2020-03-20 08:00", "2020-03-20 09:00", "2020-03-20 10:00"], + ["2020-01-02 00:00", "2020-01-02 01:00", "2020-01-02 02:00"], + ["2020-01-02 08:00", "2020-01-02 09:00", "2020-01-02 10:00"], { - SnotelVariables.TEMPGROUND2IN.name: [-0.3, -0.4, -0.5], + SnotelVariables.TEMPGROUND2IN.name: [-1.0, -1.2, -2.0], f"{SnotelVariables.TEMPGROUND2IN.name}_units": ["degF", "degF", "degF"] }, - datetime(2020, 3, 20, 0), - datetime(2020, 3, 20, 2), + datetime(2020, 1, 2, 0), + datetime(2020, 1, 2, 2), "get_hourly_data", ), ( @@ -305,7 +203,7 @@ def test_metadata(self, mock_zeep_client): ["2020-03-20", "2020-03-21", "2020-03-22"], ["2020-03-20 08:00", "2020-03-21 08:00", "2020-03-22 08:00"], { - SnotelVariables.SWE.name: [13.19, 13.17, 13.14], + SnotelVariables.SWE.name: [11.6, 11.6, 11.8], f"{SnotelVariables.SWE.name}_units": ["in", "in", "in"] }, datetime(2020, 3, 20), @@ -313,12 +211,12 @@ def test_metadata(self, mock_zeep_client): "get_daily_data", ), ( - "538:CO:SNOW", - ["2020-01-28", "2020-02-27"], - ["2020-01-28 00:00", "2020-02-27 00:00"], + "538:CO:SNTL", + ["2020-01-16", "2020-02-01", "2020-02-16", "2020-02-27"], + ["2020-01-16 08:00", "2020-02-01 08:00", "2020-02-16 08:00", "2020-03-01 08:00"], { - SnotelVariables.SWE.name: [13.19, 13.17], - f"{SnotelVariables.SWE.name}_units": ["in", "in"] + SnotelVariables.SWE.name: [6.4, 7.3, 8.8, 9.9], + f"{SnotelVariables.SWE.name}_units": ["in", "in", "in", "in"] }, datetime(2020, 1, 20), datetime(2020, 3, 15), @@ -328,7 +226,7 @@ def test_metadata(self, mock_zeep_client): ) def test_get_data_methods( self, station_id, dts, expected_dts, vals, d1, - d2, fn_name, points, mock_zeep_client): + d2, fn_name, points, mock_requests): station = SnotelPointData(station_id, "TestSite") if 'GROUND TEMPERATURE -2IN' in list(vals.keys()): vrs = [SnotelVariables.TEMPGROUND2IN] @@ -344,22 +242,22 @@ def test_get_data_methods( result.sort_index(axis=1), expected ) - def test_get_hourly_data_multi_sensor(self, points, mock_zeep_client): + def test_get_hourly_data_multi_sensor(self, points, mock_requests): expected_dts = [ - "2020-03-20 08:00", "2020-03-20 09:00", "2020-03-20 10:00", - "2020-03-20 11:00" + "2020-01-02 08:00", "2020-01-02 09:00", "2020-01-02 10:00", + "2020-01-02 11:00" ] expected_vals_obj = { - SnotelVariables.SWE.name: [13.19, 13.17, 13.14, np.nan], + SnotelVariables.SWE.name: [6.9, 6.9, 6.8, np.nan], f"{SnotelVariables.SWE.name}_units": ["in", "in", "in", np.nan], - SnotelVariables.PRECIPITATION.name: [4.1, np.nan, 4.3, 4.4], + SnotelVariables.PRECIPITATION.name: [6, 6, 6.1, 6.5], f"{SnotelVariables.PRECIPITATION.name}_units": [ - "in", np.nan, "in", "in"], + "in", "in", "in", "in"], } station = SnotelPointData("538:CO:SNTL", "TestSite") vrs = [SnotelVariables.PRECIPITATION, SnotelVariables.SWE] result = station.get_hourly_data( - datetime(2020, 3, 20, 0), datetime(2020, 3, 20, 4), vrs + datetime(2020, 1, 2, 0), datetime(2020, 1, 2, 4), vrs ) expected = self.expected_response( expected_dts, expected_vals_obj, station, points @@ -368,7 +266,7 @@ def test_get_hourly_data_multi_sensor(self, points, mock_zeep_client): result.sort_index(axis=1), expected ) - def test_points_from_geometry(self, shape_obj, mock_zeep_client): + def test_points_from_geometry(self, shape_obj, mock_requests, mock_zeep_find): result = SnotelPointData.points_from_geometry( shape_obj, [SnotelVariables.SWE], snow_courses=True ) @@ -378,11 +276,13 @@ def test_points_from_geometry(self, shape_obj, mock_zeep_client): assert set(ids) == {"FFF:CA:SNOW", "BBB:CA:SNOW"} assert set(names) == {"Fake1", "Fake2"} - def test_points_from_geomtery_buffer(self, shape_obj, mock_zeep_client): + def test_points_from_geometry_buffer( + self, shape_obj, mock_requests, mock_zeep_find + ): SnotelPointData.points_from_geometry( shape_obj, [SnotelVariables.SWE], snow_courses=False, buffer=0.1 ) - search_kwargs = mock_zeep_client().method_calls[0][2] + search_kwargs = mock_zeep_find().method_calls[0][2] expected = { 'maxLatitude': 38.3, 'minLatitude': 37.6, 'maxLongitude': -119.1, 'minLongitude': -119.9 @@ -390,8 +290,8 @@ def test_points_from_geomtery_buffer(self, shape_obj, mock_zeep_client): for k, v in expected.items(): assert v == pytest.approx(search_kwargs[k]) - def test_points_from_geometry_fail(self, shape_obj, mock_zeep_client): - mock_zeep_client.return_value.service.getStations.return_value = [] + def test_points_from_geometry_fail(self, shape_obj, mock_requests, mock_zeep_find): + mock_zeep_find.return_value.service.getStations.return_value = [] result = SnotelPointData.points_from_geometry( shape_obj, [SnotelVariables.SWE], snow_courses=True ) diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..12267a7 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,7 @@ +from pathlib import Path +import json + + +def read_json(file: Path): + with open(file, "r") as f: + return json.load(f)