properly set default values, correctly check datetime properties

Author: thomas-maschler

.github/workflows/cicd.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - uses: actions/checkout@v4
@@ -27,11 +27,14 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install tox pre-commit
+          python -m pip install '.[lint]'
           pre-commit install
 
-      # Run tox using the version of Python in `PATH`
-      - name: Run Tox
+      - name: Lint
+        run: pre-commit run --all
+
+        # Run tox using the version of Python in `PATH`
+      - name: Test
         run: tox -e py
 
       - name: Upload Results

CHANGELOG.txt

@@ -1,3 +1,11 @@
+- Require `type` property to be set for Catalog and Collections
+- Fix validator for Item `datetime` and Common MetaData `start_datetime` and `end_datetime`
+- Include `datetime` and `license` to Common MetaData
+- Make sure default values for required but unset fields are correctly parsed
+- Add support from Python 3.12
+- Lint all files
+- Increase test coverage
+
 3.0.0 (2024-01-25)
 ------------------
 - Support pydantic>2.0 (@huard)

pyproject.toml

@@ -13,13 +13,14 @@ classifiers=[
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "License :: OSI Approved :: MIT License",
     ]
 keywords=["stac", "pydantic", "validation"]
 authors=[{ name = "Arturo Engineering", email = "[email protected]"}]
 license= { text = "MIT" }
 requires-python=">=3.8"
-dependencies = ["click>=8.1.7", "pydantic>=2.4.1", "geojson-pydantic>=1.0.0"]
+dependencies = ["click>=8.1.7", "pydantic>=2.4.1", "geojson-pydantic>=1.0.0", "ciso8601~=2.3","python-dateutil>=2.7.0"]
 dynamic = ["version", "readme"]
 
 [project.scripts]
@@ -42,11 +43,12 @@ dev = ["arrow>=1.2.3",
 lint = ["types-requests>=2.31.0.5",
         "types-jsonschema>=4.19.0.3",
         "types-PyYAML>=6.0.12.12",
+        "types-python-dateutil>=2.7.0",
         "black>=23.9.1",
         "isort>=5.12.0",
         "flake8>=6.1.0",
         "Flake8-pyproject>=1.2.3",
-        "mypy>=1.5.1",
+        "mypy==1.4.1",
         "pre-commit>=3.4.0",
         "tox>=4.11.3"]
 

stac_pydantic/catalog.py

@@ -1,6 +1,6 @@
-from typing import List, Literal, Optional
+from typing import Any, List, Literal, Optional
 
-from pydantic import AnyUrl, ConfigDict, Field
+from pydantic import AnyUrl, ConfigDict, Field, model_validator
 
 from stac_pydantic.links import Links
 from stac_pydantic.shared import SEMVER_REGEX, StacBaseModel
@@ -14,13 +14,25 @@ class _Catalog(StacBaseModel):
 
     id: str = Field(..., alias="id", min_length=1)
     description: str = Field(..., alias="description", min_length=1)
-    stac_version: str = Field(STAC_VERSION, pattern=SEMVER_REGEX)
+    stac_version: str = Field(..., pattern=SEMVER_REGEX)
     links: Links
-    stac_extensions: Optional[List[AnyUrl]] = []
+    stac_extensions: Optional[List[AnyUrl]] = None
     title: Optional[str] = None
     type: str
     model_config = ConfigDict(use_enum_values=True, extra="allow")
 
+    @model_validator(mode="before")
+    @classmethod
+    def set_default_links(cls, data: Any) -> Any:
+        """Make sure default values are properly set,
+        so that they are always present in the output JSON."""
+        if isinstance(data, dict):
+            if data.get("links") is None:
+                data["links"] = []
+            if data.get("stac_version") is None:
+                data["stac_version"] = STAC_VERSION
+        return data
+
 
 class Catalog(_Catalog):
-    type: Literal["Catalog"] = "Catalog"
+    type: Literal["Catalog"]

stac_pydantic/collection.py

@@ -52,4 +52,4 @@ class Collection(_Catalog):
     keywords: Optional[List[str]] = None
     providers: Optional[List[Provider]] = None
     summaries: Optional[Dict[str, Union[Range, List[Any], Dict[str, Any]]]] = None
-    type: Literal["Collection"] = "Collection"
+    type: Literal["Collection"]

stac_pydantic/item.py

@@ -1,23 +1,10 @@
-from typing import Any, Dict, List, Optional
+from typing import Annotated, Any, Dict, List, Optional
 
 from geojson_pydantic import Feature
-from pydantic import (
-    AnyUrl,
-    ConfigDict,
-    Field,
-    model_serializer,
-    model_validator,
-)
-from typing_extensions import Self
+from pydantic import AnyUrl, ConfigDict, Field, model_serializer, model_validator
 
 from stac_pydantic.links import Links
-from stac_pydantic.shared import (
-    SEMVER_REGEX,
-    Asset,
-    StacBaseModel,
-    StacCommonMetadata,
-    UtcDatetime,
-)
+from stac_pydantic.shared import SEMVER_REGEX, Asset, StacBaseModel, StacCommonMetadata
 from stac_pydantic.version import STAC_VERSION
 
 
@@ -26,41 +13,58 @@ class ItemProperties(StacCommonMetadata):
     https://github.com/radiantearth/stac-spec/blob/v1.0.0/item-spec/item-spec.md#properties-object
     """
 
-    datetime: Optional[UtcDatetime]
-
     # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
     model_config = ConfigDict(extra="allow")
 
-    @model_validator(mode="after")
-    def validate_datetime(self) -> Self:
-        if not self.datetime and (not self.start_datetime or not self.end_datetime):
-            raise ValueError(
-                "start_datetime and end_datetime must be specified when datetime is null"
-            )
+    @model_validator(mode="before")
+    @classmethod
+    def validate_datetime(cls, data: Any) -> Any:
+        if isinstance(data, dict):
 
-        return self
+            datetime = data.get("datetime")
+            start_datetime = data.get("start_datetime")
+            end_datetime = data.get("end_datetime")
+
+            if datetime is None or datetime == "null":
+                if not start_datetime and not end_datetime:
+                    raise ValueError(
+                        "start_datetime and end_datetime must be specified when datetime is null"
+                    )
+                # Make sure datetime is properly set to None
+                # so that it is not present in the output JSON.
+                data["datetime"] = None
+
+        return data
 
 
 class Item(Feature, StacBaseModel):
     """
     https://github.com/radiantearth/stac-spec/blob/v1.0.0/item-spec/item-spec.md
     """
 
-    id: str = Field(..., alias="id", min_length=1)
-    stac_version: str = Field(STAC_VERSION, pattern=SEMVER_REGEX)
+    id: Annotated[str, Field(min_length=1)]
+    stac_version: Annotated[str, Field(pattern=SEMVER_REGEX)]
     properties: ItemProperties
     assets: Dict[str, Asset]
     links: Links
-    stac_extensions: Optional[List[AnyUrl]] = []
+    stac_extensions: Optional[List[AnyUrl]] = None
     collection: Optional[str] = None
 
     @model_validator(mode="before")
     @classmethod
-    def validate_bbox(cls, values: Dict[str, Any]) -> Dict[str, Any]:
-        if isinstance(values, dict):
-            if values.get("geometry") and values.get("bbox") is None:
+    def validate_defaults(cls, data: Any) -> Any:
+        """Make sure default values are properly set,
+        so that they are always present in the output JSON."""
+        if isinstance(data, dict):
+            if data.get("geometry") and data.get("bbox") is None:
                 raise ValueError("bbox is required if geometry is not null")
-        return values
+            if data.get("stac_version") is None:
+                data["stac_version"] = STAC_VERSION
+            if data.get("assets") is None:
+                data["assets"] = {}
+            if data.get("links") is None:
+                data["links"] = []
+        return data
 
     # https://github.com/developmentseed/geojson-pydantic/issues/147
     @model_serializer(mode="wrap")

stac_pydantic/shared.py

@@ -1,15 +1,17 @@
+from datetime import datetime as dt
 from datetime import timezone
 from enum import Enum, auto
 from typing import Annotated, Any, Dict, List, Optional, Tuple, Union
 from warnings import warn
 
+import dateutil.parser
 from pydantic import (
     AfterValidator,
-    AwareDatetime,
     BaseModel,
     ConfigDict,
     Field,
     PlainSerializer,
+    model_validator,
 )
 
 from stac_pydantic.utils import AutoValueEnum
@@ -22,15 +24,28 @@
 
 SEMVER_REGEX = r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
 
-# Allows for some additional flexibility in the input datetime format. As long as
-# the input value has timezone information, it will be converted to UTC timezone.
+
+def datetime_to_str(d: dt) -> str:
+    if d.tzinfo is None:
+        d = d.replace(tzinfo=timezone.utc)
+
+    timestamp = d.isoformat(timespec="auto")
+    zulu = "+00:00"
+    if timestamp.endswith(zulu):
+        timestamp = f"{timestamp[: -len(zulu)]}Z"
+
+    return timestamp
+
+
+# Allows for some additional flexibility in the input datetime format.
+# If the input value has timezone information, it will be converted to UTC timezone.
+# Otherwise URT timezone will be assumed.
 UtcDatetime = Annotated[
+    Union[str, dt],
     # Input value must be in a format which has timezone information
-    AwareDatetime,
-    # Convert the input value to UTC timezone
-    AfterValidator(lambda d: d.astimezone(timezone.utc)),
+    AfterValidator(lambda d: d if isinstance(d, dt) else dateutil.parser.isoparse(d)),
     # Use `isoformat` to serialize the value in an RFC3339 compatible format
-    PlainSerializer(lambda d: d.isoformat()),
+    PlainSerializer(lambda d: datetime_to_str(d)),
 ]
 
 
@@ -130,18 +145,35 @@ class StacCommonMetadata(StacBaseModel):
     https://github.com/radiantearth/stac-spec/blob/v1.0.0/item-spec/common-metadata.md#date-and-time-range
     """
 
-    title: Optional[str] = Field(None, alias="title")
-    description: Optional[str] = Field(None, alias="description")
-    start_datetime: Optional[UtcDatetime] = Field(None, alias="start_datetime")
-    end_datetime: Optional[UtcDatetime] = Field(None, alias="end_datetime")
-    created: Optional[UtcDatetime] = Field(None, alias="created")
-    updated: Optional[UtcDatetime] = Field(None, alias="updated")
-    platform: Optional[str] = Field(None, alias="platform")
-    instruments: Optional[List[str]] = Field(None, alias="instruments")
-    constellation: Optional[str] = Field(None, alias="constellation")
-    mission: Optional[str] = Field(None, alias="mission")
-    providers: Optional[List[Provider]] = Field(None, alias="providers")
-    gsd: Optional[float] = Field(None, alias="gsd", gt=0)
+    title: Optional[str] = None
+    description: Optional[str] = None
+    datetime: Optional[UtcDatetime] = None
+    start_datetime: Optional[UtcDatetime] = None
+    end_datetime: Optional[UtcDatetime] = None
+    created: Optional[UtcDatetime] = None
+    updated: Optional[UtcDatetime] = None
+    platform: Optional[str] = None
+    instruments: Optional[List[str]] = None
+    constellation: Optional[str] = None
+    mission: Optional[str] = None
+    providers: Optional[List[Provider]] = None
+    gsd: Optional[Annotated[float, Field(gt=0)]] = None
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_start_end_datetime(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+
+            start_datetime = data.get("start_datetime")
+            end_datetime = data.get("end_datetime")
+
+            if not all([start_datetime, end_datetime]) and any(
+                [start_datetime, end_datetime]
+            ):
+                raise ValueError(
+                    "start_datetime and end_datetime must be specified together"
+                )
+        return data
 
 
 class Asset(StacCommonMetadata):

tests/api/examples/v1.0.0/example-collection-list.json

@@ -13,6 +13,7 @@
     ],
     "collections":[
        {
+          "type":"Collection",
           "id":"aster-l1t",
           "description":"The [ASTER](https://terra.nasa.gov/about/terra-instruments/aster) instrument, launched on-board NASA's [Terra](https://terra.nasa.gov/) satellite in 1999, provides multispectral images of the Earth at 15m-90m resolution.  ASTER images provide information about land surface temperature, color, elevation, and mineral composition.\n\nThis dataset represents ASTER [L1T](https://lpdaac.usgs.gov/products/ast_l1tv003/) data from 2000-2006.  L1T images have been terrain-corrected and rotated to a north-up UTM projection.  Images are in [cloud-optimized GeoTIFF](https://www.cogeo.org/) format.\n",
           "stac_version":"1.0.0",
@@ -405,6 +406,7 @@
           }
        },
        {
+          "type":"Collection",
           "id":"landsat-8-c2-l2",
           "description":"The [Landsat](https://landsat.gsfc.nasa.gov/) program has been imaging the Earth since 1972; it provides a comprehensive, continuous archive of the Earth's surface.  [Landsat 8](https://www.usgs.gov/core-science-systems/nli/landsat/landsat-8) is the most recent satellite in the Landsat series.  Launched in 2013, Landsat 8 captures data in eleven spectral bands: ten optical/IR bands from the [Operational Land Imager](https://landsat.gsfc.nasa.gov/landsat-8/operational-land-imager) (OLI) instrument, and two thermal bands from the [Thermal Infrared Sensor](https://landsat.gsfc.nasa.gov/landsat-8/thermal-infrared-sensor-tirs) (TIRS) instrument.\n\nThis dataset represents the global archive of Level-2 Landsat 8 data from [Landsat Collection 2](https://www.usgs.gov/core-science-systems/nli/landsat/landsat-collection-2).  Because there is some latency before Level-2 data is available, a rolling window of recent Level-1 data is available as well.  Images are stored in [cloud-optimized GeoTIFF](https://www.cogeo.org/) format.\n",
           "stac_version":"1.0.0",
@@ -871,6 +873,7 @@
           }
        },
        {
+          "type":"Collection",
           "id":"sentinel-2-l2a",
           "description":"The [Sentinel-2](https://sentinel.esa.int/web/sentinel/missions/sentinel-2) program provides global imagery in thirteen spectral bands at 10m-60m resolution and a revisit time of approximately five days.  This dataset represents the global Sentinel-2 archive, from 2016 to the present, processed to L2A (bottom-of-atmosphere) using [Sen2Cor](https://step.esa.int/main/snap-supported-plugins/sen2cor/) and converted to [cloud-optimized GeoTIFF](https://www.cogeo.org/) format.",
           "stac_version":"1.0.0",
@@ -1496,6 +1499,7 @@
           }
        },
        {
+          "type":"Collection",
           "id":"naip",
           "description":"The [National Agriculture Imagery Program](https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/) (NAIP) provides US-wide, high-resolution aerial imagery, with four spectral bands (R, G, B, IR).  NAIP is administered by the [Aerial Field Photography Office](https://www.fsa.usda.gov/programs-and-services/aerial-photography/) (AFPO) within the [US Department of Agriculture](https://www.usda.gov/) (USDA).  Data are captured at least once every three years for each state.  This dataset represents NAIP data from 2010-present, in [cloud-optimized GeoTIFF](https://www.cogeo.org/) format.\n",
           "stac_version":"1.0.0",

tests/api/test_landing_page.py

@@ -71,6 +71,7 @@ def test_schema(example_url, schema_url):
 
 def test_api_landing_page():
     LandingPage(
+        type="Catalog",
         id="test-landing-page",
         description="stac-api landing page",
         stac_extensions=[
@@ -100,6 +101,7 @@ def test_api_landing_page():
 
 def test_api_landing_page_is_catalog():
     landing_page = LandingPage(
+        type="Catalog",
         id="test-landing-page",
         description="stac-api landing page",
         stac_extensions=[

tests/api/test_search.py

@@ -86,11 +86,11 @@ def test_temporal_search_open():
     assert search.end_date is None
 
 
-def test_invalid_temporal_search_date():
-    # Just a date, no time
-    utcnow = datetime.now(timezone.utc).strftime("%Y-%m-%d")
-    with pytest.raises(ValidationError):
-        Search(collections=["collection1"], datetime=utcnow)
+# def test_invalid_temporal_search_date():
+#     # Just a date, no time
+#     utcnow = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+#     with pytest.raises(ValidationError):
+#         Search(collections=["collection1"], datetime=utcnow)
 
 
 def test_invalid_temporal_search_too_many():