diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 96b2fdb648..0f90cbb21d 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -19,4 +19,4 @@ > **Note:** Stale issues will be automatically closed after a period of six months with no activity. > To ensure critical issues are not closed, tag them with the Github `pinned` tag. > If you are a community member and not a maintainer please escalate this issue to maintainers via -> [GIS StackExchange](https://gis.stackexchange.com/questions/tagged/open-data-cube) or [Slack](http://slack.opendatacube.org). +> [GIS StackExchange](https://gis.stackexchange.com/questions/tagged/open-data-cube) or [Discord](https://discord.com/invite/4hhBQVas5U). diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 90e05c40d0..e115726a7b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,3 +9,7 @@ updates: directory: "/" # Location of package manifests schedule: interval: "weekly" + - package-ecosystem: docker + directory: "/docker" + schedule: + interval: "daily" diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml deleted file mode 100644 index 5a1dcc41f7..0000000000 --- a/.github/workflows/build-docs.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: build-docs - -on: - pull_request: - push: - branches: - - develop - - dra/docs-updates - -env: - DKR: opendatacube/datacube-tests:latest - -jobs: - main: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Pull Docker - run: | - docker pull ${DKR} - - - name: Build Docs - run: | - cat <> $GITHUB_OUTPUT + echo "push_pypi=yes" >> $GITHUB_OUTPUT + fi + + - uses: dorny/paths-filter@v3 id: changes if: | github.event_name == 'push' @@ -41,7 +54,7 @@ jobs: - name: Set up Docker Buildx if: steps.changes.outputs.docker == 'true' - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Cache Docker layers if: steps.changes.outputs.docker == 'true' @@ -57,14 +70,13 @@ jobs: if: | github.event_name == 'push' && github.ref == 'refs/heads/develop' - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: - username: ${{ env.DOCKER_USER }} - password: ${{ secrets.GADOCKERSVC_PASSWORD }} + username: ${{ secrets.DOCKERHUBUSER }} + password: ${{ secrets.DOCKERHUBPASSWD }} - name: Build Docker - if: steps.changes.outputs.docker == 'true' - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: file: docker/Dockerfile context: . @@ -92,12 +104,16 @@ jobs: EOF - name: DockerHub Push - if: steps.changes.outputs.docker == 'true' - uses: docker/build-push-action@v4 + if: | + github.event_name == 'push' + && github.ref == 'refs/heads/develop' + && steps.changes.outputs.docker == 'true' + uses: docker/build-push-action@v6 with: + file: docker/Dockerfile context: . push: true - tags: ${DOCKER_IMAGE} + tags: ${{ env.DOCKER_IMAGE }} - name: Build Packages run: | @@ -110,10 +126,34 @@ jobs: twine check ./dist/* EOF + - name: Publish to Test PyPi + if: | + steps.cfg.outputs.push_test_pypi == 'yes' + run: | + if [ -n "${TWINE_PASSWORD}" ]; then + docker run --rm \ + -v $(pwd):/code \ + -e SKIP_DB=yes \ + ${{ env.DOCKER_IMAGE }} \ + twine upload \ + --verbose \ + --non-interactive \ + --disable-progress-bar \ + --username=__token__ \ + --password=${TWINE_PASSWORD} \ + --repository-url=${TWINE_REPOSITORY_URL} \ + --skip-existing dist/* || true + else + echo "Skipping upload as 'TestPyPiToken' is not set" + fi + env: + TWINE_PASSWORD: ${{ secrets.TestPyPiToken }} + TWINE_REPOSITORY_URL: 'https://test.pypi.org/legacy/' + - name: Publish to PyPi if: | github.event_name == 'push' - && github.ref == 'refs/heads/pypi/publish' + && steps.cfg.outputs.push_pypi == 'yes' run: | if [ -n "${TWINE_PASSWORD}" ]; then docker run --rm \ @@ -135,8 +175,8 @@ jobs: TWINE_PASSWORD: ${{ secrets.PyPiToken }} - name: Upload coverage to Codecov - if: steps.cfg.outputs.primary == 'yes' - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: file: ./coverage.xml fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/test-conda-build.yml b/.github/workflows/test-conda-build.yml index 5ff16a4a48..3b20679bde 100644 --- a/.github/workflows/test-conda-build.yml +++ b/.github/workflows/test-conda-build.yml @@ -14,13 +14,13 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.8", "3.9"] + python-version: ["3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Cache conda - uses: actions/cache@v3 + uses: actions/cache@v4 env: # Increase this value to reset cache if setup.py has not changed CACHE_NUMBER: 0 @@ -30,7 +30,7 @@ jobs: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('conda-environment.yml') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: environment-file: conda-environment.yml auto-update-conda: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e3a0412649..78344a62ca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/adrienverge/yamllint.git - rev: v1.30.0 + rev: v1.35.1 hooks: - id: yamllint - repo: https://github.com/pre-commit/pre-commit-hooks @@ -22,3 +22,18 @@ repos: rev: v3.0.0a5 # Use the sha / tag you want to point at hooks: - id: pylint + - repo: https://github.com/Lucas-C/pre-commit-hooks + rev: v1.5.5 + hooks: + - id: forbid-crlf + - id: remove-crlf + - id: forbid-tabs + - id: remove-tabs + args: [--whitespaces-count, '2'] + - id: insert-license + files: ./(.*).py$ + args: + - --license-filepath + - license-template.txt + - --use-current-year + - --no-extra-eol diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d634443118..f5cb28b8b1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,7 +3,7 @@ Firstly, thank you for contributing! We are very grateful for your assistance in improving the Open Data Cube. When contributing to this repository, please first discuss the change you wish to make via an issue, -Slack, or any other method with the owners of this repository before proposing a change. +Discord, or any other method with the owners of this repository before proposing a change. We have a [code of conduct](code-of-conduct.md), so please follow it in all your interactions with the project. @@ -40,5 +40,5 @@ and consider major enhancements, and there are a number of [current and past enh In case you haven't found them yet, please checkout the following resources: * [Documentation](https://datacube-core.readthedocs.io/en/latest/) -* [Slack](http://slack.opendatacube.org) +* [Discord](https://discord.com/invite/4hhBQVas5U) * [GIS Stack Exchange](https://gis.stackexchange.com/questions/tagged/open-data-cube). diff --git a/README.rst b/README.rst index d64e696dc7..3f73449158 100644 --- a/README.rst +++ b/README.rst @@ -13,6 +13,10 @@ Open Data Cube Core :alt: Documentation Status :target: http://datacube-core.readthedocs.org/en/latest/ +.. image:: https://img.shields.io/discord/1212501566326571070?label=Discord&logo=discord&logoColor=white&color=7289DA)](https://discord.gg/4hhBQVas5U + :alt: Discord + :target: https://discord.com/invite/4hhBQVas5U + Overview ======== @@ -27,7 +31,7 @@ Documentation See the `user guide `__ for installation and usage of the datacube, and for documentation of the API. -`Join our Slack `__ if you need help +`Join our Discord `__ if you need help setting up or using the Open Data Cube. Please help us to keep the Open Data Cube community open and inclusive by @@ -40,7 +44,7 @@ System ~~~~~~ - PostgreSQL 10+ -- Python 3.8+ +- Python 3.9+ Developer setup =============== @@ -49,12 +53,12 @@ Developer setup - ``git clone https://github.com/opendatacube/datacube-core.git`` -2. Create a Python environment for using the ODC. We recommend `conda `__ as the +2. Create a Python environment for using the ODC. We recommend `Mambaforge `__ as the easiest way to handle Python dependencies. :: - conda create -f conda-environment.yml + mamba env create -f conda-environment.yml conda activate cubeenv 3. Install a develop version of datacube-core. @@ -72,14 +76,19 @@ Developer setup pre-commit install 5. Run unit tests + PyLint - ``./check-code.sh`` - (this script approximates what is run by Travis. You can - alternatively run ``pytest`` yourself). Some test dependencies may need to be installed, attempt to install these using: +Install test dependencies using: ``pip install --upgrade -e '.[test]'`` - If install for these fails please lodge them as issues. +If install for these fails, please lodge them as issues. + +Run unit tests with: + + ``./check-code.sh`` + + (this script approximates what is run by GitHub Actions. You can + alternatively run ``pytest`` yourself). 6. **(or)** Run all tests, including integration tests. @@ -87,11 +96,14 @@ Developer setup - Assumes a password-less Postgres database running on localhost called - ``agdcintegration`` + ``pgintegration`` - - Otherwise copy ``integration_tests/agdcintegration.conf`` to + - Otherwise copy ``integration_tests/integration.conf`` to ``~/.datacube_integration.conf`` and edit to customise. + - For instructions on setting up a password-less Postgres database, see + the `developer setup instructions `__. + Alternatively one can use the ``opendatacube/datacube-tests`` docker image to run tests. This docker includes database server pre-configured for running @@ -103,11 +115,13 @@ to ``./check-code.sh`` script. ./check-code.sh --with-docker integration_tests -To run individual test in docker container +To run individual tests in a docker container :: - docker run -ti -v /home/ubuntu/datacube-core:/code opendatacube/datacube-tests:latest pytest integration_tests/test_filename.py::test_function_name + docker build --tag=opendatacube/datacube-tests-local --no-cache --progress plain -f docker/Dockerfile . + + docker run -ti -v $(pwd):/code opendatacube/datacube-tests-local:latest pytest integration_tests/test_filename.py::test_function_name Developer setup on Ubuntu diff --git a/conda-environment.yml b/conda-environment.yml index 960faf44b1..fb867b0fbd 100644 --- a/conda-environment.yml +++ b/conda-environment.yml @@ -4,7 +4,7 @@ channels: - defaults dependencies: - pip - - python >=3.8 + - python >=3.9 - setuptools - setuptools_scm >=3.4 - toml @@ -22,7 +22,7 @@ dependencies: - dask - pyproj >=2.5 - shapely >=2.0 - - jsonschema + - jsonschema >=4.18 - lark - netcdf4 - numpy diff --git a/datacube/__init__.py b/datacube/__init__.py index 55b5d6924b..a29faa45ee 100644 --- a/datacube/__init__.py +++ b/datacube/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Datacube diff --git a/datacube/__main__.py b/datacube/__main__.py index 8af90e515d..13352473d3 100644 --- a/datacube/__main__.py +++ b/datacube/__main__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 if __name__ == "__main__": from .config import auto_config diff --git a/datacube/api/__init__.py b/datacube/api/__init__.py index 3154c299f5..6e89320380 100644 --- a/datacube/api/__init__.py +++ b/datacube/api/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Modules for the Storage and Access Query API diff --git a/datacube/api/core.py b/datacube/api/core.py index 8d99419951..eedcb1836b 100644 --- a/datacube/api/core.py +++ b/datacube/api/core.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2021 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import uuid import collections.abc @@ -113,6 +113,14 @@ def list_products(self, with_pandas=True, dataset_count=False): :return: A table or list of every product in the datacube. :rtype: pandas.DataFrame or list(dict) """ + def _get_non_default(product, col): + load_hints = product.load_hints() + if load_hints: + if col == 'crs': + return load_hints.get('output_crs', None) + return load_hints.get(col, None) + return getattr(product.grid_spec, col, None) + # Read properties from each datacube product cols = [ 'name', @@ -125,10 +133,10 @@ def list_products(self, with_pandas=True, dataset_count=False): getattr(pr, col, None) # if 'default_crs' and 'default_resolution' are not None # return 'default_crs' and 'default_resolution' - if getattr(pr, col, None) and 'default' not in col - # else try 'grid_spec.crs' and 'grid_spec.resolution' + if getattr(pr, col, None) or 'default' not in col + # else get crs and resolution from load_hints or grid_spec # as per output_geobox() handling logic - else getattr(pr.grid_spec, col.replace('default_', ''), None) + else _get_non_default(pr, col.replace('default_', '')) for col in cols] for pr in self.index.products.get_all()] @@ -191,7 +199,7 @@ def _list_measurements(self): def load(self, product=None, measurements=None, output_crs=None, resolution=None, resampling=None, skip_broken_datasets=False, dask_chunks=None, like=None, fuse_func=None, align=None, datasets=None, dataset_predicate=None, progress_cbk=None, patch_url=None, **query): - """ + r""" Load data as an ``xarray.Dataset`` object. Each measurement will be a data variable in the :class:`xarray.Dataset`. @@ -232,12 +240,17 @@ def load(self, product=None, measurements=None, output_crs=None, resolution=None x=(1516200, 1541300), y=(-3867375, -3867350), crs='EPSG:3577' - The ``time`` dimension can be specified using a tuple of datetime objects or strings with - ``YYYY-MM-DD hh:mm:ss`` format. Data will be loaded inclusive of the start and finish times. E.g:: + The ``time`` dimension can be specified using a single or tuple of datetime objects or strings with + ``YYYY-MM-DD hh:mm:ss`` format. Data will be loaded inclusive of the start and finish times. + A ``None`` value in the range indicates an open range, with the provided date serving as either the + upper or lower bound. E.g:: time=('2000-01-01', '2001-12-31') time=('2000-01', '2001-12') time=('2000', '2001') + time=('2000') + time=('2000', None) # all data from 2000 onward + time=(None, '2000') # all data up to and including 2000 For 3D datasets, where the product definition contains an ``extra_dimension`` specification, these dimensions can be queried using that dimension's name. E.g.:: @@ -862,9 +875,16 @@ def output_geobox(like=None, output_crs=None, resolution=None, align=None, assert output_crs is None, "'like' and 'output_crs' are not supported together" assert resolution is None, "'like' and 'resolution' are not supported together" assert align is None, "'like' and 'align' are not supported together" + + # If user passes a GeoBox, return as-is if isinstance(like, GeoBox): return like + # For `odc-geo` GeoBox compatibility: use "compat" attribute if + # it exists to convert to a Datacube-style GeoBox + if isinstance(compat := getattr(like, "compat", None), GeoBox): + return compat + return like.geobox if load_hints: diff --git a/datacube/api/grid_workflow.py b/datacube/api/grid_workflow.py index d61595b38d..2f1011670e 100644 --- a/datacube/api/grid_workflow.py +++ b/datacube/api/grid_workflow.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging import xarray diff --git a/datacube/api/query.py b/datacube/api/query.py index 5bf50f351a..62d9804a45 100644 --- a/datacube/api/query.py +++ b/datacube/api/query.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Storage Query and Access API module @@ -14,7 +14,6 @@ from typing import Optional, Union import pandas -from dateutil import tz from pandas import to_datetime as pandas_to_datetime import numpy as np @@ -58,7 +57,7 @@ def __init__(self, group_by_func, dimension, units, sort_key=None, group_key=Non 'source_filter') -class Query(object): +class Query: def __init__(self, index=None, product=None, geopolygon=None, like=None, **search_terms): """Parses search terms in preparation for querying the Data Cube Index. @@ -69,7 +68,7 @@ def __init__(self, index=None, product=None, geopolygon=None, like=None, **searc Use by accessing :attr:`search_terms`: >>> query.search_terms['time'] # doctest: +NORMALIZE_WHITESPACE - Range(begin=datetime.datetime(2001, 1, 1, 0, 0, tzinfo=), \ + Range(begin=datetime.datetime(2001, 1, 1, 0, 0, tzinfo=tzutc()), \ end=datetime.datetime(2002, 1, 1, 23, 59, 59, 999999, tzinfo=tzutc())) By passing in an ``index``, the search parameters will be validated as existing on the ``product``. @@ -127,9 +126,9 @@ def __init__(self, index=None, product=None, geopolygon=None, like=None, **searc time_coord = like.coords.get('time') if time_coord is not None: self.search['time'] = _time_to_search_dims( + # convert from np.datetime64 to datetime.datetime (pandas_to_datetime(time_coord.values[0]).to_pydatetime(), - pandas_to_datetime(time_coord.values[-1]).to_pydatetime() - + datetime.timedelta(milliseconds=1)) # TODO: inclusive time searches + pandas_to_datetime(time_coord.values[-1]).to_pydatetime()) ) @property @@ -304,27 +303,9 @@ def _values_to_search(**kwargs): return search -def _to_datetime(t): - if isinstance(t, (float, int)): - t = datetime.datetime.fromtimestamp(t, tz=tz.tzutc()) - - if isinstance(t, tuple): - t = datetime.datetime(*t, tzinfo=tz.tzutc()) - elif isinstance(t, str): - try: - t = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S.%fZ") - except ValueError: - pass - elif isinstance(t, datetime.datetime): - return tz_aware(t) - - return pandas_to_datetime(t, utc=True, infer_datetime_format=True).to_pydatetime() - - def _time_to_search_dims(time_range): with warnings.catch_warnings(): warnings.simplefilter("ignore", UserWarning) - tr_start, tr_end = time_range, time_range if hasattr(time_range, '__iter__') and not isinstance(time_range, str): @@ -334,20 +315,29 @@ def _time_to_search_dims(time_range): tr_start, tr_end = tmp[0], tmp[-1] + if isinstance(tr_start, (int, float)) or isinstance(tr_end, (int, float)): + raise TypeError("Time dimension must be provided as a datetime or a string") + + if tr_start is None: + start = datetime.datetime.fromtimestamp(0) + elif not isinstance(tr_start, datetime.datetime): + # convert to datetime.datetime + if hasattr(tr_start, 'isoformat'): + tr_start = tr_start.isoformat() + start = pandas_to_datetime(tr_start).to_pydatetime() + else: + start = tr_start + + if tr_end is None: + tr_end = datetime.datetime.now().strftime("%Y-%m-%d") # Attempt conversion to isoformat - # allows pandas.Period to handle - # date and datetime objects - if hasattr(tr_start, 'isoformat'): - tr_start = tr_start.isoformat() + # allows pandas.Period to handle datetime objects if hasattr(tr_end, 'isoformat'): tr_end = tr_end.isoformat() + # get end of period to ensure range is inclusive + end = pandas.Period(tr_end).end_time.to_pydatetime() - start = _to_datetime(tr_start) - end = _to_datetime(pandas.Period(tr_end) - .end_time - .to_pydatetime()) - - tr = Range(start, end) + tr = Range(tz_aware(start), tz_aware(end)) if start == end: return tr[0] @@ -377,7 +367,7 @@ def solar_day(dataset: Dataset, longitude: Optional[float] = None) -> np.datetim :param longitude: If supplied correct timestamp for this longitude, rather than mid-point of the Dataset's footprint """ - utc = dataset.center_time + utc = dataset.center_time.astimezone(datetime.timezone.utc) if longitude is None: _lon = _ds_mid_longitude(dataset) diff --git a/datacube/config.py b/datacube/config.py index 34933d2d3d..48abf61c01 100755 --- a/datacube/config.py +++ b/datacube/config.py @@ -1,11 +1,13 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ User configuration. """ +import warnings +import re import os from pathlib import Path import configparser @@ -92,6 +94,11 @@ def __init__(self, config: configparser.ConfigParser, if env: if config.has_section(env): self._env = env + if not re.fullmatch(r'^[a-z][a-z0-9]*$', self._env): + warnings.warn(f"Configuration environment names like '{self._env}' are deprecated. " + "From datacube 1.9, environment names must start with a lowercase letter and " + "consist of only lowercase letters and digits.", + category=DeprecationWarning) # All is good return else: diff --git a/datacube/drivers/__init__.py b/datacube/drivers/__init__.py index 4544150027..4242c46c92 100644 --- a/datacube/drivers/__init__.py +++ b/datacube/drivers/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ This module implements a simple plugin manager for storage and index drivers. diff --git a/datacube/drivers/_tools.py b/datacube/drivers/_tools.py index d43e3db515..9f9c256559 100644 --- a/datacube/drivers/_tools.py +++ b/datacube/drivers/_tools.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from threading import Lock from typing import Any diff --git a/datacube/drivers/_types.py b/datacube/drivers/_types.py index f462743998..f40d5fcc99 100644 --- a/datacube/drivers/_types.py +++ b/datacube/drivers/_types.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Defines abstract types for IO drivers. """ diff --git a/datacube/drivers/datasource.py b/datacube/drivers/datasource.py index 7ad98e775a..e01faceb41 100644 --- a/datacube/drivers/datasource.py +++ b/datacube/drivers/datasource.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Defines abstract types for IO reader drivers. """ diff --git a/datacube/drivers/driver_cache.py b/datacube/drivers/driver_cache.py index 75c70b2bf5..2361e215a2 100644 --- a/datacube/drivers/driver_cache.py +++ b/datacube/drivers/driver_cache.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging from typing import Dict, Any, Tuple, Iterable @@ -26,14 +26,9 @@ def load_drivers(group: str) -> Dict[str, Any]: """ def safe_load(ep): - from pkg_resources import DistributionNotFound # pylint: disable=broad-except,bare-except try: driver_init = ep.load() - except DistributionNotFound: - # This happens when entry points were marked with extra features, - # but extra feature were not requested for installation - return None except Exception as e: _LOG.warning('Failed to resolve driver %s::%s', group, ep.name) _LOG.warning('Error was: %s', repr(e)) @@ -51,8 +46,11 @@ def safe_load(ep): return driver def resolve_all(group: str) -> Iterable[Tuple[str, Any]]: - from pkg_resources import iter_entry_points - for ep in iter_entry_points(group=group, name=None): + try: + from importlib_metadata import entry_points + except ModuleNotFoundError: + from importlib.metadata import entry_points + for ep in entry_points(group=group): driver = safe_load(ep) if driver is not None: yield (ep.name, driver) diff --git a/datacube/drivers/indexes.py b/datacube/drivers/indexes.py index 5c231b22ce..7bfb09abfa 100644 --- a/datacube/drivers/indexes.py +++ b/datacube/drivers/indexes.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import List, Optional diff --git a/datacube/drivers/netcdf/__init__.py b/datacube/drivers/netcdf/__init__.py index b0148d4ca0..dc33e2ed1c 100644 --- a/datacube/drivers/netcdf/__init__.py +++ b/datacube/drivers/netcdf/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from ._write import write_dataset_to_netcdf, create_netcdf_storage_unit from . import writer as netcdf_writer diff --git a/datacube/drivers/netcdf/_safestrings.py b/datacube/drivers/netcdf/_safestrings.py index 7ac7630a3c..a99f36bbd4 100644 --- a/datacube/drivers/netcdf/_safestrings.py +++ b/datacube/drivers/netcdf/_safestrings.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Provides `SafeStringsDataset`, a replacement netCDF4.Dataset class which works diff --git a/datacube/drivers/netcdf/_write.py b/datacube/drivers/netcdf/_write.py index 126dfb27d9..3dc40c1c08 100644 --- a/datacube/drivers/netcdf/_write.py +++ b/datacube/drivers/netcdf/_write.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from pathlib import Path import logging diff --git a/datacube/drivers/netcdf/driver.py b/datacube/drivers/netcdf/driver.py index 56910c40e8..9ec5853ba0 100644 --- a/datacube/drivers/netcdf/driver.py +++ b/datacube/drivers/netcdf/driver.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from urllib.parse import urlsplit diff --git a/datacube/drivers/netcdf/writer.py b/datacube/drivers/netcdf/writer.py index 11bbbc6bcd..23f749f2e0 100644 --- a/datacube/drivers/netcdf/writer.py +++ b/datacube/drivers/netcdf/writer.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Create netCDF4 Storage Units and write data to them diff --git a/datacube/drivers/postgis/__init__.py b/datacube/drivers/postgis/__init__.py index 50348f828d..00128c5297 100644 --- a/datacube/drivers/postgis/__init__.py +++ b/datacube/drivers/postgis/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Lower-level database access. diff --git a/datacube/drivers/postgis/_api.py b/datacube/drivers/postgis/_api.py index 052d5c6b97..a15b513394 100644 --- a/datacube/drivers/postgis/_api.py +++ b/datacube/drivers/postgis/_api.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 # We often have one-arg-per column, so these checks aren't so useful. @@ -389,9 +389,9 @@ def spatial_extent(self, ids, crs): if SpatialIndex is None: return None result = self._connection.execute( - select([ + select( func.ST_AsGeoJSON(func.ST_Union(SpatialIndex.extent)) - ]).select_from( + ).select_from( SpatialIndex ).where( SpatialIndex.dataset_ref.in_(ids) @@ -528,12 +528,12 @@ def delete_dataset(self, dataset_id): def get_dataset(self, dataset_id): return self._connection.execute( - select(_dataset_select_fields()).where(Dataset.id == dataset_id) + select(*_dataset_select_fields()).where(Dataset.id == dataset_id) ).first() def get_datasets(self, dataset_ids): return self._connection.execute( - select(_dataset_select_fields()).where(Dataset.id.in_(dataset_ids)) + select(*_dataset_select_fields()).where(Dataset.id.in_(dataset_ids)) ).fetchall() def get_derived_datasets(self, dataset_id): @@ -551,7 +551,7 @@ def search_datasets_by_metadata(self, metadata): """ # Find any storage types whose 'dataset_metadata' document is a subset of the metadata. return self._connection.execute( - select(_dataset_select_fields()).where(Dataset.metadata_doc.contains(metadata)) + select(*_dataset_select_fields()).where(Dataset.metadata_doc.contains(metadata)) ).fetchall() def search_products_by_metadata(self, metadata): @@ -622,7 +622,7 @@ def search_datasets_query(self, raw_expressions = PostgisDbAPI._alchemify_expressions(expressions) join_tables = PostgisDbAPI._join_tables(expressions, select_fields) where_expr = and_(Dataset.archived == None, *raw_expressions) - query = select(select_columns).select_from(Dataset) + query = select(*select_columns).select_from(Dataset) for joins in join_tables: query = query.join(*joins) if spatialquery is not None: @@ -664,7 +664,7 @@ def bulk_simple_dataset_search(self, products=None, batch_size=0): if batch_size > 0 and not self.in_transaction: raise ValueError("Postgresql bulk reads must occur within a transaction.") query = select( - _dataset_bulk_select_fields() + *_dataset_bulk_select_fields() ).select_from(Dataset).where( Dataset.archived == None ) @@ -706,11 +706,15 @@ def search_unique_datasets(self, expressions, select_fields=None, limit=None): def get_duplicates(self, match_fields: Sequence[PgField], expressions: Sequence[PgExpression]) -> Iterable[tuple]: # TODO + if "time" in [f.name for f in match_fields]: + return self.get_duplicates_with_time(match_fields, expressions) + group_expressions = tuple(f.alchemy_expression for f in match_fields) join_tables = PostgisDbAPI._join_tables(expressions, match_fields) + cols = (func.array_agg(Dataset.id).label('ids'),) + group_expressions query = select( - (func.array_agg(Dataset.id),) + group_expressions + *cols ).select_from(Dataset) for joins in join_tables: query = query.join(*joins) @@ -724,6 +728,56 @@ def get_duplicates(self, match_fields: Sequence[PgField], expressions: Sequence[ ) return self._connection.execute(query) + def get_duplicates_with_time( + self, match_fields: Sequence[PgField], expressions: Sequence[PgExpression] + ) -> Iterable[tuple]: + fields = [] + for f in match_fields: + if f.name == "time": + time_field = f.expression_with_leniency + else: + fields.append(f.alchemy_expression) + + join_tables = PostgisDbAPI._join_tables(expressions, match_fields) + + cols = [Dataset.id, time_field.label('time'), *fields] + query = select( + *cols + ).select_from(Dataset) + for joins in join_tables: + query = query.join(*joins) + + query = query.where( + and_(Dataset.archived == None, *(PostgisDbAPI._alchemify_expressions(expressions))) + ) + + t1 = query.alias("t1") + t2 = query.alias("t2") + + time_overlap = select( + t1.c.id, + text("t1.time * t2.time as time_intersect"), + *fields + ).select_from( + t1.join( + t2, + and_(t1.c.time.overlaps(t2.c.time), t1.c.id != t2.c.id) + ) + ) + + query = select( + func.array_agg(func.distinct(time_overlap.c.id)).label("ids"), + *fields, + text("(lower(time_intersect) at time zone 'UTC', upper(time_intersect) at time zone 'UTC') as time") + ).select_from( + time_overlap + ).group_by( + *fields, text("time_intersect") + ).having( + func.count(time_overlap.c.id) > 1 + ) + return self._connection.execute(query) + def count_datasets(self, expressions): """ :type expressions: tuple[datacube.drivers.postgis._fields.PgExpression] @@ -763,24 +817,24 @@ def count_datasets_through_time(self, start, end, period, time_field, expression def count_datasets_through_time_query(self, start, end, period, time_field, expressions): raw_expressions = self._alchemify_expressions(expressions) - start_times = select(( + start_times = select( func.generate_series(start, end, cast(period, INTERVAL)).label('start_time'), - )).alias('start_times') + ).alias('start_times') time_range_select = ( - select(( + select( func.tstzrange( start_times.c.start_time, func.lead(start_times.c.start_time).over() ).label('time_period'), - )) + ) ).alias('all_time_ranges') # Exclude the trailing (end time to infinite) row. Is there a simpler way? time_ranges = ( - select(( + select( time_range_select, - )).where( + ).where( ~func.upper_inf(time_range_select.c.time_period) ) ).alias('time_ranges') @@ -797,7 +851,7 @@ def count_datasets_through_time_query(self, start, end, period, time_field, expr ) ) - return select((time_ranges.c.time_period, count_query.label('dataset_count'))) + return select(time_ranges.c.time_period, count_query.label('dataset_count')) def update_search_index(self, product_names: Sequence[str] = [], dsids: Sequence[DSID] = []): """ diff --git a/datacube/drivers/postgis/_connections.py b/datacube/drivers/postgis/_connections.py index 6d189a65ae..474799579b 100755 --- a/datacube/drivers/postgis/_connections.py +++ b/datacube/drivers/postgis/_connections.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 # We often have one-arg-per column, so these checks aren't so useful. diff --git a/datacube/drivers/postgis/_core.py b/datacube/drivers/postgis/_core.py index b0922bd652..15beda3138 100644 --- a/datacube/drivers/postgis/_core.py +++ b/datacube/drivers/postgis/_core.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Core SQL schema settings. @@ -11,6 +11,7 @@ from sqlalchemy import MetaData, inspect, text from sqlalchemy.engine import Engine from sqlalchemy.schema import CreateSchema +from sqlalchemy.sql.ddl import DropSchema from datacube.drivers.postgis.sql import (INSTALL_TRIGGER_SQL_TEMPLATE, SCHEMA_NAME, TYPES_INIT_SQL, @@ -225,7 +226,9 @@ def has_schema(engine): def drop_db(connection): - connection.execute(text(f'drop schema if exists {SCHEMA_NAME} cascade')) + # if_exists parameter seems to not be working in SQLA1.4? + if has_schema(connection.engine): + connection.execute(DropSchema(SCHEMA_NAME, cascade=True, if_exists=True)) def to_pg_role(role): diff --git a/datacube/drivers/postgis/_fields.py b/datacube/drivers/postgis/_fields.py index c0066d4149..760e608c3a 100755 --- a/datacube/drivers/postgis/_fields.py +++ b/datacube/drivers/postgis/_fields.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Build and index fields within documents. @@ -16,6 +16,7 @@ from sqlalchemy import cast, func, and_ from sqlalchemy.dialects import postgresql as postgres from sqlalchemy.dialects.postgresql import NUMRANGE, TSTZRANGE +from sqlalchemy.dialects.postgresql import INTERVAL from sqlalchemy.sql import ColumnElement from sqlalchemy.orm import aliased @@ -223,7 +224,7 @@ def __init__(self, name, description, alchemy_column, indexed, offset=None, sele @property def alchemy_expression(self): - return self._alchemy_offset_value(self.offset, self.aggregation.pg_calc) + return self._alchemy_offset_value(self.offset, self.aggregation.pg_calc).label(self.name) def __eq__(self, value): """ @@ -360,7 +361,7 @@ def value_to_alchemy(self, value): @property def alchemy_expression(self): - return self.value_to_alchemy((self.lower.alchemy_expression, self.greater.alchemy_expression)) + return self.value_to_alchemy((self.lower.alchemy_expression, self.greater.alchemy_expression)).label(self.name) def __eq__(self, value): """ @@ -439,6 +440,16 @@ def between(self, low, high): raise ValueError("Unknown comparison type for date range: " "expecting datetimes, got: (%r, %r)" % (low, high)) + @property + def expression_with_leniency(self): + return func.tstzrange( + self.lower.alchemy_expression - cast('500 milliseconds', INTERVAL), + self.greater.alchemy_expression + cast('500 milliseconds', INTERVAL), + # Inclusive on both sides. + '[]', + type_=TSTZRANGE, + ) + def _number_implies_year(v: Union[int, datetime]) -> datetime: """ diff --git a/datacube/drivers/postgis/_schema.py b/datacube/drivers/postgis/_schema.py index 1aa812e353..ce2c5b6769 100644 --- a/datacube/drivers/postgis/_schema.py +++ b/datacube/drivers/postgis/_schema.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Tables for indexing the datasets which were ingested into the AGDC. diff --git a/datacube/drivers/postgis/_spatial.py b/datacube/drivers/postgis/_spatial.py index 7481745a01..90b20e89cf 100644 --- a/datacube/drivers/postgis/_spatial.py +++ b/datacube/drivers/postgis/_spatial.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Tracking spatial indexes diff --git a/datacube/drivers/postgis/sql.py b/datacube/drivers/postgis/sql.py index 321f97b9d7..5d050ad8f0 100644 --- a/datacube/drivers/postgis/sql.py +++ b/datacube/drivers/postgis/sql.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Custom types for postgres & sqlalchemy diff --git a/datacube/drivers/postgres/__init__.py b/datacube/drivers/postgres/__init__.py index 6fe10be3c7..7ba8061e40 100644 --- a/datacube/drivers/postgres/__init__.py +++ b/datacube/drivers/postgres/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Lower-level database access. diff --git a/datacube/drivers/postgres/_api.py b/datacube/drivers/postgres/_api.py index f0d19e992c..7b616bb5e7 100644 --- a/datacube/drivers/postgres/_api.py +++ b/datacube/drivers/postgres/_api.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 # We often have one-arg-per column, so these checks aren't so useful. @@ -725,10 +725,13 @@ def search_unique_datasets(self, expressions, select_fields=None, limit=None): return self._connection.execute(select_query) def get_duplicates(self, match_fields: Iterable[PgField], expressions: Iterable[PgExpression]) -> Iterable[Tuple]: + if "time" in [f.name for f in match_fields]: + return self.get_duplicates_with_time(match_fields, expressions) + group_expressions = tuple(f.alchemy_expression for f in match_fields) select_query = select( - (func.array_agg(DATASET.c.id),) + group_expressions + (func.array_agg(DATASET.c.id).label('ids'),) + group_expressions ).select_from( PostgresDbAPI._from_expression(DATASET, expressions, match_fields) ).where( @@ -740,6 +743,61 @@ def get_duplicates(self, match_fields: Iterable[PgField], expressions: Iterable[ ) return self._connection.execute(select_query) + def get_duplicates_with_time( + self, match_fields: Iterable[PgField], expressions: Iterable[PgExpression] + ) -> Iterable[Tuple]: + """ + If considering time when searching for duplicates, we need to grant some amount of leniency + in case timestamps are not exactly the same. + From the set of datasets that are active and have the correct product (candidates), + find all those whose extended timestamp range overlap (overlapping), + then group them by the other fields. + """ + fields = [] + for f in match_fields: + if f.name == "time": + time_field = f.expression_with_leniency + else: + fields.append(f.alchemy_expression) + + candidates_table = select( + DATASET.c.id, + time_field.label('time'), + *fields + ).select_from( + PostgresDbAPI._from_expression(DATASET, expressions, match_fields) + ).where( + and_(DATASET.c.archived == None, *(PostgresDbAPI._alchemify_expressions(expressions))) + ) + + t1 = candidates_table.alias("t1") + t2 = candidates_table.alias("t2") + + overlapping = select( + t1.c.id, + text("t1.time * t2.time as time_intersect"), + *fields + ).select_from( + t1.join( + t2, + and_(t1.c.time.overlaps(t2.c.time), t1.c.id != t2.c.id) + ) + ) + + final_query = select( + func.array_agg(func.distinct(overlapping.c.id)).label("ids"), + *fields, + text("(lower(time_intersect) at time zone 'UTC', upper(time_intersect) at time zone 'UTC') as time") + ).select_from( + overlapping + ).group_by( + *fields, text("time_intersect") + ).having( + func.count(overlapping.c.id) > 1 + ) + + return self._connection.execute(final_query) + def count_datasets(self, expressions): """ :type expressions: tuple[datacube.drivers.postgres._fields.PgExpression] diff --git a/datacube/drivers/postgres/_connections.py b/datacube/drivers/postgres/_connections.py index 68f5d080a4..987513d7c8 100755 --- a/datacube/drivers/postgres/_connections.py +++ b/datacube/drivers/postgres/_connections.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 # We often have one-arg-per column, so these checks aren't so useful. diff --git a/datacube/drivers/postgres/_core.py b/datacube/drivers/postgres/_core.py index 52cd926966..31e562a5de 100644 --- a/datacube/drivers/postgres/_core.py +++ b/datacube/drivers/postgres/_core.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Core SQL schema settings. @@ -17,7 +17,7 @@ pg_column_exists) from sqlalchemy import MetaData, inspect, text from sqlalchemy.engine import Engine -from sqlalchemy.schema import CreateSchema +from sqlalchemy.schema import CreateSchema, DropSchema USER_ROLES = ('agdc_user', 'agdc_ingest', 'agdc_manage', 'agdc_admin') @@ -105,7 +105,7 @@ def ensure_db(engine, with_permissions=True): # Switch to 'agdc_admin', so that all items are owned by them. c.execute(text('set role agdc_admin')) _LOG.info('Creating schema.') - c.execute(CreateSchema(SCHEMA_NAME)) + c.execute(CreateSchema(SCHEMA_NAME, if_not_exists=True)) _LOG.info('Creating tables.') c.execute(text(TYPES_INIT_SQL)) METADATA.create_all(c) @@ -239,7 +239,9 @@ def has_schema(engine): def drop_db(connection): - connection.execute(text(f'drop schema if exists {SCHEMA_NAME} cascade;')) + # if_exists parameter seems to not be working in SQLA1.4? + if has_schema(connection.engine): + connection.execute(DropSchema(SCHEMA_NAME, cascade=True, if_exists=True)) def to_pg_role(role): diff --git a/datacube/drivers/postgres/_dynamic.py b/datacube/drivers/postgres/_dynamic.py index d7401038ba..959dbceea5 100644 --- a/datacube/drivers/postgres/_dynamic.py +++ b/datacube/drivers/postgres/_dynamic.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Methods for managing dynamic dataset field indexes and views. diff --git a/datacube/drivers/postgres/_fields.py b/datacube/drivers/postgres/_fields.py index 8e58bb020f..41c33e1859 100755 --- a/datacube/drivers/postgres/_fields.py +++ b/datacube/drivers/postgres/_fields.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Build and index fields within documents. @@ -15,6 +15,7 @@ from sqlalchemy.dialects import postgresql as postgres from sqlalchemy.dialects.postgresql import INT4RANGE from sqlalchemy.dialects.postgresql import NUMRANGE, TSTZRANGE +from sqlalchemy.dialects.postgresql import INTERVAL from sqlalchemy.sql import ColumnElement from datacube import utils @@ -193,7 +194,7 @@ def __init__(self, name, description, alchemy_column, indexed, offset=None, sele @property def alchemy_expression(self): - return self._alchemy_offset_value(self.offset, self.aggregation.pg_calc) + return self._alchemy_offset_value(self.offset, self.aggregation.pg_calc).label(self.name) def __eq__(self, value): """ @@ -322,7 +323,7 @@ def postgres_index_type(self): @property def alchemy_expression(self): - return self.value_to_alchemy((self.lower.alchemy_expression, self.greater.alchemy_expression)) + return self.value_to_alchemy((self.lower.alchemy_expression, self.greater.alchemy_expression)).label(self.name) def __eq__(self, value): """ @@ -431,6 +432,16 @@ def between(self, low, high): raise ValueError("Unknown comparison type for date range: " "expecting datetimes, got: (%r, %r)" % (low, high)) + @property + def expression_with_leniency(self): + return func.tstzrange( + self.lower.alchemy_expression - cast('500 milliseconds', INTERVAL), + self.greater.alchemy_expression + cast('500 milliseconds', INTERVAL), + # Inclusive on both sides. + '[]', + type_=TSTZRANGE, + ) + def _number_implies_year(v: Union[int, datetime]) -> datetime: """ diff --git a/datacube/drivers/postgres/_schema.py b/datacube/drivers/postgres/_schema.py index 56bedf3ed6..85ab8ad81c 100644 --- a/datacube/drivers/postgres/_schema.py +++ b/datacube/drivers/postgres/_schema.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Tables for indexing the datasets which were ingested into the AGDC. diff --git a/datacube/drivers/postgres/sql.py b/datacube/drivers/postgres/sql.py index 5801d95edb..e476fdb02f 100644 --- a/datacube/drivers/postgres/sql.py +++ b/datacube/drivers/postgres/sql.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Custom types for postgres & sqlalchemy diff --git a/datacube/drivers/readers.py b/datacube/drivers/readers.py index 16d0f9e204..55fce98183 100644 --- a/datacube/drivers/readers.py +++ b/datacube/drivers/readers.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import List, Optional, Callable from .driver_cache import load_drivers diff --git a/datacube/drivers/rio/__init__.py b/datacube/drivers/rio/__init__.py index b74ea31e59..7b1432acec 100644 --- a/datacube/drivers/rio/__init__.py +++ b/datacube/drivers/rio/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ RasterIO based driver """ diff --git a/datacube/drivers/rio/_reader.py b/datacube/drivers/rio/_reader.py index fe99c641b7..ab3b666c99 100644 --- a/datacube/drivers/rio/_reader.py +++ b/datacube/drivers/rio/_reader.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ reader """ diff --git a/datacube/drivers/writers.py b/datacube/drivers/writers.py index 970eede77a..a6a99c77ee 100644 --- a/datacube/drivers/writers.py +++ b/datacube/drivers/writers.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import List diff --git a/datacube/execution/__init__.py b/datacube/execution/__init__.py index c081ad5b41..8318282aea 100644 --- a/datacube/execution/__init__.py +++ b/datacube/execution/__init__.py @@ -1,4 +1,4 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/datacube/execution/worker.py b/datacube/execution/worker.py index 9cd20faf6c..ed4a115d50 100644 --- a/datacube/execution/worker.py +++ b/datacube/execution/worker.py @@ -1,12 +1,13 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ This app launches workers for distributed work loads """ import click +from deprecat import deprecat KNOWN_WORKER_TYPES = ['distributed', 'dask', 'celery'] @@ -25,11 +26,7 @@ def parse_executor_opt(ctx, param, value): return ex_type, host, port -def launch_celery_worker(host, port, nprocs, password=''): - from datacube import _celery_runner as cr - cr.launch_worker(host, port, password=password, nprocs=nprocs) - - +@deprecat(reason="Executors have been deprecated and will be removed in v1.9", version='1.8.14') def launch_distributed_worker(host, port, nprocs, nthreads=1): import subprocess @@ -45,13 +42,13 @@ def launch_distributed_worker(host, port, nprocs, nthreads=1): @click.command(name='worker') @click.option('--executor', type=(click.Choice(KNOWN_WORKER_TYPES), str), # type: ignore - help="(distributed|dask(alias for distributed)|celery) host:port", + help="WARNING: executors have been deprecated in v1.8.14, and will be removed in v1.9.\n" + "(distributed|dask(alias for distributed)) host:port", default=(None, None), callback=parse_executor_opt) @click.option('--nprocs', type=int, default=0, help='Number of worker processes to launch') def main(executor, nprocs): - launchers = dict(celery=launch_celery_worker, - dask=launch_distributed_worker, + launchers = dict(dask=launch_distributed_worker, distributed=launch_distributed_worker) ex_type, host, port = executor return launchers[ex_type](host, port, nprocs) diff --git a/datacube/executor.py b/datacube/executor.py index 9a86b28977..5a6603b280 100644 --- a/datacube/executor.py +++ b/datacube/executor.py @@ -1,14 +1,16 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 # # type: ignore import sys +from deprecat import deprecat _REMOTE_LOG_FORMAT_STRING = '%(asctime)s {} %(process)d %(name)s %(levelname)s %(message)s' +@deprecat(reason="Executors have been deprecated and will be removed in v1.9", version='1.8.14') class SerialExecutor(object): def __repr__(self): return 'SerialExecutor' @@ -73,6 +75,7 @@ def setup_logging(): logging.root.handlers = [handler] +@deprecat(reason="Executors have been deprecated and will be removed in v1.9", version='1.8.14') def _get_distributed_executor(scheduler): """ :param scheduler: Address of a scheduler @@ -144,6 +147,7 @@ def _run_cloud_pickled_function(f_data, *args, **kwargs): return func(*args, **kwargs) +@deprecat(reason="Executors have been deprecated and will be removed in v1.9", version='1.8.14') def _get_concurrent_executor(workers, use_cloud_pickle=False): try: from concurrent.futures import ProcessPoolExecutor, as_completed @@ -221,6 +225,7 @@ def release(future): return MultiprocessingExecutor(ProcessPoolExecutor(workers), use_cloud_pickle) +@deprecat(reason="Executors have been deprecated and will be removed in v1.9", version='1.8.14') def get_executor(scheduler, workers, use_cloud_pickle=True): """ Return a task executor based on input parameters. Falling back as required. diff --git a/datacube/helpers.py b/datacube/helpers.py index cf9c3da5b1..e2e17329b8 100644 --- a/datacube/helpers.py +++ b/datacube/helpers.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Useful functions for Datacube users diff --git a/datacube/index/__init__.py b/datacube/index/__init__.py index 1c4050cf53..b53f7aa83a 100644 --- a/datacube/index/__init__.py +++ b/datacube/index/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Modules for interfacing with the index/database. diff --git a/datacube/index/_api.py b/datacube/index/_api.py index abd7e4c656..3aa8e31e24 100644 --- a/datacube/index/_api.py +++ b/datacube/index/_api.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Access methods for indexing datasets & products. diff --git a/datacube/index/abstract.py b/datacube/index/abstract.py index 69a6842bfe..f7cc2c65db 100644 --- a/datacube/index/abstract.py +++ b/datacube/index/abstract.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import datetime import logging @@ -16,12 +16,13 @@ NamedTuple, Optional, Tuple, Union, Sequence) from uuid import UUID +from datetime import timedelta from datacube.config import LocalConfig from datacube.index.exceptions import TransactionException from datacube.index.fields import Field -from datacube.model import Dataset, MetadataType, Range -from datacube.model import DatasetType as Product +from datacube.model import Dataset, MetadataType, Range, Not +from datacube.model import Product from datacube.utils import cached_property, jsonify_document, read_documents, InvalidDocException from datacube.utils.changes import AllowPolicy, Change, Offset, DocumentMismatchError, check_doc_unchanged from datacube.utils.generic import thread_local_cache @@ -373,7 +374,7 @@ def get_all_docs(self) -> Iterable[Mapping[str, Any]]: yield mdt.definition -QueryField = Union[str, float, int, Range, datetime.datetime] +QueryField = Union[str, float, int, Range, datetime.datetime, Not] QueryDict = Mapping[str, QueryField] @@ -687,7 +688,7 @@ def search(self, **query: QueryField) -> Iterator[Product]: @abstractmethod def search_robust(self, **query: QueryField - ) -> Iterable[Tuple[Product, Mapping[str, QueryField]]]: + ) -> Iterable[Tuple[Product, QueryDict]]: """ Return dataset types that match match-able fields and dict of remaining un-matchable fields. @@ -697,7 +698,7 @@ def search_robust(self, @abstractmethod def search_by_metadata(self, - metadata: Mapping[str, QueryField] + metadata: QueryDict ) -> Iterable[Dataset]: """ Perform a search using arbitrary metadata, returning results as Product objects. @@ -825,7 +826,8 @@ def bulk_has(self, ids_: Iterable[DSID]) -> Iterable[bool]: @abstractmethod def add(self, dataset: Dataset, - with_lineage: bool = True + with_lineage: bool = True, + archive_less_mature: Optional[int] = None, ) -> Dataset: """ Add ``dataset`` to the index. No-op if it is already present. @@ -837,6 +839,10 @@ def add(self, dataset: Dataset, - ``False`` record lineage relations, but do not attempt adding lineage datasets to the db + :param archive_less_mature: if integer, search for less + mature versions of the dataset with the int value as a millisecond + delta in timestamp comparison + :return: Persisted Dataset model """ @@ -874,12 +880,14 @@ def can_update(self, @abstractmethod def update(self, dataset: Dataset, - updates_allowed: Optional[Mapping[Offset, AllowPolicy]] = None + updates_allowed: Optional[Mapping[Offset, AllowPolicy]] = None, + archive_less_mature: Optional[int] = None, ) -> Dataset: """ Update dataset metadata and location :param Dataset dataset: Dataset model with unpersisted updates :param updates_allowed: Allowed updates + :param archive_less_mature: Find and archive less mature datasets with ms delta :return: Persisted dataset model """ @@ -891,6 +899,92 @@ def archive(self, ids: Iterable[DSID]) -> None: :param Iterable[Union[str,UUID]] ids: list of dataset ids to archive """ + def archive_less_mature(self, ds: Dataset, delta: Union[int, bool] = 500) -> None: + """ + Archive less mature versions of a dataset + + :param Dataset ds: dataset to search + :param Union[int,bool] delta: millisecond delta for time range. + If True, default to 500ms. If False, do not find or archive less mature datasets. + Bool value accepted only for improving backwards compatibility, int preferred. + """ + less_mature = self.find_less_mature(ds, delta) + less_mature_ids = map(lambda x: x.id, less_mature) + + self.archive(less_mature_ids) + for lm_ds in less_mature_ids: + _LOG.info(f"Archived less mature dataset: {lm_ds}") + + def find_less_mature(self, ds: Dataset, delta: Union[int, bool] = 500) -> Iterable[Dataset]: + """ + Find less mature versions of a dataset + + :param Dataset ds: Dataset to search + :param Union[int,bool] delta: millisecond delta for time range. + If True, default to 500ms. If None or False, do not find or archive less mature datasets. + Bool value accepted only for improving backwards compatibility, int preferred. + :return: Iterable of less mature datasets + """ + if isinstance(delta, bool): + _LOG.warning("received delta as a boolean value. Int is prefered") + if delta is True: # treat True as default + delta = 500 + else: # treat False the same as None + return [] + elif isinstance(delta, int): + if delta < 0: + raise ValueError("timedelta must be a positive integer") + elif delta is None: + return [] + else: + raise TypeError("timedelta must be None, a positive integer, or a boolean") + + def check_maturity_information(dataset, props): + # check that the dataset metadata includes all maturity-related properties + # passing in the required props to enable greater extensibility should it be needed + for prop in props: + if hasattr(dataset.metadata, prop) and (getattr(dataset.metadata, prop) is not None): + return + raise ValueError( + f"Dataset {dataset.id} is missing property {prop} required for maturity check" + ) + + check_maturity_information(ds, ["region_code", "time", "dataset_maturity"]) + + # 'expand' the date range by `delta` milliseconds to give a bit more leniency in datetime comparison + expanded_time_range = Range(ds.metadata.time.begin - timedelta(milliseconds=delta), + ds.metadata.time.end + timedelta(milliseconds=delta)) + dupes = self.search(product=ds.product.name, + region_code=ds.metadata.region_code, + time=expanded_time_range) + + less_mature = [] + for dupe in dupes: + if dupe.id == ds.id: + continue + + # only need to check that dupe has dataset maturity, missing/null region_code and time + # would already have been filtered out during the search query + check_maturity_information(dupe, ["dataset_maturity"]) + + if dupe.metadata.dataset_maturity == ds.metadata.dataset_maturity: + # Duplicate has the same maturity, which one should be archived is unclear + raise ValueError( + f"A dataset with the same maturity as dataset {ds.id} already exists, " + f"with id: {dupe.id}" + ) + + if dupe.metadata.dataset_maturity < ds.metadata.dataset_maturity: + # Duplicate is more mature than dataset + # Note that "final" < "nrt" + raise ValueError( + f"A more mature version of dataset {ds.id} already exists, with id: " + f"{dupe.id} and maturity: {dupe.metadata.dataset_maturity}" + ) + + less_mature.append(dupe) + return less_mature + @abstractmethod def restore(self, ids: Iterable[DSID]) -> None: """ @@ -1021,7 +1115,7 @@ def restore_location(self, @abstractmethod def search_by_metadata(self, - metadata: Mapping[str, QueryField] + metadata: QueryDict ) -> Iterable[Dataset]: """ Perform a search using arbitrary metadata, returning results as Dataset objects. @@ -1035,7 +1129,7 @@ def search_by_metadata(self, @abstractmethod def search(self, limit: Optional[int] = None, - source_filter: Optional[Mapping[str, QueryField]] = None, + source_filter: Optional[QueryDict] = None, **query: QueryField) -> Iterable[Dataset]: """ Perform a search, returning results as Dataset objects. diff --git a/datacube/index/eo3.py b/datacube/index/eo3.py index c393c318f6..e25ceb676a 100644 --- a/datacube/index/eo3.py +++ b/datacube/index/eo3.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 # # type: ignore diff --git a/datacube/index/exceptions.py b/datacube/index/exceptions.py index e75a4135ae..31a75f1d4f 100644 --- a/datacube/index/exceptions.py +++ b/datacube/index/exceptions.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/datacube/index/fields.py b/datacube/index/fields.py index 5c2c0fe7f8..2ee64acece 100644 --- a/datacube/index/fields.py +++ b/datacube/index/fields.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Common datatypes for DB drivers. @@ -10,7 +10,7 @@ from dateutil.tz import tz from typing import List -from datacube.model import Range +from datacube.model import Range, Not from datacube.model.fields import Expression, Field __all__ = ['Field', @@ -36,14 +36,26 @@ def evaluate(self, ctx): return any(expr.evaluate(ctx) for expr in self.exprs) +class NotExpression(Expression): + def __init__(self, expr): + super(NotExpression, self).__init__() + self.expr = expr + self.field = expr.field + + def evaluate(self, ctx): + return not self.expr.evaluate(ctx) + + def as_expression(field: Field, value) -> Expression: """ - Convert a single field/value to expression, following the "simple" convensions. + Convert a single field/value to expression, following the "simple" conventions. """ if isinstance(value, Range): return field.between(value.begin, value.end) elif isinstance(value, list): return OrExpression(*(as_expression(field, val) for val in value)) + elif isinstance(value, Not): + return NotExpression(as_expression(field, value.value)) # Treat a date (day) as a time range. elif isinstance(value, date) and not isinstance(value, datetime): return as_expression( diff --git a/datacube/index/hl.py b/datacube/index/hl.py index e30d02d668..ea8f789e81 100644 --- a/datacube/index/hl.py +++ b/datacube/index/hl.py @@ -1,10 +1,12 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ High level indexing operations/utilities """ +import logging + import json import toolz from uuid import UUID @@ -17,6 +19,8 @@ from datacube.utils.changes import get_doc_changes from .eo3 import prep_eo3, is_doc_eo3, is_doc_geo # type: ignore[attr-defined] +_LOG = logging.getLogger(__name__) + class ProductRule: def __init__(self, product: Product, signature: Mapping[str, Any]): @@ -63,11 +67,10 @@ def match(doc: Mapping[str, Any]) -> bool: if matches(doc, rule): return rule.product - relevant_doc = {k: v for k, v in doc.items() if k in rule.signature} raise BadMatch('Dataset metadata did not match product signature.' '\nDataset definition:\n %s\n' '\nProduct signature:\n %s\n' - % (json.dumps(relevant_doc, indent=4), + % (json.dumps(doc, indent=4), json.dumps(rule.signature, indent=4))) return match @@ -149,6 +152,13 @@ def dataset_resolver(index: AbstractIndex, skip_lineage: bool = False) -> Callable[[SimpleDocNav, str], DatasetOrError]: match_product = product_matcher(product_matching_rules) + def check_intended_eo3(ds: SimpleDocNav, product: Product) -> None: + # warn if it looks like dataset was meant to be eo3 but is not + if not is_doc_eo3(ds.doc) and ("eo3" in product.metadata_type.name): + _LOG.warning(f"Dataset {ds.id} has a product with an eo3 metadata type, " + "but the dataset definition does not include the $schema field " + "and so will not be recognised as an eo3 dataset.") + def resolve_no_lineage(ds: SimpleDocNav, uri: str) -> DatasetOrError: doc = ds.doc_without_lineage_sources try: @@ -156,6 +166,7 @@ def resolve_no_lineage(ds: SimpleDocNav, uri: str) -> DatasetOrError: except BadMatch as e: return None, e + check_intended_eo3(ds, product) return Dataset(product, doc, uris=[uri], sources={}), None def resolve(main_ds_doc: SimpleDocNav, uri: str) -> DatasetOrError: @@ -223,6 +234,7 @@ def resolve_ds(ds: SimpleDocNav, else: product = match_product(doc) + check_intended_eo3(ds, product) return with_cache(Dataset(product, doc, uris=uris, sources=sources), ds.id, cache) try: return remap_lineage_doc(main_ds, resolve_ds, cache={}), None diff --git a/datacube/index/memory/__init__.py b/datacube/index/memory/__init__.py index bea658a528..a642d3c362 100644 --- a/datacube/index/memory/__init__.py +++ b/datacube/index/memory/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/datacube/index/memory/_datasets.py b/datacube/index/memory/_datasets.py index fafb628a50..8c20c45f3f 100755 --- a/datacube/index/memory/_datasets.py +++ b/datacube/index/memory/_datasets.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import datetime import logging @@ -72,7 +72,8 @@ def bulk_has(self, ids_: Iterable[DSID]) -> Iterable[bool]: return (self.has(id_) for id_ in ids_) def add(self, dataset: Dataset, - with_lineage: bool = True) -> Dataset: + with_lineage: bool = True, + archive_less_mature: bool = False) -> Dataset: if with_lineage is None: with_lineage = True _LOG.info('indexing %s', dataset.id) @@ -100,6 +101,8 @@ def add(self, dataset: Dataset, self.by_product[dataset.product.name].append(dataset.id) else: self.by_product[dataset.product.name] = [dataset.id] + if archive_less_mature: + _LOG.warning("archive-less-mature functionality is not implemented for memory driver") return cast(Dataset, self.get(dataset.id)) def persist_source_relationship(self, ds: Dataset, src: Dataset, classifier: str) -> None: @@ -128,10 +131,11 @@ def search_product_duplicates(self, product: Product, *args: Union[str, Field] ) -> Iterable[Tuple[Tuple, Iterable[UUID]]]: - field_names: List[str] = [arg.name if isinstance(arg, Field) else arg for arg in args] - # Typing note: mypy cannot handle dynamically created namedtuples - GroupedVals = namedtuple('search_result', field_names) # type: ignore[misc] - + """ + Find dataset ids of a given product that have duplicates of the given set of field names. + Returns each set of those field values and the datasets that have them. + Note that this implementation does not account for slight timestamp discrepancies. + """ def to_field(f: Union[str, Field]) -> Field: if isinstance(f, str): f = product.metadata_type.dataset_fields[f] @@ -139,6 +143,8 @@ def to_field(f: Union[str, Field]) -> Field: return f fields = [to_field(f) for f in args] + # Typing note: mypy cannot handle dynamically created namedtuples + GroupedVals = namedtuple('search_result', list(f.name for f in fields)) # type: ignore[misc] def values(ds: Dataset) -> GroupedVals: vals = [] @@ -146,16 +152,17 @@ def values(ds: Dataset) -> GroupedVals: vals.append(field.extract(ds.metadata_doc)) # type: ignore[attr-defined] return GroupedVals(*vals) - dups: Dict[Tuple, List[UUID]] = {} + dups: Dict[Tuple, set[UUID]] = {} for ds in self.active_by_id.values(): if ds.product.name != product.name: continue vals = values(ds) if vals in dups: - dups[vals].append(ds.id) + dups[vals].add(ds.id) else: - dups[vals] = [ds.id] - return list(dups.items()) + dups[vals] = set([ds.id]) # avoid duplicate entries + # only return entries with more than one dataset + return list({k: v for k, v in dups.items() if len(v) > 1}) def can_update(self, dataset: Dataset, @@ -186,7 +193,8 @@ def can_update(self, def update(self, dataset: Dataset, - updates_allowed: Optional[Mapping[Offset, AllowPolicy]] = None + updates_allowed: Optional[Mapping[Offset, AllowPolicy]] = None, + archive_less_mature: bool = False ) -> Dataset: existing = self.get(dataset.id) if not existing: diff --git a/datacube/index/memory/_fields.py b/datacube/index/memory/_fields.py index 1ad6a7cd63..bc61e56830 100644 --- a/datacube/index/memory/_fields.py +++ b/datacube/index/memory/_fields.py @@ -1,3 +1,7 @@ +# This file is part of the Open Data Cube, see https://opendatacube.org for more information +# +# Copyright (c) 2015-2024 ODC Contributors +# SPDX-License-Identifier: Apache-2.0 from typing import Any, Mapping, MutableMapping from datacube.model.fields import SimpleField, Field, get_dataset_fields as generic_get_dataset_fields from datacube.index.abstract import Offset diff --git a/datacube/index/memory/_metadata_types.py b/datacube/index/memory/_metadata_types.py index 83501d7cbd..5893252cbe 100644 --- a/datacube/index/memory/_metadata_types.py +++ b/datacube/index/memory/_metadata_types.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging from copy import deepcopy diff --git a/datacube/index/memory/_products.py b/datacube/index/memory/_products.py index bad72c5371..9f63f95443 100644 --- a/datacube/index/memory/_products.py +++ b/datacube/index/memory/_products.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging diff --git a/datacube/index/memory/_users.py b/datacube/index/memory/_users.py index 6e597853f1..26c81253c9 100644 --- a/datacube/index/memory/_users.py +++ b/datacube/index/memory/_users.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import Iterable, Optional, Tuple from datacube.index.abstract import AbstractUserResource diff --git a/datacube/index/memory/index.py b/datacube/index/memory/index.py index 1a1385ffe3..f4436ad868 100644 --- a/datacube/index/memory/index.py +++ b/datacube/index/memory/index.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging from threading import Lock diff --git a/datacube/index/null/__init__.py b/datacube/index/null/__init__.py index bea658a528..a642d3c362 100644 --- a/datacube/index/null/__init__.py +++ b/datacube/index/null/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/datacube/index/null/_datasets.py b/datacube/index/null/_datasets.py index f8686669d0..0fb24879b7 100755 --- a/datacube/index/null/_datasets.py +++ b/datacube/index/null/_datasets.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from datacube.index.abstract import AbstractDatasetResource, DSID @@ -28,7 +28,8 @@ def bulk_has(self, ids_): return [False for id_ in ids_] def add(self, dataset: Dataset, - with_lineage: bool = True) -> Dataset: + with_lineage: bool = True, + archive_less_mature: bool = False) -> Dataset: raise NotImplementedError() def search_product_duplicates(self, product: DatasetType, *args): @@ -37,7 +38,7 @@ def search_product_duplicates(self, product: DatasetType, *args): def can_update(self, dataset, updates_allowed=None): raise NotImplementedError() - def update(self, dataset: Dataset, updates_allowed=None): + def update(self, dataset: Dataset, updates_allowed=None, archive_less_mature=False): raise NotImplementedError() def archive(self, ids): diff --git a/datacube/index/null/_metadata_types.py b/datacube/index/null/_metadata_types.py index 3346203f4f..8c8fb213c0 100644 --- a/datacube/index/null/_metadata_types.py +++ b/datacube/index/null/_metadata_types.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from datacube.index.abstract import AbstractMetadataTypeResource diff --git a/datacube/index/null/_products.py b/datacube/index/null/_products.py index 6e4f3892b2..30ab28c91c 100644 --- a/datacube/index/null/_products.py +++ b/datacube/index/null/_products.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging diff --git a/datacube/index/null/_users.py b/datacube/index/null/_users.py index cc91c5dae0..e76b4704e4 100644 --- a/datacube/index/null/_users.py +++ b/datacube/index/null/_users.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import Iterable, Optional, Tuple from datacube.index.abstract import AbstractUserResource diff --git a/datacube/index/null/index.py b/datacube/index/null/index.py index a678f7998a..dff0a47125 100644 --- a/datacube/index/null/index.py +++ b/datacube/index/null/index.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging diff --git a/datacube/index/postgis/__init__.py b/datacube/index/postgis/__init__.py index e69de29bb2..8318282aea 100644 --- a/datacube/index/postgis/__init__.py +++ b/datacube/index/postgis/__init__.py @@ -0,0 +1,4 @@ +# This file is part of the Open Data Cube, see https://opendatacube.org for more information +# +# Copyright (c) 2015-2024 ODC Contributors +# SPDX-License-Identifier: Apache-2.0 diff --git a/datacube/index/postgis/_datasets.py b/datacube/index/postgis/_datasets.py index 8a7fdcb9e0..523964b95b 100755 --- a/datacube/index/postgis/_datasets.py +++ b/datacube/index/postgis/_datasets.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ API for dataset indexing, access and search. @@ -137,7 +137,7 @@ def bulk_has(self, ids_): map((lambda x: UUID(x) if isinstance(x, str) else x), ids_)] def add(self, dataset: Dataset, - with_lineage: bool = True) -> Dataset: + with_lineage: bool = True, archive_less_mature: Optional[int] = None) -> Dataset: """ Add ``dataset`` to the index. No-op if it is already present. @@ -148,6 +148,10 @@ def add(self, dataset: Dataset, - ``False`` record lineage relations, but do not attempt adding lineage datasets to the db + :param archive_less_mature: if integer, search for less + mature versions of the dataset with the int value as a millisecond + delta in timestamp comparison + :rtype: Dataset """ @@ -161,7 +165,12 @@ def add(self, dataset: Dataset, return dataset with self._db_connection(transaction=True) as transaction: # 1a. insert (if not already exists) - is_new = transaction.insert_dataset(dataset.metadata_doc_without_lineage(), dataset.id, dataset.product.id) + product_id = dataset.product.id + if product_id is None: + # don't assume the product has an id value since it's optional + # but we should error if the product doesn't exist in the db + product_id = self.products.get_by_name_unsafe(dataset.product.name).id + is_new = transaction.insert_dataset(dataset.metadata_doc_without_lineage(), dataset.id, product_id) if is_new: # 1b. Prepare spatial index extents transaction.update_spindex(dsids=[dataset.id]) @@ -169,6 +178,8 @@ def add(self, dataset: Dataset, # 1c. Store locations if dataset.uris is not None: self._ensure_new_locations(dataset, transaction=transaction) + if archive_less_mature is not None: + self.archive_less_mature(dataset, archive_less_mature) return dataset @@ -263,15 +274,14 @@ def load_field(f: Union[str, fields.Field]) -> fields.Field: return f group_fields: List[fields.Field] = [load_field(f) for f in args] - result_type = namedtuple('search_result', list(f.name for f in group_fields)) # type: ignore - expressions = [product.metadata_type.dataset_fields.get('product') == product.name] with self._db_connection() as connection: for record in connection.get_duplicates(group_fields, expressions): - dataset_ids = set(record[0]) - grouped_fields = tuple(record[1:]) - yield result_type(*grouped_fields), dataset_ids + as_dict = dict(record) + if 'ids' in as_dict.keys(): + ids = as_dict.pop('ids') + yield namedtuple('search_result', as_dict.keys())(**as_dict), set(ids) def can_update(self, dataset, updates_allowed=None): """ @@ -305,11 +315,12 @@ def can_update(self, dataset, updates_allowed=None): return not bad_changes, good_changes, bad_changes - def update(self, dataset: Dataset, updates_allowed=None): + def update(self, dataset: Dataset, updates_allowed=None, archive_less_mature=None): """ Update dataset metadata and location :param Dataset dataset: Dataset to update :param updates_allowed: Allowed updates + :param archive_less_mature: Find and archive less mature datasets with ms delta :rtype: Dataset """ existing = self.get(dataset.id) @@ -342,6 +353,8 @@ def update(self, dataset: Dataset, updates_allowed=None): raise ValueError("Failed to update dataset %s..." % dataset.id) transaction.update_spindex(dsids=[dataset.id]) transaction.update_search_index(dsids=[dataset.id]) + if archive_less_mature is not None: + self.archive_less_mature(dataset, archive_less_mature) self._ensure_new_locations(dataset, existing) @@ -760,7 +773,7 @@ def search_summaries(self, **query): """ for _, results in self._do_search_by_product(query, return_fields=True): for columns in results: - output = dict(columns) + output = columns._asdict() _LOG.warning("search results: %s (%s)", output["id"], output["product"]) yield output diff --git a/datacube/index/postgis/_metadata_types.py b/datacube/index/postgis/_metadata_types.py index 9c593f9b9a..cd81c238f5 100644 --- a/datacube/index/postgis/_metadata_types.py +++ b/datacube/index/postgis/_metadata_types.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging from time import monotonic diff --git a/datacube/index/postgis/_products.py b/datacube/index/postgis/_products.py index 7e3cd86b70..a941290178 100644 --- a/datacube/index/postgis/_products.py +++ b/datacube/index/postgis/_products.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging diff --git a/datacube/index/postgis/_transaction.py b/datacube/index/postgis/_transaction.py index 3fabcccb59..d3bf058ce6 100644 --- a/datacube/index/postgis/_transaction.py +++ b/datacube/index/postgis/_transaction.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from contextlib import contextmanager diff --git a/datacube/index/postgis/_users.py b/datacube/index/postgis/_users.py index c604a676b4..a10fa275d1 100644 --- a/datacube/index/postgis/_users.py +++ b/datacube/index/postgis/_users.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import Iterable, Optional, Tuple from datacube.index.abstract import AbstractUserResource diff --git a/datacube/index/postgis/index.py b/datacube/index/postgis/index.py index 8400236758..cf082f64c2 100644 --- a/datacube/index/postgis/index.py +++ b/datacube/index/postgis/index.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging from contextlib import contextmanager diff --git a/datacube/index/postgres/__init__.py b/datacube/index/postgres/__init__.py index e69de29bb2..8318282aea 100644 --- a/datacube/index/postgres/__init__.py +++ b/datacube/index/postgres/__init__.py @@ -0,0 +1,4 @@ +# This file is part of the Open Data Cube, see https://opendatacube.org for more information +# +# Copyright (c) 2015-2024 ODC Contributors +# SPDX-License-Identifier: Apache-2.0 diff --git a/datacube/index/postgres/_datasets.py b/datacube/index/postgres/_datasets.py index 0df96cc7da..51a2d606fb 100755 --- a/datacube/index/postgres/_datasets.py +++ b/datacube/index/postgres/_datasets.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ API for dataset indexing, access and search. @@ -10,7 +10,7 @@ import warnings from collections import namedtuple from time import monotonic -from typing import Iterable, List, Union, Mapping, Any +from typing import Iterable, List, Union, Mapping, Optional, Any from uuid import UUID from sqlalchemy import select, func @@ -132,7 +132,7 @@ def bulk_has(self, ids_): map((lambda x: UUID(x) if isinstance(x, str) else x), ids_)] def add(self, dataset: Dataset, - with_lineage: bool = True) -> Dataset: + with_lineage: bool = True, archive_less_mature: Optional[int] = None) -> Dataset: """ Add ``dataset`` to the index. No-op if it is already present. @@ -143,6 +143,10 @@ def add(self, dataset: Dataset, - ``False`` record lineage relations, but do not attempt adding lineage datasets to the db + :param archive_less_mature: if integer, search for less + mature versions of the dataset with the int value as a millisecond + delta in timestamp comparison + :rtype: Dataset """ @@ -151,7 +155,12 @@ def process_bunch(dss, main_ds, transaction): # First insert all new datasets for ds in dss: - is_new = transaction.insert_dataset(ds.metadata_doc_without_lineage(), ds.id, ds.product.id) + product_id = ds.product.id + if product_id is None: + # don't assume the product has an id value since it's optional + # but we should error if the product doesn't exist in the db + product_id = self.products.get_by_name_unsafe(ds.product.name).id + is_new = transaction.insert_dataset(ds.metadata_doc_without_lineage(), ds.id, product_id) sources = ds.sources if is_new and sources is not None: edges.extend((name, ds.id, src.id) @@ -187,6 +196,8 @@ def process_bunch(dss, main_ds, transaction): with self._db_connection(transaction=True) as transaction: process_bunch(dss, dataset, transaction) + if archive_less_mature is not None: + self.archive_less_mature(dataset, archive_less_mature) return dataset @@ -238,15 +249,14 @@ def load_field(f: Union[str, fields.Field]) -> fields.Field: return f group_fields: List[fields.Field] = [load_field(f) for f in args] - result_type = namedtuple('search_result', list(f.name for f in group_fields)) # type: ignore - expressions = [product.metadata_type.dataset_fields.get('product') == product.name] with self._db_connection() as connection: for record in connection.get_duplicates(group_fields, expressions): - dataset_ids = set(record[0]) - grouped_fields = tuple(record[1:]) - yield result_type(*grouped_fields), dataset_ids + as_dict = dict(record) + if "ids" in as_dict.keys(): + ids = as_dict.pop('ids') + yield namedtuple('search_result', as_dict.keys())(**as_dict), set(ids) def can_update(self, dataset, updates_allowed=None): """ @@ -278,11 +288,12 @@ def can_update(self, dataset, updates_allowed=None): return not bad_changes, good_changes, bad_changes - def update(self, dataset: Dataset, updates_allowed=None): + def update(self, dataset: Dataset, updates_allowed=None, archive_less_mature=None): """ Update dataset metadata and location :param Dataset dataset: Dataset to update :param updates_allowed: Allowed updates + :param archive_less_mature: Find and archive less mature datasets with ms delta :rtype: Dataset """ existing = self.get(dataset.id) @@ -313,6 +324,8 @@ def update(self, dataset: Dataset, updates_allowed=None): with self._db_connection(transaction=True) as transaction: if not transaction.update_dataset(dataset.metadata_doc_without_lineage(), dataset.id, product.id): raise ValueError("Failed to update dataset %s..." % dataset.id) + if archive_less_mature is not None: + self.archive_less_mature(dataset, archive_less_mature) self._ensure_new_locations(dataset, existing) diff --git a/datacube/index/postgres/_metadata_types.py b/datacube/index/postgres/_metadata_types.py index 4c1f4193f5..d56d5d516f 100644 --- a/datacube/index/postgres/_metadata_types.py +++ b/datacube/index/postgres/_metadata_types.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging diff --git a/datacube/index/postgres/_products.py b/datacube/index/postgres/_products.py index a639f63363..6b9e24013d 100644 --- a/datacube/index/postgres/_products.py +++ b/datacube/index/postgres/_products.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging diff --git a/datacube/index/postgres/_transaction.py b/datacube/index/postgres/_transaction.py index 5b02a5584c..254850fb03 100644 --- a/datacube/index/postgres/_transaction.py +++ b/datacube/index/postgres/_transaction.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from contextlib import contextmanager diff --git a/datacube/index/postgres/_users.py b/datacube/index/postgres/_users.py index 2dd4c81877..95e1172ff4 100644 --- a/datacube/index/postgres/_users.py +++ b/datacube/index/postgres/_users.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import Iterable, Optional, Tuple from datacube.index.abstract import AbstractUserResource diff --git a/datacube/index/postgres/index.py b/datacube/index/postgres/index.py index d56016bf6b..d2d93636c5 100644 --- a/datacube/index/postgres/index.py +++ b/datacube/index/postgres/index.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging from contextlib import contextmanager diff --git a/datacube/model/__init__.py b/datacube/model/__init__.py index 03352eecc2..3c9b4ff65c 100644 --- a/datacube/model/__init__.py +++ b/datacube/model/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Core classes used across modules. @@ -21,9 +21,11 @@ schema_validated, DocReader from datacube.index.eo3 import is_doc_eo3 from .fields import Field, get_dataset_fields -from ._base import Range, ranges_overlap # noqa: F401 +from ._base import Range, ranges_overlap, Not # noqa: F401 from .eo3 import validate_eo3_compatible_type +from deprecat import deprecat + _LOG = logging.getLogger(__name__) DEFAULT_SPATIAL_DIMS = ('y', 'x') # Used when product lacks grid_spec @@ -438,6 +440,7 @@ def license(self) -> str: return self.definition.get("license", None) @property + @deprecat(reason="Ingestion has been deprecated and will be removed in a future version.", version="1.8.14") def managed(self) -> bool: return self.definition.get('managed', False) @@ -713,6 +716,7 @@ def __hash__(self): DatasetType = Product +@deprecat(reason="Ingestion has been deprecated and will be removed in a future version.", version="1.8.14") @schema_validated(SCHEMA_PATH / 'ingestor-config-type-schema.yaml') class IngestorConfig: """ diff --git a/datacube/model/_base.py b/datacube/model/_base.py index 5223c18fa3..d1681e8fec 100644 --- a/datacube/model/_base.py +++ b/datacube/model/_base.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from collections import namedtuple @@ -18,3 +18,6 @@ def ranges_overlap(ra: Range, rb: Range) -> bool: if ra.begin <= rb.begin: return ra.end > rb.begin return rb.end > ra.begin + + +Not = namedtuple('Not', 'value') diff --git a/datacube/model/eo3.py b/datacube/model/eo3.py index 668f8c44cd..c276746a88 100644 --- a/datacube/model/eo3.py +++ b/datacube/model/eo3.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from datacube.utils.documents import InvalidDocException diff --git a/datacube/model/fields.py b/datacube/model/fields.py index 3f94ff2803..2b4ab44a7d 100644 --- a/datacube/model/fields.py +++ b/datacube/model/fields.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """Non-db specific implementation of metadata search fields. diff --git a/datacube/model/model.py b/datacube/model/model.py index 7634393a4d..d801ba8534 100644 --- a/datacube/model/model.py +++ b/datacube/model/model.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2023 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from pathlib import Path from typing import Dict, List, Optional, Tuple, Union diff --git a/datacube/model/properties.py b/datacube/model/properties.py index 01bec1bdfc..c5f9c8882c 100644 --- a/datacube/model/properties.py +++ b/datacube/model/properties.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2023 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import collections.abc import warnings diff --git a/datacube/model/schema/dataset-type-schema.yaml b/datacube/model/schema/dataset-type-schema.yaml index 21fceda402..4ce94cbcb2 100644 --- a/datacube/model/schema/dataset-type-schema.yaml +++ b/datacube/model/schema/dataset-type-schema.yaml @@ -36,6 +36,7 @@ properties: items: "$ref": "#/definitions/measurement" managed: + # Indicates ingested product - deprecated type: boolean required: diff --git a/datacube/model/utils.py b/datacube/model/utils.py index 90746d5197..17d3522297 100644 --- a/datacube/model/utils.py +++ b/datacube/model/utils.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import datetime import os diff --git a/datacube/scripts/__init__.py b/datacube/scripts/__init__.py index c081ad5b41..8318282aea 100644 --- a/datacube/scripts/__init__.py +++ b/datacube/scripts/__init__.py @@ -1,4 +1,4 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/datacube/scripts/cli_app.py b/datacube/scripts/cli_app.py index 820d8179eb..c4a024ba4c 100644 --- a/datacube/scripts/cli_app.py +++ b/datacube/scripts/cli_app.py @@ -2,7 +2,7 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Datacube command-line interface diff --git a/datacube/scripts/dataset.py b/datacube/scripts/dataset.py index 2a318a8545..d9052d1e15 100644 --- a/datacube/scripts/dataset.py +++ b/datacube/scripts/dataset.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import csv import datetime @@ -53,8 +53,6 @@ def _resolve_uri(uri, doc): if isinstance(loc, (list, tuple)): if len(loc) > 0: return loc[0] - else: - return uri return uri @@ -144,11 +142,13 @@ def mk_dataset(ds, uri): 'you can supply several by repeating this option with a new product name'), multiple=True) @click.option('--auto-add-lineage/--no-auto-add-lineage', is_flag=True, default=True, - help=('Default behaviour is to automatically add lineage datasets if they are missing from the database, ' + help=('WARNING: will be deprecated in datacube v1.9.\n' + 'Default behaviour is to automatically add lineage datasets if they are missing from the database, ' 'but this can be disabled if lineage is expected to be present in the DB, ' 'in this case add will abort when encountering missing lineage dataset')) @click.option('--verify-lineage/--no-verify-lineage', is_flag=True, default=True, - help=('Lineage referenced in the metadata document should be the same as in DB, ' + help=('WARNING: will be deprecated in datacube v1.9.\n' + 'Lineage referenced in the metadata document should be the same as in DB, ' 'default behaviour is to skip those top-level datasets that have lineage data ' 'different from the version in the DB. This option allows omitting verification step.')) @click.option('--dry-run', help='Check if everything is ok', is_flag=True, default=False) @@ -156,8 +156,13 @@ def mk_dataset(ds, uri): help="Pretend that there is no lineage data in the datasets being indexed", is_flag=True, default=False) @click.option('--confirm-ignore-lineage', - help="Pretend that there is no lineage data in the datasets being indexed, without confirmation", + help=('WARNING: this flag has been deprecated and will be removed in datacube v1.9.\n' + 'Pretend that there is no lineage data in the datasets being indexed, without confirmation'), is_flag=True, default=False) +@click.option('--archive-less-mature', is_flag=False, flag_value=500, default=None, + help=('Find and archive less mature versions of the dataset, will fail if more mature versions ' + 'of the dataset already exist. Can also specify a millisecond delta amount to be taken ' + 'into acount when comparing timestamps. Default delta is 500ms.')) @click.argument('dataset-paths', type=str, nargs=-1) @ui.pass_index() def index_cmd(index, product_names, @@ -167,23 +172,15 @@ def index_cmd(index, product_names, dry_run, ignore_lineage, confirm_ignore_lineage, + archive_less_mature, dataset_paths): if not dataset_paths: + click.echo('Error: no datasets provided\n') print_help_msg(index_cmd) sys.exit(1) - if confirm_ignore_lineage is False and ignore_lineage is True: - if sys.stdin.isatty(): - confirmed = click.confirm("Requested to skip lineage information, Are you sure?", default=False) - if not confirmed: - click.echo('OK aborting', err=True) - sys.exit(1) - else: - click.echo("Use --confirm-ignore-lineage from non-interactive scripts. Aborting.") - sys.exit(1) - - confirm_ignore_lineage = True + confirm_ignore_lineage = ignore_lineage try: ds_resolve = Doc2Dataset(index, @@ -196,6 +193,14 @@ def index_cmd(index, product_names, _LOG.error(e) sys.exit(2) + if archive_less_mature is not None: + if type(archive_less_mature) is not int: + click.echo('Error: millisecond delta value must be an integer') + sys.exit(1) + if archive_less_mature < 0: + click.echo('Error: millisecond delta value must be a positive integer') + sys.exit(1) + def run_it(dataset_paths): doc_stream = ui_path_doc_stream(dataset_paths, logger=_LOG, uri=True) doc_stream = remap_uri_from_doc(doc_stream) @@ -203,7 +208,7 @@ def run_it(dataset_paths): index_datasets(dss, index, auto_add_lineage=auto_add_lineage and not confirm_ignore_lineage, - dry_run=dry_run) + dry_run=dry_run, archive_less_mature=archive_less_mature) # If outputting directly to terminal, show a progress bar. if sys.stdout.isatty(): @@ -213,12 +218,13 @@ def run_it(dataset_paths): run_it(dataset_paths) -def index_datasets(dss, index, auto_add_lineage, dry_run): +def index_datasets(dss, index, auto_add_lineage, dry_run, archive_less_mature): for dataset in dss: _LOG.info('Matched %s', dataset) if not dry_run: try: - index.datasets.add(dataset, with_lineage=auto_add_lineage) + index.datasets.add(dataset, with_lineage=auto_add_lineage, + archive_less_mature=archive_less_mature) except (ValueError, MissingRecordError) as e: _LOG.error('Failed to add dataset %s: %s', dataset.local_uri, e) @@ -245,10 +251,15 @@ def parse_update_rules(keys_that_can_change): - 'keep': keep as alternative location [default] - 'archive': mark as archived - 'forget': remove from the index''')) +@click.option('--archive-less-mature', is_flag=False, flag_value=500, default=None, + help=('Find and archive less mature versions of the dataset, will fail if more mature versions ' + 'of the dataset already exist. Can also specify a millisecond delta amount to be taken ' + 'into acount when comparing timestamps. Default delta is 500ms.')) @click.argument('dataset-paths', nargs=-1) @ui.pass_index() -def update_cmd(index, keys_that_can_change, dry_run, location_policy, dataset_paths): +def update_cmd(index, keys_that_can_change, dry_run, location_policy, dataset_paths, archive_less_mature): if not dataset_paths: + click.echo('Error: no datasets provided\n') print_help_msg(update_cmd) sys.exit(1) @@ -291,6 +302,14 @@ def loc_keep(new_ds, existing_ds): doc_stream = ui_path_doc_stream(dataset_paths, logger=_LOG, uri=True) doc_stream = remap_uri_from_doc(doc_stream) + if archive_less_mature is not None: + if type(archive_less_mature) is not int: + click.echo('Error: millisecond delta value must be an integer') + sys.exit(1) + if archive_less_mature < 0: + click.echo('Error: millisecond delta value must be a positive integer') + sys.exit(1) + for dataset, existing_ds in load_datasets_for_update(doc_stream, index): _LOG.info('Matched %s', dataset) @@ -301,7 +320,8 @@ def loc_keep(new_ds, existing_ds): if not dry_run: try: - index.datasets.update(dataset, updates_allowed=updates_allowed) + index.datasets.update(dataset, updates_allowed=updates_allowed, + archive_less_mature=archive_less_mature) update_loc(dataset, existing_ds) success += 1 echo('Updated %s' % dataset.id) @@ -416,6 +436,7 @@ def info_cmd(index: Index, show_sources: bool, show_derived: bool, max_depth: int, ids: Iterable[str]) -> None: if not ids: + click.echo('Error: no datasets provided\n') print_help_msg(info_cmd) sys.exit(1) @@ -486,6 +507,7 @@ def uri_search_cmd(index: Index, paths: List[str], search_mode): PATHS may be either file paths or URIs """ if not paths: + click.echo('Error: no locations provided\n') print_help_msg(uri_search_cmd) sys.exit(1) @@ -511,6 +533,7 @@ def uri_search_cmd(index: Index, paths: List[str], search_mode): @ui.pass_index() def archive_cmd(index: Index, archive_derived: bool, dry_run: bool, all_ds: bool, ids: List[str]): if not ids and not all_ds: + click.echo('Error: no datasets provided\n') print_help_msg(archive_cmd) sys.exit(1) @@ -559,6 +582,7 @@ def archive_cmd(index: Index, archive_derived: bool, dry_run: bool, all_ds: bool def restore_cmd(index: Index, restore_derived: bool, derived_tolerance_seconds: int, dry_run: bool, all_ds: bool, ids: List[str]): if not ids and not all_ds: + click.echo('Error: no datasets provided\n') print_help_msg(restore_cmd) sys.exit(1) @@ -606,6 +630,7 @@ def within_tolerance(dataset): @ui.pass_index() def purge_cmd(index: Index, dry_run: bool, all_ds: bool, ids: List[str]): if not ids and not all_ds: + click.echo('Error: no datasets provided\n') print_help_msg(purge_cmd) sys.exit(1) @@ -643,3 +668,45 @@ def purge_cmd(index: Index, dry_run: bool, all_ds: bool, ids: List[str]): click.echo(f'{len(datasets_for_archive)} datasets not purged (dry run)') click.echo('Completed dataset purge.') + + +@dataset_cmd.command('find-duplicates', help="Search for duplicate indexed datasets") +@click.option('--product', '-p', 'product_names', + help=("Only search within product(s) specified with this option. " + "You can supply several by repeating this option with a new product name."), + multiple=True) +@click.argument('fields', nargs=-1) +@ui.pass_index() +def find_duplicates(index: Index, product_names, fields): + """ + Find dataset ids of two or more active datasets that have duplicate values in the specified fields. + If products are specified, search only within those products. Otherwise, search within any products that + have the fields. + """ + if not fields: + click.echo('Error: must provide field names to match on\n') + sys.exit(1) + + # if no products were specified, use whichever ones have the specified search fields + # if products were specified, check they all have the required fields + products_with_fields = list(index.products.get_with_fields(fields)) + if not products_with_fields: + click.echo(f'Error: no products found with fields {", ".join(fields)}\n') + sys.exit(1) + if not list(product_names): + products = products_with_fields + else: + products = [index.products.get_by_name(name) for name in product_names] + products_without_fields = set(products).difference(set(products_with_fields)) + if len(products_without_fields): + click.echo(f'Error: specified products {", ".join(p.name for p in products_without_fields)} ' + 'do not contain all required fields\n') + sys.exit(1) + + dupes = [] + for product in products: + dupes.extend(list(index.datasets.search_product_duplicates(product, *fields))) + if len(dupes): + print(dupes) + else: + click.echo('No potential duplicates found.') diff --git a/datacube/scripts/ingest.py b/datacube/scripts/ingest.py index 184edacc69..e338a4f426 100644 --- a/datacube/scripts/ingest.py +++ b/datacube/scripts/ingest.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import time import logging @@ -391,7 +391,8 @@ def get_driver_from_config(config): return driver -@cli.command('ingest', help="Ingest datasets") +@cli.command('ingest', help="WARNING: Ingestion has been deprecated in v1.8.14 and will be removed in v1.9\n" + "Ingest datasets") @click.option('--config-file', '-c', type=click.Path(exists=True, readable=True, writable=False, dir_okay=False), help='Ingest configuration file') diff --git a/datacube/scripts/metadata.py b/datacube/scripts/metadata.py index a4efd7f801..ddd5df034f 100644 --- a/datacube/scripts/metadata.py +++ b/datacube/scripts/metadata.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import json import logging diff --git a/datacube/scripts/product.py b/datacube/scripts/product.py index 294291493f..ea8c682326 100644 --- a/datacube/scripts/product.py +++ b/datacube/scripts/product.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import csv import json diff --git a/datacube/scripts/search_tool.py b/datacube/scripts/search_tool.py index aba233bb2c..d20c17b5d7 100755 --- a/datacube/scripts/search_tool.py +++ b/datacube/scripts/search_tool.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Query datasets. diff --git a/datacube/scripts/system.py b/datacube/scripts/system.py index dfece6f827..aee4faf1bd 100644 --- a/datacube/scripts/system.py +++ b/datacube/scripts/system.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging diff --git a/datacube/scripts/user.py b/datacube/scripts/user.py index 1c3b8ba5b3..f05a7b6089 100644 --- a/datacube/scripts/user.py +++ b/datacube/scripts/user.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging import click diff --git a/datacube/storage/__init__.py b/datacube/storage/__init__.py index 689d71de61..2872494d48 100644 --- a/datacube/storage/__init__.py +++ b/datacube/storage/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Modules for creating and accessing Data Store Units diff --git a/datacube/storage/_base.py b/datacube/storage/_base.py index a7da9a1659..27540d573e 100644 --- a/datacube/storage/_base.py +++ b/datacube/storage/_base.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import Optional, Dict, Any, Tuple, Callable from urllib.parse import urlparse diff --git a/datacube/storage/_hdf5.py b/datacube/storage/_hdf5.py index 4029c054e9..43dae4da58 100644 --- a/datacube/storage/_hdf5.py +++ b/datacube/storage/_hdf5.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from threading import RLock HDF5_LOCK = RLock() diff --git a/datacube/storage/_load.py b/datacube/storage/_load.py index 8f071641f1..2c411aab07 100644 --- a/datacube/storage/_load.py +++ b/datacube/storage/_load.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Important functions are: diff --git a/datacube/storage/_read.py b/datacube/storage/_read.py index b3e8ae88e6..af1517e927 100644 --- a/datacube/storage/_read.py +++ b/datacube/storage/_read.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Dataset -> Raster """ @@ -182,13 +182,32 @@ def norm_read_args(roi, shape, extra_dim_index): pix = rdr.read(*norm_read_args(rr.roi_src, src_gbox.shape, extra_dim_index)) + # XSCALE and YSCALE are (currently) undocumented arguments that rasterio passed through to + # GDAL. Not using them results in very inaccurate warping in images with highly + # non-square (i.e. long and thin) aspect ratios. + # + # See https://github.com/OSGeo/gdal/issues/7750 as well as + # https://github.com/opendatacube/datacube-core/pull/1450 and + # https://github.com/opendatacube/datacube-core/issues/1456 + # + # In theory we might be able to get better results for queries with significantly + # different vertical and horizontal scales, but explicitly using XSCALE=1, YSCALE=1 + # appears to be most appropriate for most requests, and is demonstrably better + # than not setting them at all. + gdal_scale_params = { + "XSCALE": 1, + "YSCALE": 1, + } if rr.transform.linear is not None: A = (~src_gbox.transform)*dst_gbox.transform warp_affine(pix, dst, A, resampling, - src_nodata=rdr.nodata, dst_nodata=dst_nodata) + src_nodata=rdr.nodata, dst_nodata=dst_nodata, + **gdal_scale_params) else: + rio_reproject(pix, dst, src_gbox, dst_gbox, resampling, - src_nodata=rdr.nodata, dst_nodata=dst_nodata) + src_nodata=rdr.nodata, dst_nodata=dst_nodata, + **gdal_scale_params) return rr.roi_dst diff --git a/datacube/storage/_rio.py b/datacube/storage/_rio.py index 8f3e0a3bd8..e18a8839cc 100644 --- a/datacube/storage/_rio.py +++ b/datacube/storage/_rio.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Driver implementation for Rasterio based reader. diff --git a/datacube/testutils/__init__.py b/datacube/testutils/__init__.py index 097e89c589..91755a8830 100644 --- a/datacube/testutils/__init__.py +++ b/datacube/testutils/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Useful methods for tests (particularly: reading/writing and checking files) @@ -415,11 +415,12 @@ def gen_tiff_dataset(bands, **kwargs) gbox = meta.gbox - - mm.append(dict(name=name, - path=fname, - layer=1, - dtype=meta.dtype)) + bdict = dict(name=name, + path=fname, + layer=1, + nodata=band.nodata, + dtype=meta.dtype) + mm.append(bdict) uri = Path(base_folder_of_record/'metadata.yaml').absolute().as_uri() ds = mk_sample_dataset(mm, diff --git a/datacube/testutils/geom.py b/datacube/testutils/geom.py index 06b51e7f43..f1928c3af9 100644 --- a/datacube/testutils/geom.py +++ b/datacube/testutils/geom.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import numpy as np from affine import Affine diff --git a/datacube/testutils/io.py b/datacube/testutils/io.py index 614dc2da26..d46c51611a 100644 --- a/datacube/testutils/io.py +++ b/datacube/testutils/io.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import numpy as np import toolz diff --git a/datacube/testutils/iodriver.py b/datacube/testutils/iodriver.py index 4accaadb51..ddc8ced470 100644 --- a/datacube/testutils/iodriver.py +++ b/datacube/testutils/iodriver.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Reader driver construction for tests """ diff --git a/datacube/testutils/threads.py b/datacube/testutils/threads.py index 17ae13bba6..ab37711f5f 100644 --- a/datacube/testutils/threads.py +++ b/datacube/testutils/threads.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ threads related stuff """ diff --git a/datacube/ui/__init__.py b/datacube/ui/__init__.py index 51ff28cf46..c9bf934ba6 100644 --- a/datacube/ui/__init__.py +++ b/datacube/ui/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ User Interface Utilities diff --git a/datacube/ui/click.py b/datacube/ui/click.py index c339b8ae8e..49bf887736 100644 --- a/datacube/ui/click.py +++ b/datacube/ui/click.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Common functions for click-based cli scripts. @@ -285,7 +285,9 @@ def _setup_executor(ctx, param, value): executor_cli_options = click.option('--executor', # type: ignore type=(click.Choice(list(EXECUTOR_TYPES)), str), default=['serial', None], - help="Run parallelized, either locally or distributed. eg:\n" + help="WARNING: executors have been deprecated in v1.8.14, " + "and will be removed in v1.9.\n" + "Run parallelized, either locally or distributed. eg:\n" "--executor multiproc 4 (OR)\n" "--executor distributed 10.0.0.8:8888", callback=_setup_executor) @@ -340,16 +342,19 @@ def parsed_search_expressions(f): FIELD = VALUE FIELD in DATE-RANGE FIELD in [START, END] + TIME < DATE + TIME > DATE \b - DATE-RANGE is one of YYYY, YYYY-MM or YYYY-MM-DD START and END can be either numbers or dates + Dates follow YYYY, YYYY-MM, or YYYY-MM-DD format FIELD: x, y, lat, lon, time, product, ... \b eg. 'time in [1996-01-01, 1996-12-31]' 'time in 1996' + 'time > 2020-01' 'lon in [130, 140]' 'lat in [-40, -30]' product=ls5_nbar_albers diff --git a/datacube/ui/common.py b/datacube/ui/common.py index 33bc5933c5..b7552d7929 100644 --- a/datacube/ui/common.py +++ b/datacube/ui/common.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Common methods for UI code. @@ -89,34 +89,24 @@ def ui_path_doc_stream(paths, logger=None, uri=True, raw=False): """ - def on_error1(p, e): - if logger is not None: - logger.error(str(e)) + def _resolve_doc_files(paths): + for p in paths: + try: + yield get_metadata_path(p) + except ValueError as e: + if logger is not None: + logger.error(str(e)) - def on_error2(p, e): - if logger is not None: - logger.error('Failed reading documents from %s', str(p)) + def _path_doc_stream(files, uri=True, raw=False): + maybe_wrap = identity if raw else SimpleDocNav - yield from _path_doc_stream(_resolve_doc_files(paths, on_error=on_error1), - on_error=on_error2, uri=uri, raw=raw) + for fname in files: + try: + for p, doc in read_documents(fname, uri=uri): + yield p, maybe_wrap(doc) + except InvalidDocException as e: + if logger is not None: + logger.error('Failed reading documents from %s', str(fname)) -def _resolve_doc_files(paths, on_error): - for p in paths: - try: - yield get_metadata_path(p) - except ValueError as e: - on_error(p, e) - - -def _path_doc_stream(files, on_error, uri=True, raw=False): - """See :func:`ui_path_doc_stream` for documentation""" - maybe_wrap = identity if raw else SimpleDocNav - - for fname in files: - try: - for p, doc in read_documents(fname, uri=uri): - yield p, maybe_wrap(doc) - - except InvalidDocException as e: - on_error(fname, e) + yield from _path_doc_stream(_resolve_doc_files(paths), uri=uri, raw=raw) diff --git a/datacube/ui/expression.py b/datacube/ui/expression.py index c369631628..f7253612cb 100644 --- a/datacube/ui/expression.py +++ b/datacube/ui/expression.py @@ -1,17 +1,19 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Search expression parsing for command line applications. -Three types of expressions are available: +Four types of expressions are available: FIELD = VALUE FIELD in DATE-RANGE FIELD in [START, END] + TIME > DATE + TIME < DATE -Where DATE-RANGE is one of YYYY, YYYY-MM or YYYY-MM-DD +Where DATE or DATE-RANGE is one of YYYY, YYYY-MM or YYYY-MM-DD and START, END are either numbers or dates. """ # flake8: noqa @@ -27,10 +29,14 @@ ?expression: equals_expr | time_in_expr | field_in_expr + | time_gt_expr + | time_lt_expr equals_expr: field "=" value time_in_expr: time "in" date_range field_in_expr: field "in" "[" orderable "," orderable "]" + time_gt_expr: time ">" date_gt + time_lt_expr: time "<" date_lt field: FIELD time: TIME @@ -48,6 +54,10 @@ ?date_range: date -> single_date | "[" date "," date "]" -> date_pair + date_gt: date -> range_lower_bound + + date_lt: date -> range_upper_bound + date: YEAR ["-" MONTH ["-" DAY ]] TIME: "time" @@ -86,6 +96,12 @@ def field_in_expr(self, field, lower, upper): def time_in_expr(self, time_field, date_range): return {str(time_field): date_range} + + def time_gt_expr(self, time_field, date_gt): + return {str(time_field): date_gt} + + def time_lt_expr(self, time_field, date_lt): + return {str(time_field): date_lt} # Convert the literals def string(self, val): @@ -101,6 +117,12 @@ def single_date(self, date): def date_pair(self, start, end): return _time_to_search_dims((start, end)) + + def range_lower_bound(self, date): + return _time_to_search_dims((date, None)) + + def range_upper_bound(self, date): + return _time_to_search_dims((None, date)) def date(self, y, m=None, d=None): return "-".join(x for x in [y, m, d] if x is not None) diff --git a/datacube/ui/task_app.py b/datacube/ui/task_app.py index 789ebddb4c..386d9a284c 100644 --- a/datacube/ui/task_app.py +++ b/datacube/ui/task_app.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging import os diff --git a/datacube/utils/__init__.py b/datacube/utils/__init__.py index 8a6e2c7ef7..2ab5b51566 100644 --- a/datacube/utils/__init__.py +++ b/datacube/utils/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Utility functions diff --git a/datacube/utils/_misc.py b/datacube/utils/_misc.py index b797b3ace6..c4fc098b28 100644 --- a/datacube/utils/_misc.py +++ b/datacube/utils/_misc.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Utility functions diff --git a/datacube/utils/aws/__init__.py b/datacube/utils/aws/__init__.py index 48e36676ac..10722c677f 100644 --- a/datacube/utils/aws/__init__.py +++ b/datacube/utils/aws/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Helper methods for working with AWS diff --git a/datacube/utils/changes.py b/datacube/utils/changes.py index 74654de8bd..34eb9806e2 100644 --- a/datacube/utils/changes.py +++ b/datacube/utils/changes.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Validation of document/dictionary changes. diff --git a/datacube/utils/cog.py b/datacube/utils/cog.py index 9a4bd77aeb..5e1b661548 100644 --- a/datacube/utils/cog.py +++ b/datacube/utils/cog.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import warnings import toolz # type: ignore[import] @@ -133,6 +133,10 @@ def _write_cog( compress="DEFLATE", ) + # If nodata is not set, but the array is of floating point type, force nodata=nan + if nodata is None and np.issubdtype(pix.dtype, np.floating): + nodata = np.nan + if nodata is not None: rio_opts.update(nodata=nodata) diff --git a/datacube/utils/dask.py b/datacube/utils/dask.py index 9bb51a076b..60ece9f548 100644 --- a/datacube/utils/dask.py +++ b/datacube/utils/dask.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Dask Distributed Tools diff --git a/datacube/utils/dates.py b/datacube/utils/dates.py index fb6671106c..c3979ada98 100644 --- a/datacube/utils/dates.py +++ b/datacube/utils/dates.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Date and time utility functions diff --git a/datacube/utils/documents.py b/datacube/utils/documents.py index 1fe8724121..c15fbf33ed 100644 --- a/datacube/utils/documents.py +++ b/datacube/utils/documents.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Functions for working with YAML documents and configurations @@ -202,6 +202,7 @@ def read_strings_from_netcdf(path, variable): def validate_document(document, schema, schema_folder=None): import jsonschema + import referencing try: # Allow schemas to reference other schemas in the given folder. @@ -210,14 +211,14 @@ def doc_reference(path): if not path.exists(): raise ValueError("Reference not found: %s" % path) referenced_schema = next(iter(read_documents(path)))[1] - return referenced_schema + return referencing.Resource(referenced_schema, referencing.jsonschema.DRAFT4) + if schema_folder: + registry = referencing.Registry(retrieve=doc_reference) + else: + registry = referencing.Registry() jsonschema.Draft4Validator.check_schema(schema) - ref_resolver = jsonschema.RefResolver.from_schema( - schema, - handlers={'': doc_reference} if schema_folder else () - ) - validator = jsonschema.Draft4Validator(schema, resolver=ref_resolver) + validator = jsonschema.Draft4Validator(schema, registry=registry) validator.validate(document) except jsonschema.ValidationError as e: raise InvalidDocException(e) diff --git a/datacube/utils/generic.py b/datacube/utils/generic.py index 327a10f9d0..0146fe3152 100644 --- a/datacube/utils/generic.py +++ b/datacube/utils/generic.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import itertools import threading diff --git a/datacube/utils/geometry/__init__.py b/datacube/utils/geometry/__init__.py index dbe6a6110f..7c1f716c1f 100644 --- a/datacube/utils/geometry/__init__.py +++ b/datacube/utils/geometry/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Geometric shapes and operations on them """ diff --git a/datacube/utils/geometry/_base.py b/datacube/utils/geometry/_base.py index 7647ca9661..0753dd4666 100644 --- a/datacube/utils/geometry/_base.py +++ b/datacube/utils/geometry/_base.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import functools import itertools @@ -853,9 +853,6 @@ def multipoint(coords: CoordList, crs: MaybeCRS) -> Geometry: """ Create a 2D MultiPoint Geometry - >>> multipoint([(10, 10), (20, 20)], None) - Geometry(MULTIPOINT (10 10, 20 20), None) - :param coords: list of x,y coordinate tuples """ return Geometry({'type': 'MultiPoint', 'coordinates': coords}, crs=crs) diff --git a/datacube/utils/geometry/_warp.py b/datacube/utils/geometry/_warp.py index bbe2862a92..307f09fad8 100644 --- a/datacube/utils/geometry/_warp.py +++ b/datacube/utils/geometry/_warp.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import Union, Optional import rasterio.warp # type: ignore[import] diff --git a/datacube/utils/geometry/gbox.py b/datacube/utils/geometry/gbox.py index 11a2efab61..ddedb8e3e1 100644 --- a/datacube/utils/geometry/gbox.py +++ b/datacube/utils/geometry/gbox.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Geometric operations on GeoBox class """ diff --git a/datacube/utils/geometry/tools.py b/datacube/utils/geometry/tools.py index e6dc8516ee..c4c7018ee3 100644 --- a/datacube/utils/geometry/tools.py +++ b/datacube/utils/geometry/tools.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import numpy as np import collections.abc diff --git a/datacube/utils/io.py b/datacube/utils/io.py index 85d50eb640..78d2da1fe3 100644 --- a/datacube/utils/io.py +++ b/datacube/utils/io.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import os from pathlib import Path diff --git a/datacube/utils/masking.py b/datacube/utils/masking.py index 3c35b1c422..a3446ae67c 100644 --- a/datacube/utils/masking.py +++ b/datacube/utils/masking.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Tools for masking data based on a bit-mask variable with attached definition. diff --git a/datacube/utils/math.py b/datacube/utils/math.py index 0378450857..1d62f22955 100644 --- a/datacube/utils/math.py +++ b/datacube/utils/math.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import Tuple, Union, Optional, Any, cast from math import ceil, fmod @@ -178,6 +178,7 @@ def num2numpy(x, dtype, ignore_range=None): :param x int|float: Numerical value to convert to numpy.type :param dtype str|numpy.dtype|numpy.type: Destination dtype :param ignore_range: If set to True skip range check and cast anyway (for example: -1 -> 255) + (Not supported in numpy 2.0+) :returns: None if x is None :returns: None if x is outside the valid range of dtype and ignore_range is not set diff --git a/datacube/utils/py.py b/datacube/utils/py.py index 456f24bf28..33e570bce5 100644 --- a/datacube/utils/py.py +++ b/datacube/utils/py.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import importlib import logging diff --git a/datacube/utils/rio/__init__.py b/datacube/utils/rio/__init__.py index 55db80b776..1fc438239f 100644 --- a/datacube/utils/rio/__init__.py +++ b/datacube/utils/rio/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ This will move into IO driver eventually. diff --git a/datacube/utils/rio/_rio.py b/datacube/utils/rio/_rio.py index 5d176bbdc3..c2aba0a441 100644 --- a/datacube/utils/rio/_rio.py +++ b/datacube/utils/rio/_rio.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ rasterio environment management tools """ diff --git a/datacube/utils/serialise.py b/datacube/utils/serialise.py index 8f180d7a2c..2001e46d5c 100644 --- a/datacube/utils/serialise.py +++ b/datacube/utils/serialise.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Serialise function used in YAML output diff --git a/datacube/utils/uris.py b/datacube/utils/uris.py index 0abfb6cde8..97d8140ce0 100644 --- a/datacube/utils/uris.py +++ b/datacube/utils/uris.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import os diff --git a/datacube/utils/xarray_geoextensions.py b/datacube/utils/xarray_geoextensions.py index 3a671f52f4..708286f01e 100644 --- a/datacube/utils/xarray_geoextensions.py +++ b/datacube/utils/xarray_geoextensions.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Add geometric extensions to :class:`xarray.Dataset` and :class:`xarray.DataArray` for use diff --git a/datacube/virtual/__init__.py b/datacube/virtual/__init__.py index 16793105c3..51a1e1933a 100644 --- a/datacube/virtual/__init__.py +++ b/datacube/virtual/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import Mapping, Any, cast import copy diff --git a/datacube/virtual/catalog.py b/datacube/virtual/catalog.py index aab9006043..2847f0b7aa 100644 --- a/datacube/virtual/catalog.py +++ b/datacube/virtual/catalog.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Catalog of virtual products. diff --git a/datacube/virtual/expr.py b/datacube/virtual/expr.py index 23fb9f08a6..3044b07c3c 100644 --- a/datacube/virtual/expr.py +++ b/datacube/virtual/expr.py @@ -1,3 +1,7 @@ +# This file is part of the Open Data Cube, see https://opendatacube.org for more information +# +# Copyright (c) 2015-2024 ODC Contributors +# SPDX-License-Identifier: Apache-2.0 import lark import numpy diff --git a/datacube/virtual/impl.py b/datacube/virtual/impl.py index 6a7bb23e3d..7388177cca 100644 --- a/datacube/virtual/impl.py +++ b/datacube/virtual/impl.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Implementation of virtual products. Provides an interface for the products in the datacube @@ -356,9 +356,12 @@ def group(self, datasets: VirtualDatasetBag, **group_settings: Dict[str, Any]) - merged = merge_search_terms(self, group_settings) try: - geobox = output_geobox(datasets=selected, - grid_spec=datasets.product_definitions[self._product].grid_spec, - geopolygon=geopolygon, **select_keys(merged, self._GEOBOX_KEYS)) + if isinstance(merged.get("like"), GeoBox): + geobox = merged["like"] + else: + geobox = output_geobox(datasets=selected, + grid_spec=datasets.product_definitions[self._product].grid_spec, + geopolygon=geopolygon, **select_keys(merged, self._GEOBOX_KEYS)) load_natively = False except ValueError: diff --git a/datacube/virtual/transformations.py b/datacube/virtual/transformations.py index 9c929eacc7..6c40478411 100644 --- a/datacube/virtual/transformations.py +++ b/datacube/virtual/transformations.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import Optional, Collection import warnings diff --git a/datacube/virtual/utils.py b/datacube/virtual/utils.py index 27cb0acada..0f59795a48 100644 --- a/datacube/virtual/utils.py +++ b/datacube/virtual/utils.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Utilities to facilitate virtual product implementation. """ diff --git a/docker/Dockerfile b/docker/Dockerfile index 226df3bf4f..84e9b82e79 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,12 +1,14 @@ ## ## This file is part of the Open Data Cube, see https://opendatacube.org for more information ## -## Copyright (c) 2015-2020 ODC Contributors +## Copyright (c) 2015-2024 ODC Contributors ## SPDX-License-Identifier: Apache-2.0 ## -FROM osgeo/gdal:ubuntu-small-latest -ARG V_PG=14 -ARG V_PGIS=14-postgis-3 +# gdal:ubuntu-small no longer comes with netcdf support compiled into gdal +FROM ghcr.io/osgeo/gdal:ubuntu-full-3.9.0 AS builder +FROM ghcr.io/osgeo/gdal:ubuntu-full-3.9.0 +ARG V_PG=16 +ARG V_PGIS=16-postgis-3 # Update and install Ubuntu packages @@ -17,7 +19,8 @@ RUN apt-get update -y \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-change-held-packages --fix-missing --no-install-recommends \ git \ libpq-dev libudunits2-dev libproj-dev \ - python3-dev python3-distutils python3-pip \ + libhdf5-dev libnetcdf-dev libgeos-dev libudunits2-dev \ + python3-dev virtualenv \ build-essential \ postgresql \ redis-server \ @@ -30,21 +33,28 @@ RUN apt-get update -y \ # Build constrained python environment +RUN virtualenv /env # Set the locale, this is required for some of the Python packages -ENV LC_ALL C.UTF-8 +ENV PYENV=/env \ + GDAL_CONFIG=/usr/bin/gdal-config \ + LC_ALL=C.UTF-8 + +# Needed to build cf-units wheels. +ARG UDUNITS2_XML_PATH=/usr/share/xml/udunits/udunits2-common.xml COPY docker/constraints.in /conf/requirements.txt COPY docker/constraints.txt docker/nobinary.txt /conf/ -RUN python3 -m pip install \ - -r /conf/requirements.txt \ - -c /conf/constraints.txt +RUN . /env/bin/activate && python3 -m pip install --upgrade pip setuptools +RUN . /env/bin/activate && python3 -m pip install -r /conf/requirements.txt \ + -c /conf/constraints.txt \ + -c /conf/nobinary.txt # Copy datacube-core source code into container and install from source (with addons for tests). COPY . /code -RUN python3 -m pip install '/code/[all]' \ +RUN . /env/bin/activate && python3 -m pip install '/code/[all]' \ && python3 -m pip install /code/examples/io_plugin \ && python3 -m pip install /code/tests/drivers/fail_drivers @@ -56,16 +66,12 @@ RUN install --owner postgres --group postgres -D -d /var/run/postgresql /srv/po && sudo -u postgres "$(find /usr/lib/postgresql/ -type f -name initdb)" -D "/srv/postgresql" --auth-host=md5 --encoding=UTF8 # users and groups. -RUN groupadd --gid 1000 odc \ - && useradd --gid 1000 \ - --uid 1000 \ - --create-home \ - --shell /bin/bash -N odc \ +RUN groupmod ubuntu -n odc \ + && usermod ubuntu -l odc \ && adduser odc users \ && adduser odc sudo \ && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \ - && install -d -o odc -g odc /env \ - && install -d -o odc -g odc /code \ + && chown -R odc:odc /env \ && true USER root diff --git a/docker/assets/with_bootstrap b/docker/assets/with_bootstrap index 5da8866f65..e2a3a1a518 100755 --- a/docker/assets/with_bootstrap +++ b/docker/assets/with_bootstrap @@ -6,7 +6,7 @@ launch_db () { local bin=$(find /usr/lib/postgresql/ -type d -name bin) [ -e "${pgdata}/PG_VERSION" ] || { - sudo -u postgres "${bin}/initdb" -D "${pgdata}" --auth-host=md5 --encoding=UTF8 + sudo -u postgres "${bin}/initdb" -A -D "${pgdata}" --auth-host=md5 --encoding=UTF8 } sudo -u postgres "${bin}/pg_ctl" -D "${pgdata}" -l "${pgdata}/pg.log" start @@ -14,8 +14,8 @@ launch_db () { sudo -u postgres createuser --superuser "${dbuser}" sudo -u postgres createdb "${dbuser}" sudo -u postgres createdb datacube - sudo -u postgres createdb agdcintegration - sudo -u postgres createdb odcintegration + sudo -u postgres createdb pgintegration + sudo -u postgres createdb pgisintegration } # Become `odc` user with UID/GID compatible to datacube-core volume @@ -58,12 +58,12 @@ launch_db () { cat < $HOME/.datacube_integration.conf [datacube] db_hostname: -db_database: agdcintegration +db_database: pgintegration index_driver: default [experimental] db_hostname: -db_database: odcintegration +db_database: pgisintegration index_driver: postgis [no_such_driver_env] @@ -79,6 +79,7 @@ env="${PYENV:-/env}" if [ -e "${env}/bin/activate" ]; then [ -n "${VIRTUAL_ENV:-}" ] || { + echo "Activating virtual environment" source "${env}/bin/activate" # if there is no .egg-info then we need to install in diff --git a/docker/constraints.in b/docker/constraints.in index ffb015b5c5..4d271e7243 100644 --- a/docker/constraints.in +++ b/docker/constraints.in @@ -6,37 +6,57 @@ attrs>=18.1 boto3>1.17.0 bottleneck cachetools +# For Python 3.12 support +cf-units>3.1.1 +# For Python 3.12 support +cffi>=1.16.0 ciso8601 click>=8.0 cloudpickle>=0.4 -compliance-checker>=4.0.0,<5 +# For Python 3.12 support +compliance-checker>=5.1.0 dask>=2021.10.1 distributed>=2021.10.0 fiona geoalchemy2 -jsonschema -# Was lark-parser>=0.6.7 +# For Python 3.12 support +greenlet>=3.0.3 +# New reference resolution API +jsonschema>=4.18 lark +# For Python 3.12 support +lxml>=5.0.0 matplotlib -moto +moto<5 netcdf4>=1.5.8 -numpy>=1.22.2 +# 1.26.0 is the first version to support Python 3.12 +numpy>=1.26.0 +pandas>=2.0 +# For Python 3.12 support +pendulum>=3.0 psycopg2 pycodestyle -pylint -pyproj>=2.5 +# For Python 3.12 support +pyproj>=3.5 python-dateutil -pyyaml -rasterio>=1.3.2 +# For Python 3.12 support +pyyaml>=6.0.1 +# For Python 3.12 support +rasterio>=1.3.9 +# For Python 3.12 support +typing-extensions>=4.7 recommonmark redis ruamel.yaml -shapely>=2.0 +# cython3 support +shapely>=2.0.2 sphinx-click sphinx_autodoc_typehints sphinx_rtd_theme sqlalchemy<2.0 toolz +# For Python 3.12 support +wrapt>=1.16.0 xarray>=0.9 # Previous pins were to very old versions @@ -47,9 +67,8 @@ hypothesis>=6 pytest-httpserver pytest-timeout -# every new version finds new errors, so we pin it (so old though!) -pylint==2.4.4 -pycodestyle==2.5.0 +pylint>=3 +pycodestyle # for packaging setuptools>=42 @@ -57,3 +76,5 @@ setuptools_scm>=3.4 toml wheel twine + +deprecat diff --git a/docker/constraints.txt b/docker/constraints.txt index 35c1848e94..712563e750 100644 --- a/docker/constraints.txt +++ b/docker/constraints.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: # -# pip-compile --strip-extras constraints.in +# pip-compile --output-file=constraints.txt --strip-extras constraints.in # affine==2.4.0 # via @@ -12,7 +12,7 @@ alabaster==0.7.13 # via sphinx antlr4-python3-runtime==4.7.2 # via cf-units -astroid==2.3.3 +astroid==3.2.2 # via pylint async-timeout==4.0.2 # via redis @@ -24,6 +24,7 @@ attrs==22.2.0 # jsonschema # pytest # rasterio + # referencing babel==2.11.0 # via sphinx bleach==6.0.0 @@ -47,10 +48,14 @@ certifi==2022.12.7 # pyproj # rasterio # requests -cf-units==3.1.1 - # via compliance-checker -cffi==1.15.1 - # via cryptography +cf-units==3.2.0 + # via + # -r constraints.in + # compliance-checker +cffi==1.16.0 + # via + # -r constraints.in + # cryptography cftime==1.6.2 # via # cf-units @@ -85,7 +90,7 @@ cloudpickle==2.2.1 # distributed commonmark==0.9.1 # via recommonmark -compliance-checker==4.3.4 +compliance-checker==5.1.0 # via -r constraints.in contourpy==1.0.7 # via matplotlib @@ -103,6 +108,10 @@ dask==2023.2.0 # distributed decorator==5.1.1 # via validators +deprecat==2.1.1 + # via -r constraints.in +dill==0.3.8 + # via pylint distributed==2023.2.0 # via -r constraints.in docutils==0.18.1 @@ -112,10 +121,6 @@ docutils==0.18.1 # sphinx # sphinx-click # sphinx-rtd-theme -exceptiongroup==1.1.0 - # via - # hypothesis - # pytest fiona==1.9.1 # via -r constraints.in fonttools==4.38.0 @@ -124,8 +129,10 @@ fsspec==2023.1.0 # via dask geoalchemy2==0.13.1 # via -r constraints.in -greenlet==2.0.2 - # via sqlalchemy +greenlet==3.0.3 + # via + # -r constraints.in + # sqlalchemy heapdict==1.0.1 # via zict hypothesis==6.68.1 @@ -161,22 +168,24 @@ jmespath==1.0.1 # via # boto3 # botocore -jsonschema==4.17.3 +jsonschema==4.18.4 # via -r constraints.in +jsonschema-specifications==2023.7.1 + # via jsonschema keyring==23.13.1 # via twine kiwisolver==1.4.4 # via matplotlib lark==1.1.5 # via -r constraints.in -lazy-object-proxy==1.4.3 - # via astroid locket==1.0.0 # via # distributed # partd -lxml==4.9.2 - # via compliance-checker +lxml==5.0.2 + # via + # -r constraints.in + # compliance-checker markdown-it-py==2.1.0 # via rich markupsafe==2.1.2 @@ -201,7 +210,7 @@ netcdf4==1.6.2 # via # -r constraints.in # compliance-checker -numpy==1.24.2 +numpy==1.26.4 # via # -r constraints.in # bottleneck @@ -227,16 +236,22 @@ packaging==23.0 # setuptools-scm # sphinx # xarray -pandas==1.5.3 - # via xarray +pandas==2.1.4 + # via + # -r constraints.in + # xarray partd==1.3.0 # via dask -pendulum==2.1.2 - # via compliance-checker +pendulum==3.0.0 + # via + # -r constraints.in + # compliance-checker pillow==9.4.0 # via matplotlib pkginfo==1.9.6 # via twine +platformdirs==4.2.2 + # via pylint pluggy==1.0.0 # via pytest psutil==5.9.4 @@ -254,18 +269,16 @@ pygments==2.14.0 # readme-renderer # rich # sphinx -pylint==2.4.4 +pylint==3.2.2 # via -r constraints.in pyparsing==3.0.9 # via # matplotlib # snuggs -pyproj==3.4.1 +pyproj==3.6.1 # via # -r constraints.in # compliance-checker -pyrsistent==0.19.3 - # via jsonschema pytest==7.2.1 # via # -r constraints.in @@ -286,20 +299,19 @@ python-dateutil==2.8.2 # owslib # pandas # pendulum + # time-machine pytz==2022.7.1 # via # babel # owslib # pandas -pytzdata==2020.1 - # via pendulum -pyyaml==6.0 +pyyaml==6.0.1 # via # -r constraints.in # dask # distributed # owslib -rasterio==1.3.6 +rasterio==1.3.10 # via -r constraints.in readme-renderer==37.3 # via twine @@ -307,6 +319,10 @@ recommonmark==0.7.1 # via -r constraints.in redis==4.5.1 # via -r constraints.in +referencing==0.30.0 + # via + # jsonschema + # jsonschema-specifications regex==2022.10.31 # via compliance-checker requests==2.28.2 @@ -326,21 +342,24 @@ rfc3986==2.0.0 # via twine rich==13.3.1 # via twine +rpds-py==0.9.2 + # via + # jsonschema + # referencing ruamel-yaml==0.17.21 # via -r constraints.in -ruamel-yaml-clib==0.2.7 - # via ruamel-yaml s3transfer==0.6.0 # via boto3 secretstorage==3.3.3 # via keyring setuptools-scm==7.1.0 # via -r constraints.in -shapely==2.0.1 - # via -r constraints.in +shapely==2.0.4 + # via + # -r constraints.in + # compliance-checker six==1.16.0 # via - # astroid # bleach # isodate # munch @@ -386,15 +405,14 @@ sqlalchemy==1.4.46 # geoalchemy2 tblib==1.7.0 # via distributed +time-machine==2.14.1 + # via pendulum toml==0.10.2 # via # -r constraints.in # responses -tomli==2.0.1 - # via - # coverage - # pytest - # setuptools-scm +tomlkit==0.12.5 + # via pylint toolz==0.12.0 # via # -r constraints.in @@ -407,10 +425,15 @@ twine==4.0.2 # via -r constraints.in types-toml==0.10.8.3 # via responses -typing-extensions==4.4.0 +typing-extensions==4.12.0 # via + # -r constraints.in # pygeoif # setuptools-scm +tzdata==2023.4 + # via + # pandas + # pendulum urllib3==1.26.14 # via # botocore @@ -428,9 +451,11 @@ werkzeug==2.2.2 # pytest-httpserver wheel==0.38.4 # via -r constraints.in -wrapt==1.11.2 - # via astroid -xarray==2023.2.0 +wrapt==1.16.0 + # via + # -r constraints.in + # deprecat +xarray==2023.12.0 # via -r constraints.in xmltodict==0.13.0 # via moto diff --git a/docker/nobinary.txt b/docker/nobinary.txt index fd69f0b5d6..57b77ce03c 100644 --- a/docker/nobinary.txt +++ b/docker/nobinary.txt @@ -1 +1,5 @@ -# Empty for now +# libraries to compile with local gdal +--no-binary rasterio +--no-binary fiona +--no-binary shapely +--no-binary cf-units diff --git a/docs/README.rst b/docs/README.rst index 31a0a05bb3..7e4725eb86 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -8,22 +8,60 @@ Developing Locally Requires a Unix like system that includes ``make``. -#. Install NodeJS + NPM -#. Install Browser Sync +#. Clone the datacube-core repository. If you don't have permissions to push to the datacube-core library, you will need to fork the repo and clone your fork. .. code-block:: bash - npm install -g browser-sync + git clone https://github.com/opendatacube/datacube-core.git -#. Install Python dependencies +#. Check out a new branch for the documentation feature you're working on .. code-block:: bash - pip install -r requirements.txt - pip install git+https://github.com/carrotandcompany/sphinx-autobuild.git@feature_event_delay + git switch -c docs- -#. Start the auto-building development server. +#. Change directory to the docs folder + +.. code-block:: bash + + cd docs + +#. Create a conda environment for python 3.11, with conda-forge as the channel + +.. code-block:: bash + + conda create --name datacubecoredocs -c conda-forge python=3.11 + +#. Activate the conda environment + +.. code-block:: bash + + conda activate datacubecoredocs + +#. Install pandoc + +.. code-block:: bash + + conda install pandoc + +#. Install requirements with pip + +.. code-block:: bash + + pip install -r requirements.txt + +#. Run the autobuild. .. code-block:: bash sphinx-autobuild . _build/html + +#. Open a browser and navigate to the URL provided by the autobuild + +#. Make changes to the docs. The terminal with the autobuild will continue to update the docs view in the browser. + +#. When finished, quit the autobuild process using ``ctrl-c`` in the terminal. + +#. Stage and commit your changes. + +#. When ready for review, push your changes and create a pull request. diff --git a/docs/_static/custom.css b/docs/_static/custom.css index c759ee20f6..581a39cbb2 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -105,6 +105,10 @@ padding: 40px; } +.bd-container { + padding-top: var(--pst-header-height); +} + .section p { line-height: 1.8rem; } @@ -185,3 +189,24 @@ dt:target, span.highlighted { max-width: 100%; font-size: 0.9rem; } + +html[data-theme="dark"] { + --pst-color-text-base: var(--pst-color-text-muted); + --pst-color-sidebar-link-active: var(--pst-color-link); +} + +html[data-theme="dark"] .container-xl { + background-color: #3B3B3B; +} + +html[data-theme="dark"] .editthispage a { + color: var(--pst-color-link); +} + +html[data-theme="dark"] .bd-sidebar { + border-right: 3px solid #ffffff1a; +} + +html[data-theme="dark"] .navbar-light>.container-xl { + background-color: var(--pst-color-surface); +} diff --git a/docs/about-core-concepts/existing-deployments.rst b/docs/about-core-concepts/existing-deployments.rst index 42df01a6ae..be004917be 100644 --- a/docs/about-core-concepts/existing-deployments.rst +++ b/docs/about-core-concepts/existing-deployments.rst @@ -7,10 +7,10 @@ If you are using `Digital Earth Australia`_, see the `Digital Earth Australia User Guide`_ for instructions on accessing data on the `NCI`_, `AWS`_ and the `DEA Sandbox`_. .. _`Digital Earth Australia`: https://www.ga.gov.au/dea -.. _`Digital Earth Australia User Guide`: https://docs.dea.ga.gov.au/ -.. _`NCI`: https://docs.dea.ga.gov.au/setup/NCI/README.html -.. _`AWS`: https://docs.dea.ga.gov.au/setup/AWS/data_and_metadata.html -.. _`DEA Sandbox`: https://docs.dea.ga.gov.au/setup/sandbox.html +.. _`Digital Earth Australia User Guide`: https://knowledge.dea.ga.gov.au/ +.. _`NCI`: https://knowledge.dea.ga.gov.au/setup/NCI/README.html +.. _`AWS`: https://knowledge.dea.ga.gov.au/setup/AWS/data_and_metadata.html +.. _`DEA Sandbox`: https://knowledge.dea.ga.gov.au/setup/sandbox.html Digital Earth Africa diff --git a/docs/about-core-concepts/metadata-types.rst b/docs/about-core-concepts/metadata-types.rst index 323f62e470..a101af4203 100644 --- a/docs/about-core-concepts/metadata-types.rst +++ b/docs/about-core-concepts/metadata-types.rst @@ -8,4 +8,7 @@ Metadata Types Metadata type yaml file must contain name, description and dataset keys. - Dataset key must contain id, sources, creation_dt, label and search_fields keys. + Dataset key must contain id, sources, creation_dt, label, and search_fields keys. + + For metadata types of spatial datasets, the dataset key must also contain grid_spatial, measurements, and format keys. + Support for non-spatial datasets is likely to be dropped in version 2.0. diff --git a/docs/about/release_process.rst b/docs/about/release_process.rst index 42df2085e9..c4a652fe08 100644 --- a/docs/about/release_process.rst +++ b/docs/about/release_process.rst @@ -1,7 +1,7 @@ Release Process *************** -#. Decide to do a release, and check with regular contributors on Slack that +#. Decide to do a release, and check with regular contributors on `Discord `_ that they don't have anything pending. #. Ensure version pins in setup.py and conda-environment.yml are in sync and up to date. diff --git a/docs/about/whats_new.rst b/docs/about/whats_new.rst index 1df3863343..2a35acff47 100644 --- a/docs/about/whats_new.rst +++ b/docs/about/whats_new.rst @@ -7,9 +7,93 @@ What's New v1.8.next ========= +- Don't error when adding a dataset whose product doesn't have an id value (:pull:`1630`) -- Documentation fixes (:pull:`1417`, :pull:`1418`) +v1.8.19 (2nd July 2024) +======================= + +- Update whats_new.rst for 1.8.19 release (:pull:`1612`) +- Always write floating point bands to cogs with nodata=nan for ESRI and GDAL compatibility (:pull:`1602`) +- Add deprecation warning for config environment names that will not be supported in 1.9 (:pull:`1592`) +- Update docker image to GDAL 3.9/Python 3.12/Ubuntu 24.04 (:pull:`1587`) +- Update readthedocs stylesheet for dark theme (:pull:`1579`) + +v1.8.18 (27th March 2024) +========================= + +- Add dataset cli tool ``find-duplicates`` to identify duplicate indexed datasets (:pull:`1517`) +- Make solar_day() timezone aware (:pull:`1521`) +- Warn if non-eo3 dataset has eo3 metadata type (:pull:`1523`) +- Update pandas version in docker image to be consistent with conda environment and default to stdlib + timezone instead of pytz when converting timestamps; automatically update copyright years (:pull:`1527`) +- Update github-Dockerhub credential-passing mechanism. (:pull:`1528`) +- Tweak ``list_products`` logic for getting crs and resolution values (:pull:`1535`) +- Add new ODC Cheatsheet reference doc to Data Access & Analysis documentation page (:pull:`1543`) +- Compatibility fix to allow users to supply ``odc.geo``-style GeoBoxes to ``dc.load(like=...)`` (:pull:`1551`) +- Fix broken codecov github action. (:pull:`1554`) +- Update documentation links to DEA Knowledge Hub (:pull:`1559`) +- Throw error if ``time`` dimension is provided as an int or float to Query construction + instead of assuming it to be seconds since epoch (:pull:`1561`) +- Add generic NOT operator and for ODC queries and ``Not`` type wrapper (:pull:`1563`) +- Update whats_new.rst for release (:pull:`1568`) + +v1.8.17 (8th November 2023) +=========================== +- Fix schema creation with postgres driver when initialising system with ``--no-init-users`` (:pull:`1504`) +- Switch to new jsonschema 'referencing' API and repin jsonschema to >=4.18 (:pull:`1477`) +- Update whats_new.rst for release (:pull:`1510`) + +v1.8.16 (17th October 2023) +=========================== +- Improve error message for mismatch between dataset metadata and product signature (:pull:`1472`) +- Mark ``--confirm-ignore-lineage``, ``--auto-add-lineage``, and ``--verify-lineage`` as deprecated + or to be deprecated (:pull:`1472`) +- Default delta values in ``archive_less_mature`` and ``find_less_mature`` (:pull:`1472`) +- Fix SQLAlchemy calls and pin jsonschema version to suppress deprecation warnings (:pull:`1476`) +- Throw a better error if a dataset is not compatible with ``archive_less_mature`` logic (:pull:`1491`) +- Fix broken Github action workflow (:pull:`1496`) +- Support ``like=`` in virtual product ``load`` (:pull:`1497`) +- Don't archive less mature if archive_less_mature is provided as `False` instead of `None` (:pull:`1498`) +- Raise minimum supported Python version to 3.9 (:pull:`1500`) +- Manually apply Dependabot updates, and update whats_new.rst for 1.8.16 release (:pull:`1501`) + +v1.8.15 (11th July 2023) +======================== +- Replace `importlib_metadata` for python <3.10 compatibility + (:pull:`1469`) +- Update whats_new.rst for release (:pull:`1470`) + +v1.8.14 (28th June 2023) +======================== + +- Second attempt to address unexpected handling of image aspect ratios in rasterio and + GDAL. (:pull:`1457`) +- Fix broken pypi publishing Github action (:pull:`1454`) +- Documentation improvements (:pull:`1455`) +- Increase default maturity leniency to +-500ms (:pull:`1458`) +- Add option to specify maturity timedelta when using ``--archive-less-mature`` option (:pull:`1460`) +- Mark executors as deprecated (:pull:`1461`) +- Mark ingestion as deprecated (:pull:`1463`) +- Replace deprecated ``pkg_resources`` with ``importlib.resources`` and ``importlib.metadata`` (:pull:`1466`) +- Update whats_new.rst for release (:pull:`1467`) + +v1.8.13 (6th June 2023) +======================= +- Fix broken Github action workflows (:pull:`1425`, :pull:`1427`, :pull:`1433`) +- Setup Dependabot, and Dependabot-generated updates (:pull:`1416`, :pull:`1420`, :pull:`1423`, + :pull:`1428`, :pull:`1436`, :pull:`1447`) +- Documentation fixes (:pull:`1417`, :pull:`1418`, :pull:`1430`) +- ``datacube dataset`` cli commands print error message if missing argument (:pull:`1437`) +- Add pre-commit hook to verify license headers (:pull:`1438`) +- Support open-ended date ranges in `datacube dataset search`, `dc.load`, and `dc.find_datasets` (:pull:`1439`, :pull:`1443`) +- Pass Y and Y Scale factors through to rasterio.warp.reproject, to eliminate projection bug affecting + non-square Areas Of Interest (See `Issue #1448`_) (:pull:`1450`) +- Add `archive_less_mature` option to `datacube dataset add` and `datacube dataset update` (:pull:`1451`) +- Allow for +-1ms leniency in finding other maturity versions of a dataset (:pull:`1452`) +- Update whats_new.rst for release (:pull:`1453`) + +.. _`Issue #1448`: https://github.com/opendatacube/datacube-core/issues/1448 v1.8.12 (7th March 2023) ======================== diff --git a/docs/cheatsheets/ODC_Cheatsheet.jpg b/docs/cheatsheets/ODC_Cheatsheet.jpg new file mode 100644 index 0000000000..599c54dad7 Binary files /dev/null and b/docs/cheatsheets/ODC_Cheatsheet.jpg differ diff --git a/docs/cheatsheets/ODC_Cheatsheet.pdf b/docs/cheatsheets/ODC_Cheatsheet.pdf new file mode 100644 index 0000000000..3a0ced1e59 Binary files /dev/null and b/docs/cheatsheets/ODC_Cheatsheet.pdf differ diff --git a/docs/cheatsheets/README.rst b/docs/cheatsheets/README.rst new file mode 100644 index 0000000000..173dcc9565 --- /dev/null +++ b/docs/cheatsheets/README.rst @@ -0,0 +1,9 @@ + +Open Data Cube Cheatsheets +========================== + +This directory contains useful "cheatsheets" designed to make it easier to get started with Open Data Cube, including demonstrating how to perform common analysis tasks including loading data, data preparation, plotting and exporting, and geospatial manipulation. + +Can be modified via `Google Docs`_. + +.. _`Google Docs`: https://docs.google.com/presentation/d/1aSk2JSK1uGuGdZQQDYhqN0b5bY7wSgIIApLG6k229QA/edit?usp=sharing diff --git a/docs/click_utils.py b/docs/click_utils.py index 0b658d8eef..ee722544cd 100644 --- a/docs/click_utils.py +++ b/docs/click_utils.py @@ -1,8 +1,7 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 -import pkg_resources from docutils.nodes import literal_block, section, title, make_id from sphinx.domains import Domain from docutils.parsers.rst import Directive @@ -34,8 +33,12 @@ def find_script_callable_from_env(name, env): def find_script_callable(name): - return list(pkg_resources.iter_entry_points( - 'console_scripts', name))[0].load() + try: + from importlib_metadata import entry_points + except ModuleNotFoundError: + from importlib.metadata import entry_points + return list(entry_points( + group='console_scripts', name=name))[0].load() def generate_help_text(command, prefix): diff --git a/docs/conf.py b/docs/conf.py index bbcbec7b12..e3f4bd8d38 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import os import sys @@ -130,9 +130,9 @@ "icon": "fab fa-github", }, { - "name": "Slack", - "url": "http://slack.opendatacube.org/", - "icon": "fab fa-slack", + "name": "Discord", + "url": "https://discord.com/invite/4hhBQVas5U", + "icon": "fab fa-discord", }, ], } diff --git a/docs/config_samples/ingester/ls5_nbar_albers.yaml b/docs/config_samples/ingester/ls5_nbar_albers.yaml index 42cfc44127..d37747848e 100644 --- a/docs/config_samples/ingester/ls5_nbar_albers.yaml +++ b/docs/config_samples/ingester/ls5_nbar_albers.yaml @@ -39,10 +39,10 @@ global_attributes: Surface Reflectance Correction Models Image radiance values recorded by passive EO sensors are a composite of - - surface reflectance; - - atmospheric condition; - - interaction between surface land cover, solar radiation and sensor view angle; - - land surface orientation relative to the imaging sensor. + - surface reflectance; + - atmospheric condition; + - interaction between surface land cover, solar radiation and sensor view angle; + - land surface orientation relative to the imaging sensor. It has been traditionally assumed that Landsat imagery display negligible variation in sun and sensor view angles, however these can vary significantly both within and between scenes, especially in different seasons and geographic regions (Li et al., 2012). The SR product delivers modeled surface reflectance from Landsat TM/ETM+/OLI @@ -57,11 +57,11 @@ global_attributes: Given the growing time series of EO imagery, this landmark facility will streamline the process of reliably monitoring long-term chnges in land and water resources. source: SR-N_25_2 history: | - - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing - - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. - - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. - - DEM: 1 second SRTM DSM is used for Ortho-rectification. - - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. + - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing + - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. + - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. + - DEM: 1 second SRTM DSM is used for Ortho-rectification. + - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. institution: Commonwealth of Australia (Geoscience Australia) instrument: TM keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,REFLECTANCE,ETM+,TM,OLI,EARTH SCIENCE @@ -77,30 +77,30 @@ global_attributes: product_suite: Surface Reflectance NBAR+ 25m acknowledgment: Landsat data is provided by the United States Geological Survey (USGS) through direct reception of the data at Geoscience Australias satellite reception facility or download. references: | - - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. - - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. - - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. - - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. - - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. - - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. - - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. - - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. - - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. - - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. - - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. - - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. - - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. - - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. - - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. - - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. - - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf - - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. - - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. - - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. - - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. - - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. - - http://dx.doi.org/10.4225/25/5487CC0D4F40B - - http://dx.doi.org/10.1109/JSTARS.2010.2042281 + - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. + - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. + - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. + - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. + - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. + - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. + - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. + - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. + - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. + - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. + - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. + - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. + - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. + - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. + - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. + - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. + - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf + - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. + - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. + - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. + - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. + - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. + - http://dx.doi.org/10.4225/25/5487CC0D4F40B + - http://dx.doi.org/10.1109/JSTARS.2010.2042281 storage: driver: NetCDF CF diff --git a/docs/config_samples/ingester/ls5_nbart_albers.yaml b/docs/config_samples/ingester/ls5_nbart_albers.yaml index 89d942cb8d..a848a6af79 100644 --- a/docs/config_samples/ingester/ls5_nbart_albers.yaml +++ b/docs/config_samples/ingester/ls5_nbart_albers.yaml @@ -45,10 +45,10 @@ global_attributes: Surface Reflectance Correction Models Image radiance values recorded by passive EO sensors are a composite of - - surface reflectance; - - atmospheric condition; - - interaction between surface land cover, solar radiation and sensor view angle; - - land surface orientation relative to the imaging sensor. + - surface reflectance; + - atmospheric condition; + - interaction between surface land cover, solar radiation and sensor view angle; + - land surface orientation relative to the imaging sensor. It has been traditionally assumed that Landsat imagery display negligible variation in sun and sensor view angles, however these can vary significantly both within and between scenes, especially in different seasons and geographic regions (Li et al., 2012). The SR product delivers modeled surface reflectance from Landsat TM/ETM+/OLI @@ -63,11 +63,11 @@ global_attributes: Given the growing time series of EO imagery, this landmark facility will streamline the process of reliably monitoring long-term chnges in land and water resources. source: SR-NT_25_2 history: | - - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing - - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. - - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. - - DEM: 1 second SRTM DSM is used for Ortho-rectification. - - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. + - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing + - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. + - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. + - DEM: 1 second SRTM DSM is used for Ortho-rectification. + - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. institution: Commonwealth of Australia (Geoscience Australia) instrument: TM keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,REFLECTANCE,ETM+,TM,OLI,EARTH SCIENCE @@ -83,30 +83,30 @@ global_attributes: product_suite: Surface Reflectance NBAR+T 25m acknowledgment: Landsat data is provided by the United States Geological Survey (USGS) through direct reception of the data at Geoscience Australias satellite reception facility or download. references: | - - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. - - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. - - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. - - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. - - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. - - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. - - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. - - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. - - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. - - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. - - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. - - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. - - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. - - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. - - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. - - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. - - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf - - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. - - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. - - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. - - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. - - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. - - http://dx.doi.org/10.4225/25/5487CC0D4F40B - - http://dx.doi.org/10.1109/JSTARS.2010.2042281 + - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. + - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. + - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. + - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. + - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. + - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. + - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. + - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. + - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. + - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. + - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. + - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. + - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. + - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. + - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. + - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. + - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf + - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. + - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. + - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. + - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. + - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. + - http://dx.doi.org/10.4225/25/5487CC0D4F40B + - http://dx.doi.org/10.1109/JSTARS.2010.2042281 storage: driver: NetCDF CF diff --git a/docs/config_samples/ingester/ls5_pq_albers.yaml b/docs/config_samples/ingester/ls5_pq_albers.yaml index 491f30b020..4eda5e55e6 100644 --- a/docs/config_samples/ingester/ls5_pq_albers.yaml +++ b/docs/config_samples/ingester/ls5_pq_albers.yaml @@ -19,11 +19,11 @@ global_attributes: or only sea pixels depending on their analytical requirements, leading to enhanced computationally efficient. PQ provides an assessment of the quality of observations at a pixel level and includes information about: - - Spectral Contiguity (lack of signal in any band) - - Saturation in any band - - Presence of cloud - - Presence of cloud shadow - - Land or sea + - Spectral Contiguity (lack of signal in any band) + - Saturation in any band + - Presence of cloud + - Presence of cloud shadow + - Land or sea As Landsat Imagery becomes more readily available, there has been a rapid increase in the amount of analyses undertaken by researchers around the globe. Most researchers use some form of quality masking schema in order to remove undesirable @@ -43,11 +43,11 @@ global_attributes: with OLI data, however it uses additional algorithms to detect cloud and cloud shadow, and is available for Landsat 5, 7 and 8. source: PQ_25_2 history: | - - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing - - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. - - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. - - DEM: 1 second SRTM DSM is used for Ortho-rectification. - - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. + - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing + - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. + - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. + - DEM: 1 second SRTM DSM is used for Ortho-rectification. + - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. institution: Commonwealth of Australia (Geoscience Australia) instrument: TM keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,ETM+,TM,OLI,EARTH SCIENCE diff --git a/docs/config_samples/ingester/ls7_nbar_albers.yaml b/docs/config_samples/ingester/ls7_nbar_albers.yaml index 3ddba85437..fd3aa6b9e3 100644 --- a/docs/config_samples/ingester/ls7_nbar_albers.yaml +++ b/docs/config_samples/ingester/ls7_nbar_albers.yaml @@ -39,10 +39,10 @@ global_attributes: Surface Reflectance Correction Models Image radiance values recorded by passive EO sensors are a composite of - - surface reflectance; - - atmospheric condition; - - interaction between surface land cover, solar radiation and sensor view angle; - - land surface orientation relative to the imaging sensor. + - surface reflectance; + - atmospheric condition; + - interaction between surface land cover, solar radiation and sensor view angle; + - land surface orientation relative to the imaging sensor. It has been traditionally assumed that Landsat imagery display negligible variation in sun and sensor view angles, however these can vary significantly both within and between scenes, especially in different seasons and geographic regions (Li et al., 2012). The SR product delivers modeled surface reflectance from Landsat TM/ETM+/OLI @@ -57,11 +57,11 @@ global_attributes: Given the growing time series of EO imagery, this landmark facility will streamline the process of reliably monitoring long-term chnges in land and water resources. source: SR-N_25_2 history: | - - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing - - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. - - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. - - DEM: 1 second SRTM DSM is used for Ortho-rectification. - - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. + - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing + - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. + - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. + - DEM: 1 second SRTM DSM is used for Ortho-rectification. + - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. institution: Commonwealth of Australia (Geoscience Australia) instrument: ETM keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,REFLECTANCE,ETM+,TM,OLI,EARTH SCIENCE @@ -77,30 +77,30 @@ global_attributes: product_suite: Surface Reflectance NBAR+ 25m acknowledgment: Landsat data is provided by the United States Geological Survey (USGS) through direct reception of the data at Geoscience Australias satellite reception facility or download. references: | - - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. - - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. - - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. - - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. - - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. - - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. - - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. - - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. - - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. - - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. - - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. - - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. - - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. - - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. - - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. - - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. - - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf - - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. - - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. - - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. - - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. - - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. - - http://dx.doi.org/10.4225/25/5487CC0D4F40B - - http://dx.doi.org/10.1109/JSTARS.2010.2042281 + - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. + - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. + - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. + - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. + - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. + - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. + - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. + - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. + - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. + - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. + - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. + - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. + - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. + - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. + - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. + - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. + - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf + - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. + - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. + - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. + - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. + - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. + - http://dx.doi.org/10.4225/25/5487CC0D4F40B + - http://dx.doi.org/10.1109/JSTARS.2010.2042281 storage: driver: NetCDF CF diff --git a/docs/config_samples/ingester/ls7_nbart_albers.yaml b/docs/config_samples/ingester/ls7_nbart_albers.yaml index 136927ad22..0f31c91004 100644 --- a/docs/config_samples/ingester/ls7_nbart_albers.yaml +++ b/docs/config_samples/ingester/ls7_nbart_albers.yaml @@ -45,10 +45,10 @@ global_attributes: Surface Reflectance Correction Models Image radiance values recorded by passive EO sensors are a composite of - - surface reflectance; - - atmospheric condition; - - interaction between surface land cover, solar radiation and sensor view angle; - - land surface orientation relative to the imaging sensor. + - surface reflectance; + - atmospheric condition; + - interaction between surface land cover, solar radiation and sensor view angle; + - land surface orientation relative to the imaging sensor. It has been traditionally assumed that Landsat imagery display negligible variation in sun and sensor view angles, however these can vary significantly both within and between scenes, especially in different seasons and geographic regions (Li et al., 2012). The SR product delivers modeled surface reflectance from Landsat TM/ETM+/OLI @@ -63,11 +63,11 @@ global_attributes: Given the growing time series of EO imagery, this landmark facility will streamline the process of reliably monitoring long-term chnges in land and water resources. source: SR-NT_25_2 history: | - - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing - - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. - - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. - - DEM: 1 second SRTM DSM is used for Ortho-rectification. - - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. + - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing + - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. + - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. + - DEM: 1 second SRTM DSM is used for Ortho-rectification. + - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. institution: Commonwealth of Australia (Geoscience Australia) instrument: ETM keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,REFLECTANCE,ETM+,TM,OLI,EARTH SCIENCE @@ -83,30 +83,30 @@ global_attributes: product_suite: Surface Reflectance NBAR+T 25m acknowledgment: Landsat data is provided by the United States Geological Survey (USGS) through direct reception of the data at Geoscience Australias satellite reception facility or download. references: | - - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. - - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. - - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. - - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. - - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. - - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. - - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. - - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. - - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. - - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. - - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. - - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. - - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. - - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. - - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. - - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. - - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf - - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. - - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. - - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. - - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. - - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. - - http://dx.doi.org/10.4225/25/5487CC0D4F40B - - http://dx.doi.org/10.1109/JSTARS.2010.2042281 + - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. + - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. + - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. + - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. + - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. + - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. + - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. + - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. + - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. + - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. + - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. + - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. + - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. + - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. + - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. + - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. + - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf + - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. + - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. + - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. + - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. + - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. + - http://dx.doi.org/10.4225/25/5487CC0D4F40B + - http://dx.doi.org/10.1109/JSTARS.2010.2042281 storage: driver: NetCDF CF diff --git a/docs/config_samples/ingester/ls7_pq_albers.yaml b/docs/config_samples/ingester/ls7_pq_albers.yaml index 02b4a4bde1..ecd6c2deeb 100644 --- a/docs/config_samples/ingester/ls7_pq_albers.yaml +++ b/docs/config_samples/ingester/ls7_pq_albers.yaml @@ -19,11 +19,11 @@ global_attributes: or only sea pixels depending on their analytical requirements, leading to enhanced computationally efficient. PQ provides an assessment of the quality of observations at a pixel level and includes information about: - - Spectral Contiguity (lack of signal in any band) - - Saturation in any band - - Presence of cloud - - Presence of cloud shadow - - Land or sea + - Spectral Contiguity (lack of signal in any band) + - Saturation in any band + - Presence of cloud + - Presence of cloud shadow + - Land or sea As Landsat Imagery becomes more readily available, there has been a rapid increase in the amount of analyses undertaken by researchers around the globe. Most researchers use some form of quality masking schema in order to remove undesirable @@ -43,11 +43,11 @@ global_attributes: with OLI data, however it uses additional algorithms to detect cloud and cloud shadow, and is available for Landsat 5, 7 and 8. source: PQ_25_2 history: | - - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing - - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. - - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. - - DEM: 1 second SRTM DSM is used for Ortho-rectification. - - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. + - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing + - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. + - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. + - DEM: 1 second SRTM DSM is used for Ortho-rectification. + - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. institution: Commonwealth of Australia (Geoscience Australia) instrument: ETM keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,ETM+,TM,OLI,EARTH SCIENCE diff --git a/docs/config_samples/ingester/ls8_nbar_albers.yaml b/docs/config_samples/ingester/ls8_nbar_albers.yaml index 3139007794..0e7b6205f9 100644 --- a/docs/config_samples/ingester/ls8_nbar_albers.yaml +++ b/docs/config_samples/ingester/ls8_nbar_albers.yaml @@ -39,10 +39,10 @@ global_attributes: Surface Reflectance Correction Models Image radiance values recorded by passive EO sensors are a composite of - - surface reflectance; - - atmospheric condition; - - interaction between surface land cover, solar radiation and sensor view angle; - - land surface orientation relative to the imaging sensor. + - surface reflectance; + - atmospheric condition; + - interaction between surface land cover, solar radiation and sensor view angle; + - land surface orientation relative to the imaging sensor. It has been traditionally assumed that Landsat imagery display negligible variation in sun and sensor view angles, however these can vary significantly both within and between scenes, especially in different seasons and geographic regions (Li et al., 2012). The SR product delivers modeled surface reflectance from Landsat TM/ETM+/OLI @@ -57,11 +57,11 @@ global_attributes: Given the growing time series of EO imagery, this landmark facility will streamline the process of reliably monitoring long-term chnges in land and water resources. source: SR-N_25_2 history: | - - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing - - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. - - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. - - DEM: 1 second SRTM DSM is used for Ortho-rectification. - - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. + - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing + - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. + - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. + - DEM: 1 second SRTM DSM is used for Ortho-rectification. + - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. institution: Commonwealth of Australia (Geoscience Australia) instrument: OLI keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,REFLECTANCE,ETM+,TM,OLI,EARTH SCIENCE @@ -77,30 +77,30 @@ global_attributes: product_suite: Surface Reflectance NBAR+ 25m acknowledgment: Landsat data is provided by the United States Geological Survey (USGS) through direct reception of the data at Geoscience Australias satellite reception facility or download. references: | - - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. - - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. - - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. - - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. - - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. - - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. - - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. - - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. - - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. - - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. - - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. - - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. - - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. - - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. - - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. - - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. - - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf - - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. - - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. - - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. - - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. - - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. - - http://dx.doi.org/10.4225/25/5487CC0D4F40B - - http://dx.doi.org/10.1109/JSTARS.2010.2042281 + - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. + - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. + - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. + - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. + - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. + - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. + - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. + - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. + - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. + - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. + - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. + - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. + - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. + - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. + - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. + - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. + - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf + - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. + - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. + - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. + - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. + - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. + - http://dx.doi.org/10.4225/25/5487CC0D4F40B + - http://dx.doi.org/10.1109/JSTARS.2010.2042281 storage: driver: NetCDF CF diff --git a/docs/config_samples/ingester/ls8_nbart_albers.yaml b/docs/config_samples/ingester/ls8_nbart_albers.yaml index 42899fd4c9..27f22bf8cb 100644 --- a/docs/config_samples/ingester/ls8_nbart_albers.yaml +++ b/docs/config_samples/ingester/ls8_nbart_albers.yaml @@ -45,10 +45,10 @@ global_attributes: Surface Reflectance Correction Models Image radiance values recorded by passive EO sensors are a composite of - - surface reflectance; - - atmospheric condition; - - interaction between surface land cover, solar radiation and sensor view angle; - - land surface orientation relative to the imaging sensor. + - surface reflectance; + - atmospheric condition; + - interaction between surface land cover, solar radiation and sensor view angle; + - land surface orientation relative to the imaging sensor. It has been traditionally assumed that Landsat imagery display negligible variation in sun and sensor view angles, however these can vary significantly both within and between scenes, especially in different seasons and geographic regions (Li et al., 2012). The SR product delivers modeled surface reflectance from Landsat TM/ETM+/OLI @@ -63,11 +63,11 @@ global_attributes: Given the growing time series of EO imagery, this landmark facility will streamline the process of reliably monitoring long-term chnges in land and water resources. source: SR-NT_25_2 history: | - - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing - - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. - - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. - - DEM: 1 second SRTM DSM is used for Ortho-rectification. - - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. + - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing + - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. + - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. + - DEM: 1 second SRTM DSM is used for Ortho-rectification. + - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. institution: Commonwealth of Australia (Geoscience Australia) instrument: OLI keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,REFLECTANCE,ETM+,TM,OLI,EARTH SCIENCE @@ -83,30 +83,30 @@ global_attributes: product_suite: Surface Reflectance NBAR+T 25m acknowledgment: Landsat data is provided by the United States Geological Survey (USGS) through direct reception of the data at Geoscience Australias satellite reception facility or download. references: | - - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. - - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. - - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. - - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. - - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. - - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. - - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. - - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. - - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. - - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. - - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. - - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. - - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. - - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. - - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. - - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. - - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf - - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. - - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. - - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. - - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. - - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. - - http://dx.doi.org/10.4225/25/5487CC0D4F40B - - http://dx.doi.org/10.1109/JSTARS.2010.2042281 + - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. + - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. + - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. + - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. + - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. + - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. + - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. + - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. + - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. + - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. + - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. + - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. + - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. + - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. + - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. + - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. + - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf + - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. + - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. + - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. + - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. + - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. + - http://dx.doi.org/10.4225/25/5487CC0D4F40B + - http://dx.doi.org/10.1109/JSTARS.2010.2042281 storage: driver: NetCDF CF diff --git a/docs/config_samples/ingester/ls8_pq_albers.yaml b/docs/config_samples/ingester/ls8_pq_albers.yaml index 184cb81983..6caad8979f 100644 --- a/docs/config_samples/ingester/ls8_pq_albers.yaml +++ b/docs/config_samples/ingester/ls8_pq_albers.yaml @@ -19,11 +19,11 @@ global_attributes: or only sea pixels depending on their analytical requirements, leading to enhanced computationally efficient. PQ provides an assessment of the quality of observations at a pixel level and includes information about: - - Spectral Contiguity (lack of signal in any band) - - Saturation in any band - - Presence of cloud - - Presence of cloud shadow - - Land or sea + - Spectral Contiguity (lack of signal in any band) + - Saturation in any band + - Presence of cloud + - Presence of cloud shadow + - Land or sea As Landsat Imagery becomes more readily available, there has been a rapid increase in the amount of analyses undertaken by researchers around the globe. Most researchers use some form of quality masking schema in order to remove undesirable @@ -43,11 +43,11 @@ global_attributes: with OLI data, however it uses additional algorithms to detect cloud and cloud shadow, and is available for Landsat 5, 7 and 8. source: PQ_25_2 history: | - - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing - - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. - - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. - - DEM: 1 second SRTM DSM is used for Ortho-rectification. - - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. + - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing + - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. + - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. + - DEM: 1 second SRTM DSM is used for Ortho-rectification. + - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. institution: Commonwealth of Australia (Geoscience Australia) instrument: OLI keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,ETM+,TM,OLI,EARTH SCIENCE diff --git a/docs/config_samples/ingester/s2amsil1c_albers_10.yaml b/docs/config_samples/ingester/s2amsil1c_albers_10.yaml index c9701a6d2a..d7612e8b72 100644 --- a/docs/config_samples/ingester/s2amsil1c_albers_10.yaml +++ b/docs/config_samples/ingester/s2amsil1c_albers_10.yaml @@ -81,5 +81,5 @@ measurements: src_varname: '08' zlib: True attrs: - long_name: " Level 1 Top of Atmosphere Reflectance 842 nanometers (Near infrared)" + long_name: " Level 1 Top of Atmosphere Reflectance 842 nanometers (Near infrared)" alias: "Band8" diff --git a/docs/config_samples/metadata_types/bare_bone.yaml b/docs/config_samples/metadata_types/bare_bone.yaml index 6f6c3eda32..8ffc642482 100644 --- a/docs/config_samples/metadata_types/bare_bone.yaml +++ b/docs/config_samples/metadata_types/bare_bone.yaml @@ -2,10 +2,16 @@ name: barebone description: A minimalist metadata type file dataset: - id: [id] # No longer configurable in newer ODCs. - sources: [lineage, source_datasets] # No longer configurable in newer ODCs. + id: [id] # No longer configurable in newer ODCs. + sources: [lineage, source_datasets] # No longer configurable in newer ODCs. + creation_dt: [properties, 'odc:processing_datetime'] label: [label] + # The following keys are necessary if describing spatial datasets + # grid_spatial: [grid_spatial, projection] + # measurements: [measurements] + # format: [properties, 'odc:file_format'] + search_fields: platform: description: Platform code diff --git a/docs/data-access-analysis/advanced-topics/virtual-products.rst b/docs/data-access-analysis/advanced-topics/virtual-products.rst index a5439f99ab..cf9f2b0fb5 100644 --- a/docs/data-access-analysis/advanced-topics/virtual-products.rst +++ b/docs/data-access-analysis/advanced-topics/virtual-products.rst @@ -21,7 +21,7 @@ pixel-by-pixel. The source code for virtual products is in the :mod:`datacube.virtual` module. An example notebook using virtual products can be found in the `DEA`_ notebooks collection. -.. _DEA: https://docs.dea.ga.gov.au/notebooks/Frequently_used_code/Virtual_products.html +.. _DEA: https://knowledge.dea.ga.gov.au/notebooks/Frequently_used_code/Virtual_products.html Using virtual products diff --git a/docs/data-access-analysis/index.rst b/docs/data-access-analysis/index.rst index ca0070f6ee..6ffce76d0e 100644 --- a/docs/data-access-analysis/index.rst +++ b/docs/data-access-analysis/index.rst @@ -13,6 +13,16 @@ The Sandbox prepares a JupyterLab environment for you. All necessary software is Alternatively, you can also use the `Cube In a Box`_ to setup an Open Data Cube instance locally using Docker. This instance can be configured to index a selection Sentinel-2 level 2 data. +Open Data Cube Cheatsheet +######################### + +To make it easier to get started with Open Data Cube, the following reference poster demonstrates how to perform common analysis tasks including loading data, data preparation, plotting and exporting, and geospatial manipulation: + +.. image:: ../cheatsheets/ODC_Cheatsheet.jpg + :target: https://raw.githubusercontent.com/opendatacube/datacube-core/develop/docs/cheatsheets/ODC_Cheatsheet.pdf + :alt: ODC Cheatsheet + :width: 400px + .. _`Digital Earth Australia Sandbox`: https://www.dea.ga.gov.au/developers/sandbox .. _`Digital Earth Africa Sandbox`: https://sandbox.digitalearth.africa/ diff --git a/docs/data-access-analysis/tools-exploring-data/using-juypter.rst b/docs/data-access-analysis/tools-exploring-data/using-juypter.rst index 64a9d9dde0..b5192115ca 100644 --- a/docs/data-access-analysis/tools-exploring-data/using-juypter.rst +++ b/docs/data-access-analysis/tools-exploring-data/using-juypter.rst @@ -23,9 +23,9 @@ Digital Earth Australia Notebooks * `Real world examples`_: More complex workflows demonstrating how Open Data Cube can be used to address real-world challenges .. _`Digital Earth Australia Notebooks`: https://github.com/GeoscienceAustralia/dea-notebooks/ -.. _`Beginners guide`: https://docs.dea.ga.gov.au/notebooks/Beginners_guide/README.html -.. _`Frequently used code`: https://docs.dea.ga.gov.au/notebooks/Frequently_used_code/README.html -.. _`Real world examples`: https://docs.dea.ga.gov.au/notebooks/Real_world_examples/README.html +.. _`Beginners guide`: https://knowledge.dea.ga.gov.au/notebooks/Beginners_guide/README.html +.. _`Frequently used code`: https://knowledge.dea.ga.gov.au/notebooks/Frequently_used_code/README.html +.. _`Real world examples`: https://knowledge.dea.ga.gov.au/notebooks/Real_world_examples/README.html .. _Jupyter Notebooks: https://jupyter.org/ diff --git a/docs/installation/data-preparation-scripts.rst b/docs/installation/data-preparation-scripts.rst deleted file mode 100644 index b80dcde151..0000000000 --- a/docs/installation/data-preparation-scripts.rst +++ /dev/null @@ -1,147 +0,0 @@ -Data Preperation Scripts -======================== - -.. note:: - - Much of the below content is not updated and has not been tested recently. - - -Sometimes data to load into an Open Data Cube will come packaged with -compatible :ref:`dataset-metadata-doc` and be ready to :ref:`index ` -immediately. - -In other cases you will need to generate these :ref:`dataset-metadata-doc` yourself. -This is done using a ``Dataset Preparation Script``, which reads whatever format metadata -has been supplied with the data, and either writes out ODC compatible files, or adds -records directly to an ODC database. - -For common distribution formats there is likely to be a script already, but -other formats will require one to be modified. - -.. admonition:: Existing dataset-metadata-docs - :class: tip - - Examples of prepare scripts are found in the `datacube-dataset-config `_ repository - on Github. - - -Landsat Samples -=============== - -The two examples below show preparing USGS Landsat data for indexing into an Open Data Cube: - - -1. Preparing USGS Landsat Collection 1 - LEVEL1 -=============================================== - -Download the USGS Collection 1 landsat scenes from any of the links below: - -* `Earth-Explorer `_ -* `GloVis `_ -* `ESPA ordering `_ - -The prepare script for collection 1 - level 1 data is available in -`ls_usgs_prepare.py -`_. - -:: - - $ wget https://github.com/opendatacube/datacube-dataset-config/raw/master/old-prep-scripts/ls_usgs_prepare.py - $ python ls_usgs_prepare.py --help - Usage: ls_usgs_prepare.py [OPTIONS] [DATASETS]... - - Prepare USGS Landsat Collection 1 data for ingestion into the Data Cube. - This prepare script supports only for MTL.txt metadata file - To Set the Path for referring the datasets - - Download the Landsat scene data from Earth Explorer or GloVis into - 'some_space_available_folder' and unpack the file. - For example: yourscript.py --output [Yaml- which writes datasets into this file for indexing] - [Path for dataset as : /home/some_space_available_folder/] - - Options: - --output PATH Write datasets into this file - --help Show this message and exit. - - $ python ls_usgs_prepare.py --output ls8_usgs_lv1 ~/earth_explorer/Collection1/LANDSAT8 - -*ls8_usgs_lv1* is the output for required dataset for landsat 8 scene. - -To add the product definitions: - -For Landsat collection 1 level 1 product: - -:: - - $ datacube product add docs/config_samples/dataset_types/ls_usgs.yaml - Added "ls8_level1_usgs" - Added "ls7_level1_usgs" - Added "ls5_level1_usgs" - Added "ls8_l1_pc_usgs" - - -2. Preparing USGS Landsat Surface Reflectance - LEDAPS -====================================================== - -To prepare downloaded USGS LEDAPS Landsat scenes for use with the Data Cube, use -the script provided in -`usgs_ls_ard_prepare.py -`_ - -The following example generates the required Dataset Metadata files, named -`agdc-metadata.yaml` for three landsat scenes. - -:: - - $ wget https://github.com/opendatacube/datacube-dataset-config/raw/master/agdcv2-ingest/prepare_scripts/landsat_collection/usgs_ls_ard_prepare.py - $ python USGS_precollection_oldscripts/usgslsprepare.py --help - Usage: usgslsprepare.py [OPTIONS] [DATASETS]... - - Prepare USGS LS dataset for ingestion into the Data Cube. - - Options: - --help Show this message and exit. - - $ python usgslsprepare.py ~/USGS_LandsatLEDAPS/*/ - 2016-06-09 15:32:51,641 INFO Processing ~/USGS_LandsatLEDAPS/LC80960852015365-SC20160211222236 - 2016-06-09 15:32:52,096 INFO Writing ~/USGS_LandsatLEDAPS/LC80960852015365-SC20160211222236/agdc-metadata.yaml - 2016-06-09 15:32:52,119 INFO Processing ~/USGS_LandsatLEDAPS/LE70960852016024-SC20160211221824 - 2016-06-09 15:32:52,137 INFO Writing ~/USGS_LandsatLEDAPS/LE70960852016024-SC20160211221824/agdc-metadata.yaml - 2016-06-09 15:32:52,151 INFO Processing ~/USGS_LandsatLEDAPS/LT50960852011290-SC20160211221617 - 2016-06-09 15:32:52,157 INFO Writing ~/USGS_LandsatLEDAPS/LT50960852011290-SC20160211221617/agdc-metadata.yaml - - -The scenes are now ready to be :ref:`indexed ` and accessed using -the Data Cube. - -For Landsat Surface reflectance LEDAPS add: - -:: - - $ datacube product add docs/config_samples/dataset_types/* - ... - Added "ls5_ledaps_scene" - ... - Added "ls7_ledaps_scene" - ... - Added "ls8_ledaps_scene" - ... - -Then :ref:`index the data `. - -3. Indexing data on AWS, an example using Sentinel-2 -==================================================== - -To view an example of how to `index Sentinel-2 data from S3`_ check out the documentation -available in the datacube-dataset-config_ repository. - -.. _`index Sentinel-2 data from S3`: https://github.com/opendatacube/datacube-dataset-config/blob/master/sentinel-2-l2a-cogs.md -.. _datacube-dataset-config: https://github.com/opendatacube/datacube-dataset-config/ - -Custom Prepare Scripts -====================== - -We expect that many new Data Cube instances will require custom prepare scripts -to be written. It is generally a straightforward task of mapping metadata from -one form to another and writing out a YAML document. The code need not even be -written in Python, although starting with one of our examples is generally -the easiest way. diff --git a/docs/installation/database/setup.rst b/docs/installation/database/setup.rst index 74f8154cfd..0fefa33f86 100644 --- a/docs/installation/database/setup.rst +++ b/docs/installation/database/setup.rst @@ -48,7 +48,7 @@ Datacube looks for a configuration file in ~/.datacube.conf or in the location s # A blank host will use a local socket. Specify a hostname (such as localhost) to use TCP. db_hostname: - + # Port is optional. The default port is 5432. # db_port: @@ -66,13 +66,14 @@ Datacube looks for a configuration file in ~/.datacube.conf or in the location s # A "null" environment for working with no index. index_driver: null - [local_memory] + [localmemory] # A local non-persistent in-memory index. # Compatible with the default index driver, but resides purely in memory with no persistent database. # Note that each new invocation will receive a new, empty index. index_driver: memory -Uncomment and fill in lines as required. +Uncomment and fill in lines as required. Environment names should start with a lower case letter and contain +only lower case letters and numbers. This will be strictly enforced from 1.9.0. Alternately, you can configure the ODC connection to Postgres using environment variables:: @@ -81,6 +82,14 @@ Alternately, you can configure the ODC connection to Postgres using environment DB_PASSWORD DB_DATABASE +To configure a database as a single connection url instead of individual environment variables:: + + export DATACUBE_DB_URL=postgresql://[username]:[password]@[hostname]:[port]/[database] + +Alternatively, for password-less access to a database on localhost:: + + export DATACUBE_DB_URL=postgresql:///[database] + The desired environment can be specified: 1. in code, with the ``env`` argument to the ``datacube.Datacube`` constructor; diff --git a/docs/installation/index.rst b/docs/installation/index.rst index f2422569ce..b5e7c634bb 100644 --- a/docs/installation/index.rst +++ b/docs/installation/index.rst @@ -3,7 +3,7 @@ Installing and managing the Open Data Cube ****************************************** This section contains information on setting up and managing the Open Data Cube. - + .. admonition:: Note :class: info @@ -11,6 +11,13 @@ This section contains information on setting up and managing the Open Data Cube. For people using an existing Open Data Cube, such as Digital Earth Australia, you are not required to know most of the information contained here. This material is designed for people wanting to set up a new Open Data Cube instance. +Set Up +------ + +To learn more about setting up the Open Data Cube, see the relevant page: + +* :ref:`Ubuntu Setup ` + .. toctree:: :caption: Setting up an Open Data Cube :maxdepth: 5 @@ -53,5 +60,4 @@ This section contains information on setting up and managing the Open Data Cube. .. toctree:: :caption: Legacy Approaches - data-preparation-scripts ingesting-data/index diff --git a/docs/installation/indexing-data/step-guide.rst b/docs/installation/indexing-data/step-guide.rst index 48010c57d3..d005d8265e 100644 --- a/docs/installation/indexing-data/step-guide.rst +++ b/docs/installation/indexing-data/step-guide.rst @@ -72,16 +72,9 @@ for searching, querying and accessing the data. The data from Geoscience Australia already comes with relevant files (named ``ga-metadata.yaml``), so no further steps are required for indexing them. -For third party datasets, see :ref:`prepare-scripts`. - - -.. admonition:: Note - - :class: info - - Some metadata requires cleanup before they are ready to be loaded. - -For more information see :ref:`dataset-metadata-doc`. +For third party datasets, see the examples detailed `here `__. +For common distribution formats, data can be indexed using one of the tools from `odc-apps-dc-tools `__. +In other cases, the metadata may need to be mapped to an ODC-compatible format. You can find examples of data preparation scripts `here `__. Step 3. Run the Indexing process diff --git a/docs/installation/ingesting-data/index.rst b/docs/installation/ingesting-data/index.rst index 021a2a3ce9..737be508d0 100644 --- a/docs/installation/ingesting-data/index.rst +++ b/docs/installation/ingesting-data/index.rst @@ -7,25 +7,9 @@ Ingesting Data .. note:: Ingestion is no longer recommended. While it was used as an optimised on-disk - storage mechanism, there are a range of reasons why this is no longer ideal. For example + storage mechanism, there are a range of reasons why this is no longer ideal. For example, the emergence of cloud optimised storage formats means that software such - as GDAL and Rasterio are optimised for reading many files over the network. Additionally - the limitation of NetCDF reading to a single thread means that reading from .TIF - files on disk could be faster in some situations. - - In addition to limited performance improvements, ingestion leads to duplication - of data and opinionated decisions, such as reprojection of data, which can lead - to a loss of data fidelity. - - The section below is being retained for completion, but should be considered optional. - - -.. note:: - - Ingestion is no longer recommended. While it was used as an optimised on-disk - storage mechanism, there are a range of reasons why this is no longer ideal. For example - the emergence of cloud optimised storage formats means that software such - as GDAL and Rasterio are optimised for reading many files over the network. Additionally + as GDAL and Rasterio are optimised for reading many files over the network. Additionally, the limitation of NetCDF reading to a single thread means that reading from .TIF files on disk could be faster in some situations. diff --git a/docs/installation/metadata-types.rst b/docs/installation/metadata-types.rst index 0f206016bf..9a4ac538a2 100644 --- a/docs/installation/metadata-types.rst +++ b/docs/installation/metadata-types.rst @@ -5,15 +5,17 @@ A Metadata Type defines which fields should be searchable in your product or dat Three metadata types are added by default called ``eo``, ``telemetry`` and ``eo3``. +You can see the default metadata types in the repository at `datacube/index/default-metadata-types.yaml `_. + You would create a new metadata type if you want custom fields to be searchable for your products, or if you want to structure your metadata documents differently. -You can see the default metadata type in the repository at `datacube/index/default-metadata-types.yaml `_. - To add or alter metadata types, you can use commands like: ``datacube metadata add `` and to update: ``datacube metadata update ``. Using ``--allow-unsafe`` will allow you to update metadata types where the changes may have unexpected consequences. +Note that the postgis driver only supports eo3-compatible metadata types, and from version 2.0 onward, support for non-eo3-compatible metadata types +will be fully deprecated. .. literalinclude:: ../config_samples/metadata_types/bare_bone.yaml :language: yaml @@ -22,4 +24,7 @@ you to update metadata types where the changes may have unexpected consequences. Metadata type yaml file must contain name, description and dataset keys. - Dataset key must contain id, sources, creation_dt, label and search_fields keys. + Dataset key must contain id, sources, creation_dt, label, and search_fields keys. + + For metadata types of spatial datasets, the dataset key must also contain grid_spatial, measurements, and format keys. + Support for non-spatial datasets is likely to be dropped in version 2.0. diff --git a/docs/installation/setup/common_install.rst b/docs/installation/setup/common_install.rst index 5e5966fd0d..49deaa20a3 100644 --- a/docs/installation/setup/common_install.rst +++ b/docs/installation/setup/common_install.rst @@ -7,36 +7,32 @@ Python and packages Python 3.8+ is required. -Anaconda Python ---------------- - -`Install Anaconda Python `_ - -Add conda-forge to package channels:: - - conda config --append channels conda-forge +Conda environment setup +----------------------- Conda environments are recommended for use in isolating your ODC development environment from your system installation and other Python environments. -Install required Python packages and create a conda environment named ``odc_env``. +We recommend you use Mambaforge to set up your conda virtual environment, as all the required packages are obtained from the conda-forge channel. +Download and install it from `here `_. -Python:: +Download the latest version of the Open Data Cube from the `repository `_:: - conda create --name odc_env python=3.8 datacube + git clone https://github.com/opendatacube/datacube-core + cd datacube-core -Activate the ``odc_env`` conda environment:: +Create a conda environment named ``cubeenv``:: - conda activate odc_env + mamba env create -f conda-environment.yml -Find out more about conda environments `here `_. +Activate the ``cubeenv`` conda environment:: -Install other packages:: + conda activate cubeenv - conda install jupyter matplotlib scipy pytest-cov hypothesis +Find out more about conda environments `here `_. -Postgres database configuration -=============================== +Postgres testing database configuration +======================================= This configuration supports local development using your login name. @@ -52,7 +48,7 @@ Set a password for the "postgres" database role using the command:: and set the password when prompted. The password text will be hidden from the console for security purposes. -Type **Control+D** or **\q** to exit the posgreSQL prompt. +Type **Control+D** or **\\q** to exit the posgreSQL prompt. By default in Ubuntu, Postgresql is configured to use ``ident sameuser`` authentication for any connections from the same machine which is useful for development. Check out the excellent Postgresql documentation for more information, but essentially this means that if your Ubuntu username is ``foo`` and you add ``foo`` as a Postgresql user then you can connect to a database without requiring a password for many functions. @@ -61,51 +57,19 @@ Since the only user who can connect to a fresh install is the postgres user, her sudo -u postgres createuser --superuser $USER sudo -u postgres psql - postgres=# \password $USER + postgres=# \password -Now we can create an ``agdcintegration`` database for testing:: +Now we can create databases for integration testing. You will need 2 databases - one for the Postgres driver and one for the PostGIS driver. +By default, these databases are called ``pgintegration`` and ``pgisintegration``, but you can name them however you want:: - createdb agdcintegration + postgres=# create database pgintegration; + postgres=# create database pgisintegration; + +Or, directly from the bash terminal:: -Connecting to your own database to try out some SQL should now be as easy as:: - - psql -d agdcintegration - - -Open Data Cube source and development configuration -=================================================== + createdb pgintegration + createdb pgisintegration -Download the latest version of the software from the `repository `_ :: - - git clone https://github.com/opendatacube/datacube-core - cd datacube-core - -We need to specify the database user and password for the ODC integration testing. To do this:: - - cp integration_tests/agdcintegration.conf ~/.datacube_integration.conf - -Then edit the ``~/.datacube_integration.conf`` with a text editor and add the following lines replacing ```` with your username and ```` with the database user password you set above (not the postgres one, your ```` one):: - - [datacube] - db_hostname: localhost - db_database: agdcintegration - db_username: - db_password: - -Note: For Ubuntu Setup the db_hostname should be set to "/var/run/postgresql". For more refer: https://github.com/opendatacube/datacube-core/issues/1329 - -Verify it all works -=================== - -Run the integration tests:: - - cd datacube-core - ./check-code.sh integration_tests - -Build the documentation:: - - cd datacube-core/docs - pip install -r requirements.txt - make html +Connecting to your own database to try out some SQL should now be as easy as:: -Then open :file:`_build/html/index.html` in your browser to view the Documentation. + psql -d pgintegration diff --git a/docs/installation/setup/macosx.rst b/docs/installation/setup/macosx.rst index e1dfe79ca9..09deab701c 100644 --- a/docs/installation/setup/macosx.rst +++ b/docs/installation/setup/macosx.rst @@ -23,3 +23,50 @@ Postgres: .. include:: common_install.rst + + +You can now specify the database user and password for ODC integration testing. To do this:: + + cp integration_tests/integration.conf ~/.datacube_integration.conf + +Then edit the ``~/.datacube_integration.conf`` with a text editor and add the following lines, replacing ```` with your username and ```` with the database user password you set above (not the postgres one, your ```` one):: + + [datacube] + db_hostname: localhost + db_database: pgintegration + index_driver: default + db_username: + db_password: + + [experimental] + db_hostname: localhost + db_database: pgisintegration + index_driver: postgis + db_username: + db_password: + + +Verify it all works +=================== + +Install additional test dependencies:: + + cd datacube-core + pip install --upgrade -e '.[test]' + +Run the integration tests:: + + ./check-code.sh integration_tests + +Note: if moto-based AWS-mock tests fail, you may need to unset all AWS environment variables. + +Build the documentation:: + + pip install --upgrade -e '.[doc]' + cd docs + pip install -r requirements.txt + sudo apt install make + sudo apt install pandoc + make html + +Then open :file:`_build/html/index.html` in your browser to view the Documentation. diff --git a/docs/installation/setup/ubuntu.rst b/docs/installation/setup/ubuntu.rst index 56c19ed4b7..3d4d94a5c8 100644 --- a/docs/installation/setup/ubuntu.rst +++ b/docs/installation/setup/ubuntu.rst @@ -1,3 +1,4 @@ +.. _setup-ubuntu: Ubuntu Developer Setup ********************** @@ -6,7 +7,7 @@ Base OS: Ubuntu 20.04 LTS This guide will setup an ODC core development environment and includes: - - Anaconda python using conda environments to isolate the odc development environment + - Mambaforge using conda environments to isolate the odc development environment - installation of required software and useful developer manuals for those libraries - Postgres database installation with a local user configuration - Integration tests to confirm both successful development setup and for ongoing testing @@ -20,13 +21,13 @@ GDAL, HDF5, and netCDF4:: sudo apt-get install libgdal-dev libhdf5-serial-dev libnetcdf-dev -Install the latest Postgres version `available ` for your +Install the latest Postgres version `available `_ for your Ubuntu distribution, eg:: - sudo apt-get install postgresql-12 + sudo apt-get install postgresql-14 - # Optionally, Postgis too - sudo apt-get install postgresql-12-postgis-3 + # Optionally, Postgis too (required for the postgis/experimental index driver) + sudo apt-get install postgresql-14-postgis-3 Ubuntu's official repositories usually ship older versions of Postgres. You can alternatively get the most recent version from `the official PostgreSQL repository `_. @@ -37,3 +38,58 @@ Optional packages (useful utilities, docs):: sudo apt-get install hdf5-tools netcdf-bin gdal-bin pgadmin3 .. include:: common_install.rst + + +If createdb or psql cannot connect to server, check which postgresql installation is being run:: + + which psql + +If it is running the mambaforge installation, you may need to run the global installation:: + + /usr/bin/psql -d pgintegration + + +You can now specify the database user and password for ODC integration testing. To do this:: + + cp integration_tests/integration.conf ~/.datacube_integration.conf + +Then edit the ``~/.datacube_integration.conf`` with a text editor and add the following lines, replacing ```` with your username and ```` with the database user password you set above (not the postgres one, your ```` one):: + + [datacube] + db_hostname: /var/run/postgresql + db_database: pgintegration + index_driver: default + db_username: + db_password: + + [experimental] + db_hostname: /var/run/postgresql + db_database: pgisintegration + index_driver: postgis + db_username: + db_password: + +Verify it all works +=================== + +Install additional test dependencies:: + + cd datacube-core + pip install --upgrade -e '.[test]' + +Run the integration tests:: + + ./check-code.sh integration_tests + +Note: if moto-based AWS-mock tests fail, you may need to unset all AWS environment variables. + +Build the documentation:: + + pip install --upgrade -e '.[doc]' + cd docs + pip install -r requirements.txt + sudo apt install make + sudo apt install pandoc + make html + +Then open :file:`_build/html/index.html` in your browser to view the Documentation. diff --git a/docs/installation/setup/windows.rst b/docs/installation/setup/windows.rst index c4a467f3dd..2a1f558d89 100644 --- a/docs/installation/setup/windows.rst +++ b/docs/installation/setup/windows.rst @@ -62,14 +62,22 @@ Download the latest version of the software from the `repository `` with your username and ```` with the database user password you set above (not the postgres one, your ```` one):: [datacube] db_hostname: localhost - db_database: agdcintegration + db_database: pgintegration + index_driver: default + db_username: + db_password: + + [experimental] + db_hostname: localhost + db_database: pgisintegration + index_driver: postgis db_username: db_password: diff --git a/docs/make.bat b/docs/make.bat index 4d607534cd..34f6c8c045 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -5,7 +5,7 @@ echo %cd% REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build + set SPHINXBUILD=sphinx-build ) set SOURCEDIR=. set BUILDDIR=_build @@ -17,15 +17,15 @@ if "%1" == "livehtml" goto livehtml %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 ) %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% diff --git a/docs/requirements.txt b/docs/requirements.txt index ce8a5dcc8d..417d3cfd0b 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --extra=doc,s3 --output-file=docs/requirements.txt @@ -70,6 +70,8 @@ dask[array]==2023.1.1 # distributed defusedxml==0.7.1 # via nbconvert +deprecat==2.1.1 + # via datacube (setup.py) distributed==2023.1.1 # via datacube (setup.py) docutils==0.17.1 @@ -177,7 +179,7 @@ platformdirs==3.0.0 # via jupyter-core psutil==5.9.4 # via distributed -psycopg2==2.9.5 +#psycopg2==2.9.5 # via datacube (setup.py) pydata-sphinx-theme==0.9.0 # via datacube (setup.py) @@ -216,8 +218,6 @@ requests==2.28.2 # via sphinx ruamel-yaml==0.17.21 # via datacube (setup.py) -ruamel-yaml-clib==0.2.7 - # via ruamel-yaml s3transfer==0.6.0 # via boto3 shapely==2.0.1 @@ -294,6 +294,8 @@ webencodings==0.5.1 # via # bleach # tinycss2 +wrapt==1.15.0 + # via deprecat xarray==2023.1.0 # via datacube (setup.py) zict==2.2.0 diff --git a/examples/io_plugin/dcio_example/__init__.py b/examples/io_plugin/dcio_example/__init__.py index c081ad5b41..8318282aea 100644 --- a/examples/io_plugin/dcio_example/__init__.py +++ b/examples/io_plugin/dcio_example/__init__.py @@ -1,4 +1,4 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/examples/io_plugin/dcio_example/pickles.py b/examples/io_plugin/dcio_example/pickles.py index 2f6eed8df8..ec1a2b82bc 100644 --- a/examples/io_plugin/dcio_example/pickles.py +++ b/examples/io_plugin/dcio_example/pickles.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Example reader plugin """ diff --git a/examples/io_plugin/dcio_example/xarray_3d.py b/examples/io_plugin/dcio_example/xarray_3d.py index d38948824b..76ac2c61bd 100644 --- a/examples/io_plugin/dcio_example/xarray_3d.py +++ b/examples/io_plugin/dcio_example/xarray_3d.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ xarray 3D driver plugin for 3D support testing """ diff --git a/examples/io_plugin/dcio_example/zeros.py b/examples/io_plugin/dcio_example/zeros.py index 1cd088e5c0..2e221c0b7c 100644 --- a/examples/io_plugin/dcio_example/zeros.py +++ b/examples/io_plugin/dcio_example/zeros.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Sample plugin "reads" zeros each time every time diff --git a/examples/io_plugin/setup.py b/examples/io_plugin/setup.py index e5f41105ca..698eaf6b95 100644 --- a/examples/io_plugin/setup.py +++ b/examples/io_plugin/setup.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from setuptools import setup, find_packages diff --git a/integration_tests/__init__.py b/integration_tests/__init__.py index d0e5e97db3..a642d3c362 100644 --- a/integration_tests/__init__.py +++ b/integration_tests/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py index e6db0b7ab3..e527b3c000 100644 --- a/integration_tests/conftest.py +++ b/integration_tests/conftest.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Common methods for index integration tests. @@ -47,7 +47,7 @@ PROJECT_ROOT = Path(__file__).parents[1] CONFIG_SAMPLES = PROJECT_ROOT / 'docs' / 'config_samples' -CONFIG_FILE_PATHS = [str(INTEGRATION_TESTS_DIR / 'agdcintegration.conf'), +CONFIG_FILE_PATHS = [str(INTEGRATION_TESTS_DIR / 'integration.conf'), os.path.expanduser('~/.datacube_integration.conf')] # Configure Hypothesis to allow slower tests, because we're testing datasets @@ -182,6 +182,11 @@ def extended_eo3_metadata_type_doc(): return get_eo3_test_data_doc("eo3_landsat_ard.odc-type.yaml") +@pytest.fixture +def eo3_sentinel_metadata_type_doc(): + return get_eo3_test_data_doc("eo3_sentinel_ard.odc-type.yaml") + + @pytest.fixture def extended_eo3_product_doc(): return get_eo3_test_data_doc("ard_ls8.odc-product.yaml") @@ -197,6 +202,47 @@ def africa_s2_product_doc(): return get_eo3_test_data_doc("s2_africa_product.yaml") +@pytest.fixture +def s2_ard_product_doc(): + return get_eo3_test_data_doc("ga_s2am_ard_3.odc-product.yaml") + + +@pytest.fixture +def final_dataset_doc(): + return ( + get_eo3_test_data_doc("final_dataset.yaml"), + 's3://dea-public-data/baseline/ga_ls8c_ard_3/090/086/2023/04/30' + 'ga_ls8c_ard_3-2-1_090086_2023-04-30_final.stac-item.json' + ) + + +@pytest.fixture +def nrt_dataset_doc(): + return ( + get_eo3_test_data_doc("nrt_dataset.yaml"), + 's3://dea-public-data/baseline/ga_ls8c_ard_3/090/086/2023/04/30_nrt' + 'ga_ls8c_ard_3-2-1_090086_2023-04-30_nrt.stac-item.json' + ) + + +@pytest.fixture +def ga_s2am_ard_3_interim_doc(): + return ( + get_eo3_test_data_doc("ga_s2am_ard_3_interim.yaml"), + 's3://dea-public-data/baseline/ga_s2am_ard_3/53/JNN/2021/07/24_interim' + '20230222T235626/ga_s2am_ard_3-2-1_53JNN_2021-07-24_interim.odc-metadata.yaml' + ) + + +@pytest.fixture +def ga_s2am_ard_3_final_doc(): + return ( + get_eo3_test_data_doc("ga_s2am_ard_3_final.yaml"), + 's3://dea-public-data/baseline/ga_s2am_ard_3/53/JNN/2021/07/24' + '20210724T023436/ga_s2am_ard_3-2-1_53JNN_2021-07-24_final.odc-metadata.yaml' + ) + + def doc_to_ds(index, product_name, ds_doc, ds_path): from datacube.index.hl import Doc2Dataset resolver = Doc2Dataset(index, products=[product_name], verify_lineage=False) @@ -206,6 +252,14 @@ def doc_to_ds(index, product_name, ds_doc, ds_path): return index.datasets.get(ds.id) +def doc_to_ds_no_add(index, product_name, ds_doc, ds_path): + from datacube.index.hl import Doc2Dataset + resolver = Doc2Dataset(index, products=[product_name], verify_lineage=False) + ds, err = resolver(ds_doc, ds_path) + assert err is None and ds is not None + return ds + + @pytest.fixture def extended_eo3_metadata_type(index, extended_eo3_metadata_type_doc): return index.metadata_types.add( @@ -213,6 +267,13 @@ def extended_eo3_metadata_type(index, extended_eo3_metadata_type_doc): ) +@pytest.fixture +def eo3_sentinel_metadata_type(index, eo3_sentinel_metadata_type_doc): + return index.metadata_types.add( + index.metadata_types.from_doc(eo3_sentinel_metadata_type_doc) + ) + + @pytest.fixture def ls8_eo3_product(index, extended_eo3_metadata_type, extended_eo3_product_doc): return index.products.add_document(extended_eo3_product_doc) @@ -228,6 +289,11 @@ def africa_s2_eo3_product(index, africa_s2_product_doc): return index.products.add_document(africa_s2_product_doc) +@pytest.fixture +def ga_s2am_ard_3_product(index, eo3_sentinel_metadata_type, s2_ard_product_doc): + return index.products.add_document(s2_ard_product_doc) + + @pytest.fixture def eo3_products(index, extended_eo3_metadata_type, ls8_eo3_product, wo_eo3_product, @@ -281,6 +347,48 @@ def africa_eo3_dataset(index, africa_s2_eo3_product, eo3_africa_dataset_doc): *eo3_africa_dataset_doc) +@pytest.fixture +def nrt_dataset(index, extended_eo3_metadata_type, ls8_eo3_product, nrt_dataset_doc): + return doc_to_ds_no_add( + index, + ls8_eo3_product.name, + *nrt_dataset_doc) + + +@pytest.fixture +def final_dataset(index, extended_eo3_metadata_type, ls8_eo3_product, final_dataset_doc): + return doc_to_ds_no_add( + index, + ls8_eo3_product.name, + *final_dataset_doc) + + +@pytest.fixture +def ds_no_region(index, extended_eo3_metadata_type, ls8_eo3_product, final_dataset_doc): + doc_no_region = deepcopy(final_dataset_doc) + doc_no_region[0]["properties"]["odc:region_code"] = None + return doc_to_ds_no_add( + index, + ls8_eo3_product.name, + *doc_no_region) + + +@pytest.fixture +def ga_s2am_ard3_final(index, eo3_sentinel_metadata_type, ga_s2am_ard_3_product, ga_s2am_ard_3_final_doc): + return doc_to_ds_no_add( + index, + ga_s2am_ard_3_product.name, + *ga_s2am_ard_3_final_doc) + + +@pytest.fixture +def ga_s2am_ard3_interim(index, eo3_sentinel_metadata_type, ga_s2am_ard_3_product, ga_s2am_ard_3_interim_doc): + return doc_to_ds_no_add( + index, + ga_s2am_ard_3_product.name, + *ga_s2am_ard_3_interim_doc) + + @pytest.fixture def mem_index_fresh(in_memory_config): from datacube import Datacube @@ -357,14 +465,14 @@ def local_config_pair(datacube_env_name_pair): def null_config(): """Provides a :class:`LocalConfig` configured with null index driver """ - return LocalConfig.find(CONFIG_FILE_PATHS, env="null_driver") + return LocalConfig.find(CONFIG_FILE_PATHS, env="nulldriver") @pytest.fixture def in_memory_config(): """Provides a :class:`LocalConfig` configured with memory index driver """ - return LocalConfig.find(CONFIG_FILE_PATHS, env="local_memory") + return LocalConfig.find(CONFIG_FILE_PATHS, env="localmemory") @pytest.fixture @@ -417,7 +525,7 @@ def cleanup_db(local_cfg, db): db.close() -@pytest.fixture(params=["US/Pacific", "UTC", ]) +@pytest.fixture(params=["America/Los_Angeles", "UTC", ]) def uninitialised_postgres_db(local_config, request): """ Return a connection to an empty PostgreSQL or PostGIS database diff --git a/integration_tests/data/eo3/eo3_sentinel_ard.odc-type.yaml b/integration_tests/data/eo3/eo3_sentinel_ard.odc-type.yaml new file mode 100644 index 0000000000..307eb9dd43 --- /dev/null +++ b/integration_tests/data/eo3/eo3_sentinel_ard.odc-type.yaml @@ -0,0 +1,361 @@ +--- +# Metadata Type +# url: https://explorer-aws.dea.ga.gov.au/metadata-types/eo3_sentinel_ard.odc-type.yaml +name: eo3_sentinel_ard +description: EO3 for Sentinel 2 ARD +dataset: + id: + - id + label: + - label + format: + - properties + - odc:file_format + sources: + - lineage + - source_datasets + creation_dt: + - properties + - odc:processing_datetime + grid_spatial: + - grid_spatial + - projection + measurements: + - measurements + search_fields: + lat: + type: double-range + max_offset: + - - extent + - lat + - end + min_offset: + - - extent + - lat + - begin + description: Latitude range + lon: + type: double-range + max_offset: + - - extent + - lon + - end + min_offset: + - - extent + - lon + - begin + description: Longitude range + time: + type: datetime-range + max_offset: + - - properties + - dtr:end_datetime + - - properties + - datetime + min_offset: + - - properties + - dtr:start_datetime + - - properties + - datetime + description: Acquisition time range + eo_gsd: + type: double + offset: + - properties + - eo:gsd + indexed: false + description: "Ground sampling distance of the sensor’s best resolution band\n\ + in metres; represents the size (or spatial resolution) of one pixel.\n" + crs_raw: + offset: + - crs + indexed: false + description: "The raw CRS string as it appears in metadata\n\n(e.g. ‘epsg:32654’)\n" + platform: + offset: + - properties + - eo:platform + indexed: false + description: Platform code + gqa_abs_x: + type: double + offset: + - properties + - gqa:abs_x + indexed: false + description: "Absolute value of the x-axis (east-to-west) GCP residuals, in\ + \ pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5\ + \ metres)\n" + gqa_abs_y: + type: double + offset: + - properties + - gqa:abs_y + indexed: false + description: "Absolute value of the y-axis (north-to-south) GCP residuals, in\ + \ pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5\ + \ metres)\n" + gqa_cep90: + type: double + offset: + - properties + - gqa:cep90 + indexed: false + description: "Circular error probable (90%) of the values of the GCP residuals,\ + \ in pixel units based on a 25 metre resolution reference image (i.e. 0.2\ + \ = 5 metres)\n" + fmask_snow: + type: double + offset: + - properties + - fmask:snow + indexed: false + description: "The proportion (from 0 to 100) of the dataset's valid data area\ + \ that contains clear snow pixels according to the Fmask algorithm\n" + gqa_abs_xy: + type: double + offset: + - properties + - gqa:abs_xy + indexed: false + description: "Absolute value of the total GCP residuals, in pixel units based\ + \ on a 25 metre resolution reference image (i.e. 0.2 = 5 metres)\n" + gqa_mean_x: + type: double + offset: + - properties + - gqa:mean_x + indexed: false + description: "Mean of the values of the x-axis (east-to-west) GCP residuals,\ + \ in pixel units based on a 25 metre resolution reference image (i.e. 0.2\ + \ = 5 metres)\n" + gqa_mean_y: + type: double + offset: + - properties + - gqa:mean_y + indexed: false + description: "Mean of the values of the y-axis (north-to-south) GCP residuals,\ + \ in pixel units based on a 25 metre resolution reference image (i.e. 0.2\ + \ = 5 metres)\n" + instrument: + offset: + - properties + - eo:instrument + indexed: false + description: Instrument name + cloud_cover: + type: double + offset: + - properties + - eo:cloud_cover + description: "The proportion (from 0 to 100) of the dataset's valid data area\ + \ that contains cloud pixels.\n\nFor these ARD products, this value comes\ + \ from the Fmask algorithm.\n" + fmask_clear: + type: double + offset: + - properties + - fmask:clear + indexed: false + description: "The proportion (from 0 to 100) of the dataset's valid data area\ + \ that contains clear land pixels according to the Fmask algorithm\n" + fmask_water: + type: double + offset: + - properties + - fmask:water + indexed: false + description: "The proportion (from 0 to 100) of the dataset's valid data area\ + \ that contains clear water pixels according to the Fmask algorithm\n" + gqa_mean_xy: + type: double + offset: + - properties + - gqa:mean_xy + indexed: false + description: "Mean of the values of the GCP residuals, in pixel units based\ + \ on a 25 metre resolution reference image (i.e. 0.2 = 5 metres)\n" + region_code: + offset: + - properties + - odc:region_code + description: "Spatial reference code from the provider.\nFor Sentinel it is\ + \ MGRS code.\n" + gqa_stddev_x: + type: double + offset: + - properties + - gqa:stddev_x + indexed: false + description: "Standard Deviation of the values of the x-axis (east-to-west)\ + \ GCP residuals, in pixel units based on a 25 metre resolution reference image\ + \ (i.e. 0.2 = 5 metres)\n" + gqa_stddev_y: + type: double + offset: + - properties + - gqa:stddev_y + indexed: false + description: "Standard Deviation of the values of the y-axis (north-to-south)\ + \ GCP residuals, in pixel units based on a 25 metre resolution reference image\ + \ (i.e. 0.2 = 5 metres)\n" + gqa_stddev_xy: + type: double + offset: + - properties + - gqa:stddev_xy + indexed: false + description: "Standard Deviation of the values of the GCP residuals, in pixel\ + \ units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres)\n" + eo_sun_azimuth: + type: double + offset: + - properties + - eo:sun_azimuth + indexed: false + description: "The azimuth angle of the sun at the moment of acquisition, in\ + \ degree units measured clockwise from due north\n" + product_family: + offset: + - properties + - odc:product_family + indexed: false + description: Product family code + dataset_maturity: + offset: + - properties + - dea:dataset_maturity + indexed: false + description: One of - final|interim|nrt (near real time) + eo_sun_elevation: + type: double + offset: + - properties + - eo:sun_elevation + indexed: false + description: "The elevation angle of the sun at the moment of acquisition, in\ + \ degree units relative to the horizon.\n" + sentinel_tile_id: + offset: + - properties + - sentinel:sentinel_tile_id + indexed: false + description: "Granule ID according to the ESA naming convention\n\n(e.g. ‘S2A_OPER_MSI_L1C_TL_SGS__20161214T040601_A007721_T53KRB_N02.04’)\n" + s2cloudless_clear: + type: double + offset: + - properties + - s2cloudless:clear + description: "The proportion (from 0 to 100) of the dataset's valid data area\ + \ that contains clear land pixels according to s3cloudless\n" + s2cloudless_cloud: + type: double + offset: + - properties + - s2cloudless:cloud + description: "The proportion (from 0 to 100) of the dataset's valid data area\ + \ that contains cloud land pixels according to s3cloudless\n" + fmask_cloud_shadow: + type: double + offset: + - properties + - fmask:cloud_shadow + indexed: false + description: "The proportion (from 0 to 100) of the dataset's valid data area\ + \ that contains cloud shadow pixels according to the Fmask algorithm\n" + gqa_iterative_mean_x: + type: double + offset: + - properties + - gqa:iterative_mean_x + indexed: false + description: "Mean of the values of the x-axis (east-to-west) GCP residuals\ + \ after removal of outliers, in pixel units based on a 25 metre resolution\ + \ reference image (i.e. 0.2 = 5 metres)\n" + gqa_iterative_mean_y: + type: double + offset: + - properties + - gqa:iterative_mean_y + indexed: false + description: "Mean of the values of the y-axis (north-to-south) GCP residuals\ + \ after removal of outliers, in pixel units based on a 25 metre resolution\ + \ reference image (i.e. 0.2 = 5 metres)\n" + gqa_iterative_mean_xy: + type: double + offset: + - properties + - gqa:iterative_mean_xy + indexed: false + description: "Mean of the values of the GCP residuals after removal of outliers,\ + \ in pixel units based on a 25 metre resolution reference image (i.e. 0.2\ + \ = 5 metres)\n" + sentinel_datastrip_id: + offset: + - properties + - sentinel:datastrip_id + indexed: false + description: "Unique identifier for a datastrip relative to a given Datatake.\n\ + \n(e.g. ‘S2A_OPER_MSI_L1C_DS_SGS__20161214T040601_S20161214T005840_N02.04’)\n" + sentinel_product_name: + offset: + - properties + - sentinel:product_name + indexed: false + description: "ESA product URI, with the '.SAFE' ending removed.\n\n(e.g. 'S2A_MSIL1C_20220303T000731_N0400_R073_T56LNM_20220303T012845')\n" + gqa_iterative_stddev_x: + type: double + offset: + - properties + - gqa:iterative_stddev_x + indexed: false + description: "Standard Deviation of the values of the x-axis (east-to-west)\ + \ GCP residuals after removal of outliers, in pixel units based on a 25 metre\ + \ resolution reference image (i.e. 0.2 = 5 metres)\n" + gqa_iterative_stddev_y: + type: double + offset: + - properties + - gqa:iterative_stddev_y + indexed: false + description: "Standard Deviation of the values of the y-axis (north-to-south)\ + \ GCP residuals after removal of outliers, in pixel units based on a 25 metre\ + \ resolution reference image (i.e. 0.2 = 5 metres)\n" + gqa_iterative_stddev_xy: + type: double + offset: + - properties + - gqa:iterative_stddev_xy + indexed: false + description: "Standard Deviation of the values of the GCP residuals after removal\ + \ of outliers, in pixel units based on a 25 metre resolution reference image\ + \ (i.e. 0.2 = 5 metres)\n" + gqa_abs_iterative_mean_x: + type: double + offset: + - properties + - gqa:abs_iterative_mean_x + indexed: false + description: "Mean of the absolute values of the x-axis (east-to-west) GCP residuals\ + \ after removal of outliers, in pixel units based on a 25 metre resolution\ + \ reference image (i.e. 0.2 = 5 metres)\n" + gqa_abs_iterative_mean_y: + type: double + offset: + - properties + - gqa:abs_iterative_mean_y + indexed: false + description: "Mean of the absolute values of the y-axis (north-to-south) GCP\ + \ residuals after removal of outliers, in pixel units based on a 25 metre\ + \ resolution reference image (i.e. 0.2 = 5 metres)\n" + gqa_abs_iterative_mean_xy: + type: double + offset: + - properties + - gqa:abs_iterative_mean_xy + indexed: false + description: "Mean of the absolute values of the GCP residuals after removal\ + \ of outliers, in pixel units based on a 25 metre resolution reference image\ + \ (i.e. 0.2 = 5 metres)\n" +... diff --git a/integration_tests/data/eo3/final_dataset.yaml b/integration_tests/data/eo3/final_dataset.yaml new file mode 100644 index 0000000000..19c27ffca3 --- /dev/null +++ b/integration_tests/data/eo3/final_dataset.yaml @@ -0,0 +1,139 @@ +--- +# Dataset +$schema: https://schemas.opendatacube.org/dataset +id: 6241395f-f341-41b0-96b7-60f486c577d0 + +label: ga_ls8c_ard_3-2-1_090086_2023-04-30_final +product: + name: ga_ls8c_ard_3 + href: https://collections.dea.ga.gov.au/product/ga_ls8c_ard_3 + +crs: epsg:32655 +geometry: + type: Polygon + coordinates: [[[559816.0052888468, -4219857.620084257], [559721.8933982822, -4219833.106601718], + [559732.9800465275, -4219728.735567618], [560962.9813034026, -4214988.730722999], + [571612.9871046449, -4174023.7084477567], [604522.9880076076, -4048158.7049931744], + [608708.0157636222, -4032408.6003978983], [608918.1498917936, -4031628.1325757634], + [609157.5, -4030897.5], [609243.8473913191, -4030914.6899256567], [609255.0, + -4030905.0], [609449.0888722979, -4030955.549131081], [609461.1380343755, + -4030957.947862498], [611246.2821331235, -4031422.984647127], [614491.7279428138, + -4032268.87029345], [795172.5726569144, -4079325.971481828], [796033.5623962873, + -4079585.522912585], [796173.1066017178, -4079621.893398282], [747082.5, -4268617.5], + [746957.9666764413, -4268587.428373786], [746857.7239312489, -4268594.104275004], + [746137.4401493662, -4268414.031855924], [560077.4396544178, -4219964.0317270355], + [559816.0052888468, -4219857.620084257]]] +grids: + default: + shape: [7931, 7901] + transform: [30.0, 0.0, 559485.0, 0.0, -30.0, -4030785.0, 0.0, 0.0, 1.0] + panchromatic: + shape: [15861, 15801] + transform: [15.0, 0.0, 559492.5, 0.0, -15.0, -4030792.5, 0.0, 0.0, 1.0] + +properties: + datetime: 2023-04-30 23:50:34.384549Z + dea:dataset_maturity: final + dea:product_maturity: stable + dtr:end_datetime: 2023-04-30 23:50:48.771747Z + dtr:start_datetime: 2023-04-30 23:50:19.818416Z + eo:cloud_cover: 93.24450316290117 + eo:gsd: 15.0 # Ground sample distance (m) + eo:instrument: OLI_TIRS + eo:platform: landsat-8 + eo:sun_azimuth: 36.68926293 + eo:sun_elevation: 29.26893394 + fmask:clear: 6.177487638246016 + fmask:cloud: 93.24450316290117 + fmask:cloud_shadow: 0.14004371437490343 + fmask:snow: 0.006964025007482182 + fmask:water: 0.43100145947042295 + gqa:abs_iterative_mean_x: 0.19 + gqa:abs_iterative_mean_xy: 0.26 + gqa:abs_iterative_mean_y: 0.18 + gqa:abs_x: 0.65 + gqa:abs_xy: 0.73 + gqa:abs_y: 0.33 + gqa:cep90: 0.4 + gqa:iterative_mean_x: -0.15 + gqa:iterative_mean_xy: 0.18 + gqa:iterative_mean_y: 0.1 + gqa:iterative_stddev_x: 0.16 + gqa:iterative_stddev_xy: 0.24 + gqa:iterative_stddev_y: 0.18 + gqa:mean_x: 0.23 + gqa:mean_xy: 0.23 + gqa:mean_y: -0.02 + gqa:stddev_x: 1.42 + gqa:stddev_xy: 1.48 + gqa:stddev_y: 0.42 + landsat:collection_category: T1 + landsat:collection_number: 2 + landsat:landsat_product_id: LC08_L1TP_090086_20230430_20230509_02_T1 + landsat:landsat_scene_id: LC80900862023120LGN00 + landsat:wrs_path: 90 + landsat:wrs_row: 86 + odc:dataset_version: 3.2.1 + odc:file_format: GeoTIFF + odc:processing_datetime: 2023-05-10 10:57:11.431704Z + odc:producer: ga.gov.au + odc:product_family: ard + odc:region_code: '090086' + +measurements: + nbart_blue: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_final_band02.tif + nbart_coastal_aerosol: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_final_band01.tif + nbart_green: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_final_band03.tif + nbart_nir: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_final_band05.tif + nbart_panchromatic: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_final_band08.tif + grid: panchromatic + nbart_red: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_final_band04.tif + nbart_swir_1: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_final_band06.tif + nbart_swir_2: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_final_band07.tif + oa_azimuthal_exiting: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_azimuthal-exiting.tif + oa_azimuthal_incident: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_azimuthal-incident.tif + oa_combined_terrain_shadow: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_combined-terrain-shadow.tif + oa_exiting_angle: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_exiting-angle.tif + oa_fmask: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_fmask.tif + oa_incident_angle: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_incident-angle.tif + oa_nbart_contiguity: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_nbart-contiguity.tif + oa_relative_azimuth: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_relative-azimuth.tif + oa_relative_slope: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_relative-slope.tif + oa_satellite_azimuth: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_satellite-azimuth.tif + oa_satellite_view: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_satellite-view.tif + oa_solar_azimuth: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_solar-azimuth.tif + oa_solar_zenith: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_solar-zenith.tif + oa_time_delta: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_time-delta.tif + +accessories: + thumbnail:nbart: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_final_thumbnail.jpg + metadata:processor: + path: ga_ls8c_ard_3-2-1_090086_2023-04-30_final.proc-info.yaml + checksum:sha1: + path: ga_ls8c_ard_3-2-1_090086_2023-04-30_final.sha1 + +lineage: {} +... diff --git a/integration_tests/data/eo3/ga_s2am_ard_3.odc-product.yaml b/integration_tests/data/eo3/ga_s2am_ard_3.odc-product.yaml new file mode 100644 index 0000000000..edc86a770a --- /dev/null +++ b/integration_tests/data/eo3/ga_s2am_ard_3.odc-product.yaml @@ -0,0 +1,240 @@ +--- +name: ga_s2am_ard_3 +license: CC-BY-4.0 +metadata_type: eo3_sentinel_ard +description: Geoscience Australia Sentinel 2A MSI Analysis Ready Data Collection 3 +metadata: + product: + name: ga_s2am_ard_3 + properties: + eo:platform: sentinel-2a + odc:producer: ga.gov.au + eo:instrument: MSI + odc:product_family: ard + dea:product_maturity: stable +measurements: +- name: nbart_coastal_aerosol + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band01 + - coastal_aerosol +- name: nbart_blue + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band02 + - blue +- name: nbart_green + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band03 + - green +- name: nbart_red + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band04 + - red +- name: nbart_red_edge_1 + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band05 + - red_edge_1 +- name: nbart_red_edge_2 + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band06 + - red_edge_2 +- name: nbart_red_edge_3 + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band07 + - red_edge_3 +- name: nbart_nir_1 + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band08 + - nir_1 + - nbart_common_nir +- name: nbart_nir_2 + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band8a + - nir_2 +- name: nbart_swir_2 + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band11 + - swir_2 + - nbart_common_swir_1 + - swir2 +- name: nbart_swir_3 + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band12 + - swir_3 + - nbart_common_swir_2 +- name: oa_fmask + dtype: uint8 + units: '1' + nodata: 0 + aliases: + - fmask + flags_definition: + fmask: + bits: + - 0 + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + values: + '0': nodata + '1': valid + '2': cloud + '3': shadow + '4': snow + '5': water + description: Fmask +- name: oa_nbart_contiguity + dtype: uint8 + units: '1' + nodata: 255 + aliases: + - nbart_contiguity + flags_definition: + contiguous: + bits: + - 0 + values: + '0': false + '1': true +- name: oa_azimuthal_exiting + dtype: float32 + units: '1' + nodata: NaN + aliases: + - azimuthal_exiting +- name: oa_azimuthal_incident + dtype: float32 + units: '1' + nodata: NaN + aliases: + - azimuthal_incident +- name: oa_combined_terrain_shadow + dtype: uint8 + units: '1' + nodata: 255 + aliases: + - combined_terrain_shadow +- name: oa_exiting_angle + dtype: float32 + units: '1' + nodata: NaN + aliases: + - exiting_angle +- name: oa_incident_angle + dtype: float32 + units: '1' + nodata: NaN + aliases: + - incident_angle +- name: oa_relative_azimuth + dtype: float32 + units: '1' + nodata: NaN + aliases: + - relative_azimuth +- name: oa_relative_slope + dtype: float32 + units: '1' + nodata: NaN + aliases: + - relative_slope +- name: oa_satellite_azimuth + dtype: float32 + units: '1' + nodata: NaN + aliases: + - satellite_azimuth +- name: oa_satellite_view + dtype: float32 + units: '1' + nodata: NaN + aliases: + - satellite_view +- name: oa_solar_azimuth + dtype: float32 + units: '1' + nodata: NaN + aliases: + - solar_azimuth +- name: oa_solar_zenith + dtype: float32 + units: '1' + nodata: NaN + aliases: + - solar_zenith +- name: oa_time_delta + dtype: float32 + units: '1' + nodata: NaN + aliases: + - time_delta +- name: oa_s2cloudless_mask + dtype: uint8 + units: '1' + nodata: 0 + aliases: + - s2cloudless_mask + flags_definition: + s2cloudless_mask: + bits: + - 0 + - 1 + - 2 + values: + '0': nodata + '1': valid + '2': cloud + description: s2cloudless mask +- name: oa_s2cloudless_prob + dtype: float64 + units: '1' + nodata: NaN + aliases: + - s2cloudless_prob +# Product +# url: https://explorer-aws.dea.ga.gov.au/products/ga_s2am_ard_3.odc-product.yaml +load: + crs: EPSG:3577 + align: + x: 0 + y: 0 + resolution: + x: 10 + y: -10 +... diff --git a/integration_tests/data/eo3/ga_s2am_ard_3_final.yaml b/integration_tests/data/eo3/ga_s2am_ard_3_final.yaml new file mode 100644 index 0000000000..0ee56c2ad9 --- /dev/null +++ b/integration_tests/data/eo3/ga_s2am_ard_3_final.yaml @@ -0,0 +1,150 @@ +--- +# Dataset +$schema: https://schemas.opendatacube.org/dataset +id: 4123766b-5779-4f96-b71f-28e28a326434 + +label: ga_s2am_ard_3-2-1_53JNN_2021-07-24_final +product: + name: ga_s2am_ard_3 + href: https://collections.dea.ga.gov.au/product/ga_s2am_ard_3 + +crs: epsg:32753 +geometry: + type: Polygon + coordinates: [[[499980.0, 7300000.0], [609780.0, 7300000.0], [609780.0, 7190200.0], + [499980.0, 7190200.0], [499980.0, 7300000.0]]] +grids: + default: + shape: [5490, 5490] + transform: [20.0, 0.0, 499980.0, 0.0, -20.0, 7300000.0, 0.0, 0.0, 1.0] + '10': + shape: [10980, 10980] + transform: [10.0, 0.0, 499980.0, 0.0, -10.0, 7300000.0, 0.0, 0.0, 1.0] + '60': + shape: [1830, 1830] + transform: [60.0, 0.0, 499980.0, 0.0, -60.0, 7300000.0, 0.0, 0.0, 1.0] + +properties: + datetime: 2021-07-24 01:14:10.195987Z + dea:dataset_maturity: final + dea:product_maturity: stable + eo:cloud_cover: 0.0 + eo:constellation: sentinel-2 + eo:gsd: 10.0 # Ground sample distance (m) + eo:instrument: MSI + eo:platform: sentinel-2a + eo:sun_azimuth: 33.4140301515032 + eo:sun_elevation: 52.1651784839186 + fmask:clear: 99.9999469145756 + fmask:cloud: 0.0 + fmask:cloud_shadow: 0.0 + fmask:snow: 0.0 + fmask:water: 5.30854244013789e-05 + gqa:abs_iterative_mean_x: 0.88 + gqa:abs_iterative_mean_xy: 2.17 + gqa:abs_iterative_mean_y: 1.98 + gqa:abs_x: 1.48 + gqa:abs_xy: 4.03 + gqa:abs_y: 3.74 + gqa:cep90: 4.78 + gqa:iterative_mean_x: 0.52 + gqa:iterative_mean_xy: 0.55 + gqa:iterative_mean_y: 0.18 + gqa:iterative_stddev_x: 1.41 + gqa:iterative_stddev_xy: 4.21 + gqa:iterative_stddev_y: 3.97 + gqa:mean_x: 0.69 + gqa:mean_xy: 0.74 + gqa:mean_y: -0.25 + gqa:stddev_x: 14.77 + gqa:stddev_xy: 39.88 + gqa:stddev_y: 37.04 + odc:dataset_version: 3.2.1 + odc:file_format: GeoTIFF + odc:processing_datetime: 2022-07-12 17:54:39.369087Z + odc:producer: ga.gov.au + odc:product_family: ard + odc:region_code: 53JNN + s2cloudless:clear: 100.0 + s2cloudless:cloud: 0.0 + sat:orbit_state: descending + sat:relative_orbit: 45 + sentinel:datastrip_id: S2A_OPER_MSI_L1C_DS_VGS4_20210724T023436_S20210724T010731_N03.01 + sentinel:datatake_start_datetime: 2021-07-24 02:34:36Z + sentinel:product_name: S2A_MSIL1C_20210724T010731_N0301_R045_T53JNN_20210724T023436 + sentinel:sentinel_tile_id: S2A_OPER_MSI_L1C_TL_VGS4_20210724T023436_A031788_T53JNN_N03.01 + +measurements: + nbart_blue: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band02.tif + grid: '10' + nbart_coastal_aerosol: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band01.tif + grid: '60' + nbart_green: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band03.tif + grid: '10' + nbart_nir_1: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band08.tif + grid: '10' + nbart_nir_2: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band08a.tif + nbart_red: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band04.tif + grid: '10' + nbart_red_edge_1: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band05.tif + nbart_red_edge_2: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band06.tif + nbart_red_edge_3: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band07.tif + nbart_swir_2: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band11.tif + nbart_swir_3: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band12.tif + oa_azimuthal_exiting: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_azimuthal-exiting.tif + oa_azimuthal_incident: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_azimuthal-incident.tif + oa_combined_terrain_shadow: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_combined-terrain-shadow.tif + oa_exiting_angle: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_exiting-angle.tif + oa_fmask: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_fmask.tif + oa_incident_angle: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_incident-angle.tif + oa_nbart_contiguity: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_nbart-contiguity.tif + grid: '10' + oa_relative_azimuth: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_relative-azimuth.tif + oa_relative_slope: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_relative-slope.tif + oa_s2cloudless_mask: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_s2cloudless-mask.tif + grid: '60' + oa_s2cloudless_prob: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_s2cloudless-prob.tif + grid: '60' + oa_satellite_azimuth: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_satellite-azimuth.tif + oa_satellite_view: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_satellite-view.tif + oa_solar_azimuth: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_solar-azimuth.tif + oa_solar_zenith: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_solar-zenith.tif + oa_time_delta: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_time-delta.tif + +accessories: + thumbnail:nbart: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_thumbnail.jpg + metadata:processor: + path: ga_s2am_ard_3-2-1_53JNN_2021-07-24_final.proc-info.yaml + checksum:sha1: + path: ga_s2am_ard_3-2-1_53JNN_2021-07-24_final.sha1 + +lineage: {} +... diff --git a/integration_tests/data/eo3/ga_s2am_ard_3_interim.yaml b/integration_tests/data/eo3/ga_s2am_ard_3_interim.yaml new file mode 100644 index 0000000000..41e828da2d --- /dev/null +++ b/integration_tests/data/eo3/ga_s2am_ard_3_interim.yaml @@ -0,0 +1,151 @@ +--- +# Dataset +$schema: https://schemas.opendatacube.org/dataset +id: c0324cbb-f499-4e8d-8001-a38206ff6904 + +label: ga_s2am_ard_3-2-1_53JNN_2021-07-24_interim +product: + name: ga_s2am_ard_3 + href: https://collections.dea.ga.gov.au/product/ga_s2am_ard_3 + +crs: epsg:32753 +geometry: + type: Polygon + coordinates: [[[499980.0, 7300000.0], [609780.0, 7300000.0], [609780.0, 7190200.0], + [499980.0, 7190200.0], [499980.0, 7300000.0]]] +grids: + default: + shape: [5490, 5490] + transform: [20.0, 0.0, 499980.0, 0.0, -20.0, 7300000.0, 0.0, 0.0, 1.0] + '10': + shape: [10980, 10980] + transform: [10.0, 0.0, 499980.0, 0.0, -10.0, 7300000.0, 0.0, 0.0, 1.0] + '60': + shape: [1830, 1830] + transform: [60.0, 0.0, 499980.0, 0.0, -60.0, 7300000.0, 0.0, 0.0, 1.0] + +properties: + datetime: 2021-07-24 01:14:10.195809Z + dea:dataset_maturity: interim + dea:product_maturity: stable + eo:cloud_cover: 0.0 + eo:constellation: sentinel-2 + eo:gsd: 10.0 # Ground sample distance (m) + eo:instrument: MSI + eo:platform: sentinel-2a + eo:sun_azimuth: 33.4134402697606 + eo:sun_elevation: 52.1647723037447 + fmask:clear: 99.99995355025365 + fmask:cloud: 0.0 + fmask:cloud_shadow: 0.0 + fmask:snow: 0.0 + fmask:water: 4.6449746351206534e-05 + gqa:abs_iterative_mean_x: 0.48 + gqa:abs_iterative_mean_xy: 1.02 + gqa:abs_iterative_mean_y: 0.91 + gqa:abs_x: 0.81 + gqa:abs_xy: 2.04 + gqa:abs_y: 1.87 + gqa:cep90: 2.56 + gqa:iterative_mean_x: 0.33 + gqa:iterative_mean_xy: 0.41 + gqa:iterative_mean_y: 0.24 + gqa:iterative_stddev_x: 0.6 + gqa:iterative_stddev_xy: 1.7 + gqa:iterative_stddev_y: 1.6 + gqa:mean_x: 0.41 + gqa:mean_xy: 0.41 + gqa:mean_y: 0.03 + gqa:stddev_x: 3.58 + gqa:stddev_xy: 10.73 + gqa:stddev_y: 10.12 + odc:dataset_version: 3.2.1 + odc:file_format: GeoTIFF + odc:processing_datetime: 2023-05-31 21:12:25.979068Z + odc:producer: ga.gov.au + odc:product_family: ard + odc:region_code: 53JNN + s2cloudless:clear: 100.0 + s2cloudless:cloud: 0.0 + sentinel:datastrip_id: S2A_OPER_MSI_L1C_DS_S2RP_20230222T235626_S20210724T010731_N05.00 + sentinel:datatake_start_datetime: 2023-02-22 23:56:26Z + sentinel:grid_square: NN + sentinel:latitude_band: J + sentinel:product_name: S2A_MSIL1C_20210724T010731_N0500_R045_T53JNN_20230222T235626.SAFE + sentinel:sentinel_tile_id: S2A_OPER_MSI_L1C_TL_S2RP_20230222T235626_A031788_T53JNN_N05.00 + sentinel:utm_zone: 53 + +measurements: + nbart_blue: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band02.tif + grid: '10' + nbart_coastal_aerosol: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band01.tif + grid: '60' + nbart_green: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band03.tif + grid: '10' + nbart_nir_1: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band08.tif + grid: '10' + nbart_nir_2: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band08a.tif + nbart_red: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band04.tif + grid: '10' + nbart_red_edge_1: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band05.tif + nbart_red_edge_2: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band06.tif + nbart_red_edge_3: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band07.tif + nbart_swir_2: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band11.tif + nbart_swir_3: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band12.tif + oa_azimuthal_exiting: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_azimuthal-exiting.tif + oa_azimuthal_incident: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_azimuthal-incident.tif + oa_combined_terrain_shadow: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_combined-terrain-shadow.tif + oa_exiting_angle: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_exiting-angle.tif + oa_fmask: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_fmask.tif + oa_incident_angle: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_incident-angle.tif + oa_nbart_contiguity: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_nbart-contiguity.tif + grid: '10' + oa_relative_azimuth: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_relative-azimuth.tif + oa_relative_slope: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_relative-slope.tif + oa_s2cloudless_mask: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_s2cloudless-mask.tif + grid: '60' + oa_s2cloudless_prob: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_s2cloudless-prob.tif + grid: '60' + oa_satellite_azimuth: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_satellite-azimuth.tif + oa_satellite_view: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_satellite-view.tif + oa_solar_azimuth: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_solar-azimuth.tif + oa_solar_zenith: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_solar-zenith.tif + oa_time_delta: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_time-delta.tif + +accessories: + thumbnail:nbart: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_thumbnail.jpg + metadata:processor: + path: ga_s2am_ard_3-2-1_53JNN_2021-07-24_interim.proc-info.yaml + checksum:sha1: + path: ga_s2am_ard_3-2-1_53JNN_2021-07-24_interim.sha1 + +lineage: {} +... diff --git a/integration_tests/data/eo3/nrt_dataset.yaml b/integration_tests/data/eo3/nrt_dataset.yaml new file mode 100644 index 0000000000..2388e62fa0 --- /dev/null +++ b/integration_tests/data/eo3/nrt_dataset.yaml @@ -0,0 +1,136 @@ +--- +# Dataset +$schema: https://schemas.opendatacube.org/dataset +id: 94db51ab-4279-4402-8bcf-ec2f4ec494e3 + +label: ga_ls8c_ard_3-2-1_090086_2023-04-30_nrt +product: + name: ga_ls8c_ard_3 + href: https://collections.dea.ga.gov.au/product/ga_ls8c_ard_3 + +crs: epsg:32655 +geometry: + type: Polygon + coordinates: [[[559721.8933982822, -4219833.106601718], [559732.9800465275, -4219728.735567618], + [560962.9813034026, -4214988.730722999], [571612.9871046449, -4174023.7084477567], + [604522.9880076076, -4048158.7049931744], [608903.006475906, -4031688.6350602414], + [609157.5, -4030897.5], [609243.8827279789, -4030914.696960433], [609255.0, + -4030905.0], [609449.0888722979, -4030955.549131081], [609461.1380343755, + -4030957.947862498], [611246.2821331235, -4031422.984647127], [614491.7279428138, + -4032268.87029345], [795172.5726569144, -4079325.971481828], [796033.5623962873, + -4079585.522912585], [796173.1066017178, -4079621.893398282], [747082.5, -4268617.5], + [746957.9666764413, -4268587.428373786], [746857.7239312489, -4268594.104275004], + [746137.4401493662, -4268414.031855924], [560077.4396544178, -4219964.0317270355], + [559816.0052888468, -4219857.620084257], [559721.8933982822, -4219833.106601718]]] +grids: + default: + shape: [7931, 7901] + transform: [30.0, 0.0, 559485.0, 0.0, -30.0, -4030785.0, 0.0, 0.0, 1.0] + nbart:panchromatic: + shape: [15861, 15801] + transform: [15.0, 0.0, 559492.5, 0.0, -15.0, -4030792.5, 0.0, 0.0, 1.0] + +properties: + datetime: 2023-04-30 23:50:34.384549Z + dea:dataset_maturity: nrt + dea:product_maturity: stable + eo:cloud_cover: 93.23672818080765 + eo:gsd: 15.0 # Ground sample distance (m) + eo:instrument: OLI_TIRS + eo:platform: landsat-8 + eo:sun_azimuth: 36.68926293 + eo:sun_elevation: 29.26893394 + fmask:clear: 6.182802588011784 + fmask:cloud: 93.23672818080765 + fmask:cloud_shadow: 0.14184073083015938 + fmask:snow: 0.006917692167185185 + fmask:water: 0.4317108081832226 + gqa:abs_iterative_mean_x: 0.19 + gqa:abs_iterative_mean_xy: 0.26 + gqa:abs_iterative_mean_y: 0.18 + gqa:abs_x: 0.65 + gqa:abs_xy: 0.73 + gqa:abs_y: 0.33 + gqa:cep90: 0.4 + gqa:iterative_mean_x: -0.15 + gqa:iterative_mean_xy: 0.18 + gqa:iterative_mean_y: 0.1 + gqa:iterative_stddev_x: 0.16 + gqa:iterative_stddev_xy: 0.24 + gqa:iterative_stddev_y: 0.18 + gqa:mean_x: 0.23 + gqa:mean_xy: 0.23 + gqa:mean_y: -0.02 + gqa:stddev_x: 1.42 + gqa:stddev_xy: 1.48 + gqa:stddev_y: 0.42 + landsat:collection_category: RT + landsat:collection_number: 2 + landsat:landsat_product_id: LC08_L1TP_090086_20230430_20230501_02_RT + landsat:landsat_scene_id: LC80900862023120LGN00 + landsat:wrs_path: 90 + landsat:wrs_row: 86 + odc:dataset_version: 3.2.1 + odc:file_format: GeoTIFF + odc:processing_datetime: 2023-05-01 08:53:27.948360Z + odc:producer: ga.gov.au + odc:product_family: ard + odc:region_code: '090086' + +measurements: + nbart_blue: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_nrt_band02.tif + nbart_coastal_aerosol: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_nrt_band01.tif + nbart_green: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_nrt_band03.tif + nbart_nir: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_nrt_band05.tif + nbart_panchromatic: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_nrt_band08.tif + grid: nbart:panchromatic + nbart_red: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_nrt_band04.tif + nbart_swir_1: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_nrt_band06.tif + nbart_swir_2: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_nrt_band07.tif + oa_azimuthal_exiting: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_azimuthal-exiting.tif + oa_azimuthal_incident: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_azimuthal-incident.tif + oa_combined_terrain_shadow: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_combined-terrain-shadow.tif + oa_exiting_angle: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_exiting-angle.tif + oa_fmask: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_fmask.tif + oa_incident_angle: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_incident-angle.tif + oa_nbart_contiguity: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_nbart-contiguity.tif + oa_relative_azimuth: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_relative-azimuth.tif + oa_relative_slope: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_relative-slope.tif + oa_satellite_azimuth: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_satellite-azimuth.tif + oa_satellite_view: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_satellite-view.tif + oa_solar_azimuth: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_solar-azimuth.tif + oa_solar_zenith: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_solar-zenith.tif + oa_time_delta: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_time-delta.tif + +accessories: + thumbnail:nbart: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_nrt_thumbnail.jpg + metadata:processor: + path: ga_ls8c_ard_3-2-1_090086_2023-04-30_nrt.proc-info.yaml + checksum:sha1: + path: ga_ls8c_ard_3-2-1_090086_2023-04-30_nrt.sha1 + +lineage: {} +... diff --git a/integration_tests/data_utils.py b/integration_tests/data_utils.py index 64cefbf27d..d92c9daaa9 100644 --- a/integration_tests/data_utils.py +++ b/integration_tests/data_utils.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ The start of some general purpose utilities for generating test data. diff --git a/integration_tests/index/__init__.py b/integration_tests/index/__init__.py index d0e5e97db3..a642d3c362 100644 --- a/integration_tests/index/__init__.py +++ b/integration_tests/index/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/integration_tests/index/search_utils.py b/integration_tests/index/search_utils.py index 62595bb735..66fe06b5f1 100644 --- a/integration_tests/index/search_utils.py +++ b/integration_tests/index/search_utils.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import csv import io diff --git a/integration_tests/index/test_config_docs.py b/integration_tests/index/test_config_docs.py index 0ae002e5c9..918c5623d4 100644 --- a/integration_tests/index/test_config_docs.py +++ b/integration_tests/index/test_config_docs.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module @@ -15,7 +15,7 @@ from datacube.index import Index from datacube.index.abstract import default_metadata_type_docs from datacube.model import MetadataType, DatasetType -from datacube.model import Range, Dataset +from datacube.model import Range, Not, Dataset from datacube.utils import changes from datacube.utils.documents import documents_equal from datacube.testutils import sanitise_doc @@ -447,7 +447,7 @@ def test_filter_types_by_fields(index, wo_eo3_product): assert len(res) == 0 -def test_filter_types_by_search(index, wo_eo3_product): +def test_filter_types_by_search(index, wo_eo3_product, ls8_eo3_product): """ :type ls5_telem_type: datacube.model.DatasetType :type index: datacube.index.Index @@ -456,7 +456,7 @@ def test_filter_types_by_search(index, wo_eo3_product): # No arguments, return all. res = list(index.products.search()) - assert res == [wo_eo3_product] + assert res == [ls8_eo3_product, wo_eo3_product] # Matching fields res = list(index.products.search( @@ -491,6 +491,12 @@ def test_filter_types_by_search(index, wo_eo3_product): )) assert res == [wo_eo3_product] + # Not expression test + res = list(index.products.search( + product_family=Not("wo"), + )) + assert res == [ls8_eo3_product] + # Mismatching fields res = list(index.products.search( product_family='spam', diff --git a/integration_tests/index/test_index_cloning.py b/integration_tests/index/test_index_cloning.py index cacb5de4d3..a53adaefe5 100644 --- a/integration_tests/index/test_index_cloning.py +++ b/integration_tests/index/test_index_cloning.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2023 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/integration_tests/index/test_index_data.py b/integration_tests/index/test_index_data.py index f69af1b1c3..a8ff47456b 100755 --- a/integration_tests/index/test_index_data.py +++ b/integration_tests/index/test_index_data.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Test database methods. @@ -18,7 +18,8 @@ from datacube.index.exceptions import MissingRecordError from datacube.index import Index -from datacube.model import Dataset, MetadataType +from datacube.model import Dataset, Product, MetadataType +from datacube.index.eo3 import prep_eo3 _telemetry_uuid = UUID('4ec8fe97-e8b9-11e4-87ff-1040f381a756') _telemetry_dataset = { @@ -94,6 +95,59 @@ def test_archive_datasets(index, local_config, ls8_eo3_dataset): assert not indexed_dataset.is_archived +def test_archive_less_mature(index, final_dataset, nrt_dataset, ds_no_region): + # case 1: add nrt then final; nrt should get archived + index.datasets.add(nrt_dataset, with_lineage=False, archive_less_mature=0) + index.datasets.get(nrt_dataset.id).is_active + index.datasets.add(final_dataset, with_lineage=False, archive_less_mature=0) + assert index.datasets.get(nrt_dataset.id).is_archived + assert index.datasets.get(final_dataset.id).is_active + + # case 2: purge nrt; re-add with final already there + index.datasets.purge([nrt_dataset.id]) + assert index.datasets.get(nrt_dataset.id) is None + with pytest.raises(ValueError): + # should error as more mature version of dataset already exists + index.datasets.add(nrt_dataset, with_lineage=False, archive_less_mature=0) + + +def test_cannot_search_for_less_mature(index, nrt_dataset, ds_no_region): + # if a dataset is missing a property required for finding less mature datasets, + # it should error + index.datasets.add(nrt_dataset, with_lineage=False, archive_less_mature=0) + index.datasets.get(nrt_dataset.id).is_active + assert ds_no_region.metadata.region_code is None + with pytest.raises(ValueError, match="region_code"): + index.datasets.add(ds_no_region, with_lineage=False, archive_less_mature=0) + + +def test_archive_less_mature_approx_timestamp(index, ga_s2am_ard3_final, ga_s2am_ard3_interim): + # test archive_less_mature where there's a slight difference in timestamps + index.datasets.add(ga_s2am_ard3_interim, with_lineage=False) + index.datasets.get(ga_s2am_ard3_interim.id).is_active + index.datasets.add(ga_s2am_ard3_final, with_lineage=False, archive_less_mature=1) + assert index.datasets.get(ga_s2am_ard3_interim.id).is_archived + assert index.datasets.get(ga_s2am_ard3_final.id).is_active + + +def test_dont_archive_less_mature(index, final_dataset, nrt_dataset): + # ensure datasets aren't archive if no archive_less_mature value is provided + index.datasets.add(nrt_dataset, with_lineage=False) + index.datasets.get(nrt_dataset.id).is_active + index.datasets.add(final_dataset, with_lineage=False, archive_less_mature=None) + assert index.datasets.get(nrt_dataset.id).is_active + assert index.datasets.get(final_dataset.id).is_active + + +def test_archive_less_mature_bool(index, final_dataset, nrt_dataset): + # if archive_less_mature value gets passed as a bool via an outdated script + index.datasets.add(nrt_dataset, with_lineage=False) + index.datasets.get(nrt_dataset.id).is_active + index.datasets.add(final_dataset, with_lineage=False, archive_less_mature=False) + assert index.datasets.get(nrt_dataset.id).is_active + assert index.datasets.get(final_dataset.id).is_active + + def test_purge_datasets(index, ls8_eo3_dataset): assert index.datasets.has(ls8_eo3_dataset.id) datasets = index.datasets.search_eager() @@ -205,6 +259,14 @@ def test_get_dataset(index: Index, ls8_eo3_dataset: Dataset) -> None: 'f226a278-e422-11e6-b501-185e0f80a5c1']) == [] +def test_add_dataset_no_product_id(index: Index, extended_eo3_metadata_type, ls8_eo3_product, eo3_ls8_dataset_doc): + product_no_id = Product(extended_eo3_metadata_type, ls8_eo3_product.definition) + assert product_no_id.id is None + dataset_doc, _ = eo3_ls8_dataset_doc + dataset = Dataset(product_no_id, prep_eo3(dataset_doc)) + assert index.datasets.add(dataset, with_lineage=False) + + def test_transactions_api_ctx_mgr(index, extended_eo3_metadata_type_doc, ls8_eo3_product, diff --git a/integration_tests/index/test_memory_index.py b/integration_tests/index/test_memory_index.py index 41758bdb80..a19ae12c36 100644 --- a/integration_tests/index/test_memory_index.py +++ b/integration_tests/index/test_memory_index.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import datetime @@ -201,9 +201,7 @@ def test_mem_ds_search_dups(mem_eo3_data): dc, ls8_id, wo_id = mem_eo3_data ls8_ds = dc.index.datasets.get(ls8_id) dup_results = dc.index.datasets.search_product_duplicates(ls8_ds.product, "cloud_cover", "dataset_maturity") - assert len(dup_results) == 1 - assert dup_results[0][0].cloud_cover == ls8_ds.metadata.cloud_cover - assert ls8_id in dup_results[0][1] + assert len(dup_results) == 0 def test_mem_ds_locations(mem_eo3_data): diff --git a/integration_tests/index/test_null_index.py b/integration_tests/index/test_null_index.py index 8c235f0058..dd248724ba 100644 --- a/integration_tests/index/test_null_index.py +++ b/integration_tests/index/test_null_index.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest from unittest.mock import MagicMock diff --git a/integration_tests/index/test_pluggable_indexes.py b/integration_tests/index/test_pluggable_indexes.py index c2e8e1d112..06f6231995 100644 --- a/integration_tests/index/test_pluggable_indexes.py +++ b/integration_tests/index/test_pluggable_indexes.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest diff --git a/integration_tests/index/test_postgis_index.py b/integration_tests/index/test_postgis_index.py index 5bc3e47264..4a6d07c8e7 100644 --- a/integration_tests/index/test_postgis_index.py +++ b/integration_tests/index/test_postgis_index.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest diff --git a/integration_tests/index/test_search_eo3.py b/integration_tests/index/test_search_eo3.py index dfd40f14f0..942bc9517e 100644 --- a/integration_tests/index/test_search_eo3.py +++ b/integration_tests/index/test_search_eo3.py @@ -1,12 +1,13 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module """ import datetime from typing import Any +from collections import namedtuple import pytest import yaml @@ -738,13 +739,14 @@ def test_find_duplicates_eo3(index, assert len(all_datasets) == 5 # First two ls8 datasets have the same path/row, last two have a different row. + dupe_fields = namedtuple('search_result', ['region_code', 'dataset_maturity']) expected_ls8_path_row_duplicates = [ ( - ('090086', 'final'), + dupe_fields('090086', 'final'), {ls8_eo3_dataset.id, ls8_eo3_dataset2.id} ), ( - ('101077', 'final'), + dupe_fields('101077', 'final'), {ls8_eo3_dataset3.id, ls8_eo3_dataset4.id} ), @@ -772,6 +774,30 @@ def test_find_duplicates_eo3(index, assert sat_res == [] +def test_find_duplicates_with_time(index, nrt_dataset, final_dataset, ls8_eo3_dataset): + index.datasets.add(nrt_dataset, with_lineage=False) + index.datasets.add(final_dataset, with_lineage=False) + index.datasets.get(nrt_dataset.id).is_active + index.datasets.get(final_dataset.id).is_active + + all_datasets = index.datasets.search_eager() + assert len(all_datasets) == 3 + + dupe_fields = namedtuple('search_result', ['region_code', 'time']) + expected_result = [ + ( + dupe_fields('090086', '("2023-04-30 23:50:33.884549","2023-04-30 23:50:34.884549")'), + {nrt_dataset.id, final_dataset.id} + ) + ] + res = sorted(index.datasets.search_product_duplicates( + nrt_dataset.product, + 'region_code', 'time', + )) + + assert res == expected_result + + def test_csv_search_via_cli_eo3(clirunner: Any, ls8_eo3_dataset: Dataset, ls8_eo3_dataset2: Dataset) -> None: diff --git a/integration_tests/index/test_search_legacy.py b/integration_tests/index/test_search_legacy.py index 5d9c7151e5..abac793a9e 100644 --- a/integration_tests/index/test_search_legacy.py +++ b/integration_tests/index/test_search_legacy.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/integration_tests/index/test_update_columns.py b/integration_tests/index/test_update_columns.py index e64b518246..92f84a3d07 100644 --- a/integration_tests/index/test_update_columns.py +++ b/integration_tests/index/test_update_columns.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Test creation of added/updated columns during diff --git a/integration_tests/agdcintegration.conf b/integration_tests/integration.conf similarity index 61% rename from integration_tests/agdcintegration.conf rename to integration_tests/integration.conf index ac3df9a70d..36c09ebbde 100644 --- a/integration_tests/agdcintegration.conf +++ b/integration_tests/integration.conf @@ -1,18 +1,18 @@ [datacube] db_hostname: -db_database: agdcintegration +db_database: pgintegration index_driver: default [experimental] db_hostname: -db_database: odcintegration +db_database: pgisintegration index_driver: postgis -[no_such_driver_env] +[nosuchdriverenv] index_driver: no_such_driver -[null_driver] +[nulldriver] index_driver: null -[local_memory] +[localmemory] index_driver: memory diff --git a/integration_tests/test_3d.py b/integration_tests/test_3d.py index b8c6c7aa88..1f14f0db74 100644 --- a/integration_tests/test_3d.py +++ b/integration_tests/test_3d.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging from copy import deepcopy @@ -265,7 +265,7 @@ def test_indexing(clirunner, index, product_def): "-v", "dataset", "add", - "--confirm-ignore-lineage", + "--ignore-lineage", str(index_yaml), ] ) @@ -315,7 +315,7 @@ def test_indexing_with_spectral_map(clirunner, index, dataset_types): clirunner(["-v", "product", "add", str(dataset_types)]) # Index the Dataset - clirunner(["-v", "dataset", "add", '--confirm-ignore-lineage', str(index_yaml)]) + clirunner(["-v", "dataset", "add", '--ignore-lineage', str(index_yaml)]) dc = Datacube(index=index) check_open_with_dc_simple(dc, product_def, [product_id], measurement) @@ -335,7 +335,7 @@ def test_end_to_end_multitime(clirunner, index, product_def, original_data): measurement=measurement, ) # Index the Datasets - clirunner(["-v", "dataset", "add", '--confirm-ignore-lineage', str(index_yaml)]) + clirunner(["-v", "dataset", "add", '--ignore-lineage', str(index_yaml)]) if idx == 0: # Full check for the first measurement only # Check data for all product IDs diff --git a/integration_tests/test_cli_output.py b/integration_tests/test_cli_output.py index 44c6292478..ba5312a19f 100644 --- a/integration_tests/test_cli_output.py +++ b/integration_tests/test_cli_output.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 @@ -81,7 +81,33 @@ def test_cli_dataset_subcommand(index, clirunner, # Insert datasets for path in eo3_dataset_paths: - result = clirunner(['dataset', 'add', "--confirm-ignore-lineage", path]) + result = clirunner(['dataset', 'add', "--ignore-lineage", path]) + + runner = clirunner(['dataset', 'find-duplicates'], verbose_flag=False, expect_success=False) + assert "Error: must provide field names to match on" in runner.output + assert runner.exit_code == 1 + + runner = clirunner(['dataset', 'find-duplicates', 'region_code', 'fake_field'], + verbose_flag=False, expect_success=False) + assert "Error: no products found with fields region_code, fake_field" in runner.output + assert runner.exit_code == 1 + + runner = clirunner(['dataset', 'find-duplicates', 'region_code', 'landsat_scene_id', + '-p', 'ga_ls8c_ard_3', '-p', 'ga_ls_wo_3'], + verbose_flag=False, + expect_success=False) + assert "Error: specified products ga_ls_wo_3 do not contain all required fields" + assert runner.exit_code == 1 + + runner = clirunner(['dataset', 'find-duplicates', 'region_code', 'uri'], verbose_flag=False) + assert "No potential duplicates found." in runner.output + assert runner.exit_code == 0 + + runner = clirunner(['dataset', 'find-duplicates', 'region_code', 'dataset_maturity'], verbose_flag=False) + assert "No potential duplicates found." not in runner.output + assert "(region_code='090086', dataset_maturity='final')" in runner.output + assert "(region_code='101077', dataset_maturity='final')" in runner.output + assert runner.exit_code == 0 runner = clirunner(['dataset', 'archive'], verbose_flag=False, expect_success=False) assert "Completed dataset archival." not in runner.output @@ -146,8 +172,8 @@ def test_readd_and_update_metadata_product_dataset_command(index, clirunner, assert "No such dataset in the database" in update.output assert "Failure while processing" in update.output - clirunner(['dataset', 'add', '--confirm-ignore-lineage', ds_path]) - rerun_add = clirunner(['dataset', 'add', '--confirm-ignore-lineage', ds_path]) + clirunner(['dataset', 'add', '--ignore-lineage', ds_path]) + rerun_add = clirunner(['dataset', 'add', '--ignore-lineage', ds_path]) assert "WARNING Dataset" in rerun_add.output assert "is already in the database" in rerun_add.output diff --git a/integration_tests/test_config_tool.py b/integration_tests/test_config_tool.py index 991c6fb62c..81ffdc0a5c 100644 --- a/integration_tests/test_config_tool.py +++ b/integration_tests/test_config_tool.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/integration_tests/test_dataset_add.py b/integration_tests/test_dataset_add.py index 5d8e87441f..3bef05845d 100644 --- a/integration_tests/test_dataset_add.py +++ b/integration_tests/test_dataset_add.py @@ -1,12 +1,13 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import math import pytest import toolz import yaml +import logging from datacube.index import Index from datacube.index.hl import Doc2Dataset @@ -15,13 +16,15 @@ from datacube.utils import SimpleDocNav from datacube.scripts.dataset import _resolve_uri +logger = logging.getLogger(__name__) + def check_skip_lineage_test(clirunner, index): ds = SimpleDocNav(gen_dataset_test_dag(11, force_tree=True)) prefix = write_files({'agdc-metadata.yml': yaml.safe_dump(ds.doc)}) - clirunner(['dataset', 'add', '--confirm-ignore-lineage', '--product', 'A', str(prefix)]) + clirunner(['dataset', 'add', '--ignore-lineage', '--product', 'A', str(prefix)]) ds_ = index.datasets.get(ds.id, include_sources=True) assert ds_ is not None @@ -58,7 +61,7 @@ def check_no_product_match(clirunner, index): # Ignore lineage but fail to match main dataset r = clirunner(['dataset', 'add', '--product', 'B', - '--confirm-ignore-lineage', + '--ignore-lineage', str(prefix)]) assert 'ERROR' in r.output @@ -184,12 +187,6 @@ def check_missing_metadata_doc(clirunner): assert "ERROR No supported metadata docs found for dataset" in r.output -def check_no_confirm(clirunner, path): - r = clirunner(['dataset', 'add', '--ignore-lineage', str(path)], expect_success=False) - assert r.exit_code != 0 - assert 'Use --confirm-ignore-lineage from non-interactive scripts' in r.output - - def check_bad_yaml(clirunner, index): prefix = write_files({'broken.yml': '"'}) r = clirunner(['dataset', 'add', str(prefix / 'broken.yml')]) @@ -212,6 +209,29 @@ def test_dataset_add_no_id(dataset_add_configs, index_empty, clirunner): assert _err == 'No id defined in dataset doc' +@pytest.mark.parametrize('datacube_env_name', ('datacube', )) +def test_dataset_eo3_no_schema(dataset_add_configs, index_empty, clirunner, caplog): + p = dataset_add_configs + index = index_empty + ds = load_dataset_definition(p.datasets_eo3).doc + + clirunner(['metadata', 'add', p.metadata]) + clirunner(['product', 'add', p.products]) + + # no warnings if eo3 dataset has $schema + doc2ds = Doc2Dataset(index) + doc2ds(ds, 'file:///something') + warnings = [record for record in caplog.records if record.levelno == logging.WARNING] + assert len(warnings) == 0 + + # warn if eo3 metadata type but no $schema + del ds["$schema"] + doc2ds(ds, 'file:///something') + warnings = [record for record in caplog.records if record.levelno == logging.WARNING] + assert len(warnings) == 1 + assert "will not be recognised as an eo3 dataset" in warnings[0].msg + + # Current formulation of this test relies on non-EO3 test data @pytest.mark.parametrize('datacube_env_name', ('datacube', )) def test_dataset_add(dataset_add_configs, index_empty, clirunner): @@ -266,7 +286,6 @@ def test_dataset_add(dataset_add_configs, index_empty, clirunner): check_inconsistent_lineage(clirunner, index) check_missing_metadata_doc(clirunner) check_missing_lineage(clirunner, index) - check_no_confirm(clirunner, p.datasets) check_bad_yaml(clirunner, index) # check --product=nosuchproduct diff --git a/integration_tests/test_double_ingestion.py b/integration_tests/test_double_ingestion.py index 3534def553..918b5bcc6c 100644 --- a/integration_tests/test_double_ingestion.py +++ b/integration_tests/test_double_ingestion.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest import netCDF4 diff --git a/integration_tests/test_end_to_end.py b/integration_tests/test_end_to_end.py index 43bcdf425c..834a16b42c 100644 --- a/integration_tests/test_end_to_end.py +++ b/integration_tests/test_end_to_end.py @@ -1,15 +1,17 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import shutil from pathlib import Path import numpy import pytest import rasterio +from affine import Affine from datacube.api.query import query_group_by from datacube.api.core import Datacube +from datacube.utils.geometry import GeoBox from integration_tests.utils import prepare_test_ingestion_configuration @@ -76,7 +78,7 @@ def test_end_to_end(clirunner, index, testdata_dir, ingest_configs, datacube_env # - this will be no-op but with ignore lineage clirunner(['-v', 'dataset', 'add', - '--confirm-ignore-lineage', + '--ignore-lineage', str(lbg_nbar), str(lbg_pq)]) # Test no-op update @@ -116,6 +118,7 @@ def test_end_to_end(clirunner, index, testdata_dir, ingest_configs, datacube_env check_open_with_dc(index) check_open_with_grid_workflow(index) check_load_via_dss(index) + check_odcgeo_geobox_load(index) def check_open_with_dc(index): @@ -317,3 +320,45 @@ def check_legacy_open(index): xx_lazy = dc.load_data(sources, gbox, mm, dask_chunks={'time': 1}) assert xx_lazy['blue'].data.dask assert xx_lazy.blue[0, :, :].equals(xx.blue[0, :, :]) + + +def check_odcgeo_geobox_load(index): + """ + Test that users can use `dc.load(like=...)` by passing an + `odc-geo`-style GeoBox. + """ + dc = Datacube(index=index) + + # Create mock odc-geo GeoBox + class ODC_geo_geobox: + compat = GeoBox( + 500, 500, Affine(0.002, 0.0, 149.0, 0.0, -0.002, -35.0), "EPSG:4326" + ) + coords = compat.coords + + odc_geo_geobox = ODC_geo_geobox() + + # Load data using .compat method + ds_compat = dc.load( + product="ls5_nbar_albers", + measurements=["blue"], + like=odc_geo_geobox.compat, + ) + assert "blue" in ds_compat.data_vars + + # Load data using odc-geo GeoBox directly + ds_odcgeo = dc.load( + product="ls5_nbar_albers", + measurements=["blue"], + like=odc_geo_geobox, + ) + assert "blue" in ds_odcgeo.data_vars + + # Like behaves differently when time is specified; make sure this works too + ds_odcgeo_time = dc.load( + product="ls5_nbar_albers", + measurements=["blue"], + like=odc_geo_geobox, + time="1992-03-23T23:14:25.500000", + ) + assert "blue" in ds_odcgeo_time.data_vars diff --git a/integration_tests/test_environments.py b/integration_tests/test_environments.py index 2ce64f07ac..6d609dceba 100644 --- a/integration_tests/test_environments.py +++ b/integration_tests/test_environments.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest @@ -19,7 +19,7 @@ def test_multiple_environment_config(tmpdir): [default] db_hostname: db.opendatacube.test -[test_alt] +[testalt] db_hostname: alt-db.opendatacube.test """) @@ -27,7 +27,7 @@ def test_multiple_environment_config(tmpdir): config = LocalConfig.find([config_path]) assert config['db_hostname'] == 'db.opendatacube.test' - alt_config = LocalConfig.find([config_path], env='test_alt') + alt_config = LocalConfig.find([config_path], env='testalt') assert alt_config['db_hostname'] == 'alt-db.opendatacube.test' # Make sure the correct config is passed through the API @@ -42,12 +42,12 @@ def test_multiple_environment_config(tmpdir): with Datacube(config=str(config_path), validate_connection=False) as dc: assert str(dc.index.url) == db_url # When specific environment is loaded - with Datacube(config=config_path, env='test_alt', validate_connection=False) as dc: + with Datacube(config=config_path, env='testalt', validate_connection=False) as dc: assert str(dc.index.url) == alt_db_url # An environment that isn't in any config files with pytest.raises(ValueError): - with Datacube(config=config_path, env='undefined-env', validate_connection=False) as dc: + with Datacube(config=config_path, env='undefinedenv', validate_connection=False) as dc: pass diff --git a/integration_tests/test_full_ingestion.py b/integration_tests/test_full_ingestion.py index ab71d5608b..a4e3332ff8 100644 --- a/integration_tests/test_full_ingestion.py +++ b/integration_tests/test_full_ingestion.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import hashlib import warnings diff --git a/integration_tests/test_index_datasets_search.py b/integration_tests/test_index_datasets_search.py index 0a8373faf1..df572d2f02 100644 --- a/integration_tests/test_index_datasets_search.py +++ b/integration_tests/test_index_datasets_search.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest from pathlib import PurePosixPath diff --git a/integration_tests/test_index_out_of_bound.py b/integration_tests/test_index_out_of_bound.py index 8cbd343255..641662d3be 100644 --- a/integration_tests/test_index_out_of_bound.py +++ b/integration_tests/test_index_out_of_bound.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest diff --git a/integration_tests/test_model.py b/integration_tests/test_model.py index 6fe0dd06f1..69366f49eb 100644 --- a/integration_tests/test_model.py +++ b/integration_tests/test_model.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest diff --git a/integration_tests/test_validate_ingestion.py b/integration_tests/test_validate_ingestion.py index e204cb299a..83cec3a785 100644 --- a/integration_tests/test_validate_ingestion.py +++ b/integration_tests/test_validate_ingestion.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest diff --git a/integration_tests/utils.py b/integration_tests/utils.py index 4744c54efc..e15bcb700b 100644 --- a/integration_tests/utils.py +++ b/integration_tests/utils.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging import os diff --git a/license-headers.md b/license-headers.md index 92090fad06..e5cd086bbb 100644 --- a/license-headers.md +++ b/license-headers.md @@ -1,29 +1,23 @@ # Applying or updating license headers -To add or update license headers in this or other Open Data Cube -projects, you can do the following: +To add or update license headers in this or other Open Data Cube projects, you can do the following: -Download the tool from [https://github.com/johann-petrak/licenseheaders](https://github.com/johann-petrak/licenseheaders) and make it executable. +Ensure you have pre-commit hooks installed ```bash -wget https://raw.githubusercontent.com/johann-petrak/licenseheaders/master/licenseheaders.py -chmod +x licenseheaders.py +pre-commit install ``` -Change the sections on `python` files, to remove the `headerStartLine` and -`headerEndLine`, like: +Run the `insert-license` hook: -```python - "headerStartLine": "", - "headerEndLine": "", +```bash +pre-commit run insert-license --all-files ``` -Run the tool: +To remove the license headers, add the `--remove-header` arg to `.pre-commit-config.yaml` before running the hook. -```bash -python3 ./licenseheaders.py --tmpl license-template.txt --years 2015-2020 --ext py --dir datacube -python3 ./licenseheaders.py --tmpl license-template.txt --years 2015-2020 --ext py --dir integration_tests -python3 ./licenseheaders.py --tmpl license-template.txt --years 2015-2020 --ext py --dir tests -python3 ./licenseheaders.py --tmpl license-template.txt --years 2015-2020 --ext py --dir docs -python3 ./licenseheaders.py --tmpl license-template.txt --years 2015-2020 --ext py --dir examples -``` +To make updates to the license text, first remove the headers, then update `license-template.txt` before rerunning the hook as usual to add them back. + +Note that the date range is automatically updated to include the current year. + +See the full documentation here: [https://github.com/Lucas-C/pre-commit-hooks#insert-license](https://github.com/Lucas-C/pre-commit-hooks#insert-license) diff --git a/license-template.txt b/license-template.txt index 649c117012..f15b36fd20 100644 --- a/license-template.txt +++ b/license-template.txt @@ -1,4 +1,4 @@ This file is part of the Open Data Cube, see https://opendatacube.org for more information -Copyright (c) ${years} ODC Contributors +Copyright (c) 2015-2023 ODC Contributors SPDX-License-Identifier: Apache-2.0 diff --git a/setup.py b/setup.py index 3d53532bc3..43ee11fbd2 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ 'pytest-cov', 'pytest-timeout', 'pytest-httpserver', - 'moto', + 'moto<5.0', # 5.0 will require changes to some tests. ] doc_require = [ 'Sphinx', @@ -47,7 +47,7 @@ setup( name='datacube', - python_requires='>=3.8.0', + python_requires='>=3.9.0', url='https://github.com/opendatacube/datacube-core', author='Open Data Cube', @@ -70,9 +70,10 @@ "Operating System :: Microsoft :: Windows", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering :: GIS", "Topic :: Scientific/Engineering :: Information Analysis", ], @@ -96,7 +97,7 @@ 'cloudpickle>=0.4', 'dask[array]', 'distributed', - 'jsonschema', + 'jsonschema>=4.18', # New reference resolution API 'netcdf4', 'numpy', 'psycopg2', @@ -111,6 +112,8 @@ 'toolz', 'xarray>=0.9', # >0.9 fixes most problems with `crs` attributes being lost 'packaging', + 'deprecat', + 'importlib_metadata>3.5;python_version<"3.10"', ], extras_require=extras_require, tests_require=tests_require, diff --git a/tests/__init__.py b/tests/__init__.py index c081ad5b41..8318282aea 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,4 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/api/__init__.py b/tests/api/__init__.py index c081ad5b41..8318282aea 100644 --- a/tests/api/__init__.py +++ b/tests/api/__init__.py @@ -1,4 +1,4 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/api/test_core.py b/tests/api/test_core.py index c96f58824b..38b272b44f 100644 --- a/tests/api/test_core.py +++ b/tests/api/test_core.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import xarray as xr import numpy as np diff --git a/tests/api/test_grid_workflow.py b/tests/api/test_grid_workflow.py index 81ab254606..cad0f0a7a9 100644 --- a/tests/api/test_grid_workflow.py +++ b/tests/api/test_grid_workflow.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest import numpy diff --git a/tests/api/test_masking.py b/tests/api/test_masking.py index 456680a00e..378689ccb5 100644 --- a/tests/api/test_masking.py +++ b/tests/api/test_masking.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import yaml import pytest diff --git a/tests/api/test_query.py b/tests/api/test_query.py index fc982a1fca..7b6c1e4066 100644 --- a/tests/api/test_query.py +++ b/tests/api/test_query.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import datetime import pandas @@ -140,7 +140,11 @@ def format_test(start_out, end_out): ((datetime.datetime(2008, 1, 1), datetime.datetime(2008, 1, 10, 23, 59, 40)), format_test('2008-01-01T00:00:00', '2008-01-10T23:59:40.999999')), ((datetime.date(2008, 1, 1)), - format_test('2008-01-01T00:00:00', '2008-01-01T23:59:59.999999')) + format_test('2008-01-01T00:00:00', '2008-01-01T23:59:59.999999')), + ((datetime.date(2008, 1, 1), None), + format_test('2008-01-01T00:00:00', datetime.datetime.now().strftime("%Y-%m-%dT23:59:59.999999"))), + ((None, '2008'), + format_test(datetime.datetime.fromtimestamp(0).strftime("%Y-%m-%dT%H:%M:%S"), '2008-12-31T23:59:59.999999')), ] @@ -151,6 +155,11 @@ def test_time_handling(time_param, expected): assert query.search_terms['time'] == expected +def test_time_handling_int(): + with pytest.raises(TypeError): + Query(time=2008) + + def test_solar_day(): _s = SimpleNamespace ds = _s(center_time=parse_time('1987-05-22 23:07:44.2270250Z'), @@ -167,6 +176,13 @@ def test_solar_day(): assert 'Cannot compute solar_day: dataset is missing spatial info' in str(e.value) + # Test with Non-UTC timestamp in index. + ds = _s(center_time=parse_time('1997-05-22 22:07:44.2270250-7:00'), + metadata=_s(lon=Range(begin=-136.615, + end=-134.325))) + assert solar_day(ds) == np.datetime64('1997-05-22', 'D') + assert solar_day(ds, longitude=0) == np.datetime64('1997-05-23', 'D') + def test_solar_offset(): from datacube.utils.geometry import point diff --git a/tests/api/test_virtual.py b/tests/api/test_virtual.py index 598095d138..03206172a1 100644 --- a/tests/api/test_virtual.py +++ b/tests/api/test_virtual.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from collections import OrderedDict from datetime import datetime diff --git a/tests/conftest.py b/tests/conftest.py index 8a52f369a3..57a2088af9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ py.test configuration fixtures diff --git a/tests/data/lbg/LS5_TM_NBAR_P54_GANBAR01-002_090_084_19920323/metadata.xml b/tests/data/lbg/LS5_TM_NBAR_P54_GANBAR01-002_090_084_19920323/metadata.xml index 6c1abf7382..880c39e0a4 100644 --- a/tests/data/lbg/LS5_TM_NBAR_P54_GANBAR01-002_090_084_19920323/metadata.xml +++ b/tests/data/lbg/LS5_TM_NBAR_P54_GANBAR01-002_090_084_19920323/metadata.xml @@ -1,233 +1,233 @@ - - - - 19920323T23:09:13 - ALSP - acquisition - - D - 19920323T23:17:56 - - 42876 - completed - instantaneous - - preProgrammed - - TM - Multi-spectral - - Landsat-5 - - - 1 - Color JPEG Image - LS5_TM_NBAR_P54_GANBAR01-002_090_084_19920323.jpg - JPG - 4 - 7 - 237.329101562 - - - - - - - - - - Ortho-rectified, processed by Geoscience Australia - - - - 20120917T01:25:07 - creation - - - - - - - - </SOURCECITATION> - <SOURCEREFERENCESYSTEM/> - <SOURCERESOURCEID/> - <SOURCESCALE/> - <SOURCESTEP/> - </LINEAGESOURCE> - <PROCESSINGSTEP> - <ALGORITHMCITATION> - <EDITION>Git version ed0cd604d3b171ded4d8d3767feedb76e1b8ce58</EDITION> - <TITLE>Pinkmatter Landsat Processor - - - NBAR version GANBARv3.00.00-dev + + + + 19920323T23:09:13 + ALSP + acquisition + + D + 19920323T23:17:56 + + 42876 + completed + instantaneous + + preProgrammed + + TM + Multi-spectral + + Landsat-5 + + + 1 + Color JPEG Image + LS5_TM_NBAR_P54_GANBAR01-002_090_084_19920323.jpg + JPG + 4 + 7 + 237.329101562 + + + + + + + + + + Ortho-rectified, processed by Geoscience Australia + + + + 20120917T01:25:07 + creation + + + + + + + + </SOURCECITATION> + <SOURCEREFERENCESYSTEM/> + <SOURCERESOURCEID/> + <SOURCESCALE/> + <SOURCESTEP/> + </LINEAGESOURCE> + <PROCESSINGSTEP> + <ALGORITHMCITATION> + <EDITION>Git version ed0cd604d3b171ded4d8d3767feedb76e1b8ce58</EDITION> + <TITLE>Pinkmatter Landsat Processor + + + NBAR version GANBARv3.00.00-dev BRDF (MODIS_BRDF) = BAND 1 [0.008570 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b09.500m_0459_0479nm_brdf_par_fgeo.hdf), 0.043934 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b07.500m_0459_0479nm_brdf_par_fiso.hdf), 0.016253 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b08.500m_0459_0479nm_brdf_par_fvol.hdf)] - BAND 2 [0.015374 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b12.500m_0545_0565nm_brdf_par_fgeo.hdf), 0.078902 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b10.500m_0545_0565nm_brdf_par_fiso.hdf), 0.037976 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b11.500m_0545_0565nm_brdf_par_fvol.hdf)] - BAND 3 [0.017559 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b03.500m_0620_0670nm_brdf_par_fgeo.hdf), 0.085431 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b01.500m_0620_0670nm_brdf_par_fiso.hdf), 0.034050 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b02.500m_0620_0670nm_brdf_par_fvol.hdf)] - BAND 4 [0.023475 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b06.500m_0841_0876nm_brdf_par_fgeo.hdf), 0.273303 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b04.500m_0841_0876nm_brdf_par_fiso.hdf), 0.201959 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b05.500m_0841_0876nm_brdf_par_fvol.hdf)] - BAND 5 [0.043315 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b18.500m_1628_1652nm_brdf_par_fgeo.hdf), 0.279249 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b16.500m_1628_1652nm_brdf_par_fiso.hdf), 0.106509 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b17.500m_1628_1652nm_brdf_par_fvol.hdf)] - BAND 7 [0.032661 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b21.500m_2105_2155nm_brdf_par_fgeo.hdf), 0.165904 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b19.500m_2105_2155nm_brdf_par_fiso.hdf), 0.045391 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b20.500m_2105_2155nm_brdf_par_fvol.hdf)] + BAND 2 [0.015374 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b12.500m_0545_0565nm_brdf_par_fgeo.hdf), 0.078902 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b10.500m_0545_0565nm_brdf_par_fiso.hdf), 0.037976 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b11.500m_0545_0565nm_brdf_par_fvol.hdf)] + BAND 3 [0.017559 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b03.500m_0620_0670nm_brdf_par_fgeo.hdf), 0.085431 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b01.500m_0620_0670nm_brdf_par_fiso.hdf), 0.034050 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b02.500m_0620_0670nm_brdf_par_fvol.hdf)] + BAND 4 [0.023475 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b06.500m_0841_0876nm_brdf_par_fgeo.hdf), 0.273303 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b04.500m_0841_0876nm_brdf_par_fiso.hdf), 0.201959 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b05.500m_0841_0876nm_brdf_par_fvol.hdf)] + BAND 5 [0.043315 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b18.500m_1628_1652nm_brdf_par_fgeo.hdf), 0.279249 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b16.500m_1628_1652nm_brdf_par_fiso.hdf), 0.106509 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b17.500m_1628_1652nm_brdf_par_fvol.hdf)] + BAND 7 [0.032661 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b21.500m_2105_2155nm_brdf_par_fgeo.hdf), 0.165904 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b19.500m_2105_2155nm_brdf_par_fiso.hdf), 0.045391 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b20.500m_2105_2155nm_brdf_par_fvol.hdf)] OZONE (GA 2 deg pixel size) = 0.261 WATER VAPOUR (NOAA) = 2.27 DEM (GA 1 deg pixel size) = 0.409 AOD (/g/data1/v10/eoancillarydata/aerosol/AATSR/2.0/aot_mean_Mar_All_Aerosols.cmp) = 0.043009 - - dataset - - - - 4326 - 151.1469421 - - - -35.6039314 - 148.4684601 - -35.5414848 - 151.1469421 - -33.5805855 - -34.5911 - 149.775 - -35.6039314 - 19920323 23:14:13 - 19920323 23:14:38 - - -33.6386375 - 148.4342346 - -33.5805855 - 151.0505676 - - - - - - 148.4342346 - - - area - - sample - 25.0 - 9721 - - - line - 25.0 - 8721 - - - 754512.500000 - 6167987.500000 - 0 - - - GDA94 - GRS80 - 633012.5 - 6058987.5 - - 876012.5 - 6058987.5 - - 633012.5 - 6276987.5 - - 876012.5 - 6276987.5 - - UpperLeft - UTM - 55 - - 2 - 1 - - - - 1, 2, 3, 4, 5, 7 - - - 23.0 - 35.8386447 - 57.6238557 - - NBAR - 90 - 84 - SAM - 0 - 0 - - - - - - - - - - notPlanned - - - - - - - - Landsat-5 NBAR - usAscii - - Landsat-5 TM NBAR x090 y084 19920323 version 1 status completed - 2014-03-17 05:53:14 - creation - - - - Medhavy Thankappan - Geoscience Australia - Science and Strategy Project Leader - pointOfContact - - - Landsat-5 TM NBAR x090 y084 19920323 version 1 status completed - - - - - - - - 9 - - eng - NCI PBS 1.0.0 | Linux r3398 2.6.32-358.23.2.el6.x86_64 #1 SMP Wed Oct 16 18:37:12 UTC 2013 x86_64 x86_64 x86_64 GNU/Linux - 161 - - GEOTIFF - 1 - - dataset - dataset - - - Medhavy Thankappan - Geoscience Australia - Science and Strategy Project Leader - pointOfContact - - LS5_TM_NBAR_P54_GANBAR01-002_090_084_19920323 - GA Metadata Profile: A Geoscience Australia Profile of AS/NZS ISO 19115:2005, Geographic information - Metadata - 1.0 - - LS5_March_1992_GA/ULA_STND_v2.1.5 - 25.0 - completed - Processed Image - - - environment - + + dataset + + + + 4326 + 151.1469421 + + + -35.6039314 + 148.4684601 + -35.5414848 + 151.1469421 + -33.5805855 + -34.5911 + 149.775 + -35.6039314 + 19920323 23:14:13 + 19920323 23:14:38 + + -33.6386375 + 148.4342346 + -33.5805855 + 151.0505676 + + + + + + 148.4342346 + + + area + + sample + 25.0 + 9721 + + + line + 25.0 + 8721 + + + 754512.500000 + 6167987.500000 + 0 + + + GDA94 + GRS80 + 633012.5 + 6058987.5 + + 876012.5 + 6058987.5 + + 633012.5 + 6276987.5 + + 876012.5 + 6276987.5 + + UpperLeft + UTM + 55 + + 2 + 1 + + + + 1, 2, 3, 4, 5, 7 + + + 23.0 + 35.8386447 + 57.6238557 + + NBAR + 90 + 84 + SAM + 0 + 0 + + + + + + + + + + notPlanned + + + + + + + + Landsat-5 NBAR + usAscii + + Landsat-5 TM NBAR x090 y084 19920323 version 1 status completed + 2014-03-17 05:53:14 + creation + + + + Medhavy Thankappan + Geoscience Australia + Science and Strategy Project Leader + pointOfContact + + + Landsat-5 TM NBAR x090 y084 19920323 version 1 status completed + + + + + + + + 9 + + eng + NCI PBS 1.0.0 | Linux r3398 2.6.32-358.23.2.el6.x86_64 #1 SMP Wed Oct 16 18:37:12 UTC 2013 x86_64 x86_64 x86_64 GNU/Linux + 161 + + GEOTIFF + 1 + + dataset + dataset + + + Medhavy Thankappan + Geoscience Australia + Science and Strategy Project Leader + pointOfContact + + LS5_TM_NBAR_P54_GANBAR01-002_090_084_19920323 + GA Metadata Profile: A Geoscience Australia Profile of AS/NZS ISO 19115:2005, Geographic information - Metadata + 1.0 + + LS5_March_1992_GA/ULA_STND_v2.1.5 + 25.0 + completed + Processed Image + + + environment + diff --git a/tests/data/lbg/LS5_TM_PQ_P55_GAPQ01-002_090_084_19920323/metadata.xml b/tests/data/lbg/LS5_TM_PQ_P55_GAPQ01-002_090_084_19920323/metadata.xml index 383b8c68a4..9116dae376 100644 --- a/tests/data/lbg/LS5_TM_PQ_P55_GAPQ01-002_090_084_19920323/metadata.xml +++ b/tests/data/lbg/LS5_TM_PQ_P55_GAPQ01-002_090_084_19920323/metadata.xml @@ -1,70 +1,70 @@ - - - - 19920323T23:09:13 - ALSP - acquisition - - - 19920323T23:17:56 - - 42876 - - instantaneous - - automatic - - TM - Multi-spectral - - Landsat-5 - - - - - - - - - - - - - - - - - - - The pixel quality algorithm uses data from both the L1T (Systematic Terrain Correction) and ARG25 (Australian Reflectance Grid 25m) products. - - - - 20120917T01:25:07 - creation - - - - - - - - </SOURCECITATION> - <SOURCEREFERENCESYSTEM/> - <SOURCERESOURCEID/> - <SOURCESCALE/> - <SOURCESTEP/> - </LINEAGESOURCE> - <PROCESSINGSTEP> - <ALGORITHMCITATION> - <EDITION>Git version 3.2.0</EDITION> - <TITLE>Pixel Quality - - - + + + + 19920323T23:09:13 + ALSP + acquisition + + + 19920323T23:17:56 + + 42876 + + instantaneous + + automatic + + TM + Multi-spectral + + Landsat-5 + + + + + + + + + + + + + + + + + + + The pixel quality algorithm uses data from both the L1T (Systematic Terrain Correction) and ARG25 (Australian Reflectance Grid 25m) products. + + + + 20120917T01:25:07 + creation + + + + + + + + </SOURCECITATION> + <SOURCEREFERENCESYSTEM/> + <SOURCERESOURCEID/> + <SOURCESCALE/> + <SOURCESTEP/> + </LINEAGESOURCE> + <PROCESSINGSTEP> + <ALGORITHMCITATION> + <EDITION>Git version 3.2.0</EDITION> + <TITLE>Pixel Quality + + + The pixel quality algorithm assesses quality aspects such as saturation, band/spectral contiguity, land/sea, cloud and cloud shadow. Saturation Band1 (Bit 0): Run Saturation Band2 (Bit 1): Run @@ -83,158 +83,158 @@ Cloud Shadow (Fmask) (Bit 13): Run Empty Test (Bit 14): Not Run Empty Test (Bit 15): Not Run - - dataset - - - - 4326 - 151.1469421 - - - -35.6039314 - 148.4684601 - -35.5414848 - 151.1469421 - -33.5805855 - -34.5911 - 149.775 - -35.6039314 - 19920323 23:14:13 - 19920323 23:14:38 - - -33.6386375 - 148.4342346 - -33.5805855 - 151.0505676 - - - - - - 148.4342346 - - - area - - sample - 25.0 - 9721 - - - line - 25.0 - 8721 - - - 754512.500000 - 6167987.500000 - 0 - - - GDA94 - GRS80 - 633012.5 - 6058987.5 - - 876012.5 - 6058987.5 - - 633012.5 - 6276987.5 - - 876012.5 - 6276987.5 - - UpperLeft - UTM - 55 - - 2 - 1 - - - - PQ - - 23.0 - 35.8386447 - 57.6238557 - - Pixel Quality - 90 - 84 - SAM - 0 - 0 - - - - - - - - - - notPlanned - - - - - - - - Landsat-5 PQ - utf8 - - Landsat-5 TM PQ x090 y084 19920323 - 20140601T01:22:30 - creation - - - - NEMO Group - Geoscience Australia - NEMO Operation - pointOfContact - - - Landsat-5 TM PQ x090 y084 19920323 - - - - - - - - 1 - - eng - NCI PBS 1.0.0 | Linux r3398 2.6.32-431.11.2.el6.x86_64 #1 SMP Tue Mar 25 19:59:55 UTC 2014 x86_64 x86_64 x86_64 GNU/Linux - 161 - - GEOTIFF - 1 - - dataset - Satellite Imagery - - - NEMO Group - Geoscience Australia - NEMO Operation - pointOfContact - - LS5_TM_PQ_P55_GAPQ01-002_090_084_19920323 - ANZLIC Metadata Profile: An Australian/New Zealand Profile of AS/NZS ISO 19115:2005, Geographic information - Metadata - 1.0 - - Pixel Quality - 25.0 - completed - Processed Image - grid - + + dataset + + + + 4326 + 151.1469421 + + + -35.6039314 + 148.4684601 + -35.5414848 + 151.1469421 + -33.5805855 + -34.5911 + 149.775 + -35.6039314 + 19920323 23:14:13 + 19920323 23:14:38 + + -33.6386375 + 148.4342346 + -33.5805855 + 151.0505676 + + + + + + 148.4342346 + + + area + + sample + 25.0 + 9721 + + + line + 25.0 + 8721 + + + 754512.500000 + 6167987.500000 + 0 + + + GDA94 + GRS80 + 633012.5 + 6058987.5 + + 876012.5 + 6058987.5 + + 633012.5 + 6276987.5 + + 876012.5 + 6276987.5 + + UpperLeft + UTM + 55 + + 2 + 1 + + + + PQ + + 23.0 + 35.8386447 + 57.6238557 + + Pixel Quality + 90 + 84 + SAM + 0 + 0 + + + + + + + + + + notPlanned + + + + + + + + Landsat-5 PQ + utf8 + + Landsat-5 TM PQ x090 y084 19920323 + 20140601T01:22:30 + creation + + + + NEMO Group + Geoscience Australia + NEMO Operation + pointOfContact + + + Landsat-5 TM PQ x090 y084 19920323 + + + + + + + + 1 + + eng + NCI PBS 1.0.0 | Linux r3398 2.6.32-431.11.2.el6.x86_64 #1 SMP Tue Mar 25 19:59:55 UTC 2014 x86_64 x86_64 x86_64 GNU/Linux + 161 + + GEOTIFF + 1 + + dataset + Satellite Imagery + + + NEMO Group + Geoscience Australia + NEMO Operation + pointOfContact + + LS5_TM_PQ_P55_GAPQ01-002_090_084_19920323 + ANZLIC Metadata Profile: An Australian/New Zealand Profile of AS/NZS ISO 19115:2005, Geographic information - Metadata + 1.0 + + Pixel Quality + 25.0 + completed + Processed Image + grid + The pixel quality algorithm uses data from both the L1T (Systematic Terrain Correction) and ARG25 (Australian Reflectance Grid 25m) products. ACCA cloud cover is reported as a percentage of the entire data grid, while Fmask is reported as a percentage of the valid image data only. @@ -245,6 +245,6 @@ Cloud shadow is reported as a percentage of the entire data grid for both analys CLOUD SHADOW PERCENTAGE ACCA 0.38 CLOUD SHADOW PERCENTAGE Fmask 0.38 - environment - + environment + diff --git a/tests/data/ls8-eods-nbar/data/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012/metadata.xml b/tests/data/ls8-eods-nbar/data/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012/metadata.xml index b0eb00c2ea..b976062191 100644 --- a/tests/data/ls8-eods-nbar/data/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012/metadata.xml +++ b/tests/data/ls8-eods-nbar/data/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012/metadata.xml @@ -1,223 +1,223 @@ - - - - 20141012T03:23:36 - LGS - acquisition - - D - 20141012T03:29:10 - - 8846 - completed - instantaneous - - preProgrammed - - OLI-TIRS - Multi-spectral - - Landsat-8 - - - 2 - Color JPEG Image - LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012.jpg - JPG - 5 - 7 - 223.657226562 - - - - - - - - - - Ortho-rectified, processed by Geoscience Australia - - - - - - - - - - - - - </SOURCECITATION> - <SOURCEREFERENCESYSTEM/> - <SOURCERESOURCEID/> - <SOURCESCALE/> - <SOURCESTEP/> - </LINEAGESOURCE> - <PROCESSINGSTEP> - <ALGORITHMCITATION> - <EDITION>Git version 3.2.1</EDITION> - <TITLE>NBAR - - - Resampling=CC,RadiometricCorrection=CPF,Orientation=NUP,LPGS_version=3.3.3104,Hemisphere=S,CPF_NAME=L8CPF20141001_20141231.01,BPF_NAME_TIRS=LT8BPF20141012002432_20141012011154.02,BPF_NAME_OLI=LO8BPF20141012002825_20141012011100.01 - - dataset - - - - 4326 - 136.24838 - - - -26.96528 - 133.96233 - -26.96338 - 136.26962 - -24.97000 - -25.967 - 135.125 - -26.96338 - 20141012 00:55:54 - 20141012 00:56:18 - - -24.97 - 133.97969 - -24.96826 - 136.24838 - - - - - - 133.96233 - - - area - - sample - 25.0 - 9161 - - - line - 25.0 - 8841 - - - 511512.500000 - 7127487.500000 - 0 - eoancillarydataGCP - GDA94 / MGA zone 53 - GDA94 - GRS80 - 397012.5 - 7016987.5 - - 626012.5 - 7016987.5 - - 397012.5 - 7237987.5 - - 626012.5 - 7237987.5 - - UpperLeft - UTM - 53 - - 2 - 1 - - - - 1, 2, 3, 4, 5, 6, 7 - - - 0.0 - 58.00268508 - 59.41814014 - - NBAR - 101 - 78 - PUSH-BROOM - 0 - 0 - - - - - - - - - - notPlanned - - - - - - - - Processed Image: Landsat Nadir BRDF-Adjusted Reflectance products generated by Geoscience Australia. Data Source: GA - usAscii - - Landsat-8 OLI_TIRS NBAR x101 y078 20141012 - 2014-11-06 00:32:06 - creation - - - - Medhavy Thankappan - Geoscience Australia - Science and Strategy Project Leader - pointOfContact - - - Landsat-8 OLI_TIRS NBAR x101 y078 20141012 - - - - - - - - 9 - - eng - NCI PBS 1.0.0 | Linux r1699 2.6.32-431.29.2.el6.x86_64 #1 SMP Tue Sep 9 21:36:05 UTC 2014 x86_64 x86_64 x86_64 GNU/Linux - 154 - - GEOTIFF - 1 - - dataset - Satellite Imagery - - - Medhavy Thankappan - Geoscience Australia - Science and Strategy Project Leader - pointOfContact - - LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012 - GA Metadata Profile: A Geoscience Australia Profile of AS/NZS ISO 19115:2005, Geographic information - Metadata - 1.0 - - - 25.0 - completed - Processed Image - - - environment - + + + + 20141012T03:23:36 + LGS + acquisition + + D + 20141012T03:29:10 + + 8846 + completed + instantaneous + + preProgrammed + + OLI-TIRS + Multi-spectral + + Landsat-8 + + + 2 + Color JPEG Image + LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012.jpg + JPG + 5 + 7 + 223.657226562 + + + + + + + + + + Ortho-rectified, processed by Geoscience Australia + + + + + + + + + + + + + </SOURCECITATION> + <SOURCEREFERENCESYSTEM/> + <SOURCERESOURCEID/> + <SOURCESCALE/> + <SOURCESTEP/> + </LINEAGESOURCE> + <PROCESSINGSTEP> + <ALGORITHMCITATION> + <EDITION>Git version 3.2.1</EDITION> + <TITLE>NBAR + + + Resampling=CC,RadiometricCorrection=CPF,Orientation=NUP,LPGS_version=3.3.3104,Hemisphere=S,CPF_NAME=L8CPF20141001_20141231.01,BPF_NAME_TIRS=LT8BPF20141012002432_20141012011154.02,BPF_NAME_OLI=LO8BPF20141012002825_20141012011100.01 + + dataset + + + + 4326 + 136.24838 + + + -26.96528 + 133.96233 + -26.96338 + 136.26962 + -24.97000 + -25.967 + 135.125 + -26.96338 + 20141012 00:55:54 + 20141012 00:56:18 + + -24.97 + 133.97969 + -24.96826 + 136.24838 + + + + + + 133.96233 + + + area + + sample + 25.0 + 9161 + + + line + 25.0 + 8841 + + + 511512.500000 + 7127487.500000 + 0 + eoancillarydataGCP + GDA94 / MGA zone 53 + GDA94 + GRS80 + 397012.5 + 7016987.5 + + 626012.5 + 7016987.5 + + 397012.5 + 7237987.5 + + 626012.5 + 7237987.5 + + UpperLeft + UTM + 53 + + 2 + 1 + + + + 1, 2, 3, 4, 5, 6, 7 + + + 0.0 + 58.00268508 + 59.41814014 + + NBAR + 101 + 78 + PUSH-BROOM + 0 + 0 + + + + + + + + + + notPlanned + + + + + + + + Processed Image: Landsat Nadir BRDF-Adjusted Reflectance products generated by Geoscience Australia. Data Source: GA + usAscii + + Landsat-8 OLI_TIRS NBAR x101 y078 20141012 + 2014-11-06 00:32:06 + creation + + + + Medhavy Thankappan + Geoscience Australia + Science and Strategy Project Leader + pointOfContact + + + Landsat-8 OLI_TIRS NBAR x101 y078 20141012 + + + + + + + + 9 + + eng + NCI PBS 1.0.0 | Linux r1699 2.6.32-431.29.2.el6.x86_64 #1 SMP Tue Sep 9 21:36:05 UTC 2014 x86_64 x86_64 x86_64 GNU/Linux + 154 + + GEOTIFF + 1 + + dataset + Satellite Imagery + + + Medhavy Thankappan + Geoscience Australia + Science and Strategy Project Leader + pointOfContact + + LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012 + GA Metadata Profile: A Geoscience Australia Profile of AS/NZS ISO 19115:2005, Geographic information - Metadata + 1.0 + + + 25.0 + completed + Processed Image + + + environment + diff --git a/tests/drivers/fail_drivers/dc_tests_io/__init__.py b/tests/drivers/fail_drivers/dc_tests_io/__init__.py index c081ad5b41..8318282aea 100644 --- a/tests/drivers/fail_drivers/dc_tests_io/__init__.py +++ b/tests/drivers/fail_drivers/dc_tests_io/__init__.py @@ -1,4 +1,4 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/drivers/fail_drivers/dc_tests_io/dummy.py b/tests/drivers/fail_drivers/dc_tests_io/dummy.py index f4efb60242..7d3befd9ed 100644 --- a/tests/drivers/fail_drivers/dc_tests_io/dummy.py +++ b/tests/drivers/fail_drivers/dc_tests_io/dummy.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/drivers/fail_drivers/setup.py b/tests/drivers/fail_drivers/setup.py index 9ca7932ceb..60be7a42da 100644 --- a/tests/drivers/fail_drivers/setup.py +++ b/tests/drivers/fail_drivers/setup.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from setuptools import setup, find_packages diff --git a/tests/drivers/test_rio_reader.py b/tests/drivers/test_rio_reader.py index 1402b5823d..c27a2b0e06 100644 --- a/tests/drivers/test_rio_reader.py +++ b/tests/drivers/test_rio_reader.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Tests for new RIO reader driver """ diff --git a/tests/index/__init__.py b/tests/index/__init__.py index d0e5e97db3..a642d3c362 100644 --- a/tests/index/__init__.py +++ b/tests/index/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/tests/index/test_api_index_dataset.py b/tests/index/test_api_index_dataset.py index 93a57a84ee..7824b86e76 100644 --- a/tests/index/test_api_index_dataset.py +++ b/tests/index/test_api_index_dataset.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import datetime from collections import namedtuple @@ -208,6 +208,9 @@ def get(self, *args, **kwargs): def get_by_name(self, *args, **kwargs): return self.type + def get_by_name_unsafe(self, *args, **kwargs): + return self.type + @contextmanager def _db_connection(self, transaction=False): yield MockDb() diff --git a/tests/index/test_fields.py b/tests/index/test_fields.py index af2cc12da0..b43284cac3 100644 --- a/tests/index/test_fields.py +++ b/tests/index/test_fields.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/tests/index/test_hl_index.py b/tests/index/test_hl_index.py index bfe051058d..8402576964 100644 --- a/tests/index/test_hl_index.py +++ b/tests/index/test_hl_index.py @@ -1,3 +1,7 @@ +# This file is part of the Open Data Cube, see https://opendatacube.org for more information +# +# Copyright (c) 2015-2024 ODC Contributors +# SPDX-License-Identifier: Apache-2.0 import pytest from unittest.mock import MagicMock diff --git a/tests/index/test_postgis_fields.py b/tests/index/test_postgis_fields.py index d5725fbdb3..e826d785f2 100644 --- a/tests/index/test_postgis_fields.py +++ b/tests/index/test_postgis_fields.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import datetime diff --git a/tests/index/test_query.py b/tests/index/test_query.py index 60f6967652..88f2324da8 100644 --- a/tests/index/test_query.py +++ b/tests/index/test_query.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/tests/index/test_validate_dataset_type.py b/tests/index/test_validate_dataset_type.py index 058c2e4655..39da10b04a 100644 --- a/tests/index/test_validate_dataset_type.py +++ b/tests/index/test_validate_dataset_type.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/tests/scripts/__init__.py b/tests/scripts/__init__.py index d0e5e97db3..a642d3c362 100644 --- a/tests/scripts/__init__.py +++ b/tests/scripts/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/tests/scripts/test_search_tool.py b/tests/scripts/test_search_tool.py index 8e2b44e2bc..5edc7dc3de 100644 --- a/tests/scripts/test_search_tool.py +++ b/tests/scripts/test_search_tool.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index f14b05c26b..c5f4839106 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest from datacube.storage import BandInfo diff --git a/tests/storage/test_netcdfwriter.py b/tests/storage/test_netcdfwriter.py index ccf30b098e..bd3b684b2f 100644 --- a/tests/storage/test_netcdfwriter.py +++ b/tests/storage/test_netcdfwriter.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import netCDF4 import numpy diff --git a/tests/storage/test_storage.py b/tests/storage/test_storage.py index 65b930e8e5..7085a24e3a 100644 --- a/tests/storage/test_storage.py +++ b/tests/storage/test_storage.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from contextlib import contextmanager @@ -233,7 +233,8 @@ def assert_same_read_results(source, dst_shape, dst_dtype, dst_transform, dst_no dst_transform=dst_transform, dst_crs=str(dst_projection), dst_nodata=dst_nodata, - resampling=resampling) + resampling=resampling, + XSCALE=1, YSCALE=1) result = np.full(dst_shape, dst_nodata, dtype=dst_dtype) H, W = dst_shape diff --git a/tests/storage/test_storage_load.py b/tests/storage/test_storage_load.py index 8aafb550dc..781560d9ca 100644 --- a/tests/storage/test_storage_load.py +++ b/tests/storage/test_storage_load.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Test New IO driver loading """ diff --git a/tests/storage/test_storage_read.py b/tests/storage/test_storage_read.py index 2acbaa3f34..8251481aea 100644 --- a/tests/storage/test_storage_read.py +++ b/tests/storage/test_storage_read.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from affine import Affine import numpy as np diff --git a/tests/test_3d.py b/tests/test_3d.py index 6b2d1a31be..90b9e9799d 100644 --- a/tests/test_3d.py +++ b/tests/test_3d.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from collections import OrderedDict @@ -68,8 +68,8 @@ def test_extra_dimensions(eo3_metadata, cover_z_dataset_type): # Check chunk size assert dt.extra_dimensions.chunk_size() == (("z",), (30,)) - # String representation - readable = ( + # String representation (numpy 1.x and numpy 2.x formats - numpy 2.x reports total size in bytes) + readable_1 = ( "ExtraDimensions(extra_dim={'z': {'name': 'z', 'values': [5, 10, 15, " "20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100, " "105, 110, 115, 120, 125, 130, 135, 140, 145, 150], 'dtype': " @@ -80,8 +80,19 @@ def test_extra_dimensions(eo3_metadata, cover_z_dataset_type): "140., 145., 150.])\nCoordinates:\n * z (z) int64 5 10 15 20 " "25 30 35 40 ... 120 125 130 135 140 145 150} )" ) - assert str(dt.extra_dimensions) == readable - assert f"{dt.extra_dimensions!r}" == readable + readable_2 = ( + "ExtraDimensions(extra_dim={'z': {'name': 'z', 'values': [5, 10, 15, " + "20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100, " + "105, 110, 115, 120, 125, 130, 135, 140, 145, 150], 'dtype': " + "'float64'}}, dim_slice={'z': (0, 30)} coords={'z': Size: 240B\narray([ 5., 10., 15., 20., 25., 30., 35., " + "40., 45., 50., 55.,\n 60., 65., 70., 75., 80., 85., " + "90., 95., 100., 105., 110.,\n 115., 120., 125., 130., 135., " + "140., 145., 150.])\nCoordinates:\n * z (z) int64 240B 5 10 15 20 " + "25 30 35 ... 120 125 130 135 140 145 150} )" + ) + assert str(dt.extra_dimensions) in (readable_1, readable_2) + assert f"{dt.extra_dimensions!r}" in (readable_1, readable_2) def test_extra_dimensions_exceptions(eo3_metadata, cover_z_dataset_type): diff --git a/tests/test_concurrent_executor.py b/tests/test_concurrent_executor.py index c8dac01326..bbab3e50bb 100644 --- a/tests/test_concurrent_executor.py +++ b/tests/test_concurrent_executor.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Tests for MultiprocessingExecutor diff --git a/tests/test_config.py b/tests/test_config.py index dc8c20dc0e..5cb83b8a03 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/tests/test_driver.py b/tests/test_driver.py index 19bf25d683..6e13ddcb73 100644 --- a/tests/test_driver.py +++ b/tests/test_driver.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest import yaml diff --git a/tests/test_dynamic_db_passwd.py b/tests/test_dynamic_db_passwd.py index 62579b99b0..cc2e502c79 100644 --- a/tests/test_dynamic_db_passwd.py +++ b/tests/test_dynamic_db_passwd.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest from sqlalchemy.exc import OperationalError diff --git a/tests/test_eo3.py b/tests/test_eo3.py index 36750ee496..e56e7470a2 100644 --- a/tests/test_eo3.py +++ b/tests/test_eo3.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from affine import Affine import pytest diff --git a/tests/test_gbox_ops.py b/tests/test_gbox_ops.py index f6deacba4c..4d8c92aa20 100644 --- a/tests/test_gbox_ops.py +++ b/tests/test_gbox_ops.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from affine import Affine import numpy as np diff --git a/tests/test_geometry.py b/tests/test_geometry.py index f123bd632a..5c5721b93c 100644 --- a/tests/test_geometry.py +++ b/tests/test_geometry.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import math import numpy as np diff --git a/tests/test_load_data.py b/tests/test_load_data.py index e4589cdd56..f9bf98d28f 100644 --- a/tests/test_load_data.py +++ b/tests/test_load_data.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from datacube import Datacube from datacube.api.query import query_group_by diff --git a/tests/test_metadata_fields.py b/tests/test_metadata_fields.py index 00baa21e3b..03a4d17c6d 100644 --- a/tests/test_metadata_fields.py +++ b/tests/test_metadata_fields.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import yaml import datetime diff --git a/tests/test_model.py b/tests/test_model.py index 0c41ecabaa..1c96030280 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest import numpy diff --git a/tests/test_testutils.py b/tests/test_testutils.py index 91ee584ce6..f625f3062a 100644 --- a/tests/test_testutils.py +++ b/tests/test_testutils.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest from datacube.testutils.threads import FakeThreadPoolExecutor diff --git a/tests/test_utils_aws.py b/tests/test_utils_aws.py index 359acfbf4b..b7e287aba1 100644 --- a/tests/test_utils_aws.py +++ b/tests/test_utils_aws.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest from unittest import mock diff --git a/tests/test_utils_changes.py b/tests/test_utils_changes.py index 5499a0e41c..5483b5d99a 100644 --- a/tests/test_utils_changes.py +++ b/tests/test_utils_changes.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest from datacube.utils.changes import ( diff --git a/tests/test_utils_cog.py b/tests/test_utils_cog.py index e2260e3e1f..5b2ab81f5e 100644 --- a/tests/test_utils_cog.py +++ b/tests/test_utils_cog.py @@ -1,8 +1,9 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest +import math from pathlib import Path import numpy as np import xarray as xr @@ -19,8 +20,8 @@ from datacube.utils.cog import write_cog, to_cog, _write_cog -def gen_test_data(prefix, dask=False, shape=None): - w, h, dtype, nodata, ndw = 96, 64, "int16", -999, 7 +def gen_test_data(prefix, dask=False, shape=None, dtype="int16", nodata=-999): + w, h, ndw = 96, 64, 7 if shape is not None: h, w = shape @@ -109,6 +110,20 @@ def test_cog_file(tmpdir, opts): with pytest.warns(UserWarning): write_cog(xx, pp / "cog_badblocksize.tif", blocksize=50) + # check writing floating point COG with no explicit nodata + zz, ds = gen_test_data(pp, dtype="float32", nodata=None) + # write to file + ff = write_cog( + zz, + pp / "cog_float.tif", + **opts + ) + assert isinstance(ff, Path) + assert ff == pp / "cog_float.tif" + assert ff.exists() + aa = rio_slurp_xarray(pp / "cog_float.tif") + assert aa.attrs["nodata"] == "nan" or math.isnan(aa.attrs["nodata"]) + def test_cog_file_dask(tmpdir): pp = Path(str(tmpdir)) diff --git a/tests/test_utils_dask.py b/tests/test_utils_dask.py index 093e629878..c0faefb423 100644 --- a/tests/test_utils_dask.py +++ b/tests/test_utils_dask.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest import moto diff --git a/tests/test_utils_dates.py b/tests/test_utils_dates.py index a6fdd0c474..c45cd90c6a 100644 --- a/tests/test_utils_dates.py +++ b/tests/test_utils_dates.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import numpy as np import pytest diff --git a/tests/test_utils_docs.py b/tests/test_utils_docs.py index 80d4f0e010..7ea7eb93d3 100644 --- a/tests/test_utils_docs.py +++ b/tests/test_utils_docs.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Test utility functions from :module:`datacube.utils` diff --git a/tests/test_utils_generic.py b/tests/test_utils_generic.py index d8c899538c..ecddb57063 100644 --- a/tests/test_utils_generic.py +++ b/tests/test_utils_generic.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from queue import Queue from datacube.utils.generic import ( diff --git a/tests/test_utils_other.py b/tests/test_utils_other.py index 76646dcb54..462a7553ed 100644 --- a/tests/test_utils_other.py +++ b/tests/test_utils_other.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Test utility functions from :module:`datacube.utils` @@ -588,7 +588,12 @@ def test_num2numpy(): assert num2numpy(256, 'uint8') is None assert num2numpy(-1, 'uint16') is None assert num2numpy(-1, 'uint32') is None - assert num2numpy(-1, 'uint8', ignore_range=True) == np.uint8(255) + try: + # Numpy 1.x supports wrapping of unsisinged types + assert num2numpy(-1, 'uint8', ignore_range=True) == np.uint8(255) + except OverflowError: + # Numpy 2.0 will throw Overflow error rather than wrapping + pass assert num2numpy(0, 'uint8') == 0 assert num2numpy(255, 'uint8') == 255 diff --git a/tests/test_utils_rio.py b/tests/test_utils_rio.py index 558b3523b3..62b9455e61 100644 --- a/tests/test_utils_rio.py +++ b/tests/test_utils_rio.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest from unittest import mock diff --git a/tests/test_warp.py b/tests/test_warp.py index b2b10d1368..9bd6b743cd 100644 --- a/tests/test_warp.py +++ b/tests/test_warp.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import numpy as np from affine import Affine diff --git a/tests/test_xarray_extension.py b/tests/test_xarray_extension.py index 3c3812f004..76bd86dc5e 100644 --- a/tests/test_xarray_extension.py +++ b/tests/test_xarray_extension.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest import xarray as xr diff --git a/tests/ui/__init__.py b/tests/ui/__init__.py index c081ad5b41..8318282aea 100644 --- a/tests/ui/__init__.py +++ b/tests/ui/__init__.py @@ -1,4 +1,4 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/ui/test_common.py b/tests/ui/test_common.py index 51c4d3865a..3abbcac98c 100644 --- a/tests/ui/test_common.py +++ b/tests/ui/test_common.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/tests/ui/test_expression_parsing.py b/tests/ui/test_expression_parsing.py index 40526ecfe2..214afcf407 100644 --- a/tests/ui/test_expression_parsing.py +++ b/tests/ui/test_expression_parsing.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from datetime import datetime from dateutil.tz import tzutc diff --git a/tests/ui/test_task_app.py b/tests/ui/test_task_app.py index eac53d2ea8..4a0f5408a8 100644 --- a/tests/ui/test_task_app.py +++ b/tests/ui/test_task_app.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2024 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/wordlist.txt b/wordlist.txt index a3d3f93d2a..fb665e107f 100644 --- a/wordlist.txt +++ b/wordlist.txt @@ -25,6 +25,7 @@ ApplyMask ARD ard arg +aSk au australia autobuild @@ -52,10 +53,14 @@ boto BoundingBox brazil Bugfixes +bY carrotandcompany cbk cd CEOS +cheatsheet +Cheatsheet +cheatsheets CircleCI CLI cli @@ -120,6 +125,7 @@ digitalearthafrica Dingley distutils Dockerfile +Dockerhub dropdb ds dsm @@ -144,6 +150,7 @@ EP EPSG epsg ESPA +ESRI evironments f'file fd @@ -164,6 +171,7 @@ geo geobase GeoBox geobox +GeoBoxes GeoboxTiles GEOGCS GeoJSON @@ -180,6 +188,7 @@ GeoTiff geotiff GeoTIFFs GeoTiffs +gg gh GIS Github @@ -195,6 +204,7 @@ gridWorkflow GroupBy HDF hdf +hhBQVas hl hoc hostname @@ -206,6 +216,7 @@ IAM ident identifer img +importlib INEGI inegi ing @@ -223,11 +234,13 @@ jfEZEOkxRXgNsAsHEC jpg JSON jsonify +jsonschema Jupyter jupyter JupyterLab Juypter juypter +JSK KeyboardInterrupt Kirill Kubernetes @@ -251,6 +264,8 @@ libyaml linux literalinclude localhost +localmemory +logoColor lon lonlat lr @@ -258,6 +273,8 @@ LTS lv macosx MakeMask +Mambaforge +mambaforge ManagingODC mapbox matplotlib @@ -269,6 +286,7 @@ mk mkdir MODIS mongolia +moto MTL multiband multigeom @@ -288,6 +306,7 @@ nc NCI nci ncml +nd ndarray ndexpr NDVI @@ -312,6 +331,7 @@ nx ny ODC odc +odcintegration ODCv OGC ogr @@ -325,12 +345,17 @@ osgeo osr OSX ows +pandoc param params pc +pdf petewa pgadmin +pgintegration +pgisintegration pixelquality +pkg pkgs Pluggable pmap @@ -346,14 +371,14 @@ pq pre precollection prefetch -Preperation -preperation PRIMEM prog Proj provence psql pts +pwd +px py pydata pyenv @@ -366,6 +391,7 @@ pyproj pyspellcheck pytest PythonAPI +pytz QGIS RasterDatasetDataSource RasterIO @@ -382,6 +408,7 @@ reampling redis Reflectance reflectance +repin Reproject reproject reprojected @@ -422,7 +449,9 @@ STAC stac stacker stacspec +stdlib str +stylesheet subcommands sudo sv @@ -434,6 +463,7 @@ Terria th TIF tif +timedelta timeslice timeslot TIRS @@ -444,6 +474,7 @@ ToFloat txt typechecking ubuntu +uGuGdZQQDYhqN UI ui uk @@ -462,10 +493,12 @@ usgs USGS's usgslsprepare UsingODC +usr utils UUID uuid UUIDs +usp ValueError VDI vdi @@ -485,6 +518,7 @@ WGS WKT WMS WPS +wSgIIApLG www XArray xarray