From e82777d17be0cd9d15a7941e48b39a3baad3dd93 Mon Sep 17 00:00:00 2001 From: Rob Emanuele Date: Wed, 19 Aug 2020 18:35:47 -0400 Subject: [PATCH] Initial commit. --- .flake8 | 15 + .github/workflows/continuous-integration.yml | 41 ++ .github/workflows/release.yml | 31 ++ .gitignore | 26 + CHANGELOG.md | 11 + LICENSE | 15 + README.md | 99 ++++ docs/Makefile | 22 + docs/Untitled.ipynb | 500 +++++++++++++++++++ docs/api.rst | 16 + docs/conf.py | 199 ++++++++ docs/index.rst | 33 ++ docs/make.bat | 35 ++ requirements-dev.txt | 8 + requirements.txt | 3 + scripts/cibuild | 27 + scripts/test | 29 ++ setup.py | 49 ++ stactools/__init__.py | 17 + stactools/__main__.py | 4 + stactools/cli.py | 16 + stactools/commands/__init__.py | 0 stactools/commands/copy.py | 27 + stactools/commands/migrate.py | 15 + stactools/version.py | 2 + tests/__init__.py | 0 tests/commands/__init__.py | 0 tests/commands/test_copy.py | 31 ++ 28 files changed, 1271 insertions(+) create mode 100644 .flake8 create mode 100644 .github/workflows/continuous-integration.yml create mode 100644 .github/workflows/release.yml create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 docs/Makefile create mode 100644 docs/Untitled.ipynb create mode 100644 docs/api.rst create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 docs/make.bat create mode 100644 requirements-dev.txt create mode 100644 requirements.txt create mode 100755 scripts/cibuild create mode 100755 scripts/test create mode 100644 setup.py create mode 100644 stactools/__init__.py create mode 100644 stactools/__main__.py create mode 100644 stactools/cli.py create mode 100644 stactools/commands/__init__.py create mode 100644 stactools/commands/copy.py create mode 100644 stactools/commands/migrate.py create mode 100644 stactools/version.py create mode 100644 tests/__init__.py create mode 100644 tests/commands/__init__.py create mode 100644 tests/commands/test_copy.py diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..9085ab0f --- /dev/null +++ b/.flake8 @@ -0,0 +1,15 @@ +[flake8] +max-line-length = 100 + +## IGNORES + +# E127: flake8 reporting incorrect continuation line indent errors +# on multi-line and multi-level indents + +# W503: flake8 reports this as incorrect, and scripts/format_code +# changes code to it, so let format_code win. + +# E126: Continuation line over-indented for hanging indent +# Conflicts with yapf formatting + +ignore = E127,W503,E126 \ No newline at end of file diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml new file mode 100644 index 00000000..91c98d54 --- /dev/null +++ b/.github/workflows/continuous-integration.yml @@ -0,0 +1,41 @@ +name: CI + +on: + push: + branches: + - develop + - master + pull_request: + +jobs: + build: + name: build + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.6", "3.7", "3.8"] + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: pip-${{ hashFiles('requirements-dev.txt') }} + restore-keys: pip- + + - name: Execute linters and test suites + run: ./scripts/cibuild + + + - name: Upload All coverage to Codecov + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: ./coverage.xml + fail_ci_if_error: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..f8aba861 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,31 @@ +name: Release + +on: + push: + tags: + - "*" + +jobs: + release: + name: release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.x + uses: actions/setup-python@v2 + with: + python-version: "3.x" + + - name: Install release dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + + - name: Build and publish package + env: + TWINE_USERNAME: ${{ secrets.PYPI_STACUTILS_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_STACUTILS_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..17002da6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +*.pyc +*.egg-info +build/ +dist/ +*.eggs +MANIFEST +.DS_Store +.coverage +.cache +data +config.json +stdout* +/integration* +.idea/ +docs/_build/ + +# Sphinx documentation +docs/_build/ + +.ipynb_checkpoints/ + +docs/tutorials/pystac-example* +docs/tutorials/spacenet-stac/ +docs/tutorials/spacenet-cog-stac/ +docs/tutorials/data/ +docs/quickstart_stac/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..d03fdb83 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,11 @@ +# Changelog + +## Unreleased + +### Added + +### Changed + +### Removed + +### Fixed diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..6e268606 --- /dev/null +++ b/LICENSE @@ -0,0 +1,15 @@ +This software is licensed under the Apache 2 license, quoted below. + +Copyright 2020 Azavea [http://www.azavea.com] + +Licensed under the Apache License, Version 2.0 (the "License"); you may not +use this file except in compliance with the License. You may obtain a copy of +the License at + + [http://www.apache.org/licenses/LICENSE-2.0] + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations under +the License. diff --git a/README.md b/README.md new file mode 100644 index 00000000..61e0a145 --- /dev/null +++ b/README.md @@ -0,0 +1,99 @@ +## stactools +![Build Status](https://github.com/stac-utils/pystac/workflows/CI/badge.svg?branch=develop) +[![PyPI version](https://badge.fury.io/py/pystac.svg)](https://badge.fury.io/py/pystac) +[![Documentation](https://readthedocs.org/projects/pystac/badge/?version=latest)](https://pystac.readthedocs.io/en/latest/) +[![codecov](https://codecov.io/gh/stac-utils/pystac/branch/develop/graph/badge.svg)](https://codecov.io/gh/stac-utils/pystac) +[![Gitter chat](https://badges.gitter.im/azavea/pystac.svg)](https://gitter.im/azavea/pystac) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +`stactools` is a command line tool and library for working with [STAC](https://stacspec.org) based on [PySTAC](https://github.com/stac-utils/pystac). + +## Installation + +```bash +> pip install stactools +``` +From source repository: + +```bash +> git clone https://github.com/stac-utils/stactools.git +> cd stactools +> pip install . +``` + +## Running + +``` +> stactools --help +``` + +#### Versions +To install a specific versions of STAC, install the matching version of stactools. + +```bash +> pip install stactools==0.1.* +``` + +The table below shows the corresponding versions between pystac and STAC: + +| stactools | STAC | +| --------- | ----- | +| 0.1.x | 1.0.x | + +## Documentation + +See the [documentation page](https://stactools.readthedocs.io/en/latest/) for the latest docs. + +## Developing + +To ensure development libraries are installed, install everything in `requirements-dev.txt`: + +``` +> pip install -r requirements-dev.txt +``` + +### Unit Tests + +Unit tests are in the `tests` folder. To run unit tests, use `unittest`: + +``` +> python -m unittest discover tests +``` + +To run linters, code formatters, and test suites all together, use `test`: + +``` +> ./scripts/test +``` + +### Code quality checks + +stactools uses [flake8](http://flake8.pycqa.org/en/latest/) and [yapf](https://github.com/google/yapf) for code formatting and style checks. + +To run the flake8 style checks: + +``` +> flake8 stactools +> flake8 tests +``` + +To format code: + +``` +> yapf -ipr stactools +> yapf -ipr tests +``` + +You can also run the `./scripts/test` script to check flake8 and yapf. + +### Documentation + +To build and develop the documentation locally, make sure sphinx is available (which is installed with `requirements-dev.txt`), and use the Makefile in the docs folder: + +``` +> cd docs +> make html +> make livehtml +``` + +Use 'make' without arguments to see a list of available commands. diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..435169d2 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,22 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +livehtml: + sphinx-autobuild -z ../stactools --host 0.0.0.0 ${SOURCEDIR} $(BUILDDIR)/html -d _build/doctrees + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/Untitled.ipynb b/docs/Untitled.ipynb new file mode 100644 index 00000000..19d0a774 --- /dev/null +++ b/docs/Untitled.ipynb @@ -0,0 +1,500 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pystac as stac" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from urllib.parse import urlparse\n", + "import boto3\n", + "from pystac import STAC_IO\n", + "\n", + "def my_read_method(uri):\n", + " parsed = urlparse(uri)\n", + " if parsed.scheme == 's3':\n", + " bucket = parsed.netloc\n", + " key = parsed.path[1:]\n", + " s3 = boto3.resource('s3')\n", + " obj = s3.Object(bucket, key)\n", + " return obj.get()['Body'].read().decode('utf-8')\n", + " else:\n", + " return STAC_IO.default_read_text_method(uri)\n", + "\n", + "def my_write_method(uri, txt):\n", + " parsed = urlparse(uri)\n", + " if parsed.scheme == 's3':\n", + " bucket = parsed.netloc\n", + " key = parsed.path[1:]\n", + " s3 = boto3.resource(\"s3\")\n", + " s3.Object(bucket, key).put(Body=txt)\n", + " else:\n", + " STAC_IO.default_write_text_method(uri, txt)\n", + "\n", + "STAC_IO.read_text_method = my_read_method\n", + "STAC_IO.write_text_method = my_write_method" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "cat = stac.Catalog.from_file('s3://raster-vision-rob-dev/stac-packer/bae/angola-cog-water-label/catalog.json')" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "items = list(cat.get_all_items())" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2404" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(items)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "label_items = [i for i in items if isinstance(i, stac.LabelItem)]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1202" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(label_items)" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "metadata": {}, + "outputs": [], + "source": [ + "targets = list(filter(lambda x: '5-S2A_MSIL1C_20190709T084601_N0208_R107_T33LZL_20190709T105334-101' in next(iter(x.get_sources())).id, label_items))\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "metadata": {}, + "outputs": [], + "source": [ + "li = targets[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'78f8f48e-a6d7-4e1a-9b1a-dd4df92115bc': }" + ] + }, + "execution_count": 78, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "li.assets" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "metadata": {}, + "outputs": [], + "source": [ + "gj = stac.STAC_IO.read_json(next(iter(li.assets.values())).href)" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [], + "source": [ + "from shapely.geometry import shape" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1" + ] + }, + "execution_count": 81, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(gj['features'])" + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 82, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "shape(gj['features'][0]['geometry'])" + ] + }, + { + "cell_type": "code", + "execution_count": 83, + "metadata": {}, + "outputs": [ + { + "ename": "IndexError", + "evalue": "list index out of range", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgj\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'features'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'geometry'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mIndexError\u001b[0m: list index out of range" + ] + } + ], + "source": [ + "shape(gj['features'][1]['geometry'])" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 84, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "shape(li.geometry).union(shape(gj['features'][0]['geometry']))" + ] + }, + { + "cell_type": "code", + "execution_count": 85, + "metadata": {}, + "outputs": [], + "source": [ + "gj['features'].append(li.to_dict())" + ] + }, + { + "cell_type": "code", + "execution_count": 86, + "metadata": {}, + "outputs": [], + "source": [ + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": 87, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"type\": \"FeatureCollection\",\n", + " \"features\": [\n", + " {\n", + " \"type\": \"Feature\",\n", + " \"tippecanoe\": {\n", + " \"layer\": \"osm\",\n", + " \"minzoom\": 12,\n", + " \"maxzoom\": 12\n", + " },\n", + " \"properties\": {\n", + " \"@type\": \"way\",\n", + " \"@id\": 160930185,\n", + " \"@version\": 7,\n", + " \"@changeset\": 76160041,\n", + " \"@uid\": 9446950,\n", + " \"@user\": \"Olentangy\",\n", + " \"@timestamp\": 1571927803,\n", + " \"boat\": \"no\",\n", + " \"name\": \"Kwango\",\n", + " \"name:fr\": \"Kwango\",\n", + " \"name:pt\": \"Kwango\",\n", + " \"waterway\": \"river\",\n", + " \"class_id\": \"1\"\n", + " },\n", + " \"geometry\": {\n", + " \"type\": \"LineString\",\n", + " \"coordinates\": [\n", + " [\n", + " 17.998117519034825,\n", + " -8.900106006634156\n", + " ],\n", + " [\n", + " 17.997234,\n", + " -8.899896\n", + " ],\n", + " [\n", + " 17.996324,\n", + " -8.898969\n", + " ],\n", + " [\n", + " 17.995584,\n", + " -8.898014\n", + " ],\n", + " [\n", + " 17.995499,\n", + " -8.896918\n", + " ],\n", + " [\n", + " 17.996096,\n", + " -8.893405\n", + " ],\n", + " [\n", + " 17.996523,\n", + " -8.891607\n", + " ],\n", + " [\n", + " 17.997234,\n", + " -8.890118\n", + " ],\n", + " [\n", + " 17.998117519034825,\n", + " -8.88922436709233\n", + " ]\n", + " ]\n", + " }\n", + " },\n", + " {\n", + " \"type\": \"Feature\",\n", + " \"stac_version\": \"0.8.1\",\n", + " \"id\": \"78f8f48e-a6d7-4e1a-9b1a-dd4df92115bc\",\n", + " \"properties\": {\n", + " \"datetime\": \"2019-07-09 09:12:59Z\",\n", + " \"label:description\": \"Label item for a scene item\",\n", + " \"label:type\": \"vector\",\n", + " \"label:property\": [\n", + " \"GIS objects\"\n", + " ],\n", + " \"label:classes\": [\n", + " {\n", + " \"name\": \"GIS objects\",\n", + " \"classes\": [\n", + " [\n", + " \"==\",\n", + " \"natural\",\n", + " \"water\"\n", + " ]\n", + " ]\n", + " }\n", + " ],\n", + " \"label:task\": [\n", + " \"classification\"\n", + " ],\n", + " \"label:method\": [\n", + " \"OSM\"\n", + " ],\n", + " \"label:properties\": [\n", + " \"GIS objects\"\n", + " ],\n", + " \"label:tasks\": [\n", + " \"classification\"\n", + " ],\n", + " \"label:methods\": [\n", + " \"OSM\"\n", + " ]\n", + " },\n", + " \"geometry\": {\n", + " \"type\": \"Polygon\",\n", + " \"coordinates\": [\n", + " [\n", + " [\n", + " 17.95987783475377,\n", + " -8.871333425075191\n", + " ],\n", + " [\n", + " 17.997778417638852,\n", + " -8.871030934717718\n", + " ],\n", + " [\n", + " 17.998117519034825,\n", + " -8.917278575706348\n", + " ],\n", + " [\n", + " 17.960249654116048,\n", + " -8.917582365280794\n", + " ],\n", + " [\n", + " 17.95987783475377,\n", + " -8.871333425075191\n", + " ]\n", + " ]\n", + " ]\n", + " },\n", + " \"bbox\": [\n", + " -9.124940692148286,\n", + " 17.72206981427617,\n", + " -8.125031654306673,\n", + " 18.726859889243226\n", + " ],\n", + " \"links\": [\n", + " {\n", + " \"rel\": \"source\",\n", + " \"href\": \"s3://raster-vision-rob-dev/stac-packer/bae/angola-cog-water-label/74a2d593-e3b8-4d42-94ac-c1b3ce98a7ac/5-S2A_MSIL1C_20190709T084601_N0208_R107_T33LZL_20190709T105334-101/5-S2A_MSIL1C_20190709T084601_N0208_R107_T33LZL_20190709T105334-101.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"collection\",\n", + " \"href\": \"s3://raster-vision-rob-dev/stac-packer/bae/angola-cog-water-label/35e2d746-2f65-4596-bad6-195bda0b5a90/collection.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"self\",\n", + " \"href\": \"s3://raster-vision-rob-dev/stac-packer/bae/angola-cog-water-label/35e2d746-2f65-4596-bad6-195bda0b5a90/78f8f48e-a6d7-4e1a-9b1a-dd4df92115bc/78f8f48e-a6d7-4e1a-9b1a-dd4df92115bc.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"root\",\n", + " \"href\": \"s3://raster-vision-rob-dev/stac-packer/bae/angola-cog-water-label/catalog.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"parent\",\n", + " \"href\": \"s3://raster-vision-rob-dev/stac-packer/bae/angola-cog-water-label/35e2d746-2f65-4596-bad6-195bda0b5a90/collection.json\",\n", + " \"type\": \"application/json\"\n", + " }\n", + " ],\n", + " \"assets\": {\n", + " \"78f8f48e-a6d7-4e1a-9b1a-dd4df92115bc\": {\n", + " \"href\": \"s3://raster-vision-rob-dev/stac-packer/bae/angola-cog-water-label/35e2d746-2f65-4596-bad6-195bda0b5a90/78f8f48e-a6d7-4e1a-9b1a-dd4df92115bc/data.geojson\",\n", + " \"type\": \"application/geo+json\",\n", + " \"title\": \"Label Data FeatureCollection\"\n", + " }\n", + " },\n", + " \"stac_extensions\": [\n", + " \"label\"\n", + " ]\n", + " }\n", + " ]\n", + "}\n" + ] + } + ], + "source": [ + "print(json.dumps(gj, indent=2))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 00000000..5f2956e2 --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,16 @@ +API Reference +============= + +This API reference is auto-generated for the Python docstrings. + +Commands +-------- + +.. autoclass:: stactools.commands.migrate + :members: + :inherited-members: + :undoc-members: + +.. autoclass:: stactools.commands.copy + :members: + :undoc-members: diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..601aa365 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,199 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +import subprocess +sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath('../')) +from stactools.version import __version__ + +# git_branch = subprocess.check_output(['git', +# 'rev-parse', +# '--abbrev-ref', +# 'HEAD']) \ +# .decode("utf-8") \ +# .strip() + +# -- Project information ----------------------------------------------------- + +project = 'stactools' +copyright = '2019, stac-utils' +author = 'stac-utils' + +# The short X.Y version +version = __version__ +# The full version, including alpha/beta/rc tags +release = __version__ + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.viewcode', + 'sphinx.ext.napoleon', + 'sphinx.ext.githubpages', + 'sphinx.ext.extlinks', + 'sphinxcontrib.fulltoc', + 'nbsphinx' +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '**.ipynb_checkpoints'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = None + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +html_theme_options = { + "show_powered_by": False, + "github_user": "stac-utils", + "github_repo": "stactools", + "github_banner": True, + "show_related": False, + "note_bg": "#FFF59C", +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +#html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'stactoolsdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'stactools.tex', 'stactools Documentation', + 'stac-utils', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'stactools', 'stactools Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'stactools', 'stactools Documentation', + author, 'stactools', 'Command line utility and library for SpatioTemporal Asset Catalogs (STAC).', + 'Miscellaneous'), +] + + +# -- Options for Epub output ------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# +# epub_identifier = '' + +# A unique identification for the text. +# +# epub_uid = '' + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + + +# -- Extension configuration ------------------------------------------------- diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..34f1fbdf --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,33 @@ +``stactools`` Documentation +########################### + +``stactools`` is a library for working with `SpatioTemporal Asset Catalogs (STAC) `_ based on `PySTAC `_. + +Requirements +============ +* `Python 3 `_ + +STAC Versions +============= + +* 0.1 -> STAC Version 1.0 + +Standard pip install +==================== + +.. code-block:: bash + + pip install stactools + +``stactools`` Features +================== + +* TODO + +Table of Contents +================= + +.. toctree:: + :maxdepth: 2 + + api diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..27f573b8 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..2b901099 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,8 @@ +pylint==1.9.1 +Sphinx==1.8.0 +sphinx-autobuild==0.7.1 +sphinxcontrib-fulltoc==1.2.0 +sphinxcontrib-napoleon==0.7 +flake8==3.8.* +yapf==0.28.* +nbsphinx==0.4.3 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..dfdf243d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +click==7.1.2 +pystac[validation]==0.5.* +fsspec==0.8.0 diff --git a/scripts/cibuild b/scripts/cibuild new file mode 100755 index 00000000..82a8bcc0 --- /dev/null +++ b/scripts/cibuild @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + +if [[ -n "${CI}" ]]; then + set -x +fi + +function usage() { + echo -n \ + "Usage: $(basename "$0") +Execute project linters and test suites in CI. +" +} + +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + if [ "${1:-}" = "--help" ]; then + usage + else + # Install/upgrade dependencies + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + pip install -e . + + ./scripts/test + fi +fi diff --git a/scripts/test b/scripts/test new file mode 100755 index 00000000..d382791f --- /dev/null +++ b/scripts/test @@ -0,0 +1,29 @@ +#!/bin/bash + +set -e + +if [[ -n "${CI}" ]]; then + set -x +fi + +function usage() { + echo -n \ + "Usage: $(basename "$0") +Execute project linters and test suites. +" +} + +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + if [ "${1:-}" = "--help" ]; then + usage + else + # Lint + flake8 stactools tests + + # Code formatting + yapf -dpr stactools tests + + # Test suite + python -m unittest discover tests/ + fi +fi diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..a2ad2f5b --- /dev/null +++ b/setup.py @@ -0,0 +1,49 @@ +import os +from imp import load_source +from setuptools import setup, find_packages +from glob import glob +import io + +__version__ = load_source('stactools.version', 'stactools/version.py').__version__ + +from os.path import ( + basename, + splitext +) + +here = os.path.abspath(os.path.dirname(__file__)) + +# get the dependencies and installs +with io.open(os.path.join(here, 'requirements.txt'), encoding='utf-8') as f: + install_requires = f.read().split('\n') + +with open(os.path.join(here, 'README.md')) as readme_file: + readme = readme_file.read() + +setup( + name='stactools', + version=__version__, + description=("Command line tool and Python library for working with STAC."), + long_description=readme, + long_description_content_type="text/markdown", + author="stac-utils", + author_email='stac@radiant.earth', + url='https://github.com/stac-utils/stactools.git', + packages=find_packages(), + py_modules=[splitext(basename(path))[0] for path in glob('stactools/*.py')], + include_package_data=False, + install_requires=install_requires, + license="Apache Software License 2.0", + keywords=[ + 'stactools', + 'psytac', + 'imagery', + 'raster', + 'catalog', + 'STAC' + ], + entry_points={ + 'console_scripts': ['stactools=stactools.cli:cli'], + }, + test_suite='tests' +) diff --git a/stactools/__init__.py b/stactools/__init__.py new file mode 100644 index 00000000..adc08b8f --- /dev/null +++ b/stactools/__init__.py @@ -0,0 +1,17 @@ +import pystac +import fsspec + + +# Use fsspec to handle IO +def fsspec_read_method(uri): + with fsspec.open(uri) as f: + return f.read() + + +def fsspec_write_method(uri, txt): + with fsspec.open(uri, 'w') as f: + return f.write(txt) + + +pystac.STAC_IO.read_text_method = fsspec_read_method +pystac.STAC_IO.read_text_method = fsspec_write_method diff --git a/stactools/__main__.py b/stactools/__main__.py new file mode 100644 index 00000000..7e345417 --- /dev/null +++ b/stactools/__main__.py @@ -0,0 +1,4 @@ +from stactools.cli import cli + +if __name__ == "__main__": + cli() diff --git a/stactools/cli.py b/stactools/cli.py new file mode 100644 index 00000000..fe0e2767 --- /dev/null +++ b/stactools/cli.py @@ -0,0 +1,16 @@ +import click + +from stactools.commands.copy import create_copy_command +from stactools.commands.migrate import create_migrate_command + + +@click.group() +def cli(): + pass + + +copy_command = create_copy_command(cli) +migrate_command = create_migrate_command(cli) + +if __name__ == "__main__": + cli() diff --git a/stactools/commands/__init__.py b/stactools/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/stactools/commands/copy.py b/stactools/commands/copy.py new file mode 100644 index 00000000..ec0393c6 --- /dev/null +++ b/stactools/commands/copy.py @@ -0,0 +1,27 @@ +import click +import pystac + + +def copy(src, dst, catalog_type, copy_assets=False): + print('copy {} {} {} {}'.format(src, dst, catalog_type, copy_assets)) + # TODO + # cat = pystac.read_file(src) + # cat.normalize_and_save(dst, catalog_type) + + +def create_copy_command(cli): + @cli.command('copy', short_help='Copy a STAC Catalog') + @click.argument('src') + @click.argument('dst') + @click.argument('catalog_type', + type=click.Choice([ + pystac.CatalogType.ABSOLUTE_PUBLISHED, + pystac.CatalogType.RELATIVE_PUBLISHED, + pystac.CatalogType.SELF_CONTAINED + ], + case_sensitive=False)) + @click.option('--copy_assets', '-a', is_flag=True) + def copy_command(src, dst, catalog_type, copy_assets): + copy(src, dst, catalog_type, copy_assets) + + return copy_command diff --git a/stactools/commands/migrate.py b/stactools/commands/migrate.py new file mode 100644 index 00000000..b4582859 --- /dev/null +++ b/stactools/commands/migrate.py @@ -0,0 +1,15 @@ +import click + + +def migrate(option): + # TODO + print(option) + + +def create_migrate_command(cli): + @cli.command('migrate', short_help='Migrate a STAC catalog') + @click.option('--option', '-o', default=1, help='A test option') + def migrate_command(option): + migrate(option) + + return migrate_command diff --git a/stactools/version.py b/stactools/version.py new file mode 100644 index 00000000..3c9925dc --- /dev/null +++ b/stactools/version.py @@ -0,0 +1,2 @@ +__version__ = '0.1.0' +"""Library version""" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/commands/__init__.py b/tests/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/commands/test_copy.py b/tests/commands/test_copy.py new file mode 100644 index 00000000..efa771f1 --- /dev/null +++ b/tests/commands/test_copy.py @@ -0,0 +1,31 @@ +import unittest + +import click +from click.testing import CliRunner + +from stactools.commands.copy import create_copy_command + + +class CopyTest(unittest.TestCase): + def setUp(self): + @click.group() + def cli(): + pass + + create_copy_command(cli) + self.cli = cli + + def test_copy(self): + runner = CliRunner() + result = runner.invoke(self.cli, [ + 'copy', '/src/catalog.json', '/dst/catalog.json', + 'ABSOLUTE_PUBLISHED' + ]) + self.assertEqual(result.exit_code, 0) + + def test_copy_fail(self): + runner = CliRunner() + result = runner.invoke(self.cli, [ + 'copy', '/src/catalog.json', '/dst/catalog.json', 'NOT_PUBLISHED' + ]) + self.assertNotEqual(result.exit_code, 0)