V5 (#12)

* sketch of v5

* fix test

* fix test 2

* add deployement bucket

* fix packaging

* fix packaging

* add porj_lib env

* update

* fix ci

.circleci/config.yml

@@ -27,21 +27,18 @@ jobs:
 
   package:
     docker:
-      - image: remotepixel/amazonlinux-gdal:2.4.2
+      - image: remotepixel/amazonlinux:gdal3.0-py3.7
     environment:
       - PACKAGE_PATH=/root/remotepixel-tiler/package.zip
-      - PACKAGE_TMP=/root/package
+      - PACKAGE_PREFIX=/tmp/python
     working_directory: ~/remotepixel-tiler
     steps:
       - checkout
       - attach_workspace:
           at: ~/remotepixel-tiler
-      - run:
-          name: install Cython (for rio-tiler-mvt)
-          command: pip3 install cython==0.28
       - run:
           name: install requirements
-          command: CFLAGS="--std=c99" pip3 install . --no-binary numpy,rasterio -t $PACKAGE_TMP -U
+          command: pip3 install . --no-binary numpy,rasterio -t $PACKAGE_PREFIX -U
       - run:
           name: create package
           command: bin/package.sh
@@ -65,13 +62,16 @@ jobs:
           at: ~/remotepixel-tiler
       - run:
           name: Deploy cogeo application
-          command: cd services/cogeo && sls deploy
+          command: cd services/cogeo && sls deploy --bucket remotepixel-us-east-1
       - run:
           name: Deploy landsat application
-          command: cd services/landsat && sls deploy
+          command: cd services/landsat && sls deploy --bucket remotepixel-us-west-2
       - run:
           name: Deploy cbers application
-          command: cd services/cbers && sls deploy
+          command: cd services/cbers && sls deploy --bucket remotepixel-us-east-1
+      # - run:
+      #     name: Deploy sentinel application
+      #     command: cd services/sentinel && sls deploy --bucket remotepixel-eu-central-1
 
 workflows:
   version: 2

Dockerfile

@@ -1,12 +1,11 @@
-FROM remotepixel/amazonlinux-gdal:2.4.2
+FROM remotepixel/amazonlinux:gdal3.0-py3.7
 
-# Install dependencies
-RUN pip3 install pip -U
-RUN pip3 install cython==0.28
+WORKDIR /tmp
 
-ENV PACKAGE_TMP /tmp/package
+ENV PACKAGE_PREFIX /tmp/python
 
 COPY setup.py setup.py
 COPY remotepixel_tiler/ remotepixel_tiler/
 
-RUN CFLAGS="--std=c99" pip3 install . --no-binary numpy,rasterio -t $PACKAGE_TMP -U
+# Install dependencies
+RUN pip3 install . --no-binary numpy,rasterio -t $PACKAGE_PREFIX -U

Makefile

@@ -10,7 +10,6 @@ package:
 		-w /tmp \
 		--volume $(shell pwd)/bin:/tmp/bin \
 		--volume $(shell pwd)/:/local \
-		--env PACKAGE_TMP=/tmp/package \
 		--env PACKAGE_PATH=/local/package.zip \
 		-itd remotepixeltiler:latest \
 		bash
@@ -27,6 +26,7 @@ test:
 		--env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
 		--env AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
 		--env GDAL_DATA=/var/task/share/gdal \
+		--env PROJ_LIB=/var/task/share/proj \
 		--env PYTHONWARNINGS=ignore \
 		--env GDAL_CACHEMAX=75% \
 		--env VSI_CACHE=TRUE \
@@ -39,19 +39,12 @@ test:
 		--env GDAL_DISABLE_READDIR_ON_OPEN=TRUE \
 		--env CPL_VSIL_CURL_ALLOWED_EXTENSIONS=".TIF,.ovr,.jp2,.tif" \
 		-itd \
-		lambci/lambda:build-python3.6 bash
+		lambci/lambda:build-python3.7 bash
 	docker exec -it lambda bash -c 'unzip -q /local/package.zip -d /var/task/'
 	docker exec -it lambda bash -c '/tmp/bin/tests.sh'
 	docker stop lambda
 	docker rm lambda
 
-STAGENAME=production
-deploy:
-	cd services/cogeo && sls deploy --stage ${STAGENAME}
-	cd services/landsat && sls deploy --stage ${STAGENAME}
-	cd services/cbers && sls deploy --stage ${STAGENAME}
-	#cd services/sentinel && sls deploy --stage ${STAGENAME}
-
 clean:
 	docker stop lambda
 	docker rm lambda

README.md

@@ -22,7 +22,6 @@ Bundle of `landsat-tiler`, `sentinel-tiler`, `cbers-tiler` and `cogeo-tiler` pow
 $ git clone https://github.com/RemotePixel/remotepixel-tiler.git
 $ cd remotepixel-tiler/
 
-
 $ docker login
 
 # Fetch Amazon linux AMI docker container + Install Python modules + create package
@@ -31,17 +30,29 @@ $ make package && make test
 
 # Install serverless and plugin
 $ npm install
-
-# Deploy landsat, cogeo and cbers lambda functions
-$ SECRET_TOKEN=mytoken make deploy
 ```
 
 You can deploy each tiler independantly
 
 ```bash
-$ SECRET_TOKEN=mytoken cd services/landsat && sls deploy --stage production
+$ SECRET_TOKEN=mytoken cd services/landsat && sls deploy --bucket my-bucket
+Note: `my-bucket` has to be in us-west-2 region
+
+$ SECRET_TOKEN=mytoken cd services/cbers && sls deploy --stage production --bucket my-bucket
+Note: `my-bucket` has to be in us-east-1 region
+
+$ SECRET_TOKEN=mytoken  cd services/sentinel && sls deploy --bucket my-bucket 
+Note: `my-bucket` has to be in eu-central-1 region
+
+$ cd services/cogeo && sls deploy --bucket my-bucket --region us-east-1
+Note: `my-bucket` has to be in the same region
 ```
 
+### API Docs:
+- cogeo: https://cogeo.remotepixel.ca/docs
+- landsat: https://landsat.remotepixel.ca/docs
+- cbers: https://cbers.remotepixel.ca/docs
+
 #### Infos & links
 - [rio-tiler](https://github.com/mapbox/rio-tiler) rasterio plugin that process Landsat data hosted on AWS S3.
 - [landsat-tiler](https://github.com/mapbox/landsat-tiler)

bin/package.sh

@@ -4,26 +4,25 @@ echo "Creating lambda package ${PACKAGE_PATH}"
 echo "-----------------------"
 echo
 echo "Remove lambda python packages"
-rm -rdf $PACKAGE_TMP/boto3/ \
-  && rm -rdf $PACKAGE_TMP/botocore/ \
-  && rm -rdf $PACKAGE_TMP/docutils/ \
-  && rm -rdf $PACKAGE_TMP/dateutil/ \
-  && rm -rdf $PACKAGE_TMP/jmespath/ \
-  && rm -rdf $PACKAGE_TMP/s3transfer/ \
-  && rm -rdf $PACKAGE_TMP/numpy/doc/
+rm -rdf $PACKAGE_PREFIX/boto3/ \
+  && rm -rdf $PACKAGE_PREFIX/botocore/ \
+  && rm -rdf $PACKAGE_PREFIX/docutils/ \
+  && rm -rdf $PACKAGE_PREFIX/dateutil/ \
+  && rm -rdf $PACKAGE_PREFIX/jmespath/ \
+  && rm -rdf $PACKAGE_PREFIX/s3transfer/ \
+  && rm -rdf $PACKAGE_PREFIX/numpy/doc/
 
 echo "Remove uncompiled python scripts"
-find $PACKAGE_TMP -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-36//'); cp $f $n; done;
-find $PACKAGE_TMP -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf
-find $PACKAGE_TMP -type d -a -name 'tests' -print0 | xargs -0 rm -rf
-find $PACKAGE_TMP -type f -a -name '*.py' -print0 | xargs -0 rm -f
+find $PACKAGE_PREFIX -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[2-3][0-9]//'); cp $f $n; done;
+find $PACKAGE_PREFIX -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf
+find $PACKAGE_PREFIX -type d -a -name 'tests' -print0 | xargs -0 rm -rf
+find $PACKAGE_PREFIX -type f -a -name '*.py' -print0 | xargs -0 rm -f
 
 echo "Strip shared libraries"
 cd $PREFIX && find lib -name \*.so\* -exec strip {} \;
-cd $PREFIX && find lib64 -name \*.so\* -exec strip {} \;
 
 echo "Create archive"
-cd $PACKAGE_TMP && zip -r9q $PACKAGE_PATH *
-cd $PREFIX && zip -r9q --symlinks $PACKAGE_PATH lib/*.so*
-cd $PREFIX && zip -r9q --symlinks $PACKAGE_PATH lib64/*.so*
-cd $PREFIX && zip -r9q $PACKAGE_PATH share
+cd $PACKAGE_PREFIX && zip -r9q /tmp/package.zip *
+cd $PREFIX && zip -r9q --symlinks /tmp/package.zip lib/*.so* share
+
+cp /tmp/package.zip ${PACKAGE_PATH}

bin/tests.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-python3 -c 'from remotepixel_tiler import __version__ as rpix_version; print(rpix_version)'
+python3 -c 'from remotepixel_tiler import version as rpix_version; print(rpix_version)'
 
 echo "Test Landsat-8"
 echo "/bounds: " && python3 -c 'from remotepixel_tiler.landsat import APP; resp = APP({"path": "/bounds/LC80230312016320LGN00", "queryStringParameters": {"access_token": "yo"}, "pathParameters": "null", "requestContext": "null", "httpMethod": "GET", "headers": {}}, None); print("OK") if resp["statusCode"] == 200 else print("NOK")'

remotepixel_tiler/__init__.py

@@ -1,4 +1,5 @@
 """Remotepixel tiler."""
-# app
 
-__version__ = "4.1.1"
+import pkg_resources
+
+version = pkg_resources.get_distribution(__package__).version

remotepixel_tiler/cogeo.py

@@ -49,7 +49,7 @@ def tilejson_handler(
 
     with rasterio.open(url) as src_dst:
         bounds = warp.transform_bounds(
-            *[src_dst.crs, "epsg:4326"] + list(src_dst.bounds), densify_pts=21
+            src_dst.crs, "epsg:4326", *src_dst.bounds, densify_pts=21
         )
         minzoom, maxzoom = get_zooms(src_dst)
         center = [(bounds[0] + bounds[2]) / 2, (bounds[1] + bounds[3]) / 2, minzoom]

remotepixel_tiler/landsat.py

@@ -13,8 +13,6 @@
 from rio_tiler.profiles import img_profiles
 from rio_tiler.utils import array_to_image, get_colormap, expression
 
-from rio_tiler_mvt.mvt import encoder as mvtEncoder
-
 from remotepixel_tiler.utils import _postprocess
 
 from lambda_proxy.proxy import API
@@ -60,16 +58,15 @@ def tilejson_handler(
         kwargs.update(dict(access_token=token[0]))
 
     qs = urllib.parse.urlencode(list(kwargs.items()))
-    if tile_format in ["pbf", "mvt"]:
-        tile_url = f"{APP.host}/tiles/{sceneid}/{{z}}/{{x}}/{{y}}.{tile_format}?{qs}"
-    else:
-        tile_url = f"{APP.host}/tiles/{sceneid}/{{z}}/{{x}}/{{y}}@{tile_scale}x.{tile_format}?{qs}"
+    tile_url = (
+        f"{APP.host}/tiles/{sceneid}/{{z}}/{{x}}/{{y}}@{tile_scale}x.{tile_format}?{qs}"
+    )
 
     scene_params = landsat8._landsat_parse_scene_id(sceneid)
     landsat_address = f"{LANDSAT_BUCKET}/{scene_params['key']}_BQA.TIF"
     with rasterio.open(landsat_address) as src_dst:
         bounds = warp.transform_bounds(
-            *[src_dst.crs, "epsg:4326"] + list(src_dst.bounds), densify_pts=21
+            src_dst.crs, "epsg:4326", *src_dst.bounds, densify_pts=21
         )
         minzoom, maxzoom = get_zooms(src_dst)
         center = [(bounds[0] + bounds[2]) / 2, (bounds[1] + bounds[3]) / 2, minzoom]
@@ -121,42 +118,6 @@ def metadata(
     return ("OK", "application/json", json.dumps(info))
 
 
-@APP.route(
-    "/tiles/<scene>/<int:z>/<int:x>/<int:y>.pbf",
-    methods=["GET"],
-    cors=True,
-    token=True,
-    payload_compression_method="gzip",
-    binary_b64encode=True,
-    ttl=3600,
-    tag=["tiles"],
-)
-def mvttiles(
-    scene: str,
-    z: int,
-    x: int,
-    y: int,
-    bands: str = None,
-    tile_size: Union[str, int] = 256,
-    pixel_selection: str = "first",
-    feature_type: str = "point",
-    resampling_method: str = "nearest",
-) -> Tuple[str, str, BinaryIO]:
-    """Handle MVT tile requests."""
-    if tile_size is not None and isinstance(tile_size, str):
-        tile_size = int(tile_size)
-
-    bands = tuple(bands.split(","))
-    tile, mask = landsat8.tile(scene, x, y, z, bands=bands, tilesize=tile_size)
-
-    band_descriptions = [f"Band_{b}" for b in bands]
-    return (
-        "OK",
-        "application/x-protobuf",
-        mvtEncoder(tile, mask, band_descriptions, "landsat", feature_type=feature_type),
-    )
-
-
 @APP.route(
     "/tiles/<scene>/<int:z>/<int:x>/<int:y>.<ext>",
     methods=["GET"],