switch back to gdal2.4 and use lambda layer

Dockerfile

@@ -1,11 +1,12 @@
-FROM remotepixel/amazonlinux:gdal3.0-py3.7
+FROM remotepixel/amazonlinux:gdal2.4-py3.7-geo
 
 WORKDIR /tmp
 
-ENV PACKAGE_PREFIX /tmp/python
+ENV PYTHONUSERBASE=/var/task
 
-COPY setup.py setup.py
 COPY remotepixel_tiler/ remotepixel_tiler/
+COPY setup.py setup.py
 
 # Install dependencies
-RUN pip3 install . --no-binary numpy,rasterio -t $PACKAGE_PREFIX -U
+RUN pip3 install . --user
+RUN rm -rf remotepixel_tiler setup.py

Makefile

@@ -17,16 +17,14 @@ package:
 	docker stop remotepixeltiler
 	docker rm remotepixeltiler
 
-test:
+test: package
 	docker run \
 		--name lambda \
 		-w /var/task/ \
 		--volume $(shell pwd)/bin:/tmp/bin \
 		--volume $(shell pwd)/:/local \
 		--env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
 		--env AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
-		--env GDAL_DATA=/var/task/share/gdal \
-		--env PROJ_LIB=/var/task/share/proj \
 		--env PYTHONWARNINGS=ignore \
 		--env GDAL_CACHEMAX=75% \
 		--env VSI_CACHE=TRUE \
@@ -39,7 +37,7 @@ test:
 		--env GDAL_DISABLE_READDIR_ON_OPEN=TRUE \
 		--env CPL_VSIL_CURL_ALLOWED_EXTENSIONS=".TIF,.ovr,.jp2,.tif" \
 		-itd \
-		lambci/lambda:build-python3.7 bash
+		remotepixel/amazonlinux:gdal2.4-py3.7-geo bash
 	docker exec -it lambda bash -c 'unzip -q /local/package.zip -d /var/task/'
 	docker exec -it lambda bash -c '/tmp/bin/tests.sh'
 	docker stop lambda

README.md

@@ -24,8 +24,6 @@ $ cd remotepixel-tiler/
 
 $ docker login
 
-# Fetch Amazon linux AMI docker container + Install Python modules + create package
-# Create package using custom GDAL install
 $ make package && make test
 
 # Install serverless and plugin

bin/package.sh

@@ -1,28 +1,15 @@
 #!/bin/bash
 echo "-----------------------"
-echo "Creating lambda package ${PACKAGE_PATH}"
+echo "Creating lambda package"
 echo "-----------------------"
-echo
-echo "Remove lambda python packages"
-rm -rdf $PACKAGE_PREFIX/boto3/ \
-  && rm -rdf $PACKAGE_PREFIX/botocore/ \
-  && rm -rdf $PACKAGE_PREFIX/docutils/ \
-  && rm -rdf $PACKAGE_PREFIX/dateutil/ \
-  && rm -rdf $PACKAGE_PREFIX/jmespath/ \
-  && rm -rdf $PACKAGE_PREFIX/s3transfer/ \
-  && rm -rdf $PACKAGE_PREFIX/numpy/doc/
-
 echo "Remove uncompiled python scripts"
-find $PACKAGE_PREFIX -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[2-3][0-9]//'); cp $f $n; done;
-find $PACKAGE_PREFIX -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf
-find $PACKAGE_PREFIX -type d -a -name 'tests' -print0 | xargs -0 rm -rf
-find $PACKAGE_PREFIX -type f -a -name '*.py' -print0 | xargs -0 rm -f
-
-echo "Strip shared libraries"
-cd $PREFIX && find lib -name \*.so\* -exec strip {} \;
+# Leave module precompiles for faster Lambda startup
+cd ${PYTHONUSERBASE}/lib/python3.7/site-packages/
+find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[2-3][0-9]//'); cp $f $n; done;
+find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf
+find . -type f -a -name '*.py' -print0 | xargs -0 rm -f
 
 echo "Create archive"
-cd $PACKAGE_PREFIX && zip -r9q /tmp/package.zip *
-cd $PREFIX && zip -r9q --symlinks /tmp/package.zip lib/*.so* share
+zip -r9q /tmp/package.zip *
 
 cp /tmp/package.zip ${PACKAGE_PATH}

docker-compose.yml

@@ -0,0 +1,101 @@
+version: '3'
+
+services:
+  landsat:
+    build: .
+    ports:
+      - "8000:8000"
+    volumes:
+      - '.:/local'
+    environment:
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      - CPL_TMPDIR=/tmp
+      - CPL_VSIL_CURL_ALLOWED_EXTENSIONS=.TIF,.ovr
+      - GDAL_CACHEMAX=75%
+      - GDAL_DISABLE_READDIR_ON_OPEN=FALSE
+      - GDAL_HTTP_MERGE_CONSECUTIVE_RANGES=YES
+      - GDAL_HTTP_MULTIPLEX=YES
+      - GDAL_HTTP_VERSION=2
+      - TOKEN=${SECRET_TOKEN}
+      - VSI_CACHE_SIZE=536870912
+      - VSI_CACHE=TRUE
+    command: >
+      bash -c "/var/task/bin/remotepixel-tiler landsat"
+
+  cbers:
+    build: .
+    ports:
+      - "8000:8000"
+    volumes:
+      - '.:/local'
+    environment:
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - AWS_REQUEST_PAYER=requester
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      - CPL_TMPDIR=/tmp
+      - CPL_VSIL_CURL_ALLOWED_EXTENSIONS=.tif
+      - GDAL_CACHEMAX=75%
+      - GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR
+      - GDAL_HTTP_MERGE_CONSECUTIVE_RANGES=YES
+      - GDAL_HTTP_MULTIPLEX=YES
+      - GDAL_HTTP_VERSION=2
+      - TOKEN=${SECRET_TOKEN}
+      - VSI_CACHE_SIZE=536870912
+      - VSI_CACHE=TRUE
+    command: >
+      bash -c "/var/task/bin/remotepixel-tiler cbers"
+
+  sentinel:
+    build: .
+    ports:
+      - "8000:8000"
+    volumes:
+      - '.:/local'
+    environment:
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - AWS_REQUEST_PAYER=requester
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      - CPL_TMPDIR=/tmp
+      - CPL_VSIL_CURL_ALLOWED_EXTENSIONS=.jp2,.tif
+      - GDAL_CACHEMAX=75%
+      - GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR
+      - GDAL_HTTP_MERGE_CONSECUTIVE_RANGES=YES
+      - GDAL_HTTP_MULTIPLEX=YES
+      - GDAL_HTTP_VERSION=2
+      - TOKEN=${SECRET_TOKEN}
+      - VSI_CACHE_SIZE=536870912
+      - VSI_CACHE=TRUE
+    command: >
+      bash -c "/var/task/bin/remotepixel-tiler sentinel"
+
+  cogeo:
+    build: .
+    ports:
+      - "8000:8000"
+    volumes:
+      - '.:/local'
+    environment:
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      - CPL_TMPDIR=/tmp
+      - CPL_VSIL_CURL_ALLOWED_EXTENSIONS=.tif
+      - GDAL_CACHEMAX=75%
+      - GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR
+      - GDAL_HTTP_MERGE_CONSECUTIVE_RANGES=YES
+      - GDAL_HTTP_MULTIPLEX=YES
+      - GDAL_HTTP_VERSION=2
+      - TOKEN=${SECRET_TOKEN}
+      - VSI_CACHE_SIZE=536870912
+      - VSI_CACHE=TRUE
+    command: >
+      bash -c "/var/task/bin/remotepixel-tiler cogeo"
+
+  bash:
+    build: .
+    volumes:
+      - '.:/local'
+    environment:
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+    command: /bin/bash

remotepixel_tiler/scripts/cli.py

@@ -8,11 +8,17 @@
 from socketserver import ThreadingMixIn
 
 from remotepixel_tiler.landsat import APP as landsat_app
-from remotepixel_tiler.sentinel import APP as sentine_app
+from remotepixel_tiler.sentinel import APP as sentinel_app
 from remotepixel_tiler.cbers import APP as cbers_app
 from remotepixel_tiler.cogeo import APP as cogeo_app
 
 
+landsat_app.https = False
+sentinel_app.https = False
+cbers_app.https = False
+cogeo_app.https = False
+
+
 class ThreadingSimpleServer(ThreadingMixIn, HTTPServer):
     """MultiThread."""
 
@@ -115,7 +121,7 @@ def do_GET(self):
             "queryStringParameters": dict(parse_qsl(q.query)),
             "httpMethod": self.command,
         }
-        response = sentine_app(request, None)
+        response = sentinel_app(request, None)
 
         self.send_response(int(response["statusCode"]))
         for r in response["headers"]:

services/cbers/serverless.yml

@@ -23,12 +23,12 @@ provider:
     CPL_TMPDIR: /tmp
     CPL_VSIL_CURL_ALLOWED_EXTENSIONS: .tif
     GDAL_CACHEMAX: 512
-    GDAL_DATA: /var/task/share/gdal
+    GDAL_DATA: /opt/share/gdal
     GDAL_DISABLE_READDIR_ON_OPEN: EMPTY_DIR
     GDAL_HTTP_MERGE_CONSECUTIVE_RANGES: YES
     GDAL_HTTP_MULTIPLEX: YES
     GDAL_HTTP_VERSION: 2
-    PROJ_LIB: /var/task/share/proj
+    PROJ_LIB: /opt/share/proj
     PYTHONWARNINGS: ignore
     TOKEN: ${env:SECRET_TOKEN}
 
@@ -42,6 +42,8 @@ package:
 
 functions:
   tiler:
+    layers:
+      - arn:aws:lambda:${self:provider.region}:524387336408:layer:gdal24-py37-geo:2
     handler: remotepixel_tiler.cbers.APP
     memorySize: 1536
     timeout: 10

services/cogeo/serverless.yml

@@ -19,12 +19,12 @@ provider:
     VSI_CACHE_SIZE: 536870912
     CPL_TMPDIR: /tmp
     GDAL_CACHEMAX: 512
-    GDAL_DATA: /var/task/share/gdal
+    GDAL_DATA: /opt/share/gdal
     GDAL_DISABLE_READDIR_ON_OPEN: EMPTY_DIR
     GDAL_HTTP_MERGE_CONSECUTIVE_RANGES: YES
     GDAL_HTTP_MULTIPLEX: YES
     GDAL_HTTP_VERSION: 2
-    PROJ_LIB: /var/task/share/proj
+    PROJ_LIB: /opt/share/proj
     PYTHONWARNINGS: ignore
 
   apiGateway:
@@ -35,10 +35,10 @@ provider:
 package:
   artifact: ../../package.zip
 
-# deploymentBucket: remotepixel-${self:provider.region}
-
 functions:
   tiler:
+    layers:
+      - arn:aws:lambda:${self:provider.region}:524387336408:layer:gdal24-py37-geo:2
     handler: remotepixel_tiler.cogeo.APP
     memorySize: 1536
     timeout: 10

services/landsat/serverless.yml

@@ -20,12 +20,12 @@ provider:
     CPL_TMPDIR: /tmp
     CPL_VSIL_CURL_ALLOWED_EXTENSIONS: .tif,.TIF,.ovr
     GDAL_CACHEMAX: 512
-    GDAL_DATA: /var/task/share/gdal
+    GDAL_DATA: /opt/share/gdal
     GDAL_DISABLE_READDIR_ON_OPEN: FALSE
     GDAL_HTTP_MERGE_CONSECUTIVE_RANGES: YES
     GDAL_HTTP_MULTIPLEX: YES
     GDAL_HTTP_VERSION: 2
-    PROJ_LIB: /var/task/share/proj
+    PROJ_LIB: /opt/share/proj
     PYTHONWARNINGS: ignore
     TOKEN: ${env:SECRET_TOKEN}
 
@@ -39,6 +39,8 @@ package:
 
 functions:
   tiler:
+    layers:
+      - arn:aws:lambda:${self:provider.region}:524387336408:layer:gdal24-py37-geo:2
     handler: remotepixel_tiler.landsat.APP
     memorySize: 1536
     timeout: 10