Develop->master

VERSION

@@ -0,0 +1 @@
+0.1.0

dev-requirements.txt

@@ -1,7 +1,6 @@
-mypy==0.610
-bandit==1.4.0
+mypy==0.800
+bandit==1.7.0
 mccabe==0.6.1
-pyflakes==2.0.0
-flake8==3.5.0
-grequests==0.3.0
-coverage==4.5.1
+flake8==3.8.4
+grequests==0.6.0
+coverage==5.4

docker-compose.yaml

@@ -1,4 +1,4 @@
-version: '3'
+version: '3.7'
 
 services:
 
@@ -17,12 +17,15 @@ services:
       - "127.0.0.1:5000:5000"
     volumes:
       - .:/app
+    depends_on:
+      - minio
+      - auth
 
   # For running the file server
   minio:
     image: minio/minio
-    ports:
-      - "127.0.0.1:9000:9000"
+    expose:
+      - "9000"
     environment:
       - MINIO_ACCESS_KEY=minio
       - MINIO_SECRET_KEY=minio123

requirements.txt

@@ -1,8 +1,8 @@
-minio==4.0.11
-Flask==1.0.2
-gunicorn==19.9.0
-gevent==1.4.0
-simplejson==3.16.0
-python-dotenv==0.10.1
-requests==2.21.0
+minio==7.0.2
+Flask==1.1.2
+gunicorn==20.0.4
+gevent==21.1.2
+simplejson==3.17.2
+python-dotenv==0.15.0
+requests==2.25.1
 docopt==0.6.2

scripts/docker_deploy

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -euo xtrace
+
+ver=$(cat VERSION)
+export IMAGE_NAME="kbase/cachingservice:$ver"
+echo "Build hook running"
+export BRANCH=${TRAVIS_BRANCH:-`git symbolic-ref --short HEAD`}
+export DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"`
+export COMMIT=${TRAVIS_COMMIT:-`git rev-parse --short HEAD`}
+
+docker build --build-arg BUILD_DATE=$DATE \
+             --build-arg VCS_REF=$COMMIT \
+             --build-arg BRANCH=$BRANCH \
+             -t ${IMAGE_NAME} .
+docker push $IMAGE_NAME

scripts/run_tests.sh

@@ -5,5 +5,5 @@ set -e
 flake8 --max-complexity 6 src/caching_service
 flake8 src/test
 mypy --ignore-missing-imports src
-bandit -r src
+bandit -r src/caching_service
 python -m unittest discover src/test/caching_service

scripts/start_server.sh

@@ -7,11 +7,11 @@ calc_workers="$(($(nproc) * 2 + 1))"
 # Use the WORKERS environment variable, if present
 workers=${WORKERS:-$calc_workers}
 
-gunicorn \
-  --worker-class gevent \
-  --timeout 1800 \
-  --workers $workers \
-  --bind :5000 \
-  ${DEVELOPMENT:+"--reload"} \
-  src.caching_service.server:app
-  app:app
+python -m src.caching_service.utils.init_app && \
+  gunicorn \
+    --worker-class gevent \
+    --timeout 1800 \
+    --workers $workers \
+    --bind :5000 \
+    ${DEVELOPMENT:+"--reload"} \
+    src.caching_service.server:app

src/caching_service/api/api_v1.py

@@ -2,7 +2,6 @@
 import tempfile
 import json
 import flask
-import minio.error
 import shutil
 
 from ..authorization.service_token import requires_service_token
@@ -88,7 +87,6 @@ def delete(cache_id):
 # --------------
 
 @api_v1.errorhandler(exceptions.MissingCache)
-@api_v1.errorhandler(minio.error.NoSuchKey)
 def missing_cache_file(err):
     """A cache ID was not found, but was expected to exist."""
     result = {'status': 'error', 'error': 'Cache ID not found'}

src/caching_service/minio.py

@@ -4,6 +4,7 @@
 import tempfile
 import os
 import io
+import requests
 import shutil
 from werkzeug.utils import secure_filename
 
@@ -20,13 +21,39 @@
 )
 bucket_name = Config.minio_bucket_name
 
-# Create the bucket if it does not exist
-try:
-    minio_client.make_bucket(bucket_name)
-except minio.error.BucketAlreadyExists:
-    pass
-except minio.error.BucketAlreadyOwnedByYou:
-    pass
+
+def initialize_bucket():
+    """
+    Create the default bucket if it does not exist
+    """
+    print(f"Making bucket with name '{bucket_name}'")
+    try:
+        minio_client.make_bucket(bucket_name)
+    except minio.error.S3Error as err:
+        # Acceptable errors
+        errs = ["BucketAlreadyExists", "BucketAlreadyOwnedByYou"]
+        if err.code not in errs:
+            raise err
+    print(f"Done making bucket '{bucket_name}'")
+
+
+def wait_for_service():
+    """
+    Wait for the minio service to be healthy
+    """
+    url = f'http://{Config.minio_host}/minio/health/live'
+    max_time = 180
+    start = time.time()
+    while True:
+        try:
+            requests.get(url).raise_for_status()
+            print("Minio is healthy! Continuing.")
+            break
+        except Exception as err:
+            if time.time() > start + max_time:
+                raise RuntimeError("Timed out waiting for Minio")
+            print(f"Still waiting for Minio at {url} to be healthy:")
+            print(err)
 
 
 def create_placeholder(cache_id, token_id):
@@ -46,7 +73,8 @@ def create_placeholder(cache_id, token_id):
     """
     try:
         return get_metadata(cache_id)
-    except minio.error.NoSuchKey:
+    except exceptions.MissingCache:
+        # Create the cache key
         seven_days = 604800  # in seconds
         expiration = str(int(time.time() + seven_days))
         metadata = {
@@ -63,8 +91,9 @@ def authorize_access(cache_id, token_id):
     """
     Given a cache ID and token ID, authorize that the token has permission to access the cache.
 
-    This will raise caching_service.exceptions.UnauthorizedAccess if it is unauthorized.
-    This will raise minio.error.NoSuchKey if the cache ID does not exist.
+    Raises:
+        - caching_service.exceptions.UnauthorizedAccess if it is unauthorized.
+        - exceptions.MissingCache if the cache ID does not exist.
     """
     metadata = get_metadata(cache_id)
     existing_token_id = metadata['token_id']
@@ -105,7 +134,7 @@ def expire_entries():
     """
     print('Checking the expiration of all stored objects..')
     now = time.time()
-    objects = minio_client.list_objects_v2(bucket_name)
+    objects = minio_client.list_objects(bucket_name)
     removed_count = 0
     total_count = 0
     for obj in objects:
@@ -134,7 +163,13 @@ def delete_cache(cache_id, token_id):
 
 def get_metadata(cache_id):
     """Return the Minio metadata dict for a cache file."""
-    orig_metadata = minio_client.stat_object(bucket_name, cache_id).metadata
+    try:
+        orig_metadata = minio_client.stat_object(bucket_name, cache_id).metadata
+    except minio.error.S3Error as err:
+        # Catch NoSuchKey errors and raise MissingCache
+        if err.code != "NoSuchKey":
+            raise err
+        raise exceptions.MissingCache(cache_id)
     # The below keys are how metadata gets stored in minio files for 'expiration', 'filename', etc
     # For example if you set the metadata 'xyz_abc', then minio will store it as 'X-Amz-Meta-Xyz_abc'
     return {

src/caching_service/server.py

@@ -9,11 +9,11 @@
 from .exceptions import MissingHeader, InvalidContentType, UnauthorizedAccess
 from .config import Config
 
+# Initialize the server
 app = flask.Flask(__name__)
 app.config['DEBUG'] = os.environ.get('DEVELOPMENT')
 app.config['SECRET_KEY'] = Config.secret_key
 app.url_map.strict_slashes = False  # allow both `get /v1/` and `get /v1`
-
 app.register_blueprint(api_v1, url_prefix='/v1')
 
 

src/caching_service/utils/init_app.py

@@ -0,0 +1,16 @@
+"""
+Any initialization code that needs to be run before any workers start:
+    - wait for Minio to be healthy
+    - create the bucket
+"""
+import src.caching_service.minio as minio
+
+
+def init_app():
+    # Wait for minio to be healthy
+    minio.wait_for_service()
+    minio.initialize_bucket()
+
+
+if __name__ == '__main__':
+    init_app()

src/test/caching_service/test_api_v1.py

@@ -7,9 +7,9 @@
 import requests
 from uuid import uuid4
 import functools
-from minio.error import NoSuchKey
 
 import src.caching_service.minio as minio
+from src.caching_service.exceptions import MissingCache
 
 url = 'http://web:5000/v1'
 
@@ -309,7 +309,7 @@ def test_delete_valid(self):
         self.assertEqual(resp.status_code, 200, 'Status code is 200')
         self.assertEqual(json['status'], 'ok', 'Status is "deleted"')
         # Test that the cache is inaccessible
-        with self.assertRaises(NoSuchKey):
+        with self.assertRaises(MissingCache):
             minio.get_metadata(cache_id)
 
     def test_delete_unauthorized_cache(self):

src/test/caching_service/test_minio.py

@@ -4,7 +4,6 @@
 import os
 import io
 from werkzeug.datastructures import FileStorage
-from minio.error import NoSuchKey
 from uuid import uuid4
 import tempfile
 
@@ -73,7 +72,7 @@ def test_cache_delete(self):
         minio.create_placeholder(cache_id, token_id)
         minio.delete_cache(cache_id, token_id)
         tmp_dir = tempfile.mkdtemp()
-        with self.assertRaises(NoSuchKey):
+        with self.assertRaises(exceptions.MissingCache):
             minio.download_cache(cache_id, token_id, tmp_dir)
         shutil.rmtree(tmp_dir)
 
@@ -103,7 +102,7 @@ def test_missing_cache_upload(self):
         cache_id = str(uuid4())
         file_storage = self.make_test_file_storage(cache_id, token_id)
         minio.create_placeholder(cache_id, token_id)
-        with self.assertRaises(NoSuchKey):
+        with self.assertRaises(exceptions.MissingCache):
             minio.upload_cache(cache_id + 'x', token_id, file_storage)
         file_storage.stream.close()
 
@@ -113,7 +112,7 @@ def test_missing_cache_download(self):
         cache_id = str(uuid4())
         minio.create_placeholder(cache_id, token_id)
         tmp_dir = tempfile.mkdtemp()
-        with self.assertRaises(NoSuchKey):
+        with self.assertRaises(exceptions.MissingCache):
             minio.download_cache(cache_id + 'x', token_id, tmp_dir)
         shutil.rmtree(tmp_dir)