Added root path options

Makefile

@@ -0,0 +1,14 @@
+.PHONY: install reinstall setup test
+
+install:
+	pip install . --quiet
+
+reinstall:
+	pip uninstall tc_aws -y
+	pip install . --quiet
+
+setup:
+	@pip install -e .[tests] --quiet
+
+test: setup
+	@pyvows -c -l tc_aws

Readme.md

@@ -7,13 +7,14 @@ Installation
 ------------
 
 ```bash
-pip install tc_aws
+    pip install tc_aws
 ```
 
 Origin story
 ------------
 
-This is a fork of https://github.com/willtrking/thumbor_aws ; as this repository was not maintained anymore, we decided to maintain it under the ``thumbor-community`` organisation.
+This is a fork of https://github.com/willtrking/thumbor_aws ; as this repository was not maintained anymore, 
+we decided to maintain it under the ``thumbor-community`` organisation.
 
 Features
 --------
@@ -29,6 +30,9 @@ Additional Configuration values used:
     # the Amazon Web Services secret of the used access key
     AWS_SECRET_KEY = ""
 
+    # Alternatively (recommended), use Role-based connection
+    # http://docs.aws.amazon.com/IAM/latest/UserGuide/roles-assume-role.html
+    AWS_ROLE_BASED_CONNECTION = True or False (Default: False)
 
 
     # configuration settings specific for the s3_loader
@@ -38,9 +42,20 @@ Additional Configuration values used:
 
     # alternatively: set a fixed bucket, no need for bucket name in Image-Path
     S3_LOADER_BUCKET = 'thumbor-images'
+    # A root path for loading images, useful if you share the bucket
+    S3_LOADER_ROOT_PATH = 'source-images'
 
     # configuration settings specific for the storages
 
+    STORAGE_BUCKET = 'thumbor-images'
+    # A root path for the storage, useful if you share a bucket for loading / storing
+    STORAGE_AWS_STORAGE_ROOT_PATH = 'storage'
+
+    RESULT_STORAGE_BUCKET = 'thumbor-images'
+    RESULT_STORAGE_AWS_STORAGE_ROOT_PATH = 'result'
+
+    STORAGE_EXPIRATION_SECONDS
+
     # put data into S3 using the Server Side Encryption functionality to
     # encrypt data at rest in S3
     # https://aws.amazon.com/about-aws/whats-new/2011/10/04/amazon-s3-announces-server-side-encryption-support/
@@ -50,10 +65,6 @@ Additional Configuration values used:
     # https://aws.amazon.com/about-aws/whats-new/2010/05/19/announcing-amazon-s3-reduced-redundancy-storage/
     S3_STORAGE_RRS = True or False (Default: False)
 
-    # Use Role-based connection
-    # http://docs.aws.amazon.com/IAM/latest/UserGuide/roles-assume-role.html
-    AWS_ROLE_BASED_CONNECTION = True or False (Default: False)
-
 
     #Optional config value to enable the HTTP loader
     #This would allow you to load watermarks in over your images dynamically through a URI
@@ -68,4 +79,4 @@ Additional Configuration values used:
     BOTO_CONFIG = {
         'host': 'fakes3.local.dev',
         'is_secure': False
-        }
+    }

circle.yml

@@ -1,11 +1,10 @@
 dependencies:
   override:
     - gem install fpm
-    - pip install pyvows coverage tornado_pyvows thumbor boto py-dateutil moto
 
 test:
   override:
-    - pyvows -c -l tc_aws
+    - make test
   post:
     - fpm -s python -t deb --iteration 1 --no-python-dependencies -d python-dateutil -d thumbor -d python-boto --python-install-lib /usr/lib/python2.7/dist-packages -x "*.pyc" ./setup.py
     - mv ./*.deb $CIRCLE_ARTIFACTS

setup.py

@@ -3,13 +3,21 @@
 from setuptools import setup, find_packages
 
 setup(
-	name = 'tc_aws',
-	version = "1.1.0",
-	description = 'Thumbor AWS extensions',
-	author = 'William King',
-	author_email = '[email protected]',
-	zip_safe = False,
-	include_package_data = True,
-	packages=find_packages(),
-	install_requires=['python-dateutil','thumbor','boto']
+    name = 'tc_aws',
+    version = "1.1.0",
+    description = 'Thumbor AWS extensions',
+    author = 'William King',
+    author_email = '[email protected]',
+    zip_safe = False,
+    include_package_data = True,
+    packages=find_packages(),
+    install_requires=['python-dateutil','thumbor','boto'],
+    extras_require={
+        'tests': [
+            'pyvows',
+            'coverage',
+            'tornado_pyvows',
+            'moto',
+        ],
+    },
 )

tc_aws/__init__.py

@@ -1,16 +1,24 @@
 # coding: utf-8
 from thumbor.config import Config
-Config.define('STORAGE_BUCKET', 'thumbor-images','S3 bucket for Storage', 'S3')
-Config.define('RESULT_STORAGE_BUCKET', 'thumbor-result', 'S3 bucket for result Storage', 'S3')
-Config.define('S3_LOADER_BUCKET',None,'S3 bucket for loader', 'S3')
-Config.define('RESULT_STORAGE_AWS_STORAGE_ROOT_PATH','', 'S3 path prefix', 'S3')
-Config.define('STORAGE_EXPIRATION_SECONDS', 3600, 'S3 expiration', 'S3')
-Config.define('S3_STORAGE_SSE', False, 'S3 encriptipon key', 'S3')
-Config.define('S3_STORAGE_RRS', False, 'S3 redundency', 'S3')
-Config.define('S3_ALLOWED_BUCKETS', False, 'List of allowed bucket to be requeted', 'S3')
-
-Config.define('AWS_ACCESS_KEY', None, 'AWS Access key, if None use environment AWS_ACCESS_KEY_ID', 'AWS')
-Config.define('AWS_SECRET_KEY', None, 'AWS Secret key, if None use environment AWS_SECRET_ACCESS_KEY', 'AWS')
-Config.define('AWS_ROLE_BASED_CONNECTION', False, 'EC2 instance can use role that does not require AWS_ACCESS_KEY see http://docs.aws.amazon.com/IAM/latest/UserGuide/roles-usingrole-ec2instance.html', 'AWS')
-
-Config.define('BOTO_CONFIG', None, 'Additional Boto options for configuring S3 access (see http://boto.readthedocs.org/en/latest/ref/s3.html?highlight=boto.s3.connection.s3connection#boto.s3.connection.S3Connection)')
+
+
+
+Config.define('STORAGE_BUCKET',                         'thumbor-images',   'S3 bucket for Storage', 'S3')
+Config.define('RESULT_STORAGE_BUCKET',                  'thumbor-result',   'S3 bucket for result Storage', 'S3')
+Config.define('S3_LOADER_BUCKET',                       None,               'S3 bucket for loader', 'S3')
+
+Config.define('S3_LOADER_ROOT_PATH',                    '',                 'S3 path prefix for Loader bucket', 'S3')
+Config.define('STORAGE_AWS_STORAGE_ROOT_PATH',          '',                 'S3 path prefix for Storage bucket', 'S3')
+Config.define('RESULT_STORAGE_AWS_STORAGE_ROOT_PATH',   '',                 'S3 path prefix for Result storage bucket', 'S3')
+
+Config.define('STORAGE_EXPIRATION_SECONDS',             3600,               'S3 expiration', 'S3')
+
+Config.define('S3_STORAGE_SSE',                         False,              'S3 encriptipon key', 'S3')
+Config.define('S3_STORAGE_RRS',                         False,              'S3 redundency', 'S3')
+Config.define('S3_ALLOWED_BUCKETS',                     False,              'List of allowed bucket to be requeted', 'S3')
+
+Config.define('AWS_ACCESS_KEY',                         None,               'AWS Access key, if None use environment AWS_ACCESS_KEY_ID', 'AWS')
+Config.define('AWS_SECRET_KEY',                         None,               'AWS Secret key, if None use environment AWS_SECRET_ACCESS_KEY', 'AWS')
+Config.define('AWS_ROLE_BASED_CONNECTION',              False,              'EC2 instance can use role that does not require AWS_ACCESS_KEY see http://docs.aws.amazon.com/IAM/latest/UserGuide/roles-usingrole-ec2instance.html', 'AWS')
+
+Config.define('BOTO_CONFIG',                            None,               'Additional Boto options for configuring S3 access (see http://boto.readthedocs.org/en/latest/ref/s3.html?highlight=boto.s3.connection.s3connection#boto.s3.connection.S3Connection)')

tc_aws/loaders/s3_loader.py

@@ -9,15 +9,18 @@
 import thumbor.loaders.http_loader as http_loader
 
 
-def _get_bucket(url):
+def _get_bucket(url, root_path=None):
     """
     Returns a tuple containing bucket name and bucket path.
     url: A string of the format /bucket.name/file/path/in/bucket
     """
 
     url_by_piece = url.lstrip("/").split("/")
     bucket_name = url_by_piece[0]
-    bucket_path = "/".join(url_by_piece[1:])
+
+    url_by_piece[0] = root_path
+    bucket_path = "/".join(*url_by_piece)
+
     return bucket_name, bucket_path
 
 
@@ -46,7 +49,7 @@ def load(context, url, callback):
     bucket = context.config.get('S3_LOADER_BUCKET', default=None)
 
     if not bucket:
-        bucket, url = _get_bucket(url)
+        bucket, url = _get_bucket(url, root_path=context.config.S3_LOADER_ROOT_PATH)
 
     if _validate_bucket(context, bucket):
         bucket_loader = Bucket(

tc_aws/result_storages/s3_storage.py

@@ -40,6 +40,8 @@ def put(self, bytes):
             reduced_redundancy = self.context.config.S3_STORAGE_RRS
         )
 
+        return file_abspath
+
     def get(self):
         file_abspath = self.normalize_path(self.context.request.url)
         file_key = self.storage.get_key(file_abspath)

tc_aws/storages/s3_storage.py

@@ -119,9 +119,10 @@ def exists(self, path):
         return True
 
     def normalize_path(self, path):
-        root_path = self.context.config.RESULT_STORAGE_AWS_STORAGE_ROOT_PATH
-        path_segments = [path]
-        return join(root_path, *path_segments)
+        root_path = self.context.config.STORAGE_AWS_STORAGE_ROOT_PATH
+        path = path.lstrip('/')  # Remove leading '/'
+        path_segments = [root_path, path]
+        return join(*path_segments)
 
 
     def is_expired(self, key):

vows/result_storage_vows.py

@@ -0,0 +1,80 @@
+#se!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+from pyvows import Vows, expect
+
+from thumbor.context import Context
+from thumbor.config import Config
+from fixtures.storage_fixture import IMAGE_BYTES, get_server
+
+from boto.s3.connection import S3Connection
+from moto import mock_s3
+
+from tc_aws.result_storages.s3_storage import Storage
+
+s3_bucket = 'thumbor-images-test'
+
+class Request(object):
+  url = None
+
+@Vows.batch
+class S3ResultStorageVows(Vows.Context):
+
+  class CanStoreImage(Vows.Context):
+    @mock_s3
+    def topic(self):
+      self.conn = S3Connection()
+      self.conn.create_bucket(s3_bucket)
+
+      config = Config(RESULT_STORAGE_BUCKET=s3_bucket)
+      ctx = Context(config=config, server=get_server('ACME-SEC'))
+      ctx.request = Request
+      ctx.request.url = 'my-image.jpg'
+
+      storage = Storage(ctx)
+      path = storage.put(IMAGE_BYTES)
+
+      return path
+
+    def should_be_in_catalog(self, topic):
+      expect(topic).to_equal('my-image.jpg')
+
+  class CanGetImage(Vows.Context):
+    @mock_s3
+    def topic(self):
+      self.conn = S3Connection()
+      self.conn.create_bucket(s3_bucket)
+
+      config = Config(RESULT_STORAGE_BUCKET=s3_bucket)
+      ctx = Context(config=config, server=get_server('ACME-SEC'))
+      ctx.request = Request
+      ctx.request.url = 'my-image-2.jpg'
+
+      storage = Storage(ctx)
+      storage.put(IMAGE_BYTES)
+
+      return storage.get()
+
+    def should_not_be_null(self, topic):
+      expect(topic).not_to_be_null()
+      expect(topic).not_to_be_an_error()
+
+    def should_have_proper_bytes(self, topic):
+      expect(topic).to_equal(IMAGE_BYTES)
+
+  class HandlesStoragePrefix(Vows.Context):
+    @mock_s3
+    def topic(self):
+      self.conn = S3Connection()
+      self.conn.create_bucket(s3_bucket)
+
+      config = Config(RESULT_STORAGE_BUCKET=s3_bucket, RESULT_STORAGE_AWS_STORAGE_ROOT_PATH='tata')
+      ctx = Context(config=config, server=get_server('ACME-SEC'))
+
+      storage = Storage(ctx)
+
+      return storage.normalize_path('toto')
+
+    def should_return_the_same(self, topic):
+      expect(topic).to_equal("tata/toto")
+

vows/storage_vows.py

@@ -1,20 +1,9 @@
 #se!/usr/bin/python
 # -*- coding: utf-8 -*-
 
-# thumbor imaging service
-# https://github.com/globocom/thumbor/wiki
-
-# Licensed under the MIT license:
-# http://www.opensource.org/licenses/mit-license
-# Copyright (c) 2011 globo.com [email protected]
-
-
 from pyvows import Vows, expect
-from hashlib import md5
 
-from thumbor.app import ThumborServiceApp
-from thumbor.importer import Importer
-from thumbor.context import Context, ServerParameters
+from thumbor.context import Context
 from thumbor.config import Config
 from fixtures.storage_fixture import IMAGE_URL, IMAGE_BYTES, get_server
 import time
@@ -146,6 +135,20 @@ def topic(self):
     def should_return_the_same(self, topic):
       expect(topic).to_equal("toto")
 
+  class HandlesStoragePrefix(Vows.Context):
+    @mock_s3
+    def topic(self):
+      self.conn = S3Connection()
+      bucket = self.conn.create_bucket(s3_bucket)
+
+      config=Config(STORAGE_BUCKET=s3_bucket, STORAGE_AWS_STORAGE_ROOT_PATH='tata')
+      storage = Storage(Context(config=config, server=get_server('ACME-SEC')))
+
+      return storage.normalize_path('toto')
+
+    def should_return_the_same(self, topic):
+      expect(topic).to_equal("tata/toto")
+
   class CryptoVows(Vows.Context):
     class RaisesIfInvalidConfig(Vows.Context):
       @Vows.capture_error