diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..582a72b --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.pyc +build +*.egg-info +*.deb diff --git a/circle.yml b/circle.yml new file mode 100644 index 0000000..7c2f216 --- /dev/null +++ b/circle.yml @@ -0,0 +1,11 @@ +dependencies: + override: + - gem install fpm + - pip install pyvows coverage tornado_pyvows thumbor boto py-dateutil moto + +test: + override: + - pyvows -c -l thumbor_aws + post: + - fpm -s python -t deb --iteration 1 --no-python-dependencies -d python-dateutil -d thumbor -d python-boto --python-install-lib /usr/lib/python2.7/dist-packages -x "*.pyc" ./setup.py + - mv ./*.deb $CIRCLE_ARTIFACTS diff --git a/setup.py b/setup.py index 9d8ebb9..19187b2 100644 --- a/setup.py +++ b/setup.py @@ -11,5 +11,5 @@ zip_safe = False, include_package_data = True, packages=find_packages(), - requires=['dateutil','thumbor','boto'] -) \ No newline at end of file + install_requires=['py-dateutil','thumbor','boto'] +) diff --git a/thumbor_aws/__init__.py b/thumbor_aws/__init__.py index 576f56f..7167b94 100644 --- a/thumbor_aws/__init__.py +++ b/thumbor_aws/__init__.py @@ -1 +1,14 @@ -# coding: utf-8 \ No newline at end of file +# coding: utf-8 +from thumbor.config import Config +Config.define('STORAGE_BUCKET', 'thumbor-images','S3 bucket for Storage', 'S3') +Config.define('RESULT_STORAGE_BUCKET', 'thumbor-result', 'S3 bucket for result Storage', 'S3') +Config.define('S3_LOADER_BUCKET','thumbor-images','S3 bucket for loader', 'S3') +Config.define('RESULT_STORAGE_AWS_STORAGE_ROOT_PATH','', 'S3 path prefix', 'S3') +Config.define('STORAGE_EXPIRATION_SECONDS', 3600, 'S3 expiration', 'S3') +Config.define('S3_STORAGE_SSE', False, 'S3 encriptipon key', 'S3') +Config.define('S3_STORAGE_RRS', False, 'S3 redundency', 'S3') +Config.define('S3_ALLOWED_BUCKETS', False, 'List of allowed bucket to be requeted', 'S3') + +Config.define('AWS_ACCESS_KEY', None, 'AWS Access key, if None use environment AWS_ACCESS_KEY_ID', 'AWS') +Config.define('AWS_SECRET_KEY', None, 'AWS Secret key, if None use environment AWS_SECRET_ACCESS_KEY', 'AWS') +Config.define('AWS_ROLE_BASED_CONNECTION', False, 'EC2 instance can use role that does not require AWS_ACCESS_KEY see http://docs.aws.amazon.com/IAM/latest/UserGuide/roles-usingrole-ec2instance.html', 'AWS') diff --git a/thumbor_aws/connection.py b/thumbor_aws/connection.py index 106b025..96e04fa 100644 --- a/thumbor_aws/connection.py +++ b/thumbor_aws/connection.py @@ -7,12 +7,12 @@ def get_connection(context): conn = connection if conn is None: - if context.config.get('AWS_ROLE_BASED_CONNECTION', default=False): + if context.config.AWS_ROLE_BASED_CONNECTION: conn = S3Connection() else: conn = S3Connection( - context.config.get('AWS_ACCESS_KEY'), - context.config.get('AWS_SECRET_KEY') + context.config.AWS_ACCESS_KEY, + context.config.AWS_SECRET_KEY ) return conn diff --git a/thumbor_aws/result_storages/s3_storage.py b/thumbor_aws/result_storages/s3_storage.py index 8dd45da..1414212 100644 --- a/thumbor_aws/result_storages/s3_storage.py +++ b/thumbor_aws/result_storages/s3_storage.py @@ -36,8 +36,8 @@ def put(self, bytes): file_key.key = file_abspath file_key.set_contents_from_string(bytes, - encrypt_key = self.context.config.get('S3_STORAGE_SSE', default=False), - reduced_redundancy = self.context.config.get('S3_STORAGE_RRS', default=False) + encrypt_key = self.context.config.S3_STORAGE_SSE, + reduced_redundancy = self.context.config.S3_STORAGE_RRS ) def get(self): @@ -51,12 +51,11 @@ def get(self): return file_key.read() def normalize_path(self, path): - root_path = self.context.config.get('RESULT_STORAGE_AWS_STORAGE_ROOT_PATH', default='thumbor/result_storage/') + root_path = self.context.config.RESULT_STORAGE_AWS_STORAGE_ROOT_PATH path_segments = [path] if self.is_auto_webp: path_segments.append("webp") - digest = hashlib.sha1(".".join(path_segments).encode('utf-8')).hexdigest() - return os.path.join(root_path, digest) + return os.path.join(root_path, *path_segments) def is_expired(self, key): if key: diff --git a/thumbor_aws/storages/s3_storage.py b/thumbor_aws/storages/s3_storage.py index 770fa72..10e698a 100644 --- a/thumbor_aws/storages/s3_storage.py +++ b/thumbor_aws/storages/s3_storage.py @@ -5,7 +5,7 @@ import hashlib from json import loads, dumps -from os.path import splitext +from os.path import splitext, join from thumbor.storages import BaseStorage from thumbor.utils import logger @@ -36,8 +36,8 @@ def put(self, path, bytes): file_key.key = file_abspath file_key.set_contents_from_string(bytes, - encrypt_key = self.context.config.get('S3_STORAGE_SSE', default=False), - reduced_redundancy = self.context.config.get('S3_STORAGE_RRS', default=False) + encrypt_key = self.context.config.S3_STORAGE_SSE, + reduced_redundancy = self.context.config.S3_STORAGE_RRS ) return path @@ -57,8 +57,8 @@ def put_crypto(self, path): file_key.key = crypto_path file_key.set_contents_from_string(self.context.server.security_key, - encrypt_key = self.context.config.get('S3_STORAGE_SSE', default=False), - reduced_redundancy = self.context.config.get('S3_STORAGE_RRS', default=False) + encrypt_key = self.context.config.S3_STORAGE_SSE, + reduced_redundancy = self.context.config.S3_STORAGE_RRS ) return crypto_path @@ -72,8 +72,8 @@ def put_detector_data(self, path, data): file_key.key = path file_key.set_contents_from_string(dumps(data), - encrypt_key = self.context.config.get('S3_STORAGE_SSE', default=False), - reduced_redundancy = self.context.config.get('S3_STORAGE_RRS', default=False) + encrypt_key = self.context.config.S3_STORAGE_SSE, + reduced_redundancy = self.context.config.S3_STORAGE_RRS ) return path @@ -119,12 +119,14 @@ def exists(self, path): return True def normalize_path(self, path): - digest = hashlib.sha1(path.encode('utf-8')).hexdigest() - return "thumbor/storage/"+digest + root_path = self.context.config.RESULT_STORAGE_AWS_STORAGE_ROOT_PATH + path_segments = [path] + return join(root_path, *path_segments) + def is_expired(self, key): if key: - expire_in_seconds = self.context.config.get('RESULT_STORAGE_EXPIRATION_SECONDS', None) + expire_in_seconds = self.context.config.RESULT_STORAGE_EXPIRATION_SECONDS #Never expire if expire_in_seconds is None or expire_in_seconds == 0: @@ -151,3 +153,6 @@ def utc_to_local(self, utc_dt): local_dt = datetime.fromtimestamp(timestamp) assert utc_dt.resolution >= timedelta(microseconds=1) return local_dt.replace(microsecond=utc_dt.microsecond) + + def resolve_original_photo_path(self,filename): + return filename diff --git a/vows/__init__.py b/vows/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vows/fixtures/__init__.py b/vows/fixtures/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vows/fixtures/image.jpg b/vows/fixtures/image.jpg new file mode 100644 index 0000000..81a57ce Binary files /dev/null and b/vows/fixtures/image.jpg differ diff --git a/vows/fixtures/storage_fixture.py b/vows/fixtures/storage_fixture.py new file mode 100644 index 0000000..6f84649 --- /dev/null +++ b/vows/fixtures/storage_fixture.py @@ -0,0 +1,39 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# thumbor imaging service +# https://github.com/globocom/thumbor/wiki + +# Licensed under the MIT license: +# http://www.opensource.org/licenses/mit-license +# Copyright (c) 2011 globo.com timehome@corp.globo.com + +from os.path import join, abspath, dirname + +from thumbor.context import ServerParameters, Context +from thumbor.config import Config +from thumbor.importer import Importer + +IMAGE_URL = 's.glbimg.com/some/image_%s.jpg' +IMAGE_PATH = join(abspath(dirname(__file__)), 'image.jpg') + +with open(IMAGE_PATH, 'r') as img: + IMAGE_BYTES = img.read() + +def get_server(key=None): + server_params = ServerParameters(8888, 'localhost', 'thumbor.conf', None, 'info', None) + server_params.security_key = key + return server_params + +def get_context(server=None, config=None, importer=None): + if not server: + server = get_server() + + if not config: + config = Config() + + if not importer: + importer = Importer(config) + + ctx = Context(server=server, config=config, importer=importer) + return ctx diff --git a/vows/storage_vows.py b/vows/storage_vows.py new file mode 100644 index 0000000..4ad2eba --- /dev/null +++ b/vows/storage_vows.py @@ -0,0 +1,245 @@ +#se!/usr/bin/python +# -*- coding: utf-8 -*- + +# thumbor imaging service +# https://github.com/globocom/thumbor/wiki + +# Licensed under the MIT license: +# http://www.opensource.org/licenses/mit-license +# Copyright (c) 2011 globo.com timehome@corp.globo.com + + +from pyvows import Vows, expect +from hashlib import md5 + +from thumbor.app import ThumborServiceApp +from thumbor.importer import Importer +from thumbor.context import Context, ServerParameters +from thumbor.config import Config +from fixtures.storage_fixture import IMAGE_URL, IMAGE_BYTES, get_server +import time + +from boto.s3.connection import S3Connection +from boto.s3.key import Key +from moto import mock_s3 + +from thumbor_aws.storages.s3_storage import Storage + +s3_bucket = 'thumbor-images-test' + +@Vows.batch +class S3StorageVows(Vows.Context): + + class CanStoreImage(Vows.Context): + @mock_s3 + def topic(self): + self.conn = S3Connection() + bucket = self.conn.create_bucket(s3_bucket) + + thumborId = IMAGE_URL % '1' + config=Config(STORAGE_BUCKET=s3_bucket) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + store = storage.put(thumborId, IMAGE_BYTES ) + k = Key(bucket) + k.key = thumborId + result = k.get_contents_as_string() + return (store , result) + + def should_be_in_catalog(self, topic): + expect(topic[0]).to_equal(IMAGE_URL % '1') + expect(topic[1]).not_to_be_null() + expect(topic[1]).not_to_be_an_error() + expect(topic[1]).to_equal(IMAGE_BYTES) + + class CanGetImage(Vows.Context): + @mock_s3 + def topic(self): + self.conn = S3Connection() + self.conn.create_bucket(s3_bucket) + + config=Config(STORAGE_BUCKET=s3_bucket) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + storage.put(IMAGE_URL % '2', IMAGE_BYTES) + return storage.get(IMAGE_URL % '2') + + def should_not_be_null(self, topic): + expect(topic).not_to_be_null() + expect(topic).not_to_be_an_error() + + def should_have_proper_bytes(self, topic): + expect(topic).to_equal(IMAGE_BYTES) + + class CanGetImageExistance(Vows.Context): + @mock_s3 + def topic(self): + self.conn = S3Connection() + self.conn.create_bucket(s3_bucket) + + config=Config(STORAGE_BUCKET=s3_bucket) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + storage.put(IMAGE_URL % '3', IMAGE_BYTES) + return storage.exists(IMAGE_URL % '3') + + def should_exists(self, topic): + expect(topic).to_equal(True) + + class CanGetImageInexistance(Vows.Context): + @mock_s3 + def topic(self): + self.conn = S3Connection() + self.conn.create_bucket(s3_bucket) + + config=Config(STORAGE_BUCKET=s3_bucket) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + return storage.exists(IMAGE_URL % '9999') + + def should_not_exists(self, topic): + expect(topic).to_equal(False) + + class CanRemoveImage(Vows.Context): + @mock_s3 + def topic(self): + self.conn = S3Connection() + self.conn.create_bucket(s3_bucket) + + config=Config(STORAGE_BUCKET=s3_bucket) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + storage.put(IMAGE_URL % '4', IMAGE_BYTES) + created = storage.exists(IMAGE_URL % '4') + time.sleep(1) + storage.remove(IMAGE_URL % '4') + time.sleep(1) + return storage.exists(IMAGE_URL % '4') != created + + def should_be_put_and_removed(self, topic): + expect(topic).to_equal(True) + + class CanRemovethenPutImage(Vows.Context): + @mock_s3 + def topic(self): + self.conn = S3Connection() + self.conn.create_bucket(s3_bucket) + + config=Config(STORAGE_BUCKET=s3_bucket) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + storage.put(IMAGE_URL % '5', IMAGE_BYTES) + storage.remove(IMAGE_URL % '5') + time.sleep(1) + created = storage.exists(IMAGE_URL % '5') + time.sleep(1) + storage.put(IMAGE_URL % '5', IMAGE_BYTES) + return storage.exists(IMAGE_URL % '5') != created + + def should_be_put_and_removed(self, topic): + expect(topic).to_equal(True) + + class CanReturnPath(Vows.Context): + @mock_s3 + def topic(self): + self.conn = S3Connection() + self.conn.create_bucket(s3_bucket) + + config=Config(STORAGE_BUCKET=s3_bucket) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + return storage.resolve_original_photo_path("toto") + + def should_return_the_same(self, topic): + expect(topic).to_equal("toto") + + class CryptoVows(Vows.Context): + class RaisesIfInvalidConfig(Vows.Context): + @Vows.capture_error + @mock_s3 + def topic(self): + self.conn = S3Connection() + self.conn.create_bucket(s3_bucket) + + config=Config(STORAGE_BUCKET=s3_bucket, STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True) + storage = Storage(Context(config=config, server=get_server(''))) + storage.put(IMAGE_URL % '9999', IMAGE_BYTES) + storage.put_crypto(IMAGE_URL % '9999') + + def should_be_an_error(self, topic): + expect(topic).to_be_an_error_like(RuntimeError) + expect(topic).to_have_an_error_message_of("STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified") + + class GettingCryptoForANewImageReturnsNone(Vows.Context): + @mock_s3 + def topic(self): + self.conn = S3Connection() + self.conn.create_bucket(s3_bucket) + + config=Config(STORAGE_BUCKET=s3_bucket, STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + return storage.get_crypto(IMAGE_URL % '9999') + + def should_be_null(self, topic): + expect(topic).to_be_null() + + class DoesNotStoreIfConfigSaysNotTo(Vows.Context): + @mock_s3 + def topic(self): + self.conn = S3Connection() + self.conn.create_bucket(s3_bucket) + + config=Config(STORAGE_BUCKET=s3_bucket) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + storage.put(IMAGE_URL % '9998', IMAGE_BYTES) + storage.put_crypto(IMAGE_URL % '9998') + return storage.get_crypto(IMAGE_URL % '9998') + + def should_be_null(self, topic): + expect(topic).to_be_null() + + class CanStoreCrypto(Vows.Context): + @mock_s3 + def topic(self): + self.conn = S3Connection() + self.conn.create_bucket(s3_bucket) + + config=Config(STORAGE_BUCKET=s3_bucket, STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + storage.put(IMAGE_URL % '6', IMAGE_BYTES) + storage.put_crypto(IMAGE_URL % '6') + return storage.get_crypto(IMAGE_URL % '6') + + def should_not_be_null(self, topic): + expect(topic).not_to_be_null() + expect(topic).not_to_be_an_error() + + def should_have_proper_key(self, topic): + expect(topic).to_equal('ACME-SEC') + + class DetectorVows(Vows.Context): + class CanStoreDetectorData(Vows.Context): + @mock_s3 + def topic(self): + self.conn = S3Connection() + self.conn.create_bucket(s3_bucket) + + config=Config(STORAGE_BUCKET=s3_bucket) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + storage.put(IMAGE_URL % '7', IMAGE_BYTES) + storage.put_detector_data(IMAGE_URL % '7', 'some-data') + return storage.get_detector_data(IMAGE_URL % '7') + + def should_not_be_null(self, topic): + expect(topic).not_to_be_null() + expect(topic).not_to_be_an_error() + + def should_equal_some_data(self, topic): + expect(topic).to_equal('some-data') + + class ReturnsNoneIfNoDetectorData(Vows.Context): + @mock_s3 + def topic(self): + self.conn = S3Connection() + self.conn.create_bucket(s3_bucket) + + config=Config(STORAGE_BUCKET=s3_bucket) + storage = Storage(Context(config=config, server=get_server('ACME-SEC'))) + return storage.get_detector_data(IMAGE_URL % '9999') + + def should_not_be_null(self, topic): + expect(topic).to_be_null() +