Merge pull request #731 from ImageMarkup/add-proxy-storage

Add MinioS3ProxyStorage for local development with ISIC_PLACEHOLDER_IMAGES

isic/core/storage.py

@@ -1,7 +1,10 @@
 from datetime import datetime, timedelta
+import io
 from urllib.parse import urlencode
 from uuid import uuid4
 
+import boto3
+from botocore.exceptions import ClientError
 from django.utils.encoding import filepath_to_uri
 from storages.backends.s3boto3 import S3Boto3Storage
 
@@ -39,3 +42,45 @@ def url(self, name, parameters=None, expire=None, http_method=None):
                 return self.cloudfront_signer.generate_presigned_url(url, date_less_than=expiration)
 
             return url
+
+
+try:
+    from minio_storage.storage import MinioMediaStorage
+
+    class MinioS3ProxyStorage(MinioMediaStorage):
+        """
+        A storage backend that proxies to S3 if the file doesn't exist in Minio.
+
+        This is useful for local development enviroments that don't have a full copy of the
+        production images. Note that requests to the development server will be slower than
+        requests normal, since the file will be downloaded from S3 on the first request.
+
+        Enabling this also requires disabling ISIC_PLACEHOLDER_IMAGES.
+        """
+
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+
+        def url(self, name: str, *args, **kwargs) -> str:
+            exists_in_minio = super().exists(name)
+
+            if not exists_in_minio:
+                upstream_file = io.BytesIO()
+                s3 = boto3.resource("s3")
+                bucket = s3.Bucket("isic-storage")
+                upstream_obj = bucket.Object(name)
+
+                try:
+                    upstream_obj.download_fileobj(upstream_file)
+                except ClientError as e:
+                    if e.response["Error"]["Code"] != "404":
+                        raise e
+                else:
+                    size = upstream_file.tell()
+                    upstream_file.seek(0)
+                    self.client.put_object(self.bucket_name, name, upstream_file, size)
+
+            return super().url(name, *args, **kwargs)
+
+except ImportError:
+    pass

isic/settings.py

@@ -198,7 +198,10 @@ class DevelopmentConfiguration(IsicMixin, DevelopmentBaseConfiguration):
 
     ZIP_DOWNLOAD_SERVICE_URL = "http://localhost:4008"
 
-    ISIC_PLACEHOLDER_IMAGES = False
+    ISIC_PLACEHOLDER_IMAGES = True
+    # Use the MinioS3ProxyStorage for local development with ISIC_PLACEHOLDER_IMAGES
+    # set to False to view real images in development.
+    # DEFAULT_FILE_STORAGE = "isic.core.storage.MinioS3ProxyStorage"
 
 
 class TestingConfiguration(IsicMixin, TestingBaseConfiguration):