Merge pull request #709 from ImageMarkup/isic-128-fix-dev-bootstrapping

dev/.env.docker-compose

@@ -7,3 +7,5 @@ DJANGO_MINIO_STORAGE_SECRET_KEY=minioSecretKey
 DJANGO_STORAGE_BUCKET_NAME=django-storage
 DJANGO_MINIO_STORAGE_MEDIA_URL=http://localhost:9000/django-storage
 DJANGO_ISIC_ELASTICSEARCH_URI=http://elastic:elastic@elasticsearch:9200
+DJANGO_ISIC_DATACITE_USERNAME="fakeuser"
+DJANGO_ISIC_DATACITE_PASSWORD="fakepassword"

dev/.env.docker-compose-native

@@ -6,3 +6,5 @@ DJANGO_MINIO_STORAGE_ACCESS_KEY=minioAccessKey
 DJANGO_MINIO_STORAGE_SECRET_KEY=minioSecretKey
 DJANGO_STORAGE_BUCKET_NAME=django-storage
 DJANGO_ISIC_ELASTICSEARCH_URI=http://elastic:elastic@localhost:9200
+DJANGO_ISIC_DATACITE_USERNAME="fakeuser"
+DJANGO_ISIC_DATACITE_PASSWORD="fakepassword"

dev/import-db.sh

@@ -3,10 +3,14 @@ set -exuo pipefail
 
 export HEROKU_APP=isic
 
+docker-compose start postgres
+
 docker-compose exec postgres bash -c "PGPASSWORD=postgres dropdb --host localhost --username postgres --if-exists django && createdb --host localhost --username postgres django"
 
+# public.stats_imagedownload is large and not usually needed
 docker-compose exec postgres bash -c \
 "pg_dump --format=custom --no-privileges $(heroku config:get DATABASE_URL)\
+         --exclude-table-data public.stats_imagedownload \
  | PGPASSWORD=postgres pg_restore --host localhost --username postgres --format=custom --no-privileges --no-owner --dbname=django"
 
 docker-compose restart postgres

dev/node.Dockerfile

@@ -1,4 +1,4 @@
-FROM node:current-alpine
+FROM node:16.20-alpine
 
 # "@parcel/watcher" requires "node-gyp", which requires Python to build for some architectures
 RUN apk update && apk add g++ make python3

docker-compose.override.yml

@@ -14,6 +14,7 @@ services:
       - 8000:8000
     depends_on:
       - postgres
+      - elasticsearch
       - rabbitmq
       - minio
 
@@ -35,6 +36,7 @@ services:
       - .:/opt/django-project
     depends_on:
       - postgres
+      - elasticsearch
       - rabbitmq
       - minio
 

docker-compose.yml

@@ -36,3 +36,10 @@ services:
     ports:
       - ${DOCKER_MINIO_PORT-9000}:9000
       - ${DOCKER_MINIO_CONSOLE_PORT-9001}:9001
+
+  zipstreamer:
+    image: ghcr.io/imagemarkup/isic-zipstreamer:master
+    environment:
+      ZS_LISTFILE_URL_PREFIX: "http://django/api/v2/zip-download/file-descriptor/?token="
+    ports:
+      - 4008:4008

isic/core/migrations/0020_auto_20210807_0228.py

@@ -1,11 +1,33 @@
 # Generated by Django 3.2.4 on 2021-08-07 02:28
 
+import time
+
+from django.conf import settings
 from django.db import migrations
+import requests
 
 from isic.core.search import maybe_create_index
 
 
+def _block_until_elasticsearch_is_available():
+    print("Waiting for Elasticsearch to become available...")
+    time_slept = 0
+
+    while True:
+        try:
+            requests.get(f"{settings.ISIC_ELASTICSEARCH_URI}/_cluster/health", timeout=1)
+        except requests.exceptions.ConnectionError:
+            time.sleep(1)
+            time_slept += 1
+
+            if time_slept > 30:
+                raise TimeoutError("Elasticsearch did not become available in time")
+        else:
+            break
+
+
 def create_elasticsearch_index(apps, schema_editor):
+    _block_until_elasticsearch_is_available()
     maybe_create_index()