diff --git a/dev/.env.docker-compose b/dev/.env.docker-compose index 9b39e75e..3e0b1d1c 100644 --- a/dev/.env.docker-compose +++ b/dev/.env.docker-compose @@ -7,3 +7,5 @@ DJANGO_MINIO_STORAGE_SECRET_KEY=minioSecretKey DJANGO_STORAGE_BUCKET_NAME=django-storage DJANGO_MINIO_STORAGE_MEDIA_URL=http://localhost:9000/django-storage DJANGO_ISIC_ELASTICSEARCH_URI=http://elastic:elastic@elasticsearch:9200 +DJANGO_ISIC_DATACITE_USERNAME="fakeuser" +DJANGO_ISIC_DATACITE_PASSWORD="fakepassword" diff --git a/dev/.env.docker-compose-native b/dev/.env.docker-compose-native index 4115a5dc..6909a701 100644 --- a/dev/.env.docker-compose-native +++ b/dev/.env.docker-compose-native @@ -6,3 +6,5 @@ DJANGO_MINIO_STORAGE_ACCESS_KEY=minioAccessKey DJANGO_MINIO_STORAGE_SECRET_KEY=minioSecretKey DJANGO_STORAGE_BUCKET_NAME=django-storage DJANGO_ISIC_ELASTICSEARCH_URI=http://elastic:elastic@localhost:9200 +DJANGO_ISIC_DATACITE_USERNAME="fakeuser" +DJANGO_ISIC_DATACITE_PASSWORD="fakepassword" diff --git a/dev/import-db.sh b/dev/import-db.sh index 6801ac85..813da6e4 100755 --- a/dev/import-db.sh +++ b/dev/import-db.sh @@ -3,10 +3,14 @@ set -exuo pipefail export HEROKU_APP=isic +docker-compose start postgres + docker-compose exec postgres bash -c "PGPASSWORD=postgres dropdb --host localhost --username postgres --if-exists django && createdb --host localhost --username postgres django" +# public.stats_imagedownload is large and not usually needed docker-compose exec postgres bash -c \ "pg_dump --format=custom --no-privileges $(heroku config:get DATABASE_URL)\ + --exclude-table-data public.stats_imagedownload \ | PGPASSWORD=postgres pg_restore --host localhost --username postgres --format=custom --no-privileges --no-owner --dbname=django" docker-compose restart postgres diff --git a/dev/node.Dockerfile b/dev/node.Dockerfile index 5b50e1da..026bc2b6 100644 --- a/dev/node.Dockerfile +++ b/dev/node.Dockerfile @@ -1,4 +1,4 @@ -FROM node:current-alpine +FROM node:16.20-alpine # "@parcel/watcher" requires "node-gyp", which requires Python to build for some architectures RUN apk update && apk add g++ make python3 diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 115e89ea..98f0d9e5 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -14,6 +14,7 @@ services: - 8000:8000 depends_on: - postgres + - elasticsearch - rabbitmq - minio @@ -35,6 +36,7 @@ services: - .:/opt/django-project depends_on: - postgres + - elasticsearch - rabbitmq - minio diff --git a/docker-compose.yml b/docker-compose.yml index d5461f4a..69213b34 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,3 +36,10 @@ services: ports: - ${DOCKER_MINIO_PORT-9000}:9000 - ${DOCKER_MINIO_CONSOLE_PORT-9001}:9001 + + zipstreamer: + image: ghcr.io/imagemarkup/isic-zipstreamer:master + environment: + ZS_LISTFILE_URL_PREFIX: "http://django/api/v2/zip-download/file-descriptor/?token=" + ports: + - 4008:4008 diff --git a/isic/core/migrations/0020_auto_20210807_0228.py b/isic/core/migrations/0020_auto_20210807_0228.py index a678a641..51862b93 100644 --- a/isic/core/migrations/0020_auto_20210807_0228.py +++ b/isic/core/migrations/0020_auto_20210807_0228.py @@ -1,11 +1,33 @@ # Generated by Django 3.2.4 on 2021-08-07 02:28 +import time + +from django.conf import settings from django.db import migrations +import requests from isic.core.search import maybe_create_index +def _block_until_elasticsearch_is_available(): + print("Waiting for Elasticsearch to become available...") + time_slept = 0 + + while True: + try: + requests.get(f"{settings.ISIC_ELASTICSEARCH_URI}/_cluster/health", timeout=1) + except requests.exceptions.ConnectionError: + time.sleep(1) + time_slept += 1 + + if time_slept > 30: + raise TimeoutError("Elasticsearch did not become available in time") + else: + break + + def create_elasticsearch_index(apps, schema_editor): + _block_until_elasticsearch_is_available() maybe_create_index()