Docker Compose option to generate random passwords

.env.secure

@@ -0,0 +1,75 @@
+# Container names
+NGINX_CONTAINER_NAME=nginx
+REDIS_CONTAINER_NAME=redis
+POSTGRESQL_CONTAINER_NAME=db
+SOLR_CONTAINER_NAME=solr
+DATAPUSHER_CONTAINER_NAME=datapusher
+CKAN_CONTAINER_NAME=ckan
+WORKER_CONTAINER_NAME=ckan-worker
+
+# Host Ports
+CKAN_PORT_HOST=5000
+NGINX_PORT_HOST=81
+NGINX_SSLPORT_HOST=8443
+
+# CKAN databases
+POSTGRES_USER=postgres
+POSTGRES_DB=postgres
+POSTGRES_HOST=db
+CKAN_DB_USER=ckandbuser
+CKAN_DB=ckandb
+DATASTORE_READONLY_USER=datastore_ro
+DATASTORE_DB=datastore
+
+# Test database connections
+TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test
+TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test
+TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test
+
+# CKAN core
+CKAN_VERSION=2.10.0
+CKAN_SITE_ID=default
+CKAN_SITE_URL=https://localhost:8443
+CKAN_PORT=5000
+CKAN_PORT_HOST=5000
+CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
+# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings
+CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME
+CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME
+CKAN_SYSADMIN_NAME=ckan_admin
+[email protected]
+CKAN_STORAGE_PATH=/var/lib/ckan
+CKAN_SMTP_SERVER=smtp.corporateict.domain:25
+CKAN_SMTP_STARTTLS=True
+CKAN_SMTP_USER=user
+CKAN_SMTP_PASSWORD=pass
+CKAN_SMTP_MAIL_FROM=ckan@localhost
+TZ=UTC
+
+# Solr
+SOLR_IMAGE_VERSION=2.10-solr9
+CKAN_SOLR_URL=http://solr:8983/solr/ckan
+TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan
+
+# Redis
+REDIS_VERSION=6
+CKAN_REDIS_URL=redis://redis:6379/1
+TEST_CKAN_REDIS_URL=redis://redis:6379/1
+
+# Datapusher
+DATAPUSHER_VERSION=0.0.20
+CKAN_DATAPUSHER_URL=http://datapusher:8800
+CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000
+DATAPUSHER_REWRITE_RESOURCES=True
+DATAPUSHER_REWRITE_URL=http://ckan:5000
+
+# NGINX
+NGINX_PORT=80
+NGINX_SSLPORT=443
+
+# Extensions
+CKAN__PLUGINS="envvars image_view text_view recline_view datastore datapusher"
+CKAN__HARVEST__MQ__TYPE=redis
+CKAN__HARVEST__MQ__HOSTNAME=redis
+CKAN__HARVEST__MQ__PORT=6379
+CKAN__HARVEST__MQ__REDIS_DB=1

.gitignore

@@ -7,4 +7,9 @@ _service-provider/*
 _solr/schema.xml
 _src/*
 local/*
+
+# Docker Compose environment and password files
 .env
+.ckpw
+.dbpw
+docker-compose.yml.ORIG

README.md

@@ -1,6 +1,5 @@
 # Docker Compose setup for CKAN
 
-
 * [Overview](#overview)
 * [Installing Docker](#installing-docker)
 * [docker compose vs docker-compose](#docker-compose-vs-docker-compose)
@@ -45,23 +44,56 @@ versions for client and server.
 ## 3.  docker compose *vs* docker-compose
 
 All Docker Compose commands in this README will use the V2 version of Compose ie: `docker compose`. The older version (V1) 
-used the `docker-compose` command. Please see [Docker Compose](https://docs.docker.com/compose/compose-v2/) for
-more information.
+used the `docker-compose` command. Please see [Docker Compose](https://docs.docker.com/compose/compose-v2/) for more information.
 
 ## 4.  Install (build and run) CKAN plus dependencies
 
-#### Base mode
+#### Generate randomised passwords mode (secure/default mode)
+
+For additional security the default option is to run in "secure" mode. To do this a script will need to be run to generate randomised passwords for the database users and for the initial CKAN application sysadmin user (ckan_admin). It will also generate some other parameters detailed below:
+
+Here are the users that will have randomised passwords generated:
+* The Postgres system superuser (user: `POSTGRES_USER` , password: `POSTGRES_PASSWORD`)
+* The database user that owns the CKAN database (user: `CKAN_DB_USER` , password: `CKAN_DB_PASSWORD`)
+* The database user that has read-access to the Datastore database (user: `DATASTORE_READONLY_USER` , password: `DATASTORE_READONLY_PASSWORD`)
+* The CKAN application System Administrator user (user: `CKAN_SYSADMIN_NAME` ,password: `CKAN_SYSADMIN_PASSWORD`)
+
+There are other configuration parameters that will also have random secrets generated. They are as follows:
+* `CKAN___BEAKER__SESSION__SECRET`
+* `CKAN___API_TOKEN__JWT__ENCODE__SECRET`
+* `CKAN___API_TOKEN__JWT__DECODE__SECRET`
+
+Also, database URL's will need to include these new passwords so the values for this configuration parameters will be added to the generated (CKAN) file
+* `CKAN_SQLALCHEMY_URL`
+* `CKAN_DATASTORE_WRITE_URL`
+* `CKAN_DATASTORE_READ_URL`
+
+You can do this by the following:
 
-Use this if you are a maintainer and will not be making code changes to CKAN or to CKAN extensions
+* generate the passwords by running `generate_passwords.sh` this will create two files: `.ckpw` and `.dbpw`
+* by default the `docker-compose.yml` file is set to secure mode.
+* You need to make a copy of `.env.secure` as `.env`
+* if you make any subsequent changes to the `docker-compose.yml` file you can always revert back to the original file by referencing `docker-compose.secure.yml`
+* by default the `.env` file is set to secure mode.
+* if you make changes to the `.env` file you can always revert back to the original file by referencing `.env.secure`
+* build and run the docker compose stack as per normal. 
+* the `ckan_admin` user password will be located in the `.ckpw` file. It will also be displayed on the terminal when running `generate_passwords.sh`
 
-Copy the included `.env.example` and rename it to `.env`. Modify it depending on your own needs.
+#### Insecure mode (the old legacy mode)
+
+Use this if you do not wish to use secure passwords ie: how it used to be
+
+Copy the included `.env.insecure` to `.env`. Modify it depending on your own needs. It's wise to keep the original `.env.secure` file in case
+you need to revert back to it or reference from it
+
+Copy the included `docker-compose.insecure.yml` and to `docker-compose.yml`. Modify it depending on your own needs. 
 
 Please note that when accessing CKAN directly (via a browser) ie: not going through NGINX you will need to make sure you have "ckan" set up
 to be an alias to localhost in the local hosts file. Either that or you will need to change the `.env` entry for CKAN_SITE_URL
 
 Using the default values on the `.env.example` file will get you a working CKAN instance. There is a sysadmin user created by default with the values defined in `CKAN_SYSADMIN_NAME` and `CKAN_SYSADMIN_PASSWORD`(`ckan_admin` and `test1234` by default). This should be obviously changed before running this setup as a public CKAN instance.
 
-To build the images:
+### To build the images:
 
 	docker compose build
 
@@ -84,6 +116,11 @@ After this step, CKAN should be running at `CKAN_SITE_URL`.
 
 Use this mode if you are making code changes to CKAN and either creating new extensions or making code changes to existing extensions. This mode also uses the `.env` file for config options.
 
+Again, the default for Development mode is secure. Please see the section above on Secure mode. There are 2 reference compose `.yml` files
+
+* `docker-compose.dev.secure.yml` - this is the same as the default file `docker-compose.dev.yml`
+* `docker-compose.dev.insecure.yml` - to be used if the insecure (ie: legacy) config is to be used
+
 To develop local extensions use the `docker-compose.dev.yml` file:
 
 To build the images:

ckan/setup/prerun.py.override

@@ -194,6 +194,13 @@ def create_sysadmin():
         subprocess.call(command)
         print("[prerun] Made user {0} a sysadmin".format(name))
 
+        # cleanup permissions
+        # We're running as root before pivoting to uwsgi and dropping privs
+        data_dir = "%s/storage" % os.environ['CKAN_STORAGE_PATH']
+
+        command = ["chown", "-R", "ckan:ckan", data_dir]
+        subprocess.call(command)
+        print("[prerun] Ensured storage directory is owned by ckan")
 
 if __name__ == "__main__":
 

ckan/setup/start_ckan.sh.override

@@ -16,7 +16,7 @@ then
 fi
 
 # Run the prerun script to init CKAN and create the default admin user
-sudo -u ckan -EH python3 prerun.py
+python3 prerun.py
 
 echo "Set up ckan.datapusher.api_token in the CKAN config file"
 ckan config-tool $CKAN_INI "ckan.datapusher.api_token=$(ckan -c $CKAN_INI user token add ckan_admin datapusher | tail -n 1 | tr -d '\t')"
@@ -51,7 +51,7 @@ then
     # Start supervisord
     supervisord --configuration /etc/supervisord.conf &
     # Start uwsgi
-    sudo -u ckan -EH uwsgi $UWSGI_OPTS
+    uwsgi $UWSGI_OPTS
 else
   echo "[prerun] failed...not starting CKAN."
 fi

docker-compose-use-generated-passwords.yml

@@ -0,0 +1,115 @@
+version: "3"
+
+
+volumes:
+  ckan_storage:
+  pg_data:
+  solr_data:
+
+services:
+
+  nginx:
+    container_name: ${NGINX_CONTAINER_NAME}
+    build:
+      context: nginx/
+      dockerfile: Dockerfile
+    networks:
+      - webnet
+      - ckannet
+    depends_on:
+      ckan:
+        condition: service_healthy
+    ports:
+      - "0.0.0.0:${NGINX_SSLPORT_HOST}:${NGINX_SSLPORT}"
+
+  ckan:
+    container_name: ${CKAN_CONTAINER_NAME}
+    build:
+      context: ckan/
+      dockerfile: Dockerfile
+      args:
+        - TZ=${TZ}
+    networks:
+      - ckannet
+      - dbnet
+      - solrnet
+      - redisnet
+    env_file:
+      - .env
+      - .ckpw
+    depends_on:
+      db:
+        condition: service_healthy
+      solr:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    volumes:
+      - ckan_storage:/var/lib/ckan
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"]
+
+  datapusher:
+    container_name: ${DATAPUSHER_CONTAINER_NAME}
+    networks:
+      - ckannet
+      - dbnet
+    image: ckan/ckan-base-datapusher:${DATAPUSHER_VERSION}
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8800"]
+
+  db:
+    container_name: ${POSTGRESQL_CONTAINER_NAME}
+    build:
+      context: postgresql/
+    networks:
+      - dbnet
+    environment:
+      - POSTGRES_USER
+      #- POSTGRES_PASSWORD
+      - POSTGRES_DB
+      - CKAN_DB_USER
+      #- CKAN_DB_PASSWORD
+      - CKAN_DB
+      - DATASTORE_READONLY_USER
+      #- DATASTORE_READONLY_PASSWORD
+      - DATASTORE_DB
+    env_file:
+      - .ckpw
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"]
+
+  solr:
+    container_name: ${SOLR_CONTAINER_NAME}
+    networks:
+      - solrnet
+    image: ckan/ckan-solr:${SOLR_IMAGE_VERSION}
+    volumes:
+      - solr_data:/var/solr
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"]
+
+  redis:
+    container_name: ${REDIS_CONTAINER_NAME}
+    image: redis:${REDIS_VERSION}
+    networks:
+      - redisnet
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "redis-cli", "-e", "QUIT"]
+
+networks:
+  webnet:
+  ckannet:
+  solrnet:
+    internal: true
+  dbnet:
+    internal: true
+  redisnet:
+    internal: true

docker-compose.dev.insecure.yml

@@ -0,0 +1,76 @@
+version: "3"
+
+volumes:
+  ckan_storage:
+  pg_data:
+  solr_data:
+
+services:
+
+  ckan-dev:
+    container_name: ${CKAN_CONTAINER_NAME}
+    build:
+      context: ckan/
+      dockerfile: Dockerfile.dev
+      args:
+        - TZ=${TZ}
+    env_file:
+      - .env
+    depends_on:
+      db:
+        condition: service_healthy
+      solr:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    ports:
+      - "0.0.0.0:${CKAN_PORT_HOST}:${CKAN_PORT}"
+    volumes:
+      - ckan_storage:/var/lib/ckan
+      - ./src:/srv/app/src_extensions
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"]
+
+  datapusher:
+    container_name: ${DATAPUSHER_CONTAINER_NAME}
+    image: ckan/ckan-base-datapusher:${DATAPUSHER_VERSION}
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8800"]
+
+  db:
+    container_name: ${POSTGRESQL_CONTAINER_NAME}
+    build:
+      context: postgresql/
+    environment:
+      - POSTGRES_USER
+      - POSTGRES_PASSWORD
+      - POSTGRES_DB
+      - CKAN_DB_USER
+      - CKAN_DB_PASSWORD
+      - CKAN_DB
+      - DATASTORE_READONLY_USER
+      - DATASTORE_READONLY_PASSWORD
+      - DATASTORE_DB
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"]
+
+  solr:
+    container_name: ${SOLR_CONTAINER_NAME}
+    image: ckan/ckan-solr:${SOLR_IMAGE_VERSION}
+    volumes:
+      - solr_data:/var/solr
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"]
+
+  redis:
+    container_name: ${REDIS_CONTAINER_NAME}
+    image: redis:${REDIS_VERSION}
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "redis-cli", "-e", "QUIT"]