From 11d34f4501853e667679ad6a06b6b2045cfc2f34 Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 9 Aug 2023 13:25:00 +0100 Subject: [PATCH 1/8] generate_passwords.*, README and new compose --- .gitignore | 3 + README.md | 15 +++ docker-compose-use-generated-passwords.yml | 115 +++++++++++++++++++++ generate_passwords.py | 36 +++++++ generate_passwords.sh | 10 ++ 5 files changed, 179 insertions(+) create mode 100755 docker-compose-use-generated-passwords.yml create mode 100644 generate_passwords.py create mode 100755 generate_passwords.sh diff --git a/.gitignore b/.gitignore index a3d88d50..709d1b96 100755 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,7 @@ _service-provider/* _solr/schema.xml _src/* local/* + +# Docker Compose environment and password files .env +.pw diff --git a/README.md b/README.md index f48b18ba..cd823765 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,21 @@ more information. ## 4. Install (build and run) CKAN plus dependencies +#### Generate Randomised passwords mode + +For additional security it may be a better option to generate randomised passwords for the database users and the for the initial CKAN application sysadmin user. Here are the users that will have randomised + passwords generated: + +* The Postgres system superuser (user=POSTGRES_USER , password=POSTGRES_PASSWORD) +* The database user that owns the CKAN database (user=CKAN_DB_USER , password=CKAN_DB_PASSWORD) +* The database user that has read-access to the Datastore database (user=DATASTORE_READONLY_USER , password=DATASTORE_READONLY_PASSWORD) +* The CKAN application System Administrator user (user=CKAN_SYSADMIN_NAME ,password=CKAN_SYSADMIN_PASSWORD) + +You will need to generate the passwords. You can do this by running `generate_passwords.sh` before building the local images and containers. This will create a password file called .pw. +You will then need to rename the docker-compose-use-generated-passwords.yml file to docker-compose.yml. You may wish to save the docker-compose.yml file beforehand. + +You can then carry on with the Base mode instructions disregarding details about the sysadmin user + #### Base mode Use this if you are a maintainer and will not be making code changes to CKAN or to CKAN extensions diff --git a/docker-compose-use-generated-passwords.yml b/docker-compose-use-generated-passwords.yml new file mode 100755 index 00000000..3513c431 --- /dev/null +++ b/docker-compose-use-generated-passwords.yml @@ -0,0 +1,115 @@ +version: "3" + + +volumes: + ckan_storage: + pg_data: + solr_data: + +services: + + nginx: + container_name: ${NGINX_CONTAINER_NAME} + build: + context: nginx/ + dockerfile: Dockerfile + networks: + - webnet + - ckannet + depends_on: + ckan: + condition: service_healthy + ports: + - "0.0.0.0:${NGINX_SSLPORT_HOST}:${NGINX_SSLPORT}" + + ckan: + container_name: ${CKAN_CONTAINER_NAME} + build: + context: ckan/ + dockerfile: Dockerfile + args: + - TZ=${TZ} + networks: + - ckannet + - dbnet + - solrnet + - redisnet + env_file: + - .env + - .pw + depends_on: + db: + condition: service_healthy + solr: + condition: service_healthy + redis: + condition: service_healthy + volumes: + - ckan_storage:/var/lib/ckan + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] + + datapusher: + container_name: ${DATAPUSHER_CONTAINER_NAME} + networks: + - ckannet + - dbnet + image: ckan/ckan-base-datapusher:${DATAPUSHER_VERSION} + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8800"] + + db: + container_name: ${POSTGRESQL_CONTAINER_NAME} + build: + context: postgresql/ + networks: + - dbnet + environment: + - POSTGRES_USER + - POSTGRES_PASSWORD + - POSTGRES_DB + - CKAN_DB_USER + - CKAN_DB_PASSWORD + - CKAN_DB + - DATASTORE_READONLY_USER + - DATASTORE_READONLY_PASSWORD + - DATASTORE_DB + env_file: + - .pw + volumes: + - pg_data:/var/lib/postgresql/data + restart: unless-stopped + healthcheck: + test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"] + + solr: + container_name: ${SOLR_CONTAINER_NAME} + networks: + - solrnet + image: ckan/ckan-solr:${SOLR_IMAGE_VERSION} + volumes: + - solr_data:/var/solr + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"] + + redis: + container_name: ${REDIS_CONTAINER_NAME} + image: redis:${REDIS_VERSION} + networks: + - redisnet + restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "-e", "QUIT"] + +networks: + webnet: + ckannet: + solrnet: + internal: true + dbnet: + internal: true + redisnet: + internal: true diff --git a/generate_passwords.py b/generate_passwords.py new file mode 100644 index 00000000..a7bee8d7 --- /dev/null +++ b/generate_passwords.py @@ -0,0 +1,36 @@ +import base64 +import os + +fn = ".pw" +vn = {} + +pwvars = ["POSTGRES_PASSWORD", "CKAN_DB_PASSWORD", "DATASTORE_READONLY_PASSWORD","CKAN_SYSADMIN_PASSWORD"] + +print("[setup_passwords] attempting to setup secure passwords") + +with open(fn, 'w') as f: + f.truncate(0) + for pwvar in pwvars: + with open('/dev/urandom', 'rb') as rand_file: + pw = base64.urlsafe_b64encode(rand_file.read(12)).decode('utf-8') + f.write(f"{pwvar}={pw}\n") + vn[pwvar] = pw + +POSTGRES_PASSWORD = vn["POSTGRES_PASSWORD"] +CKAN_DB_PASSWORD = vn["CKAN_DB_PASSWORD"] +DATASTORE_READONLY_PASSWORD = vn["DATASTORE_READONLY_PASSWORD"] +CKAN_SYSADMIN_PASSWORD = vn["CKAN_SYSADMIN_PASSWORD"] + +CKAN_DB_USER = os.environ.get('CKAN_DB_USER') +CKAN_DB = os.environ.get('CKAN_DB') +DATASTORE_DB_USER = os.environ.get('DATASTORE_DB_USER') +DATASTORE_DB_READONLY_USER = os.environ.get('DATASTORE_DB_READONLY_USER') +DATASTORE_DB = os.environ.get('DATASTORE_DB') +POSTGRES_HOST = os.environ.get('POSTGRES_HOST') + +with open(fn, 'a') as f: + f.write(f"CKAN_SQLALCHEMY_URL=postgresql://{CKAN_DB_USER}:{CKAN_DB_PASSWORD}@{POSTGRES_HOST}/{CKAN_DB}\n") + f.write(f"CKAN_DATASTORE_WRITE_URL=postgresql://{CKAN_DB_USER}:{CKAN_DB_PASSWORD}@{POSTGRES_HOST}/{DATASTORE_DB}\n") + f.write(f"CKAN_DATASTORE_READ_URL=postgresql://{DATASTORE_DB_READONLY_USER}:{DATASTORE_READONLY_PASSWORD}@{POSTGRES_HOST}/{DATASTORE_DB}\n") + +print("[setup_passwords] password file: '.pw' created successfully") \ No newline at end of file diff --git a/generate_passwords.sh b/generate_passwords.sh new file mode 100755 index 00000000..4f910e3e --- /dev/null +++ b/generate_passwords.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -a +. ./.env +pwfile=".pw" +touch ${pwfile}; chmod 600 ${pwfile} +#truncate -s 0 ${pwfile} +python3 ./generate_passwords.py +chmod 400 ${pwfile} +sleep 1 \ No newline at end of file From 5044e988ff348f1c45df5837e79d24beb28fdc07 Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 10 Aug 2023 16:19:22 +0100 Subject: [PATCH 2/8] new .env, docker compose files...README updates --- .env.use-generated-passwords | 75 +++++++++++++++++++ .gitignore | 1 + README.md | 9 ++- docker-compose-use-generated-passwords.yml | 6 +- docker-compose.yml | 9 ++- generate_passwords.py | 6 +- ...tabases.sql => 30_setup_test_databases.sh} | 0 7 files changed, 94 insertions(+), 12 deletions(-) create mode 100644 .env.use-generated-passwords rename postgresql/docker-entrypoint-initdb.d/{30_setup_test_databases.sql => 30_setup_test_databases.sh} (100%) diff --git a/.env.use-generated-passwords b/.env.use-generated-passwords new file mode 100644 index 00000000..7c819f3f --- /dev/null +++ b/.env.use-generated-passwords @@ -0,0 +1,75 @@ +# Container names +NGINX_CONTAINER_NAME=nginx +REDIS_CONTAINER_NAME=redis +POSTGRESQL_CONTAINER_NAME=db +SOLR_CONTAINER_NAME=solr +DATAPUSHER_CONTAINER_NAME=datapusher +CKAN_CONTAINER_NAME=ckan +WORKER_CONTAINER_NAME=ckan-worker + +# Host Ports +CKAN_PORT_HOST=5000 +NGINX_PORT_HOST=81 +NGINX_SSLPORT_HOST=8443 + +# CKAN databases +POSTGRES_USER=postgres +POSTGRES_DB=postgres +POSTGRES_HOST=db +CKAN_DB_USER=ckandbuser +CKAN_DB=ckandb +DATASTORE_READONLY_USER=datastore_ro +DATASTORE_DB=datastore + +# Test database connections +TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test +TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test +TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test + +# CKAN core +CKAN_VERSION=2.10.0 +CKAN_SITE_ID=default +CKAN_SITE_URL=https://localhost:8443 +CKAN_PORT=5000 +CKAN_PORT_HOST=5000 +CKAN___BEAKER__SESSION__SECRET=CHANGE_ME +# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings +CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME +CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME +CKAN_SYSADMIN_NAME=ckan_admin +CKAN_SYSADMIN_EMAIL=your_email@example.com +CKAN_STORAGE_PATH=/var/lib/ckan +CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +CKAN_SMTP_STARTTLS=True +CKAN_SMTP_USER=user +CKAN_SMTP_PASSWORD=pass +CKAN_SMTP_MAIL_FROM=ckan@localhost +TZ=UTC + +# Solr +SOLR_IMAGE_VERSION=2.10-solr9 +CKAN_SOLR_URL=http://solr:8983/solr/ckan +TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan + +# Redis +REDIS_VERSION=6 +CKAN_REDIS_URL=redis://redis:6379/1 +TEST_CKAN_REDIS_URL=redis://redis:6379/1 + +# Datapusher +DATAPUSHER_VERSION=0.0.20 +CKAN_DATAPUSHER_URL=http://datapusher:8800 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 +DATAPUSHER_REWRITE_RESOURCES=True +DATAPUSHER_REWRITE_URL=http://ckan:5000 + +# NGINX +NGINX_PORT=80 +NGINX_SSLPORT=443 + +# Extensions +CKAN__PLUGINS="envvars image_view text_view recline_view datastore datapusher" +CKAN__HARVEST__MQ__TYPE=redis +CKAN__HARVEST__MQ__HOSTNAME=redis +CKAN__HARVEST__MQ__PORT=6379 +CKAN__HARVEST__MQ__REDIS_DB=1 diff --git a/.gitignore b/.gitignore index 709d1b96..2afe5828 100755 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ local/* # Docker Compose environment and password files .env .pw +docker-compose.yml.ORIG diff --git a/README.md b/README.md index cd823765..c6ac1c05 100644 --- a/README.md +++ b/README.md @@ -60,10 +60,13 @@ For additional security it may be a better option to generate randomised passwor * The database user that has read-access to the Datastore database (user=DATASTORE_READONLY_USER , password=DATASTORE_READONLY_PASSWORD) * The CKAN application System Administrator user (user=CKAN_SYSADMIN_NAME ,password=CKAN_SYSADMIN_PASSWORD) -You will need to generate the passwords. You can do this by running `generate_passwords.sh` before building the local images and containers. This will create a password file called .pw. -You will then need to rename the docker-compose-use-generated-passwords.yml file to docker-compose.yml. You may wish to save the docker-compose.yml file beforehand. +You can do this by the following: -You can then carry on with the Base mode instructions disregarding details about the sysadmin user +* remove all containers, images, networks, volumes plus run a `docker system prune` +* generate the passwords by running `generate_passwords.sh` this will create a file called `.pw` +* rename the `docker-compose-use-generated-passwords.yml` file to `docker-compose.yml`. You may wish to save the docker-compose.yml file beforehand. +* rename the `.env.use-generated-passwords` file to `.env`. You should save the .env file beforehand. +* build and run the docker compose stack as per normal. The ckan_admin user password will be located in the .pw file #### Base mode diff --git a/docker-compose-use-generated-passwords.yml b/docker-compose-use-generated-passwords.yml index 3513c431..d6e68e17 100755 --- a/docker-compose-use-generated-passwords.yml +++ b/docker-compose-use-generated-passwords.yml @@ -68,13 +68,13 @@ services: - dbnet environment: - POSTGRES_USER - - POSTGRES_PASSWORD + #- POSTGRES_PASSWORD - POSTGRES_DB - CKAN_DB_USER - - CKAN_DB_PASSWORD + #- CKAN_DB_PASSWORD - CKAN_DB - DATASTORE_READONLY_USER - - DATASTORE_READONLY_PASSWORD + #- DATASTORE_READONLY_PASSWORD - DATASTORE_DB env_file: - .pw diff --git a/docker-compose.yml b/docker-compose.yml index 0f5330fb..d6e68e17 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,7 @@ services: - redisnet env_file: - .env + - .pw depends_on: db: condition: service_healthy @@ -67,14 +68,16 @@ services: - dbnet environment: - POSTGRES_USER - - POSTGRES_PASSWORD + #- POSTGRES_PASSWORD - POSTGRES_DB - CKAN_DB_USER - - CKAN_DB_PASSWORD + #- CKAN_DB_PASSWORD - CKAN_DB - DATASTORE_READONLY_USER - - DATASTORE_READONLY_PASSWORD + #- DATASTORE_READONLY_PASSWORD - DATASTORE_DB + env_file: + - .pw volumes: - pg_data:/var/lib/postgresql/data restart: unless-stopped diff --git a/generate_passwords.py b/generate_passwords.py index a7bee8d7..dd4b5497 100644 --- a/generate_passwords.py +++ b/generate_passwords.py @@ -24,13 +24,13 @@ CKAN_DB_USER = os.environ.get('CKAN_DB_USER') CKAN_DB = os.environ.get('CKAN_DB') DATASTORE_DB_USER = os.environ.get('DATASTORE_DB_USER') -DATASTORE_DB_READONLY_USER = os.environ.get('DATASTORE_DB_READONLY_USER') +DATASTORE_READONLY_USER = os.environ.get('DATASTORE_READONLY_USER') DATASTORE_DB = os.environ.get('DATASTORE_DB') POSTGRES_HOST = os.environ.get('POSTGRES_HOST') with open(fn, 'a') as f: f.write(f"CKAN_SQLALCHEMY_URL=postgresql://{CKAN_DB_USER}:{CKAN_DB_PASSWORD}@{POSTGRES_HOST}/{CKAN_DB}\n") f.write(f"CKAN_DATASTORE_WRITE_URL=postgresql://{CKAN_DB_USER}:{CKAN_DB_PASSWORD}@{POSTGRES_HOST}/{DATASTORE_DB}\n") - f.write(f"CKAN_DATASTORE_READ_URL=postgresql://{DATASTORE_DB_READONLY_USER}:{DATASTORE_READONLY_PASSWORD}@{POSTGRES_HOST}/{DATASTORE_DB}\n") + f.write(f"CKAN_DATASTORE_READ_URL=postgresql://{DATASTORE_READONLY_USER}:{DATASTORE_READONLY_PASSWORD}@{POSTGRES_HOST}/{DATASTORE_DB}\n") -print("[setup_passwords] password file: '.pw' created successfully") \ No newline at end of file +print("[setup_passwords] password file: '.pw' created successfully") diff --git a/postgresql/docker-entrypoint-initdb.d/30_setup_test_databases.sql b/postgresql/docker-entrypoint-initdb.d/30_setup_test_databases.sh similarity index 100% rename from postgresql/docker-entrypoint-initdb.d/30_setup_test_databases.sql rename to postgresql/docker-entrypoint-initdb.d/30_setup_test_databases.sh From 095ac0a297286524c90a3efd6068faca4f54cc6f Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 10 Aug 2023 16:24:54 +0100 Subject: [PATCH 3/8] Updates to README --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c6ac1c05..ebe588e4 100644 --- a/README.md +++ b/README.md @@ -55,10 +55,10 @@ more information. For additional security it may be a better option to generate randomised passwords for the database users and the for the initial CKAN application sysadmin user. Here are the users that will have randomised passwords generated: -* The Postgres system superuser (user=POSTGRES_USER , password=POSTGRES_PASSWORD) -* The database user that owns the CKAN database (user=CKAN_DB_USER , password=CKAN_DB_PASSWORD) -* The database user that has read-access to the Datastore database (user=DATASTORE_READONLY_USER , password=DATASTORE_READONLY_PASSWORD) -* The CKAN application System Administrator user (user=CKAN_SYSADMIN_NAME ,password=CKAN_SYSADMIN_PASSWORD) +* The Postgres system superuser (user=`POSTGRES_USER` , password=`POSTGRES_PASSWORD`) +* The database user that owns the CKAN database (user=`CKAN_DB_USER` , password=C`KAN_DB_PASSWORD`) +* The database user that has read-access to the Datastore database (user=`DATASTORE_READONLY_USER` , password=`DATASTORE_READONLY_PASSWORD`) +* The CKAN application System Administrator user (user=`CKAN_SYSADMIN_NAME` ,password=`CKAN_SYSADMIN_PASSWORD`) You can do this by the following: @@ -66,7 +66,7 @@ You can do this by the following: * generate the passwords by running `generate_passwords.sh` this will create a file called `.pw` * rename the `docker-compose-use-generated-passwords.yml` file to `docker-compose.yml`. You may wish to save the docker-compose.yml file beforehand. * rename the `.env.use-generated-passwords` file to `.env`. You should save the .env file beforehand. -* build and run the docker compose stack as per normal. The ckan_admin user password will be located in the .pw file +* build and run the docker compose stack as per normal. The `ckan_admin user password` will be located in the `.pw` file #### Base mode From 77832207eeb8e7f175bd2c5b4e6c91678d75f254 Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 16 Aug 2023 10:59:16 +0200 Subject: [PATCH 4/8] tidy up --- README.md | 2 +- generate_passwords.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index ebe588e4..2f08ea9b 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ You can do this by the following: * generate the passwords by running `generate_passwords.sh` this will create a file called `.pw` * rename the `docker-compose-use-generated-passwords.yml` file to `docker-compose.yml`. You may wish to save the docker-compose.yml file beforehand. * rename the `.env.use-generated-passwords` file to `.env`. You should save the .env file beforehand. -* build and run the docker compose stack as per normal. The `ckan_admin user password` will be located in the `.pw` file +* build and run the docker compose stack as per normal. The `ckan_admin` user password will be located in the `.pw` file #### Base mode diff --git a/generate_passwords.sh b/generate_passwords.sh index 4f910e3e..e5ab2a94 100755 --- a/generate_passwords.sh +++ b/generate_passwords.sh @@ -4,7 +4,6 @@ set -a . ./.env pwfile=".pw" touch ${pwfile}; chmod 600 ${pwfile} -#truncate -s 0 ${pwfile} python3 ./generate_passwords.py chmod 400 ${pwfile} sleep 1 \ No newline at end of file From 5f2951b3efe160880fbc9ec5f2c6d510f4f1eb00 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 4 Sep 2023 10:57:14 +0200 Subject: [PATCH 5/8] Updates to generate-random-passwords.py --- README.md | 4 +--- generate_passwords.py | 24 ++++++++++++++++++------ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 2f08ea9b..91542939 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ # Docker Compose setup for CKAN - * [Overview](#overview) * [Installing Docker](#installing-docker) * [docker compose vs docker-compose](#docker-compose-vs-docker-compose) @@ -45,8 +44,7 @@ versions for client and server. ## 3. docker compose *vs* docker-compose All Docker Compose commands in this README will use the V2 version of Compose ie: `docker compose`. The older version (V1) -used the `docker-compose` command. Please see [Docker Compose](https://docs.docker.com/compose/compose-v2/) for -more information. +used the `docker-compose` command. Please see [Docker Compose](https://docs.docker.com/compose/compose-v2/) for more information. ## 4. Install (build and run) CKAN plus dependencies diff --git a/generate_passwords.py b/generate_passwords.py index dd4b5497..9db22f3d 100644 --- a/generate_passwords.py +++ b/generate_passwords.py @@ -1,25 +1,35 @@ import base64 import os +import secrets fn = ".pw" vn = {} -pwvars = ["POSTGRES_PASSWORD", "CKAN_DB_PASSWORD", "DATASTORE_READONLY_PASSWORD","CKAN_SYSADMIN_PASSWORD"] +pwvars = ["POSTGRES_PASSWORD", "CKAN_DB_PASSWORD", "DATASTORE_READONLY_PASSWORD","CKAN_SYSADMIN_PASSWORD", \ + "CKAN___BEAKER__SESSION__SECRET","CKAN___API_TOKEN__JWT__ENCODE__SECRET"] -print("[setup_passwords] attempting to setup secure passwords") +print("\n[setup_passwords] attempting to setup secure passwords") with open(fn, 'w') as f: + plen = 16 f.truncate(0) for pwvar in pwvars: - with open('/dev/urandom', 'rb') as rand_file: - pw = base64.urlsafe_b64encode(rand_file.read(12)).decode('utf-8') - f.write(f"{pwvar}={pw}\n") - vn[pwvar] = pw + pw = secrets.token_urlsafe(plen) + f.write(f"{pwvar}={pw}\n") + vn[pwvar] = pw + POSTGRES_PASSWORD = vn["POSTGRES_PASSWORD"] CKAN_DB_PASSWORD = vn["CKAN_DB_PASSWORD"] DATASTORE_READONLY_PASSWORD = vn["DATASTORE_READONLY_PASSWORD"] CKAN_SYSADMIN_PASSWORD = vn["CKAN_SYSADMIN_PASSWORD"] +CKAN___BEAKER__SESSION__SECRET = vn["CKAN___BEAKER__SESSION__SECRET"] +CKAN___API_TOKEN__JWT__ENCODE__SECRET = vn["CKAN___API_TOKEN__JWT__ENCODE__SECRET"] +CKAN___API_TOKEN__JWT__DECODE__SECRET = vn["CKAN___API_TOKEN__JWT__ENCODE__SECRET"] + +# Write the same secret for decoding as encoding +with open(fn, 'a') as f: + f.write(f"CKAN___API_TOKEN__JWT__DECODE__SECRET={CKAN___API_TOKEN__JWT__DECODE__SECRET}\n") CKAN_DB_USER = os.environ.get('CKAN_DB_USER') CKAN_DB = os.environ.get('CKAN_DB') @@ -28,9 +38,11 @@ DATASTORE_DB = os.environ.get('DATASTORE_DB') POSTGRES_HOST = os.environ.get('POSTGRES_HOST') +# Now write out the Database URL's with open(fn, 'a') as f: f.write(f"CKAN_SQLALCHEMY_URL=postgresql://{CKAN_DB_USER}:{CKAN_DB_PASSWORD}@{POSTGRES_HOST}/{CKAN_DB}\n") f.write(f"CKAN_DATASTORE_WRITE_URL=postgresql://{CKAN_DB_USER}:{CKAN_DB_PASSWORD}@{POSTGRES_HOST}/{DATASTORE_DB}\n") f.write(f"CKAN_DATASTORE_READ_URL=postgresql://{DATASTORE_READONLY_USER}:{DATASTORE_READONLY_PASSWORD}@{POSTGRES_HOST}/{DATASTORE_DB}\n") print("[setup_passwords] password file: '.pw' created successfully") +print("\nThe CKAN_SYSADMIN_PASSWORD password is: " + CKAN_SYSADMIN_PASSWORD + "\n") From 27bae40efb59aad19fbd0064f0e913a5c20f0c40 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 12 Sep 2023 14:19:22 +0200 Subject: [PATCH 6/8] Update overrides and generate_passwords.py --- ckan/setup/prerun.py.override | 7 +++++++ ckan/setup/start_ckan.sh.override | 4 ++-- generate_passwords.py | 16 +++++++++++----- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/ckan/setup/prerun.py.override b/ckan/setup/prerun.py.override index 3d686969..d8b79219 100644 --- a/ckan/setup/prerun.py.override +++ b/ckan/setup/prerun.py.override @@ -194,6 +194,13 @@ def create_sysadmin(): subprocess.call(command) print("[prerun] Made user {0} a sysadmin".format(name)) + # cleanup permissions + # We're running as root before pivoting to uwsgi and dropping privs + data_dir = "%s/storage" % os.environ['CKAN_STORAGE_PATH'] + + command = ["chown", "-R", "ckan:ckan", data_dir] + subprocess.call(command) + print("[prerun] Ensured storage directory is owned by ckan") if __name__ == "__main__": diff --git a/ckan/setup/start_ckan.sh.override b/ckan/setup/start_ckan.sh.override index 0c8409c9..5927deac 100755 --- a/ckan/setup/start_ckan.sh.override +++ b/ckan/setup/start_ckan.sh.override @@ -16,7 +16,7 @@ then fi # Run the prerun script to init CKAN and create the default admin user -sudo -u ckan -EH python3 prerun.py +python3 prerun.py echo "Set up ckan.datapusher.api_token in the CKAN config file" ckan config-tool $CKAN_INI "ckan.datapusher.api_token=$(ckan -c $CKAN_INI user token add ckan_admin datapusher | tail -n 1 | tr -d '\t')" @@ -51,7 +51,7 @@ then # Start supervisord supervisord --configuration /etc/supervisord.conf & # Start uwsgi - sudo -u ckan -EH uwsgi $UWSGI_OPTS + uwsgi $UWSGI_OPTS else echo "[prerun] failed...not starting CKAN." fi diff --git a/generate_passwords.py b/generate_passwords.py index 9db22f3d..3a474a38 100644 --- a/generate_passwords.py +++ b/generate_passwords.py @@ -6,7 +6,7 @@ vn = {} pwvars = ["POSTGRES_PASSWORD", "CKAN_DB_PASSWORD", "DATASTORE_READONLY_PASSWORD","CKAN_SYSADMIN_PASSWORD", \ - "CKAN___BEAKER__SESSION__SECRET","CKAN___API_TOKEN__JWT__ENCODE__SECRET"] + "CKAN___BEAKER__SESSION__SECRET"] print("\n[setup_passwords] attempting to setup secure passwords") @@ -19,18 +19,24 @@ vn[pwvar] = pw +# Set up the environment variables from the values in the .pw file POSTGRES_PASSWORD = vn["POSTGRES_PASSWORD"] CKAN_DB_PASSWORD = vn["CKAN_DB_PASSWORD"] DATASTORE_READONLY_PASSWORD = vn["DATASTORE_READONLY_PASSWORD"] CKAN_SYSADMIN_PASSWORD = vn["CKAN_SYSADMIN_PASSWORD"] CKAN___BEAKER__SESSION__SECRET = vn["CKAN___BEAKER__SESSION__SECRET"] -CKAN___API_TOKEN__JWT__ENCODE__SECRET = vn["CKAN___API_TOKEN__JWT__ENCODE__SECRET"] -CKAN___API_TOKEN__JWT__DECODE__SECRET = vn["CKAN___API_TOKEN__JWT__ENCODE__SECRET"] -# Write the same secret for decoding as encoding +# The API_TOKEN is a JWT token, which is a special case +jwtpw = secrets.token_urlsafe(plen) + with open(fn, 'a') as f: - f.write(f"CKAN___API_TOKEN__JWT__DECODE__SECRET={CKAN___API_TOKEN__JWT__DECODE__SECRET}\n") + f.write(f"CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:" + str(jwtpw) + "\n") + f.write(f"CKAN___API_TOKEN__JWT__DECODE__SECRET=string:" + str(jwtpw) + "\n") + +CKAN___API_TOKEN__JWT__ENCODE__SECRET = "string:" + str(jwtpw) +CKAN___API_TOKEN__JWT__DECODE__SECRET = "string:" + str(jwtpw) +# Now the database URL's which include the password generated above CKAN_DB_USER = os.environ.get('CKAN_DB_USER') CKAN_DB = os.environ.get('CKAN_DB') DATASTORE_DB_USER = os.environ.get('DATASTORE_DB_USER') From 5fd5d634155d61502b979164af54670b82bb8d36 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 19 Sep 2023 10:39:52 +0200 Subject: [PATCH 7/8] finishing off, including README --- .env.example => .env.insecure | 0 .env.use-generated-passwords => .env.secure | 0 .gitignore | 3 +- README.md | 52 ++++++--- docker-compose-use-generated-passwords.yml | 4 +- docker-compose.dev.insecure.yml | 76 +++++++++++++ docker-compose.dev.secure.yml | 77 +++++++++++++ docker-compose.dev.yml | 8 +- docker-compose.insecure.yml | 112 +++++++++++++++++++ docker-compose.secure.yml | 112 +++++++++++++++++++ docker-compose.yml | 9 +- generate_passwords.py | 113 +++++++++++++------- generate_passwords.sh | 8 +- 13 files changed, 506 insertions(+), 68 deletions(-) rename .env.example => .env.insecure (100%) rename .env.use-generated-passwords => .env.secure (100%) create mode 100755 docker-compose.dev.insecure.yml create mode 100755 docker-compose.dev.secure.yml create mode 100755 docker-compose.insecure.yml create mode 100755 docker-compose.secure.yml diff --git a/.env.example b/.env.insecure similarity index 100% rename from .env.example rename to .env.insecure diff --git a/.env.use-generated-passwords b/.env.secure similarity index 100% rename from .env.use-generated-passwords rename to .env.secure diff --git a/.gitignore b/.gitignore index 2afe5828..d9510f5e 100755 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,6 @@ local/* # Docker Compose environment and password files .env -.pw +.ckpw +.dbpw docker-compose.yml.ORIG diff --git a/README.md b/README.md index 91542939..b860ca36 100644 --- a/README.md +++ b/README.md @@ -48,36 +48,51 @@ used the `docker-compose` command. Please see [Docker Compose](https://docs.dock ## 4. Install (build and run) CKAN plus dependencies -#### Generate Randomised passwords mode +#### Generate randomised passwords mode (secure/default mode) -For additional security it may be a better option to generate randomised passwords for the database users and the for the initial CKAN application sysadmin user. Here are the users that will have randomised - passwords generated: +For additional security the default option is to run in "secure" mode. To do this a script will need to be run to generate randomised passwords for the database users and for the initial CKAN application sysadmin user (ckan_admin). It will also generate some other parameters detailed below: -* The Postgres system superuser (user=`POSTGRES_USER` , password=`POSTGRES_PASSWORD`) -* The database user that owns the CKAN database (user=`CKAN_DB_USER` , password=C`KAN_DB_PASSWORD`) -* The database user that has read-access to the Datastore database (user=`DATASTORE_READONLY_USER` , password=`DATASTORE_READONLY_PASSWORD`) -* The CKAN application System Administrator user (user=`CKAN_SYSADMIN_NAME` ,password=`CKAN_SYSADMIN_PASSWORD`) +Here are the users that will have randomised passwords generated: +* The Postgres system superuser (user: `POSTGRES_USER` , password: `POSTGRES_PASSWORD`) +* The database user that owns the CKAN database (user: `CKAN_DB_USER` , password: `CKAN_DB_PASSWORD`) +* The database user that has read-access to the Datastore database (user: `DATASTORE_READONLY_USER` , password: `DATASTORE_READONLY_PASSWORD`) +* The CKAN application System Administrator user (user: `CKAN_SYSADMIN_NAME` ,password: `CKAN_SYSADMIN_PASSWORD`) + +There are other configuration parameters that will also have random secrets generated. They are as follows: +* `CKAN___BEAKER__SESSION__SECRET` +* `CKAN___API_TOKEN__JWT__ENCODE__SECRET` +* `CKAN___API_TOKEN__JWT__DECODE__SECRET` + +Also, database URL's will need to include these new passwords so the values for this configuration parameters will be added to the generated (CKAN) file +* `CKAN_SQLALCHEMY_URL` +* `CKAN_DATASTORE_WRITE_URL` +* `CKAN_DATASTORE_READ_URL` You can do this by the following: -* remove all containers, images, networks, volumes plus run a `docker system prune` -* generate the passwords by running `generate_passwords.sh` this will create a file called `.pw` -* rename the `docker-compose-use-generated-passwords.yml` file to `docker-compose.yml`. You may wish to save the docker-compose.yml file beforehand. -* rename the `.env.use-generated-passwords` file to `.env`. You should save the .env file beforehand. -* build and run the docker compose stack as per normal. The `ckan_admin` user password will be located in the `.pw` file +* generate the passwords by running `generate_passwords.sh` this will create two files: `.ckpw` and `.dbpw` +* by default the `docker-compose.yml` file is set to secure mode. +* if you make any subsequent changes to the `docker-compose.yml` file you can always revert back to the original file by referencing `docker-compose.secure.yml` +* by default the `.env` file is set to secure mode. +* if you make changes to the `.env` file you can always revert back to the original file by referencing `.env.secure` +* build and run the docker compose stack as per normal. +* the `ckan_admin` user password will be located in the `.ckpw` file. It will also be displayed on the terminal when running `generate_passwords.sh` + +#### Insecure mode (the old legacy mode) -#### Base mode +Use this if you do not wish to use secure passwords ie: how it used to be -Use this if you are a maintainer and will not be making code changes to CKAN or to CKAN extensions +Copy the included `.env.insecure` to `.env`. Modify it depending on your own needs. It's wise to keep the original `.env.secure` file in case +you need to revert back to it or reference from it -Copy the included `.env.example` and rename it to `.env`. Modify it depending on your own needs. +Copy the included `docker-compose.insecure.yml` and to `docker-compose.yml`. Modify it depending on your own needs. Please note that when accessing CKAN directly (via a browser) ie: not going through NGINX you will need to make sure you have "ckan" set up to be an alias to localhost in the local hosts file. Either that or you will need to change the `.env` entry for CKAN_SITE_URL Using the default values on the `.env.example` file will get you a working CKAN instance. There is a sysadmin user created by default with the values defined in `CKAN_SYSADMIN_NAME` and `CKAN_SYSADMIN_PASSWORD`(`ckan_admin` and `test1234` by default). This should be obviously changed before running this setup as a public CKAN instance. -To build the images: +### To build the images: docker compose build @@ -100,6 +115,11 @@ After this step, CKAN should be running at `CKAN_SITE_URL`. Use this mode if you are making code changes to CKAN and either creating new extensions or making code changes to existing extensions. This mode also uses the `.env` file for config options. +Again, the default for Development mode is secure. Please see the section above on Secure mode. There are 2 reference compose `.yml` files + +* `docker-compose.dev.secure.yml` - this is the same as the default file `docker-compose.dev.yml` +* `docker-compose.dev.insecure.yml` - to be used if the insecure (ie: legacy) config is to be used + To develop local extensions use the `docker-compose.dev.yml` file: To build the images: diff --git a/docker-compose-use-generated-passwords.yml b/docker-compose-use-generated-passwords.yml index d6e68e17..61ee1ce8 100755 --- a/docker-compose-use-generated-passwords.yml +++ b/docker-compose-use-generated-passwords.yml @@ -36,7 +36,7 @@ services: - redisnet env_file: - .env - - .pw + - .ckpw depends_on: db: condition: service_healthy @@ -77,7 +77,7 @@ services: #- DATASTORE_READONLY_PASSWORD - DATASTORE_DB env_file: - - .pw + - .ckpw volumes: - pg_data:/var/lib/postgresql/data restart: unless-stopped diff --git a/docker-compose.dev.insecure.yml b/docker-compose.dev.insecure.yml new file mode 100755 index 00000000..0cf63126 --- /dev/null +++ b/docker-compose.dev.insecure.yml @@ -0,0 +1,76 @@ +version: "3" + +volumes: + ckan_storage: + pg_data: + solr_data: + +services: + + ckan-dev: + container_name: ${CKAN_CONTAINER_NAME} + build: + context: ckan/ + dockerfile: Dockerfile.dev + args: + - TZ=${TZ} + env_file: + - .env + depends_on: + db: + condition: service_healthy + solr: + condition: service_healthy + redis: + condition: service_healthy + ports: + - "0.0.0.0:${CKAN_PORT_HOST}:${CKAN_PORT}" + volumes: + - ckan_storage:/var/lib/ckan + - ./src:/srv/app/src_extensions + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] + + datapusher: + container_name: ${DATAPUSHER_CONTAINER_NAME} + image: ckan/ckan-base-datapusher:${DATAPUSHER_VERSION} + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8800"] + + db: + container_name: ${POSTGRESQL_CONTAINER_NAME} + build: + context: postgresql/ + environment: + - POSTGRES_USER + - POSTGRES_PASSWORD + - POSTGRES_DB + - CKAN_DB_USER + - CKAN_DB_PASSWORD + - CKAN_DB + - DATASTORE_READONLY_USER + - DATASTORE_READONLY_PASSWORD + - DATASTORE_DB + volumes: + - pg_data:/var/lib/postgresql/data + restart: unless-stopped + healthcheck: + test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"] + + solr: + container_name: ${SOLR_CONTAINER_NAME} + image: ckan/ckan-solr:${SOLR_IMAGE_VERSION} + volumes: + - solr_data:/var/solr + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"] + + redis: + container_name: ${REDIS_CONTAINER_NAME} + image: redis:${REDIS_VERSION} + restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "-e", "QUIT"] \ No newline at end of file diff --git a/docker-compose.dev.secure.yml b/docker-compose.dev.secure.yml new file mode 100755 index 00000000..5c664604 --- /dev/null +++ b/docker-compose.dev.secure.yml @@ -0,0 +1,77 @@ +version: "3" + + +volumes: + ckan_storage: + pg_data: + solr_data: + +services: + + ckan-dev: + container_name: ${CKAN_CONTAINER_NAME} + build: + context: ckan/ + dockerfile: Dockerfile.dev + args: + - TZ=${TZ} + env_file: + - .env + - .ckpw + depends_on: + db: + condition: service_healthy + solr: + condition: service_healthy + redis: + condition: service_healthy + ports: + - "0.0.0.0:${CKAN_PORT_HOST}:${CKAN_PORT}" + volumes: + - ckan_storage:/var/lib/ckan + - ./src:/srv/app/src_extensions + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] + + datapusher: + container_name: ${DATAPUSHER_CONTAINER_NAME} + image: ckan/ckan-base-datapusher:${DATAPUSHER_VERSION} + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8800"] + + db: + container_name: ${POSTGRESQL_CONTAINER_NAME} + build: + context: postgresql/ + environment: + - POSTGRES_USER + - POSTGRES_DB + - CKAN_DB_USER + - CKAN_DB + - DATASTORE_READONLY_USER + - DATASTORE_DB + env_file: + - .dbpw + volumes: + - pg_data:/var/lib/postgresql/data + restart: unless-stopped + healthcheck: + test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"] + + solr: + container_name: ${SOLR_CONTAINER_NAME} + image: ckan/ckan-solr:${SOLR_IMAGE_VERSION} + volumes: + - solr_data:/var/solr + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"] + + redis: + container_name: ${REDIS_CONTAINER_NAME} + image: redis:${REDIS_VERSION} + restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "-e", "QUIT"] \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 2bc7894c..0cf63126 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -44,9 +44,15 @@ services: build: context: postgresql/ environment: - - DATASTORE_READONLY_PASSWORD + - POSTGRES_USER - POSTGRES_PASSWORD + - POSTGRES_DB + - CKAN_DB_USER - CKAN_DB_PASSWORD + - CKAN_DB + - DATASTORE_READONLY_USER + - DATASTORE_READONLY_PASSWORD + - DATASTORE_DB volumes: - pg_data:/var/lib/postgresql/data restart: unless-stopped diff --git a/docker-compose.insecure.yml b/docker-compose.insecure.yml new file mode 100755 index 00000000..0f5330fb --- /dev/null +++ b/docker-compose.insecure.yml @@ -0,0 +1,112 @@ +version: "3" + + +volumes: + ckan_storage: + pg_data: + solr_data: + +services: + + nginx: + container_name: ${NGINX_CONTAINER_NAME} + build: + context: nginx/ + dockerfile: Dockerfile + networks: + - webnet + - ckannet + depends_on: + ckan: + condition: service_healthy + ports: + - "0.0.0.0:${NGINX_SSLPORT_HOST}:${NGINX_SSLPORT}" + + ckan: + container_name: ${CKAN_CONTAINER_NAME} + build: + context: ckan/ + dockerfile: Dockerfile + args: + - TZ=${TZ} + networks: + - ckannet + - dbnet + - solrnet + - redisnet + env_file: + - .env + depends_on: + db: + condition: service_healthy + solr: + condition: service_healthy + redis: + condition: service_healthy + volumes: + - ckan_storage:/var/lib/ckan + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] + + datapusher: + container_name: ${DATAPUSHER_CONTAINER_NAME} + networks: + - ckannet + - dbnet + image: ckan/ckan-base-datapusher:${DATAPUSHER_VERSION} + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8800"] + + db: + container_name: ${POSTGRESQL_CONTAINER_NAME} + build: + context: postgresql/ + networks: + - dbnet + environment: + - POSTGRES_USER + - POSTGRES_PASSWORD + - POSTGRES_DB + - CKAN_DB_USER + - CKAN_DB_PASSWORD + - CKAN_DB + - DATASTORE_READONLY_USER + - DATASTORE_READONLY_PASSWORD + - DATASTORE_DB + volumes: + - pg_data:/var/lib/postgresql/data + restart: unless-stopped + healthcheck: + test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"] + + solr: + container_name: ${SOLR_CONTAINER_NAME} + networks: + - solrnet + image: ckan/ckan-solr:${SOLR_IMAGE_VERSION} + volumes: + - solr_data:/var/solr + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"] + + redis: + container_name: ${REDIS_CONTAINER_NAME} + image: redis:${REDIS_VERSION} + networks: + - redisnet + restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "-e", "QUIT"] + +networks: + webnet: + ckannet: + solrnet: + internal: true + dbnet: + internal: true + redisnet: + internal: true diff --git a/docker-compose.secure.yml b/docker-compose.secure.yml new file mode 100755 index 00000000..de5b5f8c --- /dev/null +++ b/docker-compose.secure.yml @@ -0,0 +1,112 @@ +version: "3" + + +volumes: + ckan_storage: + pg_data: + solr_data: + +services: + + nginx: + container_name: ${NGINX_CONTAINER_NAME} + build: + context: nginx/ + dockerfile: Dockerfile + networks: + - webnet + - ckannet + depends_on: + ckan: + condition: service_healthy + ports: + - "0.0.0.0:${NGINX_SSLPORT_HOST}:${NGINX_SSLPORT}" + + ckan: + container_name: ${CKAN_CONTAINER_NAME} + build: + context: ckan/ + dockerfile: Dockerfile + args: + - TZ=${TZ} + networks: + - ckannet + - dbnet + - solrnet + - redisnet + env_file: + - .env + - .ckpw + depends_on: + db: + condition: service_healthy + solr: + condition: service_healthy + redis: + condition: service_healthy + volumes: + - ckan_storage:/var/lib/ckan + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] + + datapusher: + container_name: ${DATAPUSHER_CONTAINER_NAME} + networks: + - ckannet + - dbnet + image: ckan/ckan-base-datapusher:${DATAPUSHER_VERSION} + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8800"] + + db: + container_name: ${POSTGRESQL_CONTAINER_NAME} + build: + context: postgresql/ + networks: + - dbnet + environment: + - POSTGRES_USER + - POSTGRES_DB + - CKAN_DB_USER + - CKAN_DB + - DATASTORE_READONLY_USER + - DATASTORE_DB + env_file: + - .dbpw + volumes: + - pg_data:/var/lib/postgresql/data + restart: unless-stopped + healthcheck: + test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"] + + solr: + container_name: ${SOLR_CONTAINER_NAME} + networks: + - solrnet + image: ckan/ckan-solr:${SOLR_IMAGE_VERSION} + volumes: + - solr_data:/var/solr + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"] + + redis: + container_name: ${REDIS_CONTAINER_NAME} + image: redis:${REDIS_VERSION} + networks: + - redisnet + restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "-e", "QUIT"] + +networks: + webnet: + ckannet: + solrnet: + internal: true + dbnet: + internal: true + redisnet: + internal: true diff --git a/docker-compose.yml b/docker-compose.yml index d6e68e17..0f5330fb 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,7 +36,6 @@ services: - redisnet env_file: - .env - - .pw depends_on: db: condition: service_healthy @@ -68,16 +67,14 @@ services: - dbnet environment: - POSTGRES_USER - #- POSTGRES_PASSWORD + - POSTGRES_PASSWORD - POSTGRES_DB - CKAN_DB_USER - #- CKAN_DB_PASSWORD + - CKAN_DB_PASSWORD - CKAN_DB - DATASTORE_READONLY_USER - #- DATASTORE_READONLY_PASSWORD + - DATASTORE_READONLY_PASSWORD - DATASTORE_DB - env_file: - - .pw volumes: - pg_data:/var/lib/postgresql/data restart: unless-stopped diff --git a/generate_passwords.py b/generate_passwords.py index 3a474a38..22dcb330 100644 --- a/generate_passwords.py +++ b/generate_passwords.py @@ -1,54 +1,89 @@ import base64 import os import secrets +import logging -fn = ".pw" -vn = {} +# Password length set to 16 characters +plen = 16 -pwvars = ["POSTGRES_PASSWORD", "CKAN_DB_PASSWORD", "DATASTORE_READONLY_PASSWORD","CKAN_SYSADMIN_PASSWORD", \ - "CKAN___BEAKER__SESSION__SECRET"] +def generate_passwords(variables): + passwords = {} + for variable in variables: + password = secrets.token_urlsafe(plen) + passwords[variable] = password + return passwords -print("\n[setup_passwords] attempting to setup secure passwords") +def write_passwords_to_file(passwords, filename): + try: + with open(filename, 'w') as f: + for variable, password in passwords.items(): + if variable == "CKAN___API_TOKEN__JWT__SECRET" : + # Prepend 'string:' to the API token secret + prepended_apitoken = "string:" + passwords['CKAN___API_TOKEN__JWT__SECRET'] + f.write(f"CKAN___API_TOKEN__JWT__ENCODE__SECRET={prepended_apitoken}\n") + os.environ['CKAN___API_TOKEN__JWT__ENCODE__SECRET'] = prepended_apitoken + f.write(f"CKAN___API_TOKEN__JWT__DECODE__SECRET={prepended_apitoken}\n") + os.environ['CKAN___API_TOKEN__JWT__DECODE__SECRET'] = prepended_apitoken + else: + f.write(f"{variable}={password}\n") + os.environ[variable] = password + print(f"[generate_passwords] Passwords written to '{filename}' successfully.") + except Exception as e: + logging.error(f"Failed to write passwords to '{filename}': {str(e)}") -with open(fn, 'w') as f: - plen = 16 - f.truncate(0) - for pwvar in pwvars: - pw = secrets.token_urlsafe(plen) - f.write(f"{pwvar}={pw}\n") - vn[pwvar] = pw +def write_urls_to_file(filename): + try: + with open(filename, 'a') as f: + CKAN_DB_USER = os.environ.get('CKAN_DB_USER') + CKAN_DB_PASSWORD = os.environ.get('CKAN_DB_PASSWORD') + CKAN_DB = os.environ.get('CKAN_DB') + DATASTORE_READONLY_USER = os.environ.get('DATASTORE_READONLY_USER') + DATASTORE_READONLY_PASSWORD = os.environ.get('DATASTORE_READONLY_PASSWORD') + DATASTORE_DB = os.environ.get('DATASTORE_DB') + POSTGRES_HOST = os.environ.get('POSTGRES_HOST') + f.write(f"CKAN_SQLALCHEMY_URL=postgresql://{CKAN_DB_USER}:{CKAN_DB_PASSWORD}@{POSTGRES_HOST}/{CKAN_DB}\n") + f.write(f"CKAN_DATASTORE_WRITE_URL=postgresql://{CKAN_DB_USER}:{CKAN_DB_PASSWORD}@{POSTGRES_HOST}/{DATASTORE_DB}\n") + f.write(f"CKAN_DATASTORE_READ_URL=postgresql://{DATASTORE_READONLY_USER}:{DATASTORE_READONLY_PASSWORD}@{POSTGRES_HOST}/{DATASTORE_DB}\n") + print(f"[generate_passwords] Database URL's written to '{filename}' successfully.") + except Exception as e: + logging.error(f"Failed to write database URL's to '{filename}': {str(e)}") +def main(): + pwvars = [ + "POSTGRES_PASSWORD", + "CKAN_DB_PASSWORD", + "DATASTORE_READONLY_PASSWORD", + "CKAN_SYSADMIN_PASSWORD", + "CKAN___BEAKER__SESSION__SECRET", + "CKAN___API_TOKEN__JWT__SECRET", + ] -# Set up the environment variables from the values in the .pw file -POSTGRES_PASSWORD = vn["POSTGRES_PASSWORD"] -CKAN_DB_PASSWORD = vn["CKAN_DB_PASSWORD"] -DATASTORE_READONLY_PASSWORD = vn["DATASTORE_READONLY_PASSWORD"] -CKAN_SYSADMIN_PASSWORD = vn["CKAN_SYSADMIN_PASSWORD"] -CKAN___BEAKER__SESSION__SECRET = vn["CKAN___BEAKER__SESSION__SECRET"] + vn = generate_passwords(pwvars) -# The API_TOKEN is a JWT token, which is a special case -jwtpw = secrets.token_urlsafe(plen) + # Write database passwords to .dbpw + db_passwords = [ + "POSTGRES_PASSWORD", + "CKAN_DB_PASSWORD", + "DATASTORE_READONLY_PASSWORD", + ] + write_passwords_to_file({variable: vn[variable] for variable in db_passwords}, ".dbpw") -with open(fn, 'a') as f: - f.write(f"CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:" + str(jwtpw) + "\n") - f.write(f"CKAN___API_TOKEN__JWT__DECODE__SECRET=string:" + str(jwtpw) + "\n") + # Write CKAN passwords to .ckpw + ck_passwords = [ + "CKAN_DB_PASSWORD", + "DATASTORE_READONLY_PASSWORD", + "CKAN_SYSADMIN_PASSWORD", + "CKAN___BEAKER__SESSION__SECRET", + "CKAN___API_TOKEN__JWT__SECRET", + ] + write_passwords_to_file({variable: vn[variable] for variable in ck_passwords}, ".ckpw") -CKAN___API_TOKEN__JWT__ENCODE__SECRET = "string:" + str(jwtpw) -CKAN___API_TOKEN__JWT__DECODE__SECRET = "string:" + str(jwtpw) + write_urls_to_file(".ckpw") -# Now the database URL's which include the password generated above -CKAN_DB_USER = os.environ.get('CKAN_DB_USER') -CKAN_DB = os.environ.get('CKAN_DB') -DATASTORE_DB_USER = os.environ.get('DATASTORE_DB_USER') -DATASTORE_READONLY_USER = os.environ.get('DATASTORE_READONLY_USER') -DATASTORE_DB = os.environ.get('DATASTORE_DB') -POSTGRES_HOST = os.environ.get('POSTGRES_HOST') + print("\nThe CKAN_SYSADMIN_PASSWORD password is: " + vn['CKAN_SYSADMIN_PASSWORD'] + "\n") -# Now write out the Database URL's -with open(fn, 'a') as f: - f.write(f"CKAN_SQLALCHEMY_URL=postgresql://{CKAN_DB_USER}:{CKAN_DB_PASSWORD}@{POSTGRES_HOST}/{CKAN_DB}\n") - f.write(f"CKAN_DATASTORE_WRITE_URL=postgresql://{CKAN_DB_USER}:{CKAN_DB_PASSWORD}@{POSTGRES_HOST}/{DATASTORE_DB}\n") - f.write(f"CKAN_DATASTORE_READ_URL=postgresql://{DATASTORE_READONLY_USER}:{DATASTORE_READONLY_PASSWORD}@{POSTGRES_HOST}/{DATASTORE_DB}\n") + # Rest of your code -print("[setup_passwords] password file: '.pw' created successfully") -print("\nThe CKAN_SYSADMIN_PASSWORD password is: " + CKAN_SYSADMIN_PASSWORD + "\n") +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() \ No newline at end of file diff --git a/generate_passwords.sh b/generate_passwords.sh index e5ab2a94..6abfe49a 100755 --- a/generate_passwords.sh +++ b/generate_passwords.sh @@ -2,8 +2,10 @@ set -a . ./.env -pwfile=".pw" -touch ${pwfile}; chmod 600 ${pwfile} +pwfile=".ckpw" +dbpwfile=".dbpw" +touch ${pwfile} && touch ${dbpwfile} +chmod 600 ${pwfile} && chmod 600 ${dbpwfile} python3 ./generate_passwords.py -chmod 400 ${pwfile} +chmod 400 ${pwfile} && chmod 400 ${dbpwfile} sleep 1 \ No newline at end of file From 9010c5c086f617c797dde254c108768a59b75285 Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 5 Oct 2023 14:55:33 +0200 Subject: [PATCH 8/8] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b860ca36..1a532df6 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,7 @@ You can do this by the following: * generate the passwords by running `generate_passwords.sh` this will create two files: `.ckpw` and `.dbpw` * by default the `docker-compose.yml` file is set to secure mode. +* You need to make a copy of `.env.secure` as `.env` * if you make any subsequent changes to the `docker-compose.yml` file you can always revert back to the original file by referencing `docker-compose.secure.yml` * by default the `.env` file is set to secure mode. * if you make changes to the `.env` file you can always revert back to the original file by referencing `.env.secure`