Merge pull request #1447 from alphagov/add-pycsw-2.4.0

Add pycsw 2.4.0
alphagov · Aug 11, 2023 · 85f9d12 · 85f9d12
2 parents de8ea04 + 9689066
commit 85f9d12
Show file tree

Hide file tree

Showing 6 changed files with 167 additions and 4 deletions.
diff --git a/.github/workflows/build-image-on-tags.yaml b/.github/workflows/build-image-on-tags.yaml
@@ -1,4 +1,4 @@
-name: Build and push images
+name: Build and push images on tags
 
 on:
   push:

diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml
@@ -13,6 +13,11 @@ on:
         required: true
         type: boolean
         default: false
+      buildPycsw240Images:
+        description: Build pycsw 2.4.0 image
+        required: true
+        type: boolean
+        default: false
   push:
     branches:
       - main
@@ -30,10 +35,13 @@ jobs:
             version: 2.9.9
           - name: pycsw
             version: 2.6.1
+            skip: true
+          - name: pycsw
+            version: 2.4.0f
           - name: postgis
             version: 13-3.1
           - name: solr
-            version: 8
+            version: 8-2.9f
         arch: [ amd64 ]
     permissions:
       packages: write
@@ -56,10 +64,19 @@ jobs:
           ARCH: ${{ matrix.arch }}
         run: ./docker/build-image.sh
       - name: Build and push images
-        if: ${{ (!inputs.noPushToRegistry && !inputs.buildBaseImages) || (!inputs.noPushToRegistry && inputs.buildBaseImages && (matrix.app.name == 'ckan' || matrix.app.name == 'solr') ) }}
+        if: ${{ (!inputs.buildPycsw240Images && !inputs.noPushToRegistry && !inputs.buildBaseImages) || (!inputs.buildPycsw240Images && !inputs.noPushToRegistry && inputs.buildBaseImages && (matrix.app.name == 'ckan' || matrix.app.name == 'solr') ) }}
         env:
           APP: ${{ matrix.app.name }}
           VERSION: ${{ matrix.app.version }}
           ARCH: ${{ matrix.arch }}
           BUILD_BASE: ${{ inputs.buildBaseImages }}
+          SKIP: ${{ matrix.app.skip }}
+        run: ./docker/build-image.sh
+      - name: Build and push pycsw 2.4.0 image only
+        if: ${{ (inputs.buildPycsw240Images && matrix.app.name == 'pycsw' && matrix.app.version == '2.4.0f') }}
+        env:
+          APP: ${{ matrix.app.name }}
+          VERSION: ${{ matrix.app.version }}
+          ARCH: ${{ matrix.arch }}
+          BUILD_BASE: true
         run: ./docker/build-image.sh
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -9,7 +9,7 @@ jobs:
           - ckan: governmentdigitalservice/ckan-main:2.9.3
             solr: governmentdigitalservice/ckan-solr:6.6.2
           - ckan: ghcr.io/alphagov/ckan:2.9.9-base
-            solr: ghcr.io/alphagov/solr:8
+            solr: ghcr.io/alphagov/solr:8-2.9f
     name: test
     runs-on: ubuntu-latest
     container:

diff --git a/docker/build-image.sh b/docker/build-image.sh
@@ -10,6 +10,10 @@ build () {
   fi
 }
 
+if [[ -n ${SKIP:-} ]]; then
+  exit
+fi
+
 if [[ ${BUILD_BASE:-} = "true" ]]; then
   if [ "${APP}" = "ckan" ]; then
     build "${VERSION}-core" "${VERSION}-core"

diff --git a/docker/pycsw/2.4.0f.Dockerfile b/docker/pycsw/2.4.0f.Dockerfile
@@ -0,0 +1,126 @@
+FROM govuk/python:3.7.15 AS base
+
+# Set timezone
+ENV TZ=UTC
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+# Setting the locale
+RUN apt-get update
+RUN apt-get install --no-install-recommends -y locales
+RUN echo "LANG=en_US.utf8" > /etc/default/locale 
+
+# Install required system packages
+RUN apt-get -q -y update \
+    && DEBIAN_FRONTEND=noninteractive apt-get -q -y upgrade \
+    && apt-get -q -y install \
+        python3-dev \
+        python3-pip \
+        python3-wheel \
+        libpq-dev \
+        libxml2-dev \
+        libxslt-dev \
+        libgeos-dev \
+        libssl-dev \
+        libffi-dev \
+        postgresql-client \
+        build-essential \
+        git-core \
+        vim \
+        wget \
+        curl \
+        proj-bin \
+        libproj-dev \
+    && apt-get -q clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Define environment variables
+ENV CKAN_HOME /usr/lib/ckan
+ENV CKAN_VENV $CKAN_HOME/venv
+ENV CKAN_CONFIG /config
+ENV CKAN_STORAGE_PATH=/var/lib/ckan
+
+# Create ckan user
+RUN useradd -r -u 900 -m -c "ckan account" -d $CKAN_HOME -s /bin/false ckan
+
+# Setup virtual environment for CKAN
+RUN mkdir -p $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH && \
+    python3.7 -m venv $CKAN_VENV && \
+    ln -s $CKAN_VENV/bin/ckan /usr/local/bin/ckan
+
+# Virtual environment binaries/scripts to be used first
+ENV PATH=${CKAN_VENV}/bin:${PATH}  
+
+# copy setup_pycsw.sh, pycsw.cfg & production.ini
+COPY bin/setup_pycsw.sh /pycsw-entrypoint.sh
+COPY docker/pycsw/pycsw.cfg $CKAN_CONFIG/pycsw.cfg
+COPY production.ini $CKAN_CONFIG/production.ini
+
+# Setup additional env vars
+ENV pipopt='--exists-action=b --force-reinstall'
+ENV CKAN_INI $CKAN_CONFIG/production.ini
+ENV PYCSW_CONFIG=/config/pycsw.cfg
+ENV CKAN_DB_HOST db
+
+# ckan 2.9.3
+
+ENV ckan_sha='22613024e9a7303cb85438bd64ca67d168db5436'
+ENV ckan_fork='alphagov'
+
+# Setup CKAN
+
+RUN pip install -U pip && \
+    pip install $pipopt -U prometheus-flask-exporter==0.20.3 && \
+    pip install $pipopt -U $(curl -s https://raw.githubusercontent.com/$ckan_fork/ckan/$ckan_sha/requirement-setuptools.txt) && \
+    # pin zope to 5.0.0 as causing issues - github.com/pypa/setuptools/issues/2017
+    curl "https://raw.githubusercontent.com/$ckan_fork/ckan/$ckan_sha/requirements.txt" > requirements.txt && \
+    sed -i 's/zope.interface==4.3.2/zope.interface==5.0.0/g' ./requirements.txt && \
+    pip install --upgrade --no-cache-dir -r requirements.txt && \
+    pip install $pipopt -Ue "git+https://github.com/$ckan_fork/ckan.git@$ckan_sha#egg=ckan" && \
+    ln -s $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini && \
+    chmod +x /pycsw-entrypoint.sh && \
+    chown -R ckan:ckan $CKAN_HOME $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH
+
+FROM base AS prod
+
+RUN locale-gen en_US.UTF-8
+
+WORKDIR $CKAN_VENV/src
+
+ENTRYPOINT ["/pycsw-entrypoint.sh"]
+
+USER ckan
+EXPOSE 5000
+
+ENV ckan_spatial_sha='4a26f3bdb02c34e5dbb23658de2b5b05bc8b34a7'
+ENV ckan_spatial_fork='alphagov'
+
+ENV ckan_harvest_fork='alphagov'
+ENV ckan_harvest_sha='f52605c9a4f8ccaa0f5e83937c59ce9bf58cbc06'
+
+RUN echo "Cython==0.29.35" >> constraints.txt && \
+    export PIP_CONSTRAINT="$(pwd)/constraints.txt"
+
+RUN echo "pip install spatial extension..." && \
+
+    pip install $pipopt -U $(curl -s https://raw.githubusercontent.com/$ckan_harvest_fork/ckanext-harvest/$ckan_harvest_sha/pip-requirements.txt) && \
+    pip install $pipopt -U "git+https://github.com/$ckan_harvest_fork/ckanext-harvest.git@$ckan_harvest_sha#egg=ckanext-harvest" && \
+
+    # save spatial-requirements.txt locally before installing dependencies to work around pip error
+    curl -s https://raw.githubusercontent.com/$ckan_spatial_fork/ckanext-spatial/$ckan_spatial_sha/pip-requirements.txt > spatial-requirements.txt && \
+    pip install $pipopt -r spatial-requirements.txt && \
+    pip install $pipopt -U "git+https://github.com/$ckan_spatial_fork/ckanext-spatial.git@$ckan_spatial_sha#egg=ckanext-spatial"
+
+## 2.4.0 pycsw
+ENV pycsw_sha='64464c817248d8ff948d957dd11b955cb8cdd490'
+RUN pip install $pipopt -U $(curl -s https://raw.githubusercontent.com/geopython/pycsw/$pycsw_sha/requirements.txt) && \
+    pip install $pipopt -Ue "git+https://github.com/geopython/pycsw.git@$pycsw_sha#egg=pycsw" && \
+    (cd pycsw && python setup.py build)
+
+# need to pin pyyaml to correctly pick up config settings
+# pin sqlalchemy to use SessionExtensions
+RUN pip install $pipopt -U pyyaml==5.3.1 sqlalchemy==1.3.24 Shapely==1.8.5
+
+# upgrade pyopenssl
+RUN pip install pyopenssl --upgrade
+
+RUN ckan config-tool $CKAN_INI "ckan.plugins = harvest ckan_harvester spatial_metadata spatial_query spatial_harvest_metadata_api gemini_csw_harvester gemini_waf_harvester gemini_doc_harvester"
diff --git a/docker/solr/8-2.9f.Dockerfile b/docker/solr/8-2.9f.Dockerfile
@@ -0,0 +1,16 @@
+FROM ckan/ckan-solr:2.9-solr8-spatial
+
+# Enviroment
+ENV SOLR_CORE ckan
+
+# Giving ownership to Solr
+USER root
+
+# replace the ckan-2.10 schema with the 2.9 schema to make it compatible with CSW
+RUN sed -z 's/<schema name="ckan-2.10" version="1.6">/<schema name="ckan" version="2.9">/' -i /opt/solr/server/solr/configsets/ckan/conf/managed-schema
+
+# Add harvest field to schema
+RUN sed -z 's/<field name="urls" type="text" indexed="true" stored="false" multiValued="true"\/>/<field name="urls" type="text" indexed="true" stored="false" multiValued="true"\/>\n    <field name="harvest" type="text" indexed="true" stored="true" multiValued="true"\/>/' -i /opt/solr/server/solr/configsets/ckan/conf/managed-schema
+
+# Set to restricted User
+USER $SOLR_USER:$SOLR_USER