diff --git a/.github/workflows/build-image-on-tags.yaml b/.github/workflows/build-image-on-tags.yaml index db502ad3..060a3939 100644 --- a/.github/workflows/build-image-on-tags.yaml +++ b/.github/workflows/build-image-on-tags.yaml @@ -1,4 +1,4 @@ -name: Build and push images +name: Build and push images on tags on: push: diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 5bdb8b08..00605ac2 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -13,6 +13,11 @@ on: required: true type: boolean default: false + buildPycsw240Images: + description: Build pycsw 2.4.0 image + required: true + type: boolean + default: false push: branches: - main @@ -30,10 +35,13 @@ jobs: version: 2.9.9 - name: pycsw version: 2.6.1 + skip: true + - name: pycsw + version: 2.4.0f - name: postgis version: 13-3.1 - name: solr - version: 8 + version: 8-2.9f arch: [ amd64 ] permissions: packages: write @@ -56,10 +64,19 @@ jobs: ARCH: ${{ matrix.arch }} run: ./docker/build-image.sh - name: Build and push images - if: ${{ (!inputs.noPushToRegistry && !inputs.buildBaseImages) || (!inputs.noPushToRegistry && inputs.buildBaseImages && (matrix.app.name == 'ckan' || matrix.app.name == 'solr') ) }} + if: ${{ (!inputs.buildPycsw240Images && !inputs.noPushToRegistry && !inputs.buildBaseImages) || (!inputs.buildPycsw240Images && !inputs.noPushToRegistry && inputs.buildBaseImages && (matrix.app.name == 'ckan' || matrix.app.name == 'solr') ) }} env: APP: ${{ matrix.app.name }} VERSION: ${{ matrix.app.version }} ARCH: ${{ matrix.arch }} BUILD_BASE: ${{ inputs.buildBaseImages }} + SKIP: ${{ matrix.app.skip }} + run: ./docker/build-image.sh + - name: Build and push pycsw 2.4.0 image only + if: ${{ (inputs.buildPycsw240Images && matrix.app.name == 'pycsw' && matrix.app.version == '2.4.0f') }} + env: + APP: ${{ matrix.app.name }} + VERSION: ${{ matrix.app.version }} + ARCH: ${{ matrix.arch }} + BUILD_BASE: true run: ./docker/build-image.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c31749a..851d2722 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: - ckan: governmentdigitalservice/ckan-main:2.9.3 solr: governmentdigitalservice/ckan-solr:6.6.2 - ckan: ghcr.io/alphagov/ckan:2.9.9-base - solr: ghcr.io/alphagov/solr:8 + solr: ghcr.io/alphagov/solr:8-2.9f name: test runs-on: ubuntu-latest container: diff --git a/docker/build-image.sh b/docker/build-image.sh index 2ebac91f..67e445a4 100755 --- a/docker/build-image.sh +++ b/docker/build-image.sh @@ -10,6 +10,10 @@ build () { fi } +if [[ -n ${SKIP:-} ]]; then + exit +fi + if [[ ${BUILD_BASE:-} = "true" ]]; then if [ "${APP}" = "ckan" ]; then build "${VERSION}-core" "${VERSION}-core" diff --git a/docker/pycsw/2.4.0f.Dockerfile b/docker/pycsw/2.4.0f.Dockerfile new file mode 100644 index 00000000..a3611c73 --- /dev/null +++ b/docker/pycsw/2.4.0f.Dockerfile @@ -0,0 +1,126 @@ +FROM govuk/python:3.7.15 AS base + +# Set timezone +ENV TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Setting the locale +RUN apt-get update +RUN apt-get install --no-install-recommends -y locales +RUN echo "LANG=en_US.utf8" > /etc/default/locale + +# Install required system packages +RUN apt-get -q -y update \ + && DEBIAN_FRONTEND=noninteractive apt-get -q -y upgrade \ + && apt-get -q -y install \ + python3-dev \ + python3-pip \ + python3-wheel \ + libpq-dev \ + libxml2-dev \ + libxslt-dev \ + libgeos-dev \ + libssl-dev \ + libffi-dev \ + postgresql-client \ + build-essential \ + git-core \ + vim \ + wget \ + curl \ + proj-bin \ + libproj-dev \ + && apt-get -q clean \ + && rm -rf /var/lib/apt/lists/* + +# Define environment variables +ENV CKAN_HOME /usr/lib/ckan +ENV CKAN_VENV $CKAN_HOME/venv +ENV CKAN_CONFIG /config +ENV CKAN_STORAGE_PATH=/var/lib/ckan + +# Create ckan user +RUN useradd -r -u 900 -m -c "ckan account" -d $CKAN_HOME -s /bin/false ckan + +# Setup virtual environment for CKAN +RUN mkdir -p $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH && \ + python3.7 -m venv $CKAN_VENV && \ + ln -s $CKAN_VENV/bin/ckan /usr/local/bin/ckan + +# Virtual environment binaries/scripts to be used first +ENV PATH=${CKAN_VENV}/bin:${PATH} + +# copy setup_pycsw.sh, pycsw.cfg & production.ini +COPY bin/setup_pycsw.sh /pycsw-entrypoint.sh +COPY docker/pycsw/pycsw.cfg $CKAN_CONFIG/pycsw.cfg +COPY production.ini $CKAN_CONFIG/production.ini + +# Setup additional env vars +ENV pipopt='--exists-action=b --force-reinstall' +ENV CKAN_INI $CKAN_CONFIG/production.ini +ENV PYCSW_CONFIG=/config/pycsw.cfg +ENV CKAN_DB_HOST db + +# ckan 2.9.3 + +ENV ckan_sha='22613024e9a7303cb85438bd64ca67d168db5436' +ENV ckan_fork='alphagov' + +# Setup CKAN + +RUN pip install -U pip && \ + pip install $pipopt -U prometheus-flask-exporter==0.20.3 && \ + pip install $pipopt -U $(curl -s https://raw.githubusercontent.com/$ckan_fork/ckan/$ckan_sha/requirement-setuptools.txt) && \ + # pin zope to 5.0.0 as causing issues - github.com/pypa/setuptools/issues/2017 + curl "https://raw.githubusercontent.com/$ckan_fork/ckan/$ckan_sha/requirements.txt" > requirements.txt && \ + sed -i 's/zope.interface==4.3.2/zope.interface==5.0.0/g' ./requirements.txt && \ + pip install --upgrade --no-cache-dir -r requirements.txt && \ + pip install $pipopt -Ue "git+https://github.com/$ckan_fork/ckan.git@$ckan_sha#egg=ckan" && \ + ln -s $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini && \ + chmod +x /pycsw-entrypoint.sh && \ + chown -R ckan:ckan $CKAN_HOME $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH + +FROM base AS prod + +RUN locale-gen en_US.UTF-8 + +WORKDIR $CKAN_VENV/src + +ENTRYPOINT ["/pycsw-entrypoint.sh"] + +USER ckan +EXPOSE 5000 + +ENV ckan_spatial_sha='4a26f3bdb02c34e5dbb23658de2b5b05bc8b34a7' +ENV ckan_spatial_fork='alphagov' + +ENV ckan_harvest_fork='alphagov' +ENV ckan_harvest_sha='f52605c9a4f8ccaa0f5e83937c59ce9bf58cbc06' + +RUN echo "Cython==0.29.35" >> constraints.txt && \ + export PIP_CONSTRAINT="$(pwd)/constraints.txt" + +RUN echo "pip install spatial extension..." && \ + + pip install $pipopt -U $(curl -s https://raw.githubusercontent.com/$ckan_harvest_fork/ckanext-harvest/$ckan_harvest_sha/pip-requirements.txt) && \ + pip install $pipopt -U "git+https://github.com/$ckan_harvest_fork/ckanext-harvest.git@$ckan_harvest_sha#egg=ckanext-harvest" && \ + + # save spatial-requirements.txt locally before installing dependencies to work around pip error + curl -s https://raw.githubusercontent.com/$ckan_spatial_fork/ckanext-spatial/$ckan_spatial_sha/pip-requirements.txt > spatial-requirements.txt && \ + pip install $pipopt -r spatial-requirements.txt && \ + pip install $pipopt -U "git+https://github.com/$ckan_spatial_fork/ckanext-spatial.git@$ckan_spatial_sha#egg=ckanext-spatial" + +## 2.4.0 pycsw +ENV pycsw_sha='64464c817248d8ff948d957dd11b955cb8cdd490' +RUN pip install $pipopt -U $(curl -s https://raw.githubusercontent.com/geopython/pycsw/$pycsw_sha/requirements.txt) && \ + pip install $pipopt -Ue "git+https://github.com/geopython/pycsw.git@$pycsw_sha#egg=pycsw" && \ + (cd pycsw && python setup.py build) + +# need to pin pyyaml to correctly pick up config settings +# pin sqlalchemy to use SessionExtensions +RUN pip install $pipopt -U pyyaml==5.3.1 sqlalchemy==1.3.24 Shapely==1.8.5 + +# upgrade pyopenssl +RUN pip install pyopenssl --upgrade + +RUN ckan config-tool $CKAN_INI "ckan.plugins = harvest ckan_harvester spatial_metadata spatial_query spatial_harvest_metadata_api gemini_csw_harvester gemini_waf_harvester gemini_doc_harvester" diff --git a/docker/solr/8-2.9f.Dockerfile b/docker/solr/8-2.9f.Dockerfile new file mode 100644 index 00000000..f700261e --- /dev/null +++ b/docker/solr/8-2.9f.Dockerfile @@ -0,0 +1,16 @@ +FROM ckan/ckan-solr:2.9-solr8-spatial + +# Enviroment +ENV SOLR_CORE ckan + +# Giving ownership to Solr +USER root + +# replace the ckan-2.10 schema with the 2.9 schema to make it compatible with CSW +RUN sed -z 's///' -i /opt/solr/server/solr/configsets/ckan/conf/managed-schema + +# Add harvest field to schema +RUN sed -z 's//\n /' -i /opt/solr/server/solr/configsets/ckan/conf/managed-schema + +# Set to restricted User +USER $SOLR_USER:$SOLR_USER