[SEDONA-285] Added docker images for spark-sedona-zeppelin and spark-sedona-jupyterlab

.github/workflows/dockerhub-deploy.yml

@@ -0,0 +1,117 @@
+name: Deploy to Docker Hub
+on: [push, pull_request, workflow_dispatch]
+jobs:
+
+  base-jdk:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check Out Repo 
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build and push image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: docker/base-jdk.dockerfile
+          tags: kartikeyhadiya/base-jdk:latest
+          push: ${{ github.event_name != 'pull_request' }}
+
+  spark-base-jupyterlab:
+
+    runs-on: ubuntu-latest
+
+    needs: base-jdk
+
+    strategy:
+      fail-fast: false
+      matrix:
+        image: [spark-base, spark-sedona-jupyterlab/sedona_jupyterlab]
+
+    steps:
+      - name: Check Out Repo 
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Get branch names
+        id: branch-name
+        uses: tj-actions/branch-names@v7
+
+      - name: Sets env vars for image_tag
+        run: |
+          IMAGE_TAG=${{ steps.branch-name.outputs.tag }}
+          ! [[ "$IMAGE_TAG" ]] && IMAGE_TAG=${{ steps.branch-name.outputs.current_branch }}
+          [[ "$IMAGE_TAG" =~ ^(master|main)$ ]] && IMAGE_TAG=latest
+          image=`echo ${{ matrix.image }} | cut -d '/' -f 2`
+          [[ ${{ matrix.image }} == "spark-base" ]] && tag=3.3.2 || tag=$IMAGE_TAG
+          echo "DOCKER_IMAGE_TAG=kartikeyhadiya/$image:$tag" >> $GITHUB_ENV
+
+      - name: Build and push image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: docker/${{ matrix.image }}.dockerfile
+          tags: ${{ env.DOCKER_IMAGE_TAG }}
+          push: ${{ github.event_name != 'pull_request' }}
+
+  master_worker:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        image: [spark-master, spark-worker]
+
+    needs: spark-base-jupyterlab
+    steps:
+
+      - name: Check Out Repo 
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build and push image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: docker/${{ matrix.image }}.dockerfile
+          tags: kartikeyhadiya/${{ matrix.image }}:3.3.2
+          push: ${{ github.event_name != 'pull_request' }}

docker/base-jdk.dockerfile

@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ARG debian_buster_image_tag=8-jre-slim
+FROM openjdk:${debian_buster_image_tag}
+
+# -- Layer: OS + Python
+
+ARG shared_workspace=/opt/workspace
+ARG python_version=3.7.12
+
+RUN mkdir -p ${shared_workspace} && \
+    apt-get update -y && \
+    apt install -y curl gcc &&\ 
+    apt install -y build-essential zlib1g-dev libncurses5-dev && \
+	apt install -y libsqlite3-dev && \
+	apt install -y libgdbm-dev libnss3-dev libssl-dev libreadline-dev libffi-dev wget libjpeg-dev && \
+    rm -rf /var/lib/apt/lists/*
+RUN cd /usr/bin && \
+    curl -O https://www.python.org/ftp/python/${python_version}/Python-${python_version}.tar.xz && \
+    tar -xf Python-${python_version}.tar.xz && cd Python-${python_version} && ./configure --with-ensurepip=install && make -j 8 &&\
+    ln -s /usr/bin/Python-${python_version}/python /usr/bin/python && \
+    ln -s /usr/bin/Python-${python_version}/python /usr/bin/python3 && \
+    rm -rf Python-${python_version}.tar.xz
+
+ENV SHARED_WORKSPACE=${shared_workspace}
+
+# -- Runtime
+
+VOLUME ${shared_workspace}
+CMD ["bash"]

docker/spark-base.dockerfile

@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM kartikeyhadiya/base-jdk
+
+# -- Layer: Apache Spark
+
+ARG spark_version=3.3.2
+ARG hadoop_version=3
+ARG sedona_version=1.4.0
+ARG geotools_wrapper_version=1.4.0-28.2
+
+ENV SPARK_HOME /opt/spark
+ENV SPARK_MASTER_HOST spark-master
+ENV SPARK_MASTER_PORT 7077
+ENV PYTHONPATH=$SPARK_HOME/python
+ENV PYSPARK_PYTHON python3
+ENV PYSPARK_DRIVER_PYTHON jupyter
+
+RUN apt-get update -y && \
+    apt-get install -y curl && \
+    curl https://archive.apache.org/dist/spark/spark-${spark_version}/spark-${spark_version}-bin-hadoop${hadoop_version}.tgz -o spark.tgz && \
+    tar -xf spark.tgz && \
+    mv spark-${spark_version}-bin-hadoop${hadoop_version} /opt/spark && \
+    mkdir /opt/spark/logs && \
+    rm spark.tgz && \
+    # -- Copy sedona jars to Spark jars
+    curl https://dlcdn.apache.org/sedona/${sedona_version}/apache-sedona-${sedona_version}-bin.tar.gz -o sedona.tar.gz && \
+    tar -xf sedona.tar.gz && \
+    mv apache-sedona-${sedona_version}-bin/* /${SPARK_HOME}/jars/ && \
+    rm -r apache-sedona-${sedona_version}-bin sedona.tar.gz && \
+    # -- Copy geotools-wrapper jars to Spark jars
+    curl https://repo1.maven.org/maven2/org/datasyslab/geotools-wrapper/${geotools_wrapper_version}/geotools-wrapper-${geotools_wrapper_version}.jar -o geotools-wrapper-${geotools_wrapper_version}.jar && \
+    mv geotools-wrapper-${geotools_wrapper_version}.jar /${SPARK_HOME}/jars/
+
+# -- Runtime
+
+WORKDIR ${SPARK_HOME}

docker/spark-master.dockerfile

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ARG spark_version=3.3.2
+FROM kartikeyhadiya/spark-base:${spark_version}
+
+# -- Runtime
+
+ARG spark_master_web_ui=8080
+
+EXPOSE ${spark_master_web_ui} ${SPARK_MASTER_PORT}
+CMD bin/spark-class org.apache.spark.deploy.master.Master >> logs/spark-master.out

docker/spark-sedona-jupyterlab/.dockerignore

@@ -0,0 +1,3 @@
+Dockerfile
+compose.yml
+README.md

docker/spark-sedona-jupyterlab/.gitignore

@@ -0,0 +1,6 @@
+commands.txt
+docker-compose-orig.yml
+Dockerfile.bak
+log.txt
+examples
+.ipynb_checkpoints

docker/spark-sedona-jupyterlab/README.md

@@ -0,0 +1,21 @@
+# sedona_jupyterlab_spark
+Dockerfiles for apache sedona with JupyterLab and 1 spark master node and 2 worker node
+
+# Docker Image
+kartikeyhadiya/sedona_jupyterlab:1.4.0
+
+The image contains all the dependencies required for configuring Apache Sedona and JupyterLab. JupyterLab can be accessed at http://localhost:8888/ after pulling image.
+
+# Spark Configuration
+  - Master node can be accessed at port http://localhost:8080/ (spark://spark-master:7077). Refer link given in credit for more information.
+  - Spark-worker-1 can be accessed at port http://localhost:8081/.
+  - Spark-worker-2 can be accessed at port http://localhost:8082/.
+
+# How to build
+  - Clone the repository
+  - Run build.sh script. (If using WSL, use dos2unix command to convert line endings to unix format)
+  - Run docker-compose up
+
+# Requirements
+  - Docker
+  - Docker-compose

docker/spark-sedona-jupyterlab/build.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# -- Software Stack Version
+
+PYTHON_VERSION="3.9.17"
+SPARK_VERSION="3.3.2"
+HADOOP_VERSION="3"
+SEDONA_VERSION="1.4.0"
+GEOTOOLS_WRAPPER_VERSION="1.4.0-28.2"
+
+# -- Building the Images
+
+docker build \
+    --no-cache \
+    --build-arg python_version="${PYTHON_VERSION}" \
+    -f docker/base-jdk.dockerfile \
+    -t kartikeyhadiya/base-jdk:latest .
+
+docker build \
+    --no-cache \
+    --build-arg spark_version="${SPARK_VERSION}" \
+    --build-arg hadoop_version="${HADOOP_VERSION}" \
+    --build-arg sedona_version="${SEDONA_VERSION}" \
+    --build-arg geotools_wrapper_version="${GEOTOOLS_WRAPPER_VERSION}" \
+    -f docker/spark-base.dockerfile \
+    -t kartikeyhadiya/spark-base:latest \
+    -t kartikeyhadiya/spark-base:${SPARK_VERSION} .
+
+docker build \
+    --no-cache \
+    --build-arg spark_version="${SPARK_VERSION}" \
+    -f docker/spark-master.dockerfile \
+    -t kartikeyhadiya/spark-master:latest \
+    -t kartikeyhadiya/spark-master:${SPARK_VERSION} .
+
+docker build \
+    --no-cache \
+    --build-arg spark_version="${SPARK_VERSION}" \
+    -f docker/spark-worker.dockerfile \
+    -t kartikeyhadiya/spark-worker:latest \
+    -t kartikeyhadiya/spark-worker:${SPARK_VERSION} .
+
+docker build \
+    --no-cache \
+    --build-arg sedona_version="${SEDONA_VERSION}" \
+    --build-arg geotools_wrapper_version="${GEOTOOLS_WRAPPER_VERSION}" \
+    --build-arg python_version="${PYTHON_VERSION}" \
+    -f docker/spark-sedona-jupyterlab/sedona_jupyterlab.dockerfile \
+    -t kartikeyhadiya/sedona_jupyterlab:latest \
+    -t kartikeyhadiya/sedona_jupyterlab:${SEDONA_VERSION} .