From c642017258859f133d17832618cd1660f654c232 Mon Sep 17 00:00:00 2001 From: Sowmya U B Date: Tue, 6 Aug 2024 16:51:36 +0530 Subject: [PATCH] MOS-3162 fix Signed-off-by: Sowmya U B --- .github/workflows/push-trigger.yml | 260 +++++++++--------- .../.dockerignore | 0 build/es-kafka-connecter/Dockerfile | 5 + .../kafka_connect_transforms/Dockerfile | 6 - 4 files changed, 137 insertions(+), 134 deletions(-) rename build/es-kafka-connecter/{kafka_connect_transforms => }/.dockerignore (100%) create mode 100644 build/es-kafka-connecter/Dockerfile delete mode 100644 build/es-kafka-connecter/kafka_connect_transforms/Dockerfile diff --git a/.github/workflows/push-trigger.yml b/.github/workflows/push-trigger.yml index 171132f..a6af174 100644 --- a/.github/workflows/push-trigger.yml +++ b/.github/workflows/push-trigger.yml @@ -1,146 +1,150 @@ -name: Maven Package upon a push +name: Create Dockers on Push on: - release: - types: [published] - pull_request: - types: [opened, reopened, synchronize] workflow_dispatch: - inputs: - message: - description: 'Message for manually triggering' - required: false - default: 'Triggered for Updates' - type: string push: branches: - '!release-branch' + - release-1* - master - 1.* - develop - - MOSIP* - - release* - + - MOS-3162 + paths: + - 'build/**' jobs: - build-maven-kafka-connect-transforms: - uses: mosip/kattu/.github/workflows/maven-build.yml@master - with: - SERVICE_LOCATION: build/es-kafka-connecter/kafka_connect_transforms - BUILD_ARTIFACT: kafka-connect - secrets: - OSSRH_USER: ${{ secrets.OSSRH_USER }} - OSSRH_SECRET: ${{ secrets.OSSRH_SECRET }} - OSSRH_TOKEN: ${{ secrets.OSSRH_TOKEN }} - GPG_SECRET: ${{ secrets.GPG_SECRET }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }} + kafka_connect_transforms: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + ref: ${{ github.ref }} + java-version: 11 + + - name: Setup branch and env + run: | + # Strip git ref prefix from version + echo "BRANCH_NAME=$(echo ${{ github.ref }} | sed -e 's,.*/\(.*\),\1,')" >> $GITHUB_ENV + + - uses: actions/cache@v1 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven-${{ env.BRANCH_NAME }} + + - name: Build with Maven + run: | + cd build/es-kafka-connecter + cd kafka_connect_transforms ; mvn -B clean package -DskipTests ; cd .. + mkdir jars; cd jars + cp ../kafka_connect_transforms/target/kafka_connect_transforms*.jar . + wget https://repo.maven.apache.org/maven2/io/debezium/debezium-core/1.7.0.Final/debezium-core-1.7.0.Final.jar + + - name: Log into registry + run: echo "${{ secrets.release_docker_hub }}" | docker login -u ${{ secrets.actor_docker_hub }} --password-stdin - ## Recreate zips to provide support for docker unzip operation - recreate-zips: - needs: build-maven-kafka-connect-transforms + - name: Create Docker + run: | + NAMESPACE=${{ secrets.dev_namespace_docker_hub }} + SERVICE_NAME=cp-kafka-connect-es + IMAGE_ID=$NAMESPACE/$SERVICE_NAME + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + if [[ ${{ env.BRANCH_NAME }} == main || ${{ env.BRANCH_NAME }} == master ]]; then + VERSION=latest + else + VERSION=${{ env.BRANCH_NAME }} + fi + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + cd build/es-kafka-connecter + docker build . -t $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION + debezium-fixer: runs-on: ubuntu-latest - env: - BUILD_ARTIFACT: kafka-connect - NEW_BUILD_ARTIFACT: es-kafka-connect - SERVICE_LOCATION: build/es-kafka-connecter/kafka_connect_transforms steps: - - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 - if: ${{ ( env.BUILD_ARTIFACT != 'false' ) }} - with: - name: ${{ env.BUILD_ARTIFACT }} - path: ${{ env.SERVICE_LOCATION }} - - name: recreate zip - run: | - echo "unzip command under target directory" - - unzip -uj "${{ env.SERVICE_LOCATION }}/${{ env.BUILD_ARTIFACT }}.zip" "*/target/*" -d "${{ env.SERVICE_LOCATION }}/target/" - - cd ${{ env.SERVICE_LOCATION }}/target/ - - wget https://repo.maven.apache.org/maven2/io/debezium/debezium-core/1.7.0.Final/debezium-core-1.7.0.Final.jar - - cd - - echo "find command" - find ./ -path '*/target/*' -name '*.jar' -type f -exec zip ${{ env.NEW_BUILD_ARTIFACT }}.zip {} + + - uses: actions/checkout@v2 - - name: Upload the springboot jars - if: ${{ !contains(github.ref, 'master') || !contains(github.ref, 'main') }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.NEW_BUILD_ARTIFACT }} - path: ${{ env.NEW_BUILD_ARTIFACT }}.zip + - name: Setup branch and env + run: | + # Strip git ref prefix from version + echo "BRANCH_NAME=$(echo ${{ github.ref }} | sed -e 's,.*/\(.*\),\1,')" >> $GITHUB_ENV - build-dockers-kafka-connect-transforms: - needs: recreate-zips - strategy: - matrix: - include: - - SERVICE_LOCATION: 'build/es-kafka-connecter/kafka_connect_transforms' - SERVICE_NAME: 'es-kafka-connect' - BUILD_ARTIFACT: 'es-kafka-connect' - fail-fast: false - name: ${{ matrix.SERVICE_NAME }} - uses: mosip/kattu/.github/workflows/docker-build.yml@master - with: - SERVICE_LOCATION: ${{ matrix.SERVICE_LOCATION }} - SERVICE_NAME: ${{ matrix.SERVICE_NAME }} - BUILD_ARTIFACT: ${{ matrix.BUILD_ARTIFACT }} - secrets: - DEV_NAMESPACE_DOCKER_HUB: ${{ secrets.DEV_NAMESPACE_DOCKER_HUB }} - ACTOR_DOCKER_HUB: ${{ secrets.ACTOR_DOCKER_HUB }} - RELEASE_DOCKER_HUB: ${{ secrets.RELEASE_DOCKER_HUB }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }} + - name: Log into registry + run: echo "${{ secrets.release_docker_hub }}" | docker login -u ${{ secrets.actor_docker_hub }} --password-stdin + + - name: Create Docker + run: | + NAMESPACE=${{ secrets.dev_namespace_docker_hub }} + SERVICE_NAME=debezium-fixer + IMAGE_ID=$NAMESPACE/$SERVICE_NAME + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + if [[ ${{ env.BRANCH_NAME }} == main || ${{ env.BRANCH_NAME }} == master ]]; then + VERSION=latest + else + VERSION=${{ env.BRANCH_NAME }} + fi + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + cd build/debezium-fixer + docker build . -t $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION + reporting-init-debezium: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Setup branch and env + run: | + # Strip git ref prefix from version + echo "BRANCH_NAME=$(echo ${{ github.ref }} | sed -e 's,.*/\(.*\),\1,')" >> $GITHUB_ENV + + - name: Log into registry + run: echo "${{ secrets.release_docker_hub }}" | docker login -u ${{ secrets.actor_docker_hub }} --password-stdin + - name: Create Docker + run: | + NAMESPACE=${{ secrets.dev_namespace_docker_hub }} + SERVICE_NAME=debezium-init + IMAGE_ID=$NAMESPACE/$SERVICE_NAME + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + if [[ ${{ env.BRANCH_NAME }} == main || ${{ env.BRANCH_NAME }} == master ]]; then + VERSION=latest + else + VERSION=${{ env.BRANCH_NAME }} + fi + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + cd build/reporting-init/debezium-init + docker build . -t $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION + reporting-init-es-connect: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 - build-dockers-debezium-fixer: - strategy: - matrix: - include: - - SERVICE_LOCATION: 'build/debezium-fixer' - SERVICE_NAME: 'debezium-fixer' - fail-fast: false - name: ${{ matrix.SERVICE_NAME }} - uses: mosip/kattu/.github/workflows/docker-build.yml@master - with: - SERVICE_LOCATION: ${{ matrix.SERVICE_LOCATION }} - SERVICE_NAME: ${{ matrix.SERVICE_NAME }} - secrets: - DEV_NAMESPACE_DOCKER_HUB: ${{ secrets.DEV_NAMESPACE_DOCKER_HUB }} - ACTOR_DOCKER_HUB: ${{ secrets.ACTOR_DOCKER_HUB }} - RELEASE_DOCKER_HUB: ${{ secrets.RELEASE_DOCKER_HUB }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }} + - name: Setup branch and env + run: | + # Strip git ref prefix from version + echo "BRANCH_NAME=$(echo ${{ github.ref }} | sed -e 's,.*/\(.*\),\1,')" >> $GITHUB_ENV - build-dockers-debezium-init: - strategy: - matrix: - include: - - SERVICE_LOCATION: 'build/reporting-init/debezium-init' - SERVICE_NAME: 'debezium-init' - fail-fast: false - name: ${{ matrix.SERVICE_NAME }} - uses: mosip/kattu/.github/workflows/docker-build.yml@master - with: - SERVICE_LOCATION: ${{ matrix.SERVICE_LOCATION }} - SERVICE_NAME: ${{ matrix.SERVICE_NAME }} - secrets: - DEV_NAMESPACE_DOCKER_HUB: ${{ secrets.DEV_NAMESPACE_DOCKER_HUB }} - ACTOR_DOCKER_HUB: ${{ secrets.ACTOR_DOCKER_HUB }} - RELEASE_DOCKER_HUB: ${{ secrets.RELEASE_DOCKER_HUB }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }} + - name: Log into registry + run: echo "${{ secrets.release_docker_hub }}" | docker login -u ${{ secrets.actor_docker_hub }} --password-stdin - build-dockers-es-connect-init: - strategy: - matrix: - include: - - SERVICE_LOCATION: 'build/reporting-init/es-kafka-connect-init' - SERVICE_NAME: 'es-kafka-connect-init' - fail-fast: false - name: ${{ matrix.SERVICE_NAME }} - uses: mosip/kattu/.github/workflows/docker-build.yml@master - with: - SERVICE_LOCATION: ${{ matrix.SERVICE_LOCATION }} - SERVICE_NAME: ${{ matrix.SERVICE_NAME }} - secrets: - DEV_NAMESPACE_DOCKER_HUB: ${{ secrets.DEV_NAMESPACE_DOCKER_HUB }} - ACTOR_DOCKER_HUB: ${{ secrets.ACTOR_DOCKER_HUB }} - RELEASE_DOCKER_HUB: ${{ secrets.RELEASE_DOCKER_HUB }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }} + - name: Create Docker + run: | + NAMESPACE=${{ secrets.dev_namespace_docker_hub }} + SERVICE_NAME=es-kafka-connect-init + IMAGE_ID=$NAMESPACE/$SERVICE_NAME + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + if [[ ${{ env.BRANCH_NAME }} == main || ${{ env.BRANCH_NAME }} == master ]]; then + VERSION=latest + else + VERSION=${{ env.BRANCH_NAME }} + fi + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + cd build/reporting-init/es-kafka-connect-init + docker build . -t $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION \ No newline at end of file diff --git a/build/es-kafka-connecter/kafka_connect_transforms/.dockerignore b/build/es-kafka-connecter/.dockerignore similarity index 100% rename from build/es-kafka-connecter/kafka_connect_transforms/.dockerignore rename to build/es-kafka-connecter/.dockerignore diff --git a/build/es-kafka-connecter/Dockerfile b/build/es-kafka-connecter/Dockerfile new file mode 100644 index 0000000..c07f7a6 --- /dev/null +++ b/build/es-kafka-connecter/Dockerfile @@ -0,0 +1,5 @@ +FROM confluentinc/cp-kafka-connect:7.0.0-1-ubi8 + +RUN confluent-hub install --no-prompt confluentinc/kafka-connect-elasticsearch:11.1.3 + +ADD ./jars /usr/share/java/my_transforms diff --git a/build/es-kafka-connecter/kafka_connect_transforms/Dockerfile b/build/es-kafka-connecter/kafka_connect_transforms/Dockerfile deleted file mode 100644 index 57096bb..0000000 --- a/build/es-kafka-connecter/kafka_connect_transforms/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM confluentinc/cp-kafka-connect:7.0.0-1-ubi8 - -RUN confluent-hub install --no-prompt confluentinc/kafka-connect-elasticsearch:11.1.3 - -ADD ./target/kafka_connect_transforms-*.jar /usr/share/java/my_transforms -ADD ./target/debezium-core-*.jar /usr/share/java/my_transforms