Skip to content

Final changes from POC, adds agg support #43

Final changes from POC, adds agg support

Final changes from POC, adds agg support #43

# This workflow will build a Java project with Gradle
# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle
name: Kafka plugin integration tests
on:
push:
paths:
- 'data-prepper-plugins/kafka-plugins/**'
- '*gradle*'
pull_request_target:
types: [ opened, synchronize, reopened ]
paths:
- 'data-prepper-plugins/kafka-plugins/**'
- '*gradle*'
workflow_dispatch:
permissions:
id-token: write
contents: read
jobs:
integration-tests:
strategy:
matrix:
java: [11]
kafka: [2.8.1, 3.5.1]
fail-fast: false
runs-on: ubuntu-latest
steps:
- name: Set up JDK ${{ matrix.java }}
uses: actions/setup-java@v1
with:
java-version: ${{ matrix.java }}
- name: Checkout Data Prepper
uses: actions/checkout@v2
- name: Run Kafka Docker
run: |
echo 'KAFKA_VERSION=${{ matrix.kafka }}' > data-prepper-plugins/kafka-plugins/src/integrationTest/resources/kafka/.env
docker compose --project-directory data-prepper-plugins/kafka-plugins/src/integrationTest/resources/kafka/zookeeper --env-file data-prepper-plugins/kafka-plugins/src/integrationTest/resources/kafka/.env up -d
sleep 10
- name: Wait for Kafka
run: |
./gradlew data-prepper-plugins:kafka-plugins:integrationTest -Dtests.kafka.bootstrap_servers=localhost:9092 -Dtests.kafka.authconfig.username=admin -Dtests.kafka.authconfig.password=admin --tests KafkaStartIT
- name: Configure AWS credentials
id: aws-credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.TEST_IAM_ROLE_ARN }}
aws-region: ${{ secrets.TEST_REGION }}
output-credentials: true
- name: Configure AWS default credentials
run: |
aws configure set default.region ${{ secrets.TEST_REGION }}
aws configure set default.aws_access_key_id ${{ steps.aws-credentials.outputs.aws-access-key-id }}
aws configure set default.aws_secret_access_key ${{ steps.aws-credentials.outputs.aws-secret-access-key }}
aws configure set default.aws_session_token ${{ steps.aws-credentials.outputs.aws-session-token }}
- name: Run Kafka integration tests
run: |
./gradlew data-prepper-plugins:kafka-plugins:integrationTest \
-Dtests.kafka.bootstrap_servers=localhost:9092 \
-Dtests.kafka.authconfig.username=admin -Dtests.kafka.authconfig.password=admin \
-Dtests.kafka.kms_key=alias/DataPrepperTesting \
--tests '*kafka.buffer*' --tests KafkaSourceJsonTypeIT --tests KafkaBufferOTelIT
- name: Upload Unit Test Results
if: always()
uses: actions/upload-artifact@v3
with:
name: data-prepper-kafka-integration-tests-kafka-${{ matrix.kafka }}-java-${{ matrix.java }}
path: '**/test-results/**/*.xml'
publish-test-results:
name: "Publish Unit Tests Results"
needs: integration-tests
runs-on: ubuntu-latest
if: always()
steps:
- name: Download Artifacts
uses: actions/download-artifact@v3
with:
path: test-results
- name: Publish Unit Test Results
uses: EnricoMi/publish-unit-test-result-action@v1
with:
files: "test-results/**/*.xml"