Skip to content

SELFHOST-INTEGRATION-TEST #343

SELFHOST-INTEGRATION-TEST

SELFHOST-INTEGRATION-TEST #343

name: SELFHOST-INTEGRATION-TEST
on:
schedule:
- cron: '0 14 * * *'
workflow_dispatch:
inputs:
EXEC_TEST_TYPE:
description: 'Which tests need to be executed? The options are all, python, java, batch, cli, standalone-cli, apiserver, yarn'
required: true
default: 'all'
EXEC_VERSION:
description: 'Which version needs to be tested?'
required: true
default: 'main'
env:
GIT_SUBMODULE_STRATEGY: recursive
HYBRIDSE_SOURCE: local
E_VERSION: ${{ github.event.inputs.EXEC_VERSION || 'main'}}
ETYPE: ${{ github.event.inputs.EXEC_TEST_TYPE || 'all'}}
NPROC: 4
jobs:
build-openmldb:
runs-on: [self-hosted,generic]
if: github.repository == '4paradigm/OpenMLDB'
container:
image: ghcr.io/4paradigm/hybridsql:latest
env:
OS: linux
steps:
- uses: actions/checkout@v2
- name: build
if: ${{ env.E_VERSION == 'main' }}
run: |
make configure CMAKE_INSTALL_PREFIX=openmldb-linux
make SQL_JAVASDK_ENABLE=ON && make SQL_JAVASDK_ENABLE=ON install
mv openmldb-linux openmldb-main-linux
tar -zcf openmldb-linux.tar.gz openmldb-main-linux
mv java/openmldb-batch/target/openmldb*-SNAPSHOT.jar openmldb-batch.jar
mv java/openmldb-jdbc/target/openmldb*-SNAPSHOT.jar openmldb-jdbc.jar
mv java/openmldb-native/target/openmldb*-SNAPSHOT.jar openmldb-native.jar
- name: download
if: ${{ env.E_VERSION != 'main' }}
run: |
curl -SLO https://github.com/4paradigm/OpenMLDB/releases/download/v${{ env.E_VERSION }}/openmldb-${{ env.E_VERSION }}-linux.tar.gz
tar -zxf openmldb-${{ env.E_VERSION }}-linux.tar.gz
mv openmldb-${{ env.E_VERSION }}-linux.tar.gz openmldb-linux.tar.gz
- name: upload artifact
uses: actions/upload-artifact@v3
with:
name: openmldb-package
path: |
openmldb-batch.jar
openmldb-jdbc.jar
openmldb-native.jar
openmldb-linux.tar.gz
apiserver-cluster:
needs: build-openmldb
runs-on: [self-hosted,common-user]
steps:
- uses: actions/checkout@v3
- name: before test
if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }}
run: mkdir ${{ github.job }}
- name: download artifact
uses: actions/download-artifact@v3
with:
name: openmldb-package
- name: install openmldb
run: |
tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/
bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 20001 21000 java
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
- name: test
run: bash test/steps/openmldb-integration-test.sh -j 0.0.0 -d deploy -m apiserver
- name: stop openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
- name: remove openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
- name: TEST Results
if: always()
uses: EnricoMi/publish-unit-test-result-action@v1
with:
files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml
check_name: "SRC apiserver-cluster Report"
comment_title: "SRC apiserver-cluster Report"
- name: tar test report
if: ${{ failure() }}
run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports
- name: Send Email
if: ${{ failure() }}
uses: dawidd6/action-send-mail@master
with:
server_address: mail.4paradigm.com
server_port: 995
username: ${{ secrets.MAIL_USERNAME }}
password: ${{ secrets.MAIL_PASSWORD }}
secure: true
subject: OpenMLDB Memory Test
body: OpenMLDB Memory Test Failed
html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
to: ${{ secrets.MAIL_TO }}
from: GitHub Actions
attachments: surefire-reports.tar.gz
java-sdk-cluster-memory-0:
needs: build-openmldb
runs-on: [self-hosted,common-user]
steps:
- uses: actions/checkout@v3
- name: before test
if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }}
run: mkdir ${{ github.job }}
- name: download artifact
uses: actions/download-artifact@v3
with:
name: openmldb-package
- name: install openmldb
run: |
tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/
bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 20001 21000 java
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
- name: test
run: bash test/steps/openmldb-integration-test.sh -j 0.0.0 -c test_cluster.xml -d deploy -l "0" -s "memory"
- name: stop openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
- name: remove openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
- name: TEST Results
if: always()
uses: EnricoMi/publish-unit-test-result-action@v1
with:
files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml
check_name: "SRC java-sdk-cluster-memory-0 Report"
comment_title: "SRC java-sdk-cluster-memory-0 Report"
- name: tar test report
if: ${{ failure() }}
run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports
- name: Send Email
if: ${{ failure() }}
uses: dawidd6/action-send-mail@master
with:
server_address: mail.4paradigm.com
server_port: 995
username: ${{ secrets.MAIL_USERNAME }}
password: ${{ secrets.MAIL_PASSWORD }}
secure: true
subject: OpenMLDB Memory Test
body: OpenMLDB Memory Test Failed
html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
to: ${{ secrets.MAIL_TO }}
from: GitHub Actions
attachments: surefire-reports.tar.gz
java-sdk-cluster-memory-1:
needs: build-openmldb
runs-on: [self-hosted,common-user]
steps:
- uses: actions/checkout@v3
- name: before test
if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }}
run: mkdir ${{ github.job }}
- name: download artifact
uses: actions/download-artifact@v3
with:
name: openmldb-package
- name: install openmldb
run: |
tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/
bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 21001 22000 java
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
- name: test
run: bash test/steps/openmldb-integration-test.sh -j 0.0.0 -c test_cluster.xml -d deploy -l "1,2,3,4,5" -s "memory"
- name: stop openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
- name: remove openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
- name: TEST Results
if: always()
uses: EnricoMi/publish-unit-test-result-action@v1
with:
files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml
check_name: "SRC java-sdk-cluster-memory-1 Report"
comment_title: "SRC java-sdk-cluster-memory-1 Report"
- name: tar test report
if: ${{ failure() }}
run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports
- name: Send Email
if: ${{ failure() }}
uses: dawidd6/action-send-mail@master
with:
server_address: mail.4paradigm.com
server_port: 995
username: ${{ secrets.MAIL_USERNAME }}
password: ${{ secrets.MAIL_PASSWORD }}
secure: true
subject: OpenMLDB Memory Test
body: OpenMLDB Memory Test Failed
html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
to: ${{ secrets.MAIL_TO }}
from: GitHub Actions
attachments: surefire-reports.tar.gz
java-sdk-cluster-hdd:
needs: build-openmldb
runs-on: [self-hosted,common-user]
steps:
- uses: actions/checkout@v3
- name: before test
if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }}
run: mkdir ${{ github.job }}
- name: download artifact
uses: actions/download-artifact@v3
with:
name: openmldb-package
- name: install openmldb
run: |
tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/
bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 22001 23000 java
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
- name: test
run: bash test/steps/openmldb-integration-test.sh -j 0.0.0 -c test_cluster_disk.xml -d deploy -l "0" -s "hdd"
- name: stop openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
- name: remove openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
- name: TEST Results
if: always()
uses: EnricoMi/publish-unit-test-result-action@v1
with:
files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml
check_name: "SRC java-sdk-cluster-hdd Report"
comment_title: "SRC java-sdk-cluster-hdd Report"
- name: tar test report
if: ${{ failure() }}
run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports
- name: Send Email
if: ${{ failure() }}
uses: dawidd6/action-send-mail@master
with:
server_address: mail.4paradigm.com
server_port: 995
username: ${{ secrets.MAIL_USERNAME }}
password: ${{ secrets.MAIL_PASSWORD }}
secure: true
subject: OpenMLDB HDD Test
body: OpenMLDB HDD Test Failed
html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
to: ${{ secrets.MAIL_TO }}
from: GitHub Actions
attachments: surefire-reports.tar.gz
java-sdk-cluster-ssd:
needs: build-openmldb
runs-on: [self-hosted,common-user]
steps:
- uses: actions/checkout@v3
- name: before test
if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }}
run: mkdir ${{ github.job }}
- name: download artifact
uses: actions/download-artifact@v3
with:
name: openmldb-package
- name: install openmldb
run: |
tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/
bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 23001 24000 java ssd
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
- name: test
run: bash test/steps/openmldb-integration-test.sh -j 0.0.0 -c test_cluster_disk.xml -d deploy -l "0" -s "ssd"
- name: stop openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
- name: remove openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
- name: TEST Results
if: always()
uses: EnricoMi/publish-unit-test-result-action@v1
with:
files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml
check_name: "SRC java-sdk-cluster-ssd Report"
comment_title: "SRC java-sdk-cluster-ssd Report"
- name: tar test report
if: ${{ failure() }}
run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports
- name: Send Email
if: ${{ failure() }}
uses: dawidd6/action-send-mail@master
with:
server_address: mail.4paradigm.com
server_port: 995
username: ${{ secrets.MAIL_USERNAME }}
password: ${{ secrets.MAIL_PASSWORD }}
secure: true
subject: OpenMLDB SSD Test
body: OpenMLDB SSD Test Failed
html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
to: ${{ secrets.MAIL_TO }}
from: GitHub Actions
attachments: surefire-reports.tar.gz
java-sdk-yarn:
needs: build-openmldb
runs-on: [self-hosted,common-user]
steps:
- uses: actions/checkout@v3
- name: before test
if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }}
run: mkdir ${{ github.job }}
- name: download artifact
uses: actions/download-artifact@v3
with:
name: openmldb-package
- name: install openmldb
run: |
tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/
bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 24001 25000 java hadoop
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-taskmanagers.sh
HADOOP_CONF_DIR=/mnt/hdd0/denglong/openmldb_runner_work/hadoop ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-taskmanagers.sh
- name: test
run: bash test/steps/openmldb-integration-test.sh -j 0.0.0 -c test_yarn.xml -d deploy -l "0" -s "memory"
- name: stop openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
- name: remove openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
- name: TEST Results
if: always()
uses: EnricoMi/publish-unit-test-result-action@v1
with:
files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml
check_name: "SRC java-sdk-yarn Report"
comment_title: "SRC java-yarn Report"
- name: tar test report
if: ${{ failure() }}
run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports
- name: Send Email
if: ${{ failure() }}
uses: dawidd6/action-send-mail@master
with:
server_address: mail.4paradigm.com
server_port: 995
username: ${{ secrets.MAIL_USERNAME }}
password: ${{ secrets.MAIL_PASSWORD }}
secure: true
subject: OpenMLDB yarn Test
body: OpenMLDB yarn Test Failed
html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
to: ${{ secrets.MAIL_TO }}
from: GitHub Actions
attachments: surefire-reports.tar.gz
java-sdk-kafka:
needs: build-openmldb
runs-on: [self-hosted,common-user]
steps:
- uses: actions/checkout@v3
- name: before test
if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }}
run: mkdir ${{ github.job }}
- name: download artifact
uses: actions/download-artifact@v3
with:
name: openmldb-package
- name: install openmldb
run: |
tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/
bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 25001 26000 java kafka
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
- name: test
run: echo "a"
- name: stop openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
- name: remove openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
# - name: TEST Results
# if: always()
# uses: EnricoMi/publish-unit-test-result-action@v1
# with:
# files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml
# check_name: "SRC java-sdk-yarn Report"
# comment_title: "SRC java-yarn Report"
# - name: tar test report
# if: ${{ failure() }}
# run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports
# - name: Send Email
# if: ${{ failure() }}
# uses: dawidd6/action-send-mail@master
# with:
# server_address: mail.4paradigm.com
# server_port: 995
# secure: true
# username: ${{ secrets.MAIL_USERNAME }}
# password: ${{ secrets.MAIL_PASSWORD }}
# subject: OpenMLDB yarn Test
# body: OpenMLDB yarn Test Failed
# html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
# to: ${{ secrets.MAIL_TO }}
# from: GitHub Actions
# content_type: text/plain
# attachments: surefire-reports.tar.gz