From 1cb6184fa6ecb3812b09991c5e7d846dc54a2f4b Mon Sep 17 00:00:00 2001 From: "qiyuan.liu" <1043276694@qq.com> Date: Thu, 17 Aug 2023 15:53:55 +0800 Subject: [PATCH] ci: add selfhost muliti-machine integration test (#3283) --- .github/workflows/selfhost_intergration.yml | 396 ++++++++++++++++++ cases/integration_test/ddl/test_options.yaml | 4 +- .../expression/test_condition.yaml | 45 +- .../expression/test_like.yaml | 2 +- .../long_window/test_long_window.yaml | 2 + .../openmldb-sdk-test/pom.xml | 4 +- .../java_sdk_test/common/OpenMLDBConfig.java | 3 +- .../java_sdk_test/common/OpenMLDBTest.java | 2 +- .../yarn/TestExternalFunction.java | 2 +- .../openmldb-test-common/pom.xml | 2 +- test/steps/format_config.sh | 126 ++++++ .../openmldb-deploy/cases/test_upgrade.py | 4 +- 12 files changed, 575 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/selfhost_intergration.yml create mode 100755 test/steps/format_config.sh diff --git a/.github/workflows/selfhost_intergration.yml b/.github/workflows/selfhost_intergration.yml new file mode 100644 index 00000000000..dcaf34cce8d --- /dev/null +++ b/.github/workflows/selfhost_intergration.yml @@ -0,0 +1,396 @@ +name: SELFHOST-INTEGRATION-TEST +on: + + schedule: + - cron: '0 14 * * *' + + workflow_dispatch: + inputs: + EXEC_TEST_TYPE: + description: 'Which tests need to be executed? The options are all, python, java, batch, cli, standalone-cli, apiserver, yarn' + required: true + default: 'all' + EXEC_VERSION: + description: 'Which version needs to be tested?' + required: true + default: 'main' +env: + GIT_SUBMODULE_STRATEGY: recursive + HYBRIDSE_SOURCE: local + E_VERSION: ${{ github.event.inputs.EXEC_VERSION || 'main'}} + ETYPE: ${{ github.event.inputs.EXEC_TEST_TYPE || 'all'}} + +jobs: + build-openmldb: + + runs-on: [self-hosted,generic] + if: github.repository == '4paradigm/OpenMLDB' + container: + image: ghcr.io/4paradigm/hybridsql:latest + env: + OS: linux + steps: + - uses: actions/checkout@v2 + - name: build + if: ${{ env.E_VERSION == 'main' }} + run: | + make configure CMAKE_INSTALL_PREFIX=openmldb-linux + make SQL_JAVASDK_ENABLE=ON && make SQL_JAVASDK_ENABLE=ON install + mv openmldb-linux openmldb-main-linux + tar -zcf openmldb-linux.tar.gz openmldb-main-linux + - name: download + if: ${{ env.E_VERSION != 'main' }} + run: | + curl -SLO https://github.com/4paradigm/OpenMLDB/releases/download/v${{ env.E_VERSION }}/openmldb-${{ env.E_VERSION }}-linux.tar.gz + tar -zxf openmldb-${{ env.E_VERSION }}-linux.tar.gz + mv openmldb-${{ env.E_VERSION }}-linux.tar.gz openmldb-linux.tar.gz + - name: upload artifact + uses: actions/upload-artifact@v3 + with: + name: openmldb-package + path: openmldb-linux.tar.gz + + + + java-sdk-cluster-memory-0: + needs: build-openmldb + + runs-on: [self-hosted,common-user] + steps: + - uses: actions/checkout@v3 + - name: before test + if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }} + run: mkdir ${{ github.job }} + - name: download artifact + uses: actions/download-artifact@v3 + with: + name: openmldb-package + - name: install openmldb + run: | + tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/ + bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 20001 21000 java + bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh + bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh + - name: test + run: | + mkdir mvnrepo + export MAVEN_OPTS="-Dmaven.repo.local=$(pwd)/mvnrepo" + echo $MAVEN_OPTS + bash test/steps/openmldb-sdk-test-java-src.sh -c test_cluster.xml -d deploy -l "0" -s "memory" + - name: stop openmldb + if: always() + run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh + - name: remove openmldb + if: success() + run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh + - name: TEST Results + if: always() + uses: EnricoMi/publish-unit-test-result-action@v1 + with: + files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml + check_name: "SRC java-sdk-cluster-memory-0 Report" + comment_title: "SRC java-sdk-cluster-memory-0 Report" + - name: tar test report + if: ${{ failure() }} + run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports + - name: Send Email + if: ${{ failure() }} + uses: dawidd6/action-send-mail@master + with: + server_address: smtp.partner.outlook.cn + server_port: 587 + username: ${{ secrets.MAIL_USERNAME }} + password: ${{ secrets.MAIL_PASSWORD }} + subject: OpenMLDB Memory Test + body: OpenMLDB Memory Test Failed + html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html + to: ${{ secrets.MAIL_TO }} + from: GitHub Actions + attachments: surefire-reports.tar.gz + + + + java-sdk-cluster-memory-1: + needs: build-openmldb + runs-on: [self-hosted,common-user] + steps: + - uses: actions/checkout@v3 + - name: before test + if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }} + run: mkdir ${{ github.job }} + - name: download artifact + uses: actions/download-artifact@v3 + with: + name: openmldb-package + - name: install openmldb + run: | + tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/ + bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 21001 22000 java + bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh + bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh + - name: test + run: | + mkdir mvnrepo + export MAVEN_OPTS="-Dmaven.repo.local=$(pwd)/mvnrepo" + echo $MAVEN_OPTS + bash test/steps/openmldb-sdk-test-java-src.sh -c test_cluster.xml -d deploy -l "1,2,3,4,5" -s "memory" + - name: stop openmldb + if: always() + run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh + - name: remove openmldb + if: success() + run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh + - name: TEST Results + if: always() + uses: EnricoMi/publish-unit-test-result-action@v1 + with: + files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml + check_name: "SRC java-sdk-cluster-memory-1 Report" + comment_title: "SRC java-sdk-cluster-memory-1 Report" + - name: tar test report + if: ${{ failure() }} + run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports + - name: Send Email + if: ${{ failure() }} + uses: dawidd6/action-send-mail@master + with: + server_address: smtp.partner.outlook.cn + server_port: 587 + username: ${{ secrets.MAIL_USERNAME }} + password: ${{ secrets.MAIL_PASSWORD }} + subject: OpenMLDB Memory Test + body: OpenMLDB Memory Test Failed + html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html + to: ${{ secrets.MAIL_TO }} + from: GitHub Actions + attachments: surefire-reports.tar.gz + + + + + java-sdk-cluster-hdd: + needs: build-openmldb + runs-on: [self-hosted,common-user] + steps: + - uses: actions/checkout@v3 + - name: before test + if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }} + run: mkdir ${{ github.job }} + - name: download artifact + uses: actions/download-artifact@v3 + with: + name: openmldb-package + - name: install openmldb + run: | + tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/ + bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 22001 23000 java + bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh + bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh + - name: test + run: | + mkdir mvnrepo + export MAVEN_OPTS="-Dmaven.repo.local=$(pwd)/mvnrepo" + echo $MAVEN_OPTS + bash test/steps/openmldb-sdk-test-java-src.sh -c test_cluster_disk.xml -d deploy -l "0" -s "hdd" + - name: stop openmldb + if: always() + run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh + - name: remove openmldb + if: success() + run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh + - name: TEST Results + if: always() + uses: EnricoMi/publish-unit-test-result-action@v1 + with: + files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml + check_name: "SRC java-sdk-cluster-hdd Report" + comment_title: "SRC java-sdk-cluster-hdd Report" + - name: tar test report + if: ${{ failure() }} + run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports + - name: Send Email + if: ${{ failure() }} + uses: dawidd6/action-send-mail@master + with: + server_address: smtp.partner.outlook.cn + server_port: 587 + username: ${{ secrets.MAIL_USERNAME }} + password: ${{ secrets.MAIL_PASSWORD }} + subject: OpenMLDB HDD Test + body: OpenMLDB HDD Test Failed + html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html + to: ${{ secrets.MAIL_TO }} + from: GitHub Actions + attachments: surefire-reports.tar.gz + + + + java-sdk-cluster-ssd: + needs: build-openmldb + runs-on: [self-hosted,common-user] + steps: + - uses: actions/checkout@v3 + - name: before test + if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }} + run: mkdir ${{ github.job }} + - name: download artifact + uses: actions/download-artifact@v3 + with: + name: openmldb-package + - name: install openmldb + run: | + tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/ + bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux java-sdk-cluster-memory-0 23001 24000 java ssd + bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh + bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh + - name: test + run: | + mkdir mvnrepo + export MAVEN_OPTS="-Dmaven.repo.local=$(pwd)/mvnrepo" + echo $MAVEN_OPTS + bash test/steps/openmldb-sdk-test-java-src.sh -c test_cluster_disk.xml -d deploy -l "0" -s "ssd" + - name: stop openmldb + if: always() + run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh + - name: remove openmldb + if: success() + run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh + - name: TEST Results + if: always() + uses: EnricoMi/publish-unit-test-result-action@v1 + with: + files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml + check_name: "SRC java-sdk-cluster-ssd Report" + comment_title: "SRC java-sdk-cluster-ssd Report" + - name: tar test report + if: ${{ failure() }} + run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports + - name: Send Email + if: ${{ failure() }} + uses: dawidd6/action-send-mail@master + with: + server_address: smtp.partner.outlook.cn + server_port: 587 + username: ${{ secrets.MAIL_USERNAME }} + password: ${{ secrets.MAIL_PASSWORD }} + subject: OpenMLDB SSD Test + body: OpenMLDB SSD Test Failed + html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html + to: ${{ secrets.MAIL_TO }} + from: GitHub Actions + attachments: surefire-reports.tar.gz + + + java-sdk-yarn: + needs: build-openmldb + runs-on: [self-hosted,common-user] + steps: + - uses: actions/checkout@v3 + - name: before test + if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }} + run: mkdir ${{ github.job }} + - name: download artifact + uses: actions/download-artifact@v3 + with: + name: openmldb-package + - name: install openmldb + run: | + tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/ + bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 24001 25000 java hadoop + bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh + bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh + bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-taskmanager.sh + bash HADOOP_CONF_DIR=/mnt/hdd0/denglong/openmldb_runner_work/hadoop ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-taskmanager.sh + - name: test + run: | + mkdir mvnrepo + export MAVEN_OPTS="-Dmaven.repo.local=$(pwd)/mvnrepo" + echo $MAVEN_OPTS + bash test/steps/openmldb-sdk-test-java-src.sh -c test_yarn.xml -d deploy -l "0" -s "memory" + - name: stop openmldb + if: always() + run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh + - name: remove openmldb + if: success() + run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh + - name: TEST Results + if: always() + uses: EnricoMi/publish-unit-test-result-action@v1 + with: + files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml + check_name: "SRC java-sdk-yarn Report" + comment_title: "SRC java-yarn Report" + - name: tar test report + if: ${{ failure() }} + run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports + - name: Send Email + if: ${{ failure() }} + uses: dawidd6/action-send-mail@master + with: + server_address: smtp.partner.outlook.cn + server_port: 587 + username: ${{ secrets.MAIL_USERNAME }} + password: ${{ secrets.MAIL_PASSWORD }} + subject: OpenMLDB yarn Test + body: OpenMLDB yarn Test Failed + html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html + to: ${{ secrets.MAIL_TO }} + from: GitHub Actions + attachments: surefire-reports.tar.gz + + + + java-sdk-kafka: + needs: build-openmldb + runs-on: [self-hosted,common-user] + steps: + - uses: actions/checkout@v3 + - name: before test + if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }} + run: mkdir ${{ github.job }} + - name: download artifact + uses: actions/download-artifact@v3 + with: + name: openmldb-package + - name: install openmldb + run: | + tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/ + bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 25001 26000 java kafka + bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh + bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh + - name: test + run: | + mkdir mvnrepo + export MAVEN_OPTS="-Dmaven.repo.local=$(pwd)/mvnrepo" + echo $MAVEN_OPTS + - name: stop openmldb + if: always() + run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh + - name: remove openmldb + if: success() + run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh + # - name: TEST Results + # if: always() + # uses: EnricoMi/publish-unit-test-result-action@v1 + # with: + # files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml + # check_name: "SRC java-sdk-yarn Report" + # comment_title: "SRC java-yarn Report" + # - name: tar test report + # if: ${{ failure() }} + # run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports + # - name: Send Email + # if: ${{ failure() }} + # uses: dawidd6/action-send-mail@master + # with: + # server_address: smtp.partner.outlook.cn + # server_port: 587 + # username: ${{ secrets.MAIL_USERNAME }} + # password: ${{ secrets.MAIL_PASSWORD }} + # subject: OpenMLDB yarn Test + # body: OpenMLDB yarn Test Failed + # html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html + # to: ${{ secrets.MAIL_TO }} + # from: GitHub Actions + # content_type: text/plain + # attachments: surefire-reports.tar.gz diff --git a/cases/integration_test/ddl/test_options.yaml b/cases/integration_test/ddl/test_options.yaml index d35fb6bec31..29511b8fe30 100644 --- a/cases/integration_test/ddl/test_options.yaml +++ b/cases/integration_test/ddl/test_options.yaml @@ -358,7 +358,7 @@ cases: - id: 22 desc: test-case - mode: standalone-unsupport + mode: standalone-unsupport,disk-unsupport inputs: - columns : ["c1 string","c2 smallint","c3 int","c4 bigint","c5 float","c6 double","c7 timestamp","c8 date","c9 bool"] @@ -397,7 +397,7 @@ cases: - id: 24 desc: 没有partitionnum和replicanum，指定distribution - mode: standalone-unsupport + mode: standalone-unsupport,disk-unsupport inputs: - name: t3 sql: | diff --git a/cases/integration_test/expression/test_condition.yaml b/cases/integration_test/expression/test_condition.yaml index 54d1dd4ad4d..b63ecdf67c6 100644 --- a/cases/integration_test/expression/test_condition.yaml +++ b/cases/integration_test/expression/test_condition.yaml @@ -252,7 +252,12 @@ cases: sql: | select col1,ifnull(col2,"abc") as e1 from {0}; expect: - success: false + columns: ["col1 int", "e1 string"] + order: col1 + rows: + - [1, '0'] + - [2, 'abc'] + - [3, '1'] - id: 10 desc: IFNULL-表达式 sqlDialect: ["HybridSQL"] @@ -285,7 +290,12 @@ cases: sql: | select col1,ifnull(col2 /0 ,100) as e3 from {0}; expect: - success: false + columns: ["col1 int", "e3 double"] + order: col1 + rows: + - [1, 100] + - [2, 100] + - [3, 100] - id: 11-2 mode: cli-unsupport desc: NVL is synonyms to ifnull @@ -317,7 +327,12 @@ cases: sql: | select col1,nvl(col2 /0 ,100) as e3 from {0}; expect: - success: false + columns: ["col1 int", "e3 double"] + order: col1 + rows: + - [1, 100] + - [2, 100] + - [3, 100] - id: 12 desc: IFNULL-兼容类型 sqlDialect: ["HybridSQL"] @@ -331,7 +346,13 @@ cases: sql: | select col1,ifnull(col2,100) as e1 from {0}; expect: - success: false + columns: ["col1 int", "e1 bigint"] + order: col1 + rows: + - [1, 0] + - [2, 100] + - [3, 1] + - id: 13 desc: IFNULL-浮点型 sqlDialect: ["HybridSQL"] @@ -345,7 +366,13 @@ cases: sql: | select col1,ifnull(col2,1.1) as e2 from {0}; expect: - success: false + columns: ["col1 int", "e2 double"] + order: col1 + rows: + - [1, 0] + - [2, 1.1] + - [3, 1] + - id: NVL2-1 desc: NVL2 @@ -378,7 +405,13 @@ cases: sql: | select col1,nvl2(col2, "abc", col1 + 1) as e1 from {0}; expect: - success: false + columns: ["col1 int", "e1 string"] + order: col1 + rows: + - [1, 'abc'] + - [2, '3'] + - [3, 'abc'] + - id: NVL2-3 desc: NVL2, sub expression diff --git a/cases/integration_test/expression/test_like.yaml b/cases/integration_test/expression/test_like.yaml index d47bb57b616..7f0d47daf5f 100644 --- a/cases/integration_test/expression/test_like.yaml +++ b/cases/integration_test/expression/test_like.yaml @@ -496,7 +496,7 @@ cases: columns : ["id bigint","c1 string","c7 timestamp"] indexs: ["index1:id:c7"] rows: - - [1,'\\\%a_b',1590738990000] + - [1,'\\%a_b',1590738990000] - [2,'\\\aabb',1590738991000] - [3,"_a%_b",1590738992000] - [4,"ba_c",1590738993000] diff --git a/cases/integration_test/long_window/test_long_window.yaml b/cases/integration_test/long_window/test_long_window.yaml index 95a5b6f2594..4f49cb0736b 100644 --- a/cases/integration_test/long_window/test_long_window.yaml +++ b/cases/integration_test/long_window/test_long_window.yaml @@ -322,6 +322,7 @@ cases: id: 10 version: 0.6.1 desc: delete pk + mode: cluster-unsupport longWindow: w1:2s inputs: - @@ -359,6 +360,7 @@ cases: id: 11 version: 0.6.1 desc: delete 组合索引 + mode: cluster-unsupport longWindow: w1:2s inputs: - diff --git a/test/integration-test/openmldb-test-java/openmldb-sdk-test/pom.xml b/test/integration-test/openmldb-test-java/openmldb-sdk-test/pom.xml index b5839033256..32c81920daa 100644 --- a/test/integration-test/openmldb-test-java/openmldb-sdk-test/pom.xml +++ b/test/integration-test/openmldb-test-java/openmldb-sdk-test/pom.xml @@ -15,8 +15,8 @@ 8 8 UTF-8 - 0.7.0 - 0.7.0-allinone + 0.7.0-SNAPSHOT + 0.7.0-SNAPSHOT 2.2.0 test_suite/test_tmp.xml 1.8.9 diff --git a/test/integration-test/openmldb-test-java/openmldb-sdk-test/src/main/java/com/_4paradigm/openmldb/java_sdk_test/common/OpenMLDBConfig.java b/test/integration-test/openmldb-test-java/openmldb-sdk-test/src/main/java/com/_4paradigm/openmldb/java_sdk_test/common/OpenMLDBConfig.java index 2cdcd096c0a..03f413b71dd 100644 --- a/test/integration-test/openmldb-test-java/openmldb-sdk-test/src/main/java/com/_4paradigm/openmldb/java_sdk_test/common/OpenMLDBConfig.java +++ b/test/integration-test/openmldb-test-java/openmldb-sdk-test/src/main/java/com/_4paradigm/openmldb/java_sdk_test/common/OpenMLDBConfig.java @@ -72,7 +72,8 @@ public class OpenMLDBConfig { } public static boolean isCluster() { - return OpenMLDBGlobalVar.env.equals("cluster"); + + return OpenMLDBGlobalVar.env.equals("cluster")||OpenMLDBGlobalVar.env.equals("deploy"); } } diff --git a/test/integration-test/openmldb-test-java/openmldb-sdk-test/src/main/java/com/_4paradigm/openmldb/java_sdk_test/common/OpenMLDBTest.java b/test/integration-test/openmldb-test-java/openmldb-sdk-test/src/main/java/com/_4paradigm/openmldb/java_sdk_test/common/OpenMLDBTest.java index 09e3e895543..3e39ddbb0a0 100644 --- a/test/integration-test/openmldb-test-java/openmldb-sdk-test/src/main/java/com/_4paradigm/openmldb/java_sdk_test/common/OpenMLDBTest.java +++ b/test/integration-test/openmldb-test-java/openmldb-sdk-test/src/main/java/com/_4paradigm/openmldb/java_sdk_test/common/OpenMLDBTest.java @@ -61,7 +61,7 @@ public void beforeTest(@Optional("qa") String env,@Optional("main") String versi openMLDBDeploy.setCluster(false); OpenMLDBGlobalVar.mainInfo = openMLDBDeploy.deployCluster(2, 3); }else if(env.equalsIgnoreCase("deploy")){ - OpenMLDBGlobalVar.mainInfo = YamlUtil.getObject("out/openmldb_info.yaml",OpenMLDBInfo.class); + OpenMLDBGlobalVar.mainInfo = YamlUtil.getObject(Tool.openMLDBDir().getAbsolutePath()+"/out/openmldb_info.yaml",OpenMLDBInfo.class); } else if(env.equalsIgnoreCase("yarn")) { OpenMLDBDeploy openMLDBDeploy = new OpenMLDBDeploy(version); openMLDBDeploy.setOpenMLDBPath(openMLDBPath); diff --git a/test/integration-test/openmldb-test-java/openmldb-sdk-test/src/test/java/com/_4paradigm/openmldb/java_sdk_test/yarn/TestExternalFunction.java b/test/integration-test/openmldb-test-java/openmldb-sdk-test/src/test/java/com/_4paradigm/openmldb/java_sdk_test/yarn/TestExternalFunction.java index 104552c08c1..bd1bd229e53 100644 --- a/test/integration-test/openmldb-test-java/openmldb-sdk-test/src/test/java/com/_4paradigm/openmldb/java_sdk_test/yarn/TestExternalFunction.java +++ b/test/integration-test/openmldb-test-java/openmldb-sdk-test/src/test/java/com/_4paradigm/openmldb/java_sdk_test/yarn/TestExternalFunction.java @@ -31,7 +31,7 @@ public class TestExternalFunction extends OpenMLDBTest { @Story("ExternalFunction") - @Test(enabled = true) + @Test(enabled = false) public void testFunctionMethods() { Statement statement = executor.getStatement(); diff --git a/test/integration-test/openmldb-test-java/openmldb-test-common/pom.xml b/test/integration-test/openmldb-test-java/openmldb-test-common/pom.xml index 6741f69688e..57d6a1d0cb4 100644 --- a/test/integration-test/openmldb-test-java/openmldb-test-common/pom.xml +++ b/test/integration-test/openmldb-test-java/openmldb-test-common/pom.xml @@ -15,7 +15,7 @@ 8 8 - 0.6.4 + 0.7.0-SNAPSHOT 0.7.0-SNAPSHOT diff --git a/test/steps/format_config.sh b/test/steps/format_config.sh new file mode 100755 index 00000000000..9700d330974 --- /dev/null +++ b/test/steps/format_config.sh @@ -0,0 +1,126 @@ +#! /usr/bin/env bash + +#set DeployDir +rootPath="$1" +jobName="$2" +portFrom="$3" +portTo="$4" +Type="$5" +Dependency="$6" +version=$(grep 'OPENMLDB_VERSION' "$rootPath"/conf/openmldb-env.sh | awk -F= '{print $2}') +curTime=$(date "+%m%d%H%M") +dirName="${jobName}-${version}-${curTime}" + +#set Deploy Host and Ports +Hosts=(node-3 node-4 node-1) + +AvaNode1Ports=$(ssh "${Hosts[0]}" "comm -23 <(seq $portFrom $portTo | sort) <(sudo ss -Htan | awk '{print $4}' | cut -d':' -f2 | sort -u) | shuf | head -n 8") +AvaNode2Ports=$(ssh "${Hosts[1]}" "comm -23 <(seq $portFrom $portTo | sort) <(sudo ss -Htan | awk '{print $4}' | cut -d':' -f2 | sort -u) | shuf | head -n 2") +AvaNode3Ports=$(ssh "${Hosts[2]}" "comm -23 <(seq $portFrom $portTo | sort) <(sudo ss -Htan | awk '{print $4}' | cut -d':' -f2 | sort -u) | shuf | head -n 1") + + +taskmanagerHost=$(hostname | awk -F"." '{print $1}' ) + +taskmanagerPort=$(ssh "${taskmanagerHost}" "comm -23 <(seq $portFrom $portTo | sort) <(sudo ss -Htan | awk '{print $4}' | cut -d':' -f2 | sort -u) | shuf | head -n 1") + + +tablet1Port=$(echo "$AvaNode1Ports" | awk 'BEGIN{ RS="";FS="\n"}{print $1}') +tablet2Port=$(echo "$AvaNode2Ports" | awk 'BEGIN{ RS="";FS="\n"}{print $1}') +tablet3Port=$(echo "$AvaNode3Ports" | awk 'BEGIN{ RS="";FS="\n"}{print $1}') +ns1Port=$(echo "$AvaNode1Ports" | awk 'BEGIN{ RS="";FS="\n"}{print $2}') +ns2Port=$(echo "$AvaNode2Ports" | awk 'BEGIN{ RS="";FS="\n"}{print $2}') +apiserverPort=$(echo "$AvaNode1Ports" | awk 'BEGIN{ RS="";FS="\n"}{print $4}') +#taskmanagerPort=$(echo $AvaNode1Ports | awk '{print $5}') +zookeeperPort1=$(echo "$AvaNode1Ports" | awk 'BEGIN{ RS="";FS="\n"}{print $6}') +zookeeperPort2=$(echo "$AvaNode1Ports" | awk 'BEGIN{ RS="";FS="\n"}{print $7}') +zookeeperPort3=$(echo "$AvaNode1Ports" | awk 'BEGIN{ RS="";FS="\n"}{print $8}') + +# write addr to hosts +cat >"$rootPath"/conf/hosts<"$rootPath"/conf/openmldb-env.sh<out/openmldb_info.yaml<>"$rootPath"/conf/tablet.flags.template<"$rootPath"/conf/taskmanager.properties<$rootPath/test/integration-test/openmldb-test-java/openmldb-ecosystem/src/test/resources/kafka_test_cases.ymls< 0 + assert status.OK() status, unalive_cnt = self.get_unalive_cnt("test", table_name) assert status.OK() and unalive_cnt == 0