diff --git a/.github/workflows/hqps-db-ci.yml b/.github/workflows/hqps-db-ci.yml index 13b16950763b..90482274797a 100644 --- a/.github/workflows/hqps-db-ci.yml +++ b/.github/workflows/hqps-db-ci.yml @@ -130,9 +130,9 @@ jobs: cp ${SCHEMA_FILE} ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/indices/ cd ${GITHUB_WORKSPACE}/flex/tests/hqps - sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ./engine_config_test.yaml - bash hqps_admin_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./engine_config_test.yaml ${GS_TEST_DIR} - sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ./engine_config_test.yaml + sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ./engine_config_test_rbo.yaml + bash hqps_admin_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./engine_config_test_rbo.yaml ${GS_TEST_DIR} + sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ./engine_config_test_rbo.yaml - name: Build and test Interactive Java/Python SDK env: @@ -158,10 +158,10 @@ jobs: cp ${SCHEMA_FILE} ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/indices/ cd ${GITHUB_WORKSPACE}/flex/tests/hqps - sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ./engine_config_test.yaml - bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./engine_config_test.yaml java - bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./engine_config_test.yaml python - sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ./engine_config_test.yaml + sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ./engine_config_test_rbo.yaml + bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./engine_config_test_rbo.yaml java + bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./engine_config_test_rbo.yaml python + sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ./engine_config_test_rbo.yaml - name: Sample Query test env: @@ -187,7 +187,7 @@ jobs: for i in 1 2 3 4 5 6 7 8 9 10 11 12; do cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=/tmp/codegen/" - cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml " + cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test_rbo.yaml " cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml" echo $cmd eval ${cmd} || exit 1 @@ -196,19 +196,19 @@ jobs: for i in 1 2 3 4 5 6 7 8 9 11 12; # 10 is not supported now do cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/simple_match_${i}.cypher -w=/tmp/codegen/" - cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml " + cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test_rbo.yaml " cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml" echo $cmd eval ${cmd} || exit 1 done # test movie graph, 8,9,10 are not supported now - # change the default_graph config in ../tests/hqps/engine_config_test.yaml to movies - sed -i 's/default_graph: ldbc/default_graph: movies/g' ../tests/hqps/engine_config_test.yaml + # change the default_graph config in ../tests/hqps/engine_config_test_rbo.yaml to movies + sed -i 's/default_graph: ldbc/default_graph: movies/g' ../tests/hqps/engine_config_test_rbo.yaml for i in 1 2 3 4 5 6 7 11 12 13 14 15; do cmd="./load_plan_and_gen.sh -e=hqps -i=../tests/hqps/queries/movie/query${i}.cypher -w=/tmp/codegen/" - cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml " + cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test_rbo.yaml " cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml" echo $cmd eval ${cmd} || exit 1 @@ -225,23 +225,23 @@ jobs: cd ${GITHUB_WORKSPACE}/flex/bin ./load_plan_and_gen.sh -e=hqps -i=../interactive/examples/modern_graph/get_person_name.cypher -w=/tmp/codegen \ - --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml -o=${PLUGIN_DIR} \ + --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test_rbo.yaml -o=${PLUGIN_DIR} \ --procedure_name=get_person_name \ --graph_schema_path=../interactive/examples/modern_graph/graph.yaml ./load_plan_and_gen.sh -e=hqps -i=../interactive/examples/modern_graph/count_vertex_num.cypher -w=/tmp/codegen \ - --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml -o=${PLUGIN_DIR} \ + --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test_rbo.yaml -o=${PLUGIN_DIR} \ --procedure_name=count_vertex_num \ --graph_schema_path=../interactive/examples/modern_graph/graph.yaml cd ${GITHUB_WORKSPACE}/flex/tests/interactive/ bash test_plugin_loading.sh ./modern_graph_schema_v0_0.yaml \ ../../interactive/examples/modern_graph/bulk_load.yaml \ - ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml + ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test_rbo.yaml bash test_plugin_loading.sh ./modern_graph_schema_v0_1.yaml \ ../../interactive/examples/modern_graph/bulk_load.yaml \ - ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml + ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test_rbo.yaml - name: Run End-to-End cypher adhoc ldbc query test env: @@ -252,10 +252,10 @@ jobs: run: | cd ${GITHUB_WORKSPACE}/flex/tests/hqps/ export ENGINE_TYPE=hiactor - # change the default_graph config in ./engine_config_test.yaml to ldbc - sed -i 's/default_graph: movies/default_graph: ldbc/g' ./engine_config_test.yaml + # change the default_graph config in ./engine_config_test_rbo.yaml to ldbc + sed -i 's/default_graph: movies/default_graph: ldbc/g' ./engine_config_test_rbo.yaml bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} ldbc \ - ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml + ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test_rbo.yaml cypher - name: Run End-to-End cypher adhoc movie query test env: @@ -266,10 +266,10 @@ jobs: run: | cd ${GITHUB_WORKSPACE}/flex/tests/hqps/ export ENGINE_TYPE=hiactor - # change the default_graph config in ./engine_config_test.yaml to movies - sed -i 's/default_graph: ldbc/default_graph: movies/g' ./engine_config_test.yaml + # change the default_graph config in ./engine_config_test_rbo.yaml to movies + sed -i 's/default_graph: ldbc/default_graph: movies/g' ./engine_config_test_rbo.yaml bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} movies \ - ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml + ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test_rbo.yaml cypher - name: Run End-to-End cypher adhoc graph_algo query test env: @@ -281,9 +281,9 @@ jobs: cd ${GITHUB_WORKSPACE}/flex/tests/hqps/ export ENGINE_TYPE=hiactor # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to graph_algo - sed -i 's/default_graph: movies/default_graph: graph_algo/g' ./engine_config_test.yaml + sed -i 's/default_graph: movies/default_graph: graph_algo/g' ./engine_config_test_rbo.yaml bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} graph_algo \ - ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml + ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test_rbo.yaml cypher - name: Run Gremlin test on modern graph env: @@ -293,8 +293,8 @@ jobs: run: | cd ${GITHUB_WORKSPACE}/flex/tests/hqps/ export ENGINE_TYPE=hiactor - sed -i 's/default_graph: graph_algo/default_graph: modern_graph/g' ./engine_config_test.yaml + sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ./engine_config_test_cbo.yaml bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} modern_graph \ - ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml gremlin + ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test_rbo.yaml gremlin diff --git a/flex/interactive/docker/interactive-runtime.Dockerfile b/flex/interactive/docker/interactive-runtime.Dockerfile index 15bffe5c57d3..b1ead0ff4051 100644 --- a/flex/interactive/docker/interactive-runtime.Dockerfile +++ b/flex/interactive/docker/interactive-runtime.Dockerfile @@ -79,7 +79,7 @@ COPY --from=builder /opt/flex /opt/flex # copy the builtin graph, modern_graph RUN mkdir -p /opt/flex/share/gs_interactive_default_graph/ COPY --from=builder /home/graphscope/GraphScope/flex/interactive/examples/modern_graph/* /opt/flex/share/gs_interactive_default_graph/ -COPY --from=builder /home/graphscope/GraphScope/flex/tests/hqps/engine_config_test.yaml /opt/flex/share/engine_config.yaml +COPY --from=builder /home/graphscope/GraphScope/flex/tests/hqps/engine_config_test_rbo.yaml /opt/flex/share/engine_config.yaml COPY --from=builder /home/graphscope/GraphScope/flex/interactive/docker/entrypoint.sh /opt/flex/bin/entrypoint.sh COPY --from=builder /home/graphscope/GraphScope/flex/third_party/nlohmann-json/single_include/* /opt/flex/include/ RUN sed -i 's/name: modern_graph/name: gs_interactive_default_graph/g' /opt/flex/share/gs_interactive_default_graph/graph.yaml diff --git a/flex/interactive/examples/graph_algo/statistics.txt b/flex/interactive/examples/graph_algo/statistics.txt new file mode 100644 index 000000000000..808ffcd75be3 --- /dev/null +++ b/flex/interactive/examples/graph_algo/statistics.txt @@ -0,0 +1,14 @@ +v,0,Paper,123 +v,1,Challenge,5 +v,2,Topic,8 +v,3,Task,25 +v,4,Solution,15 +v,5,CCFField,11 +e,0,WorkOn,Paper,Task,123 +e,1,Resolve,Paper,Challenge,168 +e,2,Target,Task,Challenge,57 +e,3,Belong,Task,Topic,25 +e,4,User,Paper,Solution,170 +e,5,ApplyON,Solution,Challenge,15 +e,6,HasField,Paper,CCFField,109 +e,7,Citation,Paper,Paper,116 \ No newline at end of file diff --git a/flex/interactive/examples/modern_graph/statistics.txt b/flex/interactive/examples/modern_graph/statistics.txt new file mode 100644 index 000000000000..bb1d5a560b56 --- /dev/null +++ b/flex/interactive/examples/modern_graph/statistics.txt @@ -0,0 +1,4 @@ +v,0,person,4 +v,1,software,2 +e,0,knows,person,person,2 +e,1,created,person,software,4 \ No newline at end of file diff --git a/flex/interactive/examples/movies/statistics.txt b/flex/interactive/examples/movies/statistics.txt new file mode 100644 index 000000000000..ff2b2a3d2229 --- /dev/null +++ b/flex/interactive/examples/movies/statistics.txt @@ -0,0 +1,9 @@ +v,0,movie,38 +v,1,person,130 +v,2,user,7 +e,0,acted_in,person,movie,172 +e,1,directed,person,movie,44 +e,2,review,user,movie,16 +e,3,follows,user,person,8 +e,4,wrote,person,movie,10 +e,5,produced,person,movie,15 \ No newline at end of file diff --git a/flex/tests/hqps/engine_config_test_cbo.yaml b/flex/tests/hqps/engine_config_test_cbo.yaml new file mode 100644 index 000000000000..86671afc11b2 --- /dev/null +++ b/flex/tests/hqps/engine_config_test_cbo.yaml @@ -0,0 +1,42 @@ +directories: + workspace: /tmp/interactive_workspace + subdirs: + data: data + logs: logs + conf: conf +log_level: INFO +default_graph: modern_graph +compute_engine: + type: hiactor + workers: + - localhost:10000 + thread_num_per_worker: 1 + store: + type: cpp-mcsr + metadata_store: + type: file # file/sqlite/etcd +compiler: + planner: + is_on: true + opt: CBO + rules: + - FilterIntoJoinRule + - FilterMatchRule + - NotMatchToAntiJoinRule + - ExpandGetVFusionRule + physical: + opt: proto + endpoint: + default_listen_address: localhost + bolt_connector: + disabled: false + port: 7687 + gremlin_connector: + disabled: false + port: 8182 + query_timeout: 40000 + gremlin_script_language_name: antlr_gremlin_calcite +http_service: + default_listen_address: localhost + admin_port: 7777 + query_port: 10000 diff --git a/flex/tests/hqps/engine_config_test.yaml b/flex/tests/hqps/engine_config_test_rbo.yaml similarity index 93% rename from flex/tests/hqps/engine_config_test.yaml rename to flex/tests/hqps/engine_config_test_rbo.yaml index a5d42fca55f2..0b566865ebee 100644 --- a/flex/tests/hqps/engine_config_test.yaml +++ b/flex/tests/hqps/engine_config_test_rbo.yaml @@ -23,6 +23,9 @@ compiler: - FilterIntoJoinRule - FilterMatchRule - NotMatchToAntiJoinRule + - ExpandGetVFusionRule + physical: + opt: ffi endpoint: default_listen_address: localhost bolt_connector: diff --git a/flex/tests/hqps/hqps_adhoc_test.sh b/flex/tests/hqps/hqps_adhoc_test.sh index 3e37629b30a6..53f58aea1ee1 100644 --- a/flex/tests/hqps/hqps_adhoc_test.sh +++ b/flex/tests/hqps/hqps_adhoc_test.sh @@ -20,19 +20,18 @@ SERVER_BIN=${FLEX_HOME}/build/bin/interactive_server GIE_HOME=${FLEX_HOME}/../interactive_engine/ # -if [ $# -lt 3 ] || [ $# -gt 4 ]; then - echo "Receives: $# args, need 3 or 4 args" - echo "Usage: $0 [TEST_TYPE(cypher/gremlin/all)]" +if [ $# -lt 4 ] || [ $# -gt 5 ]; then + echo "Receives: $# args, need 4 or 5 args" + echo "Usage: $0 [STATISTIC]" exit 1 fi INTERACTIVE_WORKSPACE=$1 GRAPH_NAME=$2 ENGINE_CONFIG_PATH=$3 -if [ $# -eq 4 ]; then - TEST_TYPE=$4 -else - TEST_TYPE="cypher" # default run cypher tests +TEST_TYPE=$4 +if [ $# -eq 5 ]; then + STATISTIC_FILE=$4 fi # check TEST_TYPE is valid @@ -119,6 +118,10 @@ start_compiler_service(){ echo "try to start compiler service" pushd ${GIE_HOME}/compiler cmd="make run graph.schema=${GRAPH_SCHEMA_YAML} config.path=${ENGINE_CONFIG_PATH}" + # if STATISTIC_FILE is set, add it to cmd + if [ -n "${STATISTIC_FILE}" ]; then + cmd="${cmd} graph.planner.cbo.glogue.schema=${STATISTIC_FILE}" + fi echo "Start compiler service with command: ${cmd}" ${cmd} & sleep 5 diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java index 3e9c79f07b4c..6f7a64976de4 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java @@ -35,6 +35,14 @@ public class YamlConfigs extends Configs { static { ImmutableMap.Builder mapBuilder = ImmutableMap.builder(); mapBuilder + .put("physical.opt.config", + (Configs configs) -> { + if (configs.get("compiler.physical.opt") != null) { + return configs.get("compiler.physical.opt"); + } else { + return "ffi"; + } + }) .put( "graph.planner.is.on", (Configs configs) -> configs.get("compiler.planner.is_on"))