feat(interactive): Migrate to CBO optimizer

.github/workflows/hqps-db-ci.yml

@@ -130,9 +130,9 @@ jobs:
         cp ${SCHEMA_FILE} ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml
         GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/indices/
         cd ${GITHUB_WORKSPACE}/flex/tests/hqps
-        sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ./engine_config_test.yaml
-        bash hqps_admin_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./engine_config_test.yaml ${GS_TEST_DIR}
-        sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ./engine_config_test.yaml
+        sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ./engine_config_test_rbo.yaml
+        bash hqps_admin_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./engine_config_test_rbo.yaml ${GS_TEST_DIR}
+        sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ./engine_config_test_rbo.yaml
 
     - name: Build and test Interactive Java/Python SDK
       env: 
@@ -158,10 +158,10 @@ jobs:
         cp ${SCHEMA_FILE} ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml
         GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/indices/
         cd ${GITHUB_WORKSPACE}/flex/tests/hqps
-        sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ./engine_config_test.yaml
-        bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./engine_config_test.yaml java
-        bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./engine_config_test.yaml python
-        sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ./engine_config_test.yaml
+        sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ./engine_config_test_rbo.yaml
+        bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./engine_config_test_rbo.yaml java
+        bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./engine_config_test_rbo.yaml python
+        sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ./engine_config_test_rbo.yaml
 
     - name: Sample Query test
       env:
@@ -187,7 +187,7 @@ jobs:
         for i in 1 2 3 4 5 6 7 8 9 10 11 12;
         do
           cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=/tmp/codegen/"
-          cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
+          cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test_rbo.yaml "
           cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
           echo $cmd
           eval ${cmd} || exit 1
@@ -196,19 +196,19 @@ jobs:
         for i in 1 2 3 4 5 6 7 8 9 11 12; # 10 is not supported now
         do
           cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/simple_match_${i}.cypher -w=/tmp/codegen/"
-          cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
+          cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test_rbo.yaml "
           cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
           echo $cmd
           eval ${cmd} || exit 1
         done
 
         # test movie graph, 8,9,10 are not supported now
-        # change the default_graph config in ../tests/hqps/engine_config_test.yaml to movies
-        sed -i 's/default_graph: ldbc/default_graph: movies/g' ../tests/hqps/engine_config_test.yaml
+        # change the default_graph config in ../tests/hqps/engine_config_test_rbo.yaml to movies
+        sed -i 's/default_graph: ldbc/default_graph: movies/g' ../tests/hqps/engine_config_test_rbo.yaml
         for i in 1 2 3 4 5 6 7 11 12 13 14 15;
         do
           cmd="./load_plan_and_gen.sh -e=hqps -i=../tests/hqps/queries/movie/query${i}.cypher -w=/tmp/codegen/"
-          cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
+          cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test_rbo.yaml "
           cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml"
           echo $cmd
           eval ${cmd} || exit 1
@@ -225,23 +225,23 @@ jobs:
 
         cd ${GITHUB_WORKSPACE}/flex/bin
         ./load_plan_and_gen.sh -e=hqps -i=../interactive/examples/modern_graph/get_person_name.cypher -w=/tmp/codegen \
-          --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml -o=${PLUGIN_DIR} \
+          --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test_rbo.yaml -o=${PLUGIN_DIR} \
           --procedure_name=get_person_name \
           --graph_schema_path=../interactive/examples/modern_graph/graph.yaml
 
         ./load_plan_and_gen.sh -e=hqps -i=../interactive/examples/modern_graph/count_vertex_num.cypher -w=/tmp/codegen \
-          --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml  -o=${PLUGIN_DIR} \
+          --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test_rbo.yaml  -o=${PLUGIN_DIR} \
           --procedure_name=count_vertex_num \
           --graph_schema_path=../interactive/examples/modern_graph/graph.yaml
 
         cd ${GITHUB_WORKSPACE}/flex/tests/interactive/
         bash test_plugin_loading.sh ./modern_graph_schema_v0_0.yaml \
             ../../interactive/examples/modern_graph/bulk_load.yaml \
-           ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml
+           ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test_rbo.yaml
 
         bash test_plugin_loading.sh ./modern_graph_schema_v0_1.yaml \
             ../../interactive/examples/modern_graph/bulk_load.yaml \
-           ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml
+           ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test_rbo.yaml
 
     - name: Run End-to-End cypher adhoc ldbc query test
       env:
@@ -252,10 +252,10 @@ jobs:
       run: |
         cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
         export ENGINE_TYPE=hiactor
-        # change the default_graph config in ./engine_config_test.yaml to ldbc
-        sed -i 's/default_graph: movies/default_graph: ldbc/g' ./engine_config_test.yaml
+        # change the default_graph config in ./engine_config_test_rbo.yaml to ldbc
+        sed -i 's/default_graph: movies/default_graph: ldbc/g' ./engine_config_test_rbo.yaml
         bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} ldbc \
-              ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml
+              ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test_rbo.yaml cypher
 
     - name: Run End-to-End cypher adhoc movie query test
       env:
@@ -266,10 +266,10 @@ jobs:
       run: |
         cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
         export ENGINE_TYPE=hiactor
-        # change the default_graph config in ./engine_config_test.yaml to movies
-        sed -i 's/default_graph: ldbc/default_graph: movies/g' ./engine_config_test.yaml
+        # change the default_graph config in ./engine_config_test_rbo.yaml to movies
+        sed -i 's/default_graph: ldbc/default_graph: movies/g' ./engine_config_test_rbo.yaml
         bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} movies \
-          ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml
+          ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test_rbo.yaml cypher
 
     - name: Run End-to-End cypher adhoc graph_algo query test
       env:
@@ -281,9 +281,9 @@ jobs:
         cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
         export ENGINE_TYPE=hiactor
         # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to graph_algo
-        sed -i 's/default_graph: movies/default_graph: graph_algo/g' ./engine_config_test.yaml
+        sed -i 's/default_graph: movies/default_graph: graph_algo/g' ./engine_config_test_rbo.yaml
         bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} graph_algo \
-          ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml
+          ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test_rbo.yaml cypher
 
     - name: Run Gremlin test on modern graph
       env:
@@ -293,8 +293,8 @@ jobs:
       run: |
         cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
         export ENGINE_TYPE=hiactor
-        sed -i 's/default_graph: graph_algo/default_graph: modern_graph/g' ./engine_config_test.yaml
+        sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ./engine_config_test_cbo.yaml
         bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} modern_graph \
-          ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml gremlin
+          ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test_rbo.yaml gremlin
 
 

flex/interactive/docker/interactive-runtime.Dockerfile

@@ -79,7 +79,7 @@ COPY --from=builder /opt/flex /opt/flex
 # copy the builtin graph, modern_graph
 RUN mkdir -p /opt/flex/share/gs_interactive_default_graph/
 COPY --from=builder /home/graphscope/GraphScope/flex/interactive/examples/modern_graph/* /opt/flex/share/gs_interactive_default_graph/
-COPY --from=builder /home/graphscope/GraphScope/flex/tests/hqps/engine_config_test.yaml /opt/flex/share/engine_config.yaml
+COPY --from=builder /home/graphscope/GraphScope/flex/tests/hqps/engine_config_test_rbo.yaml /opt/flex/share/engine_config.yaml
 COPY --from=builder /home/graphscope/GraphScope/flex/interactive/docker/entrypoint.sh /opt/flex/bin/entrypoint.sh
 COPY --from=builder /home/graphscope/GraphScope/flex/third_party/nlohmann-json/single_include/* /opt/flex/include/
 RUN sed -i 's/name: modern_graph/name: gs_interactive_default_graph/g' /opt/flex/share/gs_interactive_default_graph/graph.yaml

flex/interactive/examples/graph_algo/statistics.txt

@@ -0,0 +1,14 @@
+v,0,Paper,123
+v,1,Challenge,5
+v,2,Topic,8
+v,3,Task,25
+v,4,Solution,15
+v,5,CCFField,11
+e,0,WorkOn,Paper,Task,123
+e,1,Resolve,Paper,Challenge,168
+e,2,Target,Task,Challenge,57
+e,3,Belong,Task,Topic,25
+e,4,User,Paper,Solution,170
+e,5,ApplyON,Solution,Challenge,15
+e,6,HasField,Paper,CCFField,109
+e,7,Citation,Paper,Paper,116

flex/interactive/examples/modern_graph/statistics.txt

@@ -0,0 +1,4 @@
+v,0,person,4
+v,1,software,2
+e,0,knows,person,person,2
+e,1,created,person,software,4

flex/interactive/examples/movies/statistics.txt

@@ -0,0 +1,9 @@
+v,0,movie,38
+v,1,person,130
+v,2,user,7
+e,0,acted_in,person,movie,172
+e,1,directed,person,movie,44
+e,2,review,user,movie,16
+e,3,follows,user,person,8
+e,4,wrote,person,movie,10
+e,5,produced,person,movie,15

flex/tests/hqps/engine_config_test_cbo.yaml

@@ -0,0 +1,42 @@
+directories:
+  workspace: /tmp/interactive_workspace
+  subdirs:
+    data: data
+    logs: logs
+    conf: conf
+log_level: INFO
+default_graph: modern_graph
+compute_engine:
+  type: hiactor
+  workers:
+    - localhost:10000
+  thread_num_per_worker: 1
+  store:
+    type: cpp-mcsr
+  metadata_store:
+    type: file # file/sqlite/etcd
+compiler:
+  planner:
+    is_on: true
+    opt: CBO
+    rules:
+      - FilterIntoJoinRule
+      - FilterMatchRule
+      - NotMatchToAntiJoinRule
+      - ExpandGetVFusionRule 
+  physical:
+    opt: proto
+  endpoint:
+    default_listen_address: localhost
+    bolt_connector:
+      disabled: false
+      port: 7687
+    gremlin_connector:
+      disabled: false
+      port: 8182
+  query_timeout: 40000
+  gremlin_script_language_name: antlr_gremlin_calcite
+http_service:
+  default_listen_address: localhost
+  admin_port: 7777
+  query_port: 10000

flex/tests/hqps/engine_config_test.yaml → flex/tests/hqps/engine_config_test_rbo.yaml

@@ -23,6 +23,9 @@ compiler:
       - FilterIntoJoinRule
       - FilterMatchRule
       - NotMatchToAntiJoinRule
+      - ExpandGetVFusionRule 
+  physical:
+    opt: ffi
   endpoint:
     default_listen_address: localhost
     bolt_connector:

flex/tests/hqps/hqps_adhoc_test.sh

@@ -20,19 +20,18 @@ SERVER_BIN=${FLEX_HOME}/build/bin/interactive_server
 GIE_HOME=${FLEX_HOME}/../interactive_engine/
 
 # 
-if [ $# -lt 3 ] || [ $# -gt 4 ]; then
-  echo "Receives: $# args, need 3 or 4 args"
-  echo "Usage: $0 <INTERACTIVE_WORKSPACE> <GRAPH_NAME> <ENGINE_CONFIG> [TEST_TYPE(cypher/gremlin/all)]"
+if [ $# -lt 4 ] || [ $# -gt 5 ]; then
+  echo "Receives: $# args, need 4 or 5 args"
+  echo "Usage: $0 <INTERACTIVE_WORKSPACE> <GRAPH_NAME> <ENGINE_CONFIG> <TEST_TYPE(cypher/gremlin/all)> [STATISTIC]"
   exit 1
 fi
 
 INTERACTIVE_WORKSPACE=$1
 GRAPH_NAME=$2
 ENGINE_CONFIG_PATH=$3
-if [ $# -eq 4 ]; then
-  TEST_TYPE=$4
-else
-  TEST_TYPE="cypher" # default run cypher tests
+TEST_TYPE=$4
+if [ $# -eq 5 ]; then
+  STATISTIC_FILE=$4
 fi
 
 # check TEST_TYPE is valid
@@ -119,6 +118,10 @@ start_compiler_service(){
   echo "try to start compiler service"
   pushd ${GIE_HOME}/compiler
   cmd="make run graph.schema=${GRAPH_SCHEMA_YAML} config.path=${ENGINE_CONFIG_PATH}"
+  # if STATISTIC_FILE is set, add it to cmd
+  if [ -n "${STATISTIC_FILE}" ]; then
+    cmd="${cmd} graph.planner.cbo.glogue.schema=${STATISTIC_FILE}"
+  fi
   echo "Start compiler service with command: ${cmd}"
   ${cmd} &
   sleep 5

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java

@@ -35,6 +35,14 @@ public class YamlConfigs extends Configs {
     static {
         ImmutableMap.Builder<String, ValueGetter> mapBuilder = ImmutableMap.builder();
         mapBuilder
+                .put("physical.opt.config",
+                        (Configs configs) -> {
+                            if (configs.get("compiler.physical.opt") != null) {
+                                return configs.get("compiler.physical.opt");
+                            } else {
+                                return "ffi";
+                            }
+                        })
                 .put(
                         "graph.planner.is.on",
                         (Configs configs) -> configs.get("compiler.planner.is_on"))