Merge branch 'BackportwBigDS'

aerospike-community · Jul 31, 2024 · fe49edd · fe49edd
2 parents d606fb2 + 9060f7e
commit fe49edd
Show file tree

Hide file tree

Showing 19 changed files with 912 additions and 217 deletions.
diff --git a/.gitignore b/.gitignore
@@ -20,4 +20,4 @@ venv
 aerospike/data/*
 aerospike/results/*
 aerospike/data
-aerospike/results
+aerospike/results
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -145,7 +145,8 @@
             "args": [
                     "--dataset", "glove-100-angular",
                     "--idxdrop",
-                    "--concurrency", "10000"                    
+                    "--concurrency", "10000",
+                    "--indexparams", "{\"m\":16, \"efconst\":100, \"ef\":100, \"batching_params\":{\"maxrecs\":10000, \"interval\":10000}, \"caching_params\":{\"max_entries\":\"None\", \"expiry\":\"None\"}, \"healer_params\":{\"max_scan_rate_per_node\":\"None\", \"max_scan_page_size\":\"None\", \"re_index_percent\": \"None\", \"schedule_delay\": \"None\", \"parallelism\": \"None\"}, \"merge_params\":{\"parallelism\": \"None\"}}"
                     ],
             "console": "integratedTerminal"
         },
@@ -242,9 +243,9 @@
             "args": [
                     "--dataset", "gist-960-euclidean",
                     "--logfile", "./hdfquery-gist1.log",                    
-                    "-r", "10",
+                    "-r", "1",
                     "--limit", "10",                    
-                    "--idxname", "ANN-data_euclidean_SQUARED_EUCLIDEAN_960_16_100_100_Idx"
+                    "--idxname", "gist-960-euclidean_Idx"
                     ],
             "console": "integratedTerminal"
         },

diff --git a/aerospike/AerospikeHDFDashboard.json b/aerospike/AerospikeHDFDashboard.json
@@ -752,7 +752,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   }
                 ]
               },
@@ -861,7 +862,7 @@
                   "text": "Done"
                 },
                 "Paused": {
-                  "color": "red",
+                  "color": "dark-yellow",
                   "index": 1,
                   "text": "Paused"
                 },
@@ -871,7 +872,7 @@
                   "text": "Running"
                 },
                 "Waiting": {
-                  "color": "yellow",
+                  "color": "orange",
                   "index": 2,
                   "text": "Waiting"
                 }
@@ -1044,6 +1045,76 @@
       ],
       "type": "gauge"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "description": "The current total of vectors upserted.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "light-blue",
+                "value": null
+              }
+            ]
+          },
+          "unit": "locale"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 3,
+        "x": 6,
+        "y": 11
+      },
+      "id": 670,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "sum(aerospike_hdf_populate_total{instance=~\"$client\", ns=\"$namespace\", set=\"$set\"})",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Current Upserted Vectors",
+      "type": "stat"
+    },
     {
       "datasource": {
         "type": "prometheus",
@@ -1087,7 +1158,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "purple",
                 "value": null
               }
             ]
@@ -1099,7 +1170,7 @@
       "gridPos": {
         "h": 5,
         "w": 3,
-        "x": 6,
+        "x": 9,
         "y": 11
       },
       "id": 666,
@@ -1185,7 +1256,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "purple",
                 "value": null
               }
             ]
@@ -1197,7 +1268,7 @@
       "gridPos": {
         "h": 5,
         "w": 3,
-        "x": 9,
+        "x": 12,
         "y": 11
       },
       "id": 665,
@@ -1283,7 +1354,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "purple",
                 "value": null
               }
             ]
@@ -1295,7 +1366,7 @@
       "gridPos": {
         "h": 5,
         "w": 3,
-        "x": 12,
+        "x": 15,
         "y": 11
       },
       "id": 668,
@@ -1381,7 +1452,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "purple",
                 "value": null
               }
             ]
@@ -1393,7 +1464,7 @@
       "gridPos": {
         "h": 5,
         "w": 3,
-        "x": 15,
+        "x": 18,
         "y": 11
       },
       "id": 667,
@@ -1436,6 +1507,77 @@
       "title": "After Population Index Action",
       "type": "stat"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 0,
+          "fieldMinMax": false,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "blue",
+                "value": null
+              }
+            ]
+          },
+          "unit": "locale"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 3,
+        "x": 21,
+        "y": 11
+      },
+      "id": 669,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "aerospike_hdf_vectorqueuedepth",
+          "format": "time_series",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": true,
+          "legendFormat": "__auto",
+          "range": false,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Index Merge Queue Depth",
+      "type": "stat"
+    },
     {
       "datasource": {
         "type": "prometheus",
@@ -2573,7 +2715,7 @@
       "type": "barchart"
     }
   ],
-  "refresh": "10s",
+  "refresh": "5s",
   "schemaVersion": 39,
   "tags": [
     "Monitoring",
@@ -2749,7 +2891,7 @@
     ]
   },
   "time": {
-    "from": "now-1h",
+    "from": "now-5m",
     "to": "now"
   },
   "timepicker": {
@@ -2780,6 +2922,6 @@
   "timezone": "",
   "title": "Aerospike HDF",
   "uid": "fzUPYeJIkhdf",
-  "version": 25,
+  "version": 31,
   "weekStart": ""
 }
diff --git a/aerospike/README.md b/aerospike/README.md
@@ -89,12 +89,6 @@ Note: if “—hosts” argument is used, only the first host in the list is use
 
 (default: False)
 
-\-T, --vectortls
-
-Use TLS to connect to the Vector Server
-
-(default: False)
-
 \-n NS, --namespace NS
 
 The Aerospike Namespace  
@@ -230,12 +224,6 @@ Note: if “—hosts” argument is used, only the first host in the list is use
 
 (default: False)
 
-\-T, --vectortls
-
-Use TLS to connect to the Vector Server
-
-(default: False)
-
 \-N NS, --idxnamespace NS
 
 Aerospike Namespace where the vector index will be located.  
@@ -345,10 +333,6 @@ Example: 'hosta:5000' or 'hostb' (default: ['localhost:5000'])
 
 Use Vector's DB Load Balancer (default: False)
 
-\-T, --vectortls
-
-Use TLS to connect to the Vector DB Server (default: False)
-
 \-idxns INDEXNAME, --indexnamespace INDEXNAME
 
 Vector's Index Namespace.