Enable logsdb index mode in security track (elastic#670)

This PR changes the security track so that we can enable LogsDB in index templates. Note that the failure store is only available in serverless so we gate its usage excluding it in case the deployment is not serverless. For LogsDB testing we rely on Kibana to install all other component/composable templates. This is to make sure we need limited changes to the Rally track. While testing this new configuration we discovered that installation of (component) templates done by Kibana is Serverless only happens when a user interacts with it. This means (component) templates are not installed and the `elastic/security` track execution fails as a result of using (component) templates that do not exist.
salvatore-campagna · Sep 19, 2024 · 2423ac1 · 2423ac1
1 parent d087d82
commit 2423ac1
Show file tree

Hide file tree

Showing 9 changed files with 342 additions and 0 deletions.
diff --git a/elastic/security/README.md b/elastic/security/README.md
@@ -83,6 +83,7 @@ The following parameters are available:
 
 * `wait_for_status` (default: `green`) - The track creates Data Streams prior to indexing. All created Data Streams must at least reach this status before indexing commences. Reduce to `yellow` for clusters where green isn't possible e.g. single node.
 * `corpora_uri_base` (default: `https://rally-tracks.elastic.co`) - Specify the base location of the datasets used by this track.
+* `index_mode` (default: unset) - A parameter meant to be used internally which defines one of the available indexing modes, "standard", "logsdb" or "time_series". If not set, "standard" is used.
 
 ### Data Generation Parameters
 

diff --git a/elastic/security/templates/composable-logsdb/logs-endpoint.alerts.json b/elastic/security/templates/composable-logsdb/logs-endpoint.alerts.json
@@ -0,0 +1,44 @@
+{
+  "name": "logs-endpoint.alerts",
+  "index_template": {
+    "index_patterns": [
+      "logs-endpoint.alerts-*"
+    ],
+    "template": {
+      "settings": {},
+      "mappings": {
+        "_meta": {
+          "package": {
+            "name": "endpoint"
+          },
+          "managed_by": "fleet",
+          "managed": true
+        }
+      }
+    },
+    "composed_of": [
+      "logs@mappings",
+      "logs@settings",
+      "logs-endpoint.alerts@package",
+      "ecs@mappings",
+      ".fleet_globals-1",
+      ".fleet_agent_id_verification-1",
+      "track-shared-logsdb-mode"
+    ],
+    "priority": 200,
+    "_meta": {
+      "package": {
+        "name": "endpoint"
+      },
+      "managed_by": "fleet",
+      "managed": true
+    },
+    "data_stream": {
+      "hidden": false,
+      "allow_custom_routing": false
+      {% if build_flavor != "serverless" %},
+      "failure_store": false
+      {% endif %}
+    }
+  }
+}
diff --git a/elastic/security/templates/composable-logsdb/logs-endpoint.events.file.json b/elastic/security/templates/composable-logsdb/logs-endpoint.events.file.json
@@ -0,0 +1,44 @@
+{
+  "name": "logs-endpoint.events.file",
+  "index_template": {
+    "index_patterns": [
+      "logs-endpoint.events.file-*"
+    ],
+    "template": {
+      "settings": {},
+      "mappings": {
+        "_meta": {
+          "package": {
+            "name": "endpoint"
+          },
+          "managed_by": "fleet",
+          "managed": true
+        }
+      }
+    },
+    "composed_of": [
+      "logs@mappings",
+      "logs@settings",
+      "logs-endpoint.events.file@package",
+      "ecs@mappings",
+      ".fleet_globals-1",
+      ".fleet_agent_id_verification-1",
+      "track-shared-logsdb-mode"
+    ],
+    "priority": 200,
+    "_meta": {
+      "package": {
+        "name": "endpoint"
+      },
+      "managed_by": "fleet",
+      "managed": true
+    },
+    "data_stream": {
+      "hidden": false,
+      "allow_custom_routing": false
+      {% if build_flavor != "serverless" %},
+      "failure_store": false
+      {% endif %}
+    }
+  }
+}
diff --git a/elastic/security/templates/composable-logsdb/logs-endpoint.events.library.json b/elastic/security/templates/composable-logsdb/logs-endpoint.events.library.json
@@ -0,0 +1,44 @@
+{
+  "name": "logs-endpoint.events.library",
+  "index_template": {
+    "index_patterns": [
+      "logs-endpoint.events.library-*"
+    ],
+    "template": {
+      "settings": {},
+      "mappings": {
+        "_meta": {
+          "package": {
+            "name": "endpoint"
+          },
+          "managed_by": "fleet",
+          "managed": true
+        }
+      }
+    },
+    "composed_of": [
+      "logs@mappings",
+      "logs@settings",
+      "logs-endpoint.events.library@package",
+      "ecs@mappings",
+      ".fleet_globals-1",
+      ".fleet_agent_id_verification-1",
+      "track-shared-logsdb-mode"
+    ],
+    "priority": 200,
+    "_meta": {
+      "package": {
+        "name": "endpoint"
+      },
+      "managed_by": "fleet",
+      "managed": true
+    },
+    "data_stream": {
+      "hidden": false,
+      "allow_custom_routing": false
+      {% if build_flavor != "serverless" %},
+      "failure_store": false
+      {% endif %}
+    }
+  }
+}
diff --git a/elastic/security/templates/composable-logsdb/logs-endpoint.events.network.json b/elastic/security/templates/composable-logsdb/logs-endpoint.events.network.json
@@ -0,0 +1,44 @@
+{
+  "name": "logs-endpoint.events.network",
+  "index_template": {
+    "index_patterns": [
+      "logs-endpoint.events.network-*"
+    ],
+    "template": {
+      "settings": {},
+      "mappings": {
+        "_meta": {
+          "package": {
+            "name": "endpoint"
+          },
+          "managed_by": "fleet",
+          "managed": true
+        }
+      }
+    },
+    "composed_of": [
+      "logs@mappings",
+      "logs@settings",
+      "logs-endpoint.events.network@package",
+      "ecs@mappings",
+      ".fleet_globals-1",
+      ".fleet_agent_id_verification-1",
+      "track-shared-logsdb-mode"
+    ],
+    "priority": 200,
+    "_meta": {
+      "package": {
+        "name": "endpoint"
+      },
+      "managed_by": "fleet",
+      "managed": true
+    },
+    "data_stream": {
+      "hidden": false,
+      "allow_custom_routing": false
+      {% if build_flavor != "serverless" %},
+      "failure_store": false
+      {% endif %}
+    }
+  }
+}
diff --git a/elastic/security/templates/composable-logsdb/logs-endpoint.events.process.json b/elastic/security/templates/composable-logsdb/logs-endpoint.events.process.json
@@ -0,0 +1,44 @@
+{
+  "name": "logs-endpoint.events.process",
+  "index_template": {
+    "index_patterns": [
+      "logs-endpoint.events.process-*"
+    ],
+    "template": {
+      "settings": {},
+      "mappings": {
+        "_meta": {
+          "package": {
+            "name": "endpoint"
+          },
+          "managed_by": "fleet",
+          "managed": true
+        }
+      }
+    },
+    "composed_of": [
+      "logs@mappings",
+      "logs@settings",
+      "logs-endpoint.events.process@package",
+      "ecs@mappings",
+      ".fleet_globals-1",
+      ".fleet_agent_id_verification-1",
+      "track-shared-logsdb-mode"
+    ],
+    "priority": 200,
+    "_meta": {
+      "package": {
+        "name": "endpoint"
+      },
+      "managed_by": "fleet",
+      "managed": true
+    },
+    "data_stream": {
+      "hidden": false,
+      "allow_custom_routing": false
+      {% if build_flavor != "serverless" %},
+      "failure_store": false
+      {% endif %}
+    }
+  }
+}
diff --git a/elastic/security/templates/composable-logsdb/logs-endpoint.events.registry.json b/elastic/security/templates/composable-logsdb/logs-endpoint.events.registry.json
@@ -0,0 +1,44 @@
+{
+  "name": "logs-endpoint.events.registry",
+  "index_template": {
+    "index_patterns": [
+      "logs-endpoint.events.registry-*"
+    ],
+    "template": {
+      "settings": {},
+      "mappings": {
+        "_meta": {
+          "package": {
+            "name": "endpoint"
+          },
+          "managed_by": "fleet",
+          "managed": true
+        }
+      }
+    },
+    "composed_of": [
+      "logs@mappings",
+      "logs@settings",
+      "logs-endpoint.events.registry@package",
+      "ecs@mappings",
+      ".fleet_globals-1",
+      ".fleet_agent_id_verification-1",
+      "track-shared-logsdb-mode"
+    ],
+    "priority": 200,
+    "_meta": {
+      "package": {
+        "name": "endpoint"
+      },
+      "managed_by": "fleet",
+      "managed": true
+    },
+    "data_stream": {
+      "hidden": false,
+      "allow_custom_routing": false
+      {% if build_flavor != "serverless" %},
+      "failure_store": false
+      {% endif %}
+    }
+  }
+}
diff --git a/elastic/security/templates/composable-logsdb/logs-endpoint.events.security.json b/elastic/security/templates/composable-logsdb/logs-endpoint.events.security.json
@@ -0,0 +1,44 @@
+{
+  "name": "logs-endpoint.events.security",
+  "index_template": {
+    "index_patterns": [
+      "logs-endpoint.events.security-*"
+    ],
+    "template": {
+      "settings": {},
+      "mappings": {
+        "_meta": {
+          "package": {
+            "name": "endpoint"
+          },
+          "managed_by": "fleet",
+          "managed": true
+        }
+      }
+    },
+    "composed_of": [
+      "logs@mappings",
+      "logs@settings",
+      "logs-endpoint.events.security@package",
+      "ecs@mappings",
+      ".fleet_globals-1",
+      ".fleet_agent_id_verification-1",
+      "track-shared-logsdb-mode"
+    ],
+    "priority": 200,
+    "_meta": {
+      "package": {
+        "name": "endpoint"
+      },
+      "managed_by": "fleet",
+      "managed": true
+    },
+    "data_stream": {
+      "hidden": false,
+      "allow_custom_routing": false
+      {% if build_flavor != "serverless" %},
+      "failure_store": false
+      {% endif %}
+    }
+  }
+}