Update elastic/security artifacts

elastic/security/ilm/logs.json

@@ -5,15 +5,16 @@
         "min_age": "0ms",
         "actions": {
           "rollover": {
-            "max_primary_shard_size": "50gb",
-            "max_age": "30d"
+            "max_age": "30d",
+            "max_primary_shard_size": "50gb"
           }
         }
       }
     },
     "_meta": {
-      "description": "default policy for the logs index template installed by x-pack",
-      "managed": true
-    }
+      "managed": true,
+      "description": "default policy for the logs index template installed by x-pack"
+    },
+    "deprecated": true
   }
 }

elastic/security/pipelines/.fleet_final_pipeline-1.json

@@ -1,24 +1,17 @@
 {
-  "version": 2,
+  "version": 4,
   "_meta": {
     "managed_by": "fleet",
     "managed": true
   },
   "description": "Final pipeline for processing all incoming Fleet Agent documents.\n",
   "processors": [
-    {
-      "set": {
-        "description": "Add time when event was ingested.",
-        "field": "event.ingested",
-        "copy_from": "_ingest.timestamp"
-      }
-    },
     {
       "script": {
-        "description": "Remove sub-seconds from event.ingested to improve storage efficiency.",
+        "description": "Add time when event was ingested (and remove sub-seconds to improve storage efficiency)",
         "tag": "truncate-subseconds-event-ingested",
-        "source": "ctx.event.ingested = ctx.event.ingested.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);",
-        "ignore_failure": true
+        "ignore_failure": true,
+        "source": "if (ctx?.event == null) {\n  ctx.event = [:];\n}\n\nctx.event.ingested = metadata().now.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);"
       }
     },
     {
@@ -31,6 +24,15 @@
         "ignore_missing": true
       }
     },
+    {
+      "remove": {
+        "description": "Remove event.original unless the preserve_original_event tag is set",
+        "field": "event.original",
+        "if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))",
+        "ignore_failure": true,
+        "ignore_missing": true
+      }
+    },
     {
       "set_security_user": {
         "field": "_security",

elastic/security/pipelines/logs-endpoint.events.file-8.15.1.json

@@ -0,0 +1,47 @@
+{
+  "description": "Pipeline for setting event.ingested",
+  "processors": [
+    {
+      "set": {
+        "field": "event.ingested",
+        "value": "{{ _ingest.timestamp }}",
+        "ignore_failure": true
+      }
+    },
+    {
+      "pipeline": {
+        "name": "global@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Global pipeline for all data streams"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for all data streams of type `logs`"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-endpoint.integration@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `endpoint` integration"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-endpoint.events.file@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for the `endpoint.events.file` dataset"
+      }
+    }
+  ],
+  "_meta": {
+    "managed_by": "fleet",
+    "managed": true,
+    "package": {
+      "name": "endpoint"
+    }
+  }
+}

elastic/security/pipelines/logs-endpoint.events.library-8.15.1.json

@@ -0,0 +1,47 @@
+{
+  "description": "Pipeline for setting event.ingested",
+  "processors": [
+    {
+      "set": {
+        "field": "event.ingested",
+        "value": "{{ _ingest.timestamp }}",
+        "ignore_failure": true
+      }
+    },
+    {
+      "pipeline": {
+        "name": "global@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Global pipeline for all data streams"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for all data streams of type `logs`"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-endpoint.integration@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `endpoint` integration"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-endpoint.events.library@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for the `endpoint.events.library` dataset"
+      }
+    }
+  ],
+  "_meta": {
+    "managed_by": "fleet",
+    "managed": true,
+    "package": {
+      "name": "endpoint"
+    }
+  }
+}

elastic/security/pipelines/logs-endpoint.events.network-8.2.0.json → elastic/security/pipelines/logs-endpoint.events.network-8.15.1.json

@@ -117,6 +117,34 @@
         "ignore_missing": true,
         "field": "dns.question.Ext_temp"
       }
+    },
+    {
+      "pipeline": {
+        "name": "global@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Global pipeline for all data streams"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for all data streams of type `logs`"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-endpoint.integration@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `endpoint` integration"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-endpoint.events.network@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for the `endpoint.events.network` dataset"
+      }
     }
   ],
   "_meta": {