Merge branch 'main' into add_panw_metrics_integration

.buildkite/pipeline.schedule-daily.yml

@@ -21,23 +21,36 @@ steps:
       env:
         SERVERLESS: "false"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 7.17.23-SNAPSHOT
+        STACK_VERSION: 7.17.24-SNAPSHOT
     depends_on:
       - step: "check"
         allow_failure: false
 
-  - label: "Check integrations local stacks - Stack Version v8.15"
+  - label: "Check integrations local stacks - Stack Version v8.16"
     trigger: "integrations"
     build:
       env:
         SERVERLESS: "false"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 8.15.0-SNAPSHOT
+        STACK_VERSION: 8.16.0-SNAPSHOT
         PUBLISH_COVERAGE_REPORTS: "true"
     depends_on:
       - step: "check"
         allow_failure: false
 
+  - label: "Check integrations local stacks - Stack Version v8.16 - LogsDB"
+    trigger: "integrations"
+    build:
+      env:
+        SERVERLESS: "false"
+        FORCE_CHECK_ALL: "true"
+        STACK_VERSION: 8.16.0-SNAPSHOT
+        STACK_LOGSDB_ENABLED: "true"
+        PUBLISH_COVERAGE_REPORTS: "false"
+    depends_on:
+      - step: "check"
+        allow_failure: false
+
   - label: "Check integrations in serverless - project: Observability"
     key: "trigger-integrations-serverless-obs"
     trigger: "integrations-serverless"

.buildkite/pipeline.schedule-weekly.yml

@@ -0,0 +1,29 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+name: integrations-schedule-weekly
+
+env:
+  SETUP_GVM_VERSION: "v0.5.2"
+  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
+
+# The pipeline is triggered by the scheduler every week
+steps:
+  - label: ":white_check_mark: Check go sources"
+    key: "check"
+    command: ".buildkite/scripts/check_sources.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+
+  - label: "Check integrations local stacks and Elastic Agent Ubuntu docker - Stack Version v8.16"
+    trigger: "integrations"
+    build:
+      env:
+        SERVERLESS: "false"
+        FORCE_CHECK_ALL: "true"
+        STACK_VERSION: 8.16.0-SNAPSHOT
+        PUBLISH_COVERAGE_REPORTS: "false"
+        ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "true"
+    depends_on:
+      - step: "check"
+        allow_failure: false

.buildkite/pipeline.serverless.yml

@@ -6,7 +6,7 @@ env:
   DOCKER_COMPOSE_VERSION: "v2.24.1"
   DOCKER_VERSION: "false" # not required to set since system tests are not running yet
   KIND_VERSION: 'v0.20.0'
-  K8S_VERSION: 'v1.30.0'
+  K8S_VERSION: 'v1.31.0'
   YQ_VERSION: 'v4.35.2'
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
   GH_CLI_VERSION: "2.29.0"
@@ -26,6 +26,8 @@ env:
   ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: "true"
   # Set maximum number of parallel tests to run if package allows it
   ELASTIC_PACKAGE_MAXIMUM_NUMBER_PARALLEL_TESTS: "5"
+  # Enable/Disable the usage of wolfi images for Elastic Agent
+  ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "${ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI:-false}"
 
 steps:
   - input: "Input values for the variables"
@@ -76,8 +78,10 @@ steps:
 
   - label: ":junit: Junit annotate"
     plugins:
-      - junit-annotate#v2.4.1:
+      - junit-annotate#v2.5.0:
           artifacts: "build/test-results/*.xml"
+          failed-download-exit-code: 0 # Not fail the build in case there are no XML files
+          report-skipped: true
     agents:
       provider: "gcp"  # junit plugin requires docker
 

.buildkite/pipeline.yml

@@ -1,14 +1,16 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 env:
   SETUP_GVM_VERSION: "v0.5.2"
-  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
   DOCKER_COMPOSE_VERSION: "v2.24.1"
   DOCKER_VERSION: "26.1.2"
   KIND_VERSION: 'v0.20.0'
-  K8S_VERSION: 'v1.30.0'
+  K8S_VERSION: 'v1.31.0'
   YQ_VERSION: 'v4.35.2'
   JQ_VERSION: '1.7'
   GH_CLI_VERSION: "2.29.0"
+
+  # Agent images used in pipeline steps
+  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
 
   # Elastic package settings
@@ -24,6 +26,8 @@ env:
   ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: "true"
   # Set maximum number of parallel tests to run if package allows it
   ELASTIC_PACKAGE_MAXIMUM_NUMBER_PARALLEL_TESTS: "5"
+  # Enable/Disable the usage of wolfi images for Elastic Agent
+  ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "${ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI:-false}"
 
 steps:
   - label: "Get reference from target branch"
@@ -80,9 +84,11 @@ steps:
 
   - label: ":junit: Junit annotate"
     plugins:
-      - junit-annotate#v2.4.1:
+      - junit-annotate#v2.5.0:
           artifacts: "build/test-results/*.xml"
           failed-download-exit-code: 0 # Not fail the build in case there are no XML files
+          report-skipped: true
+          always-annotate: true
     agents:
       provider: "gcp"  # junit plugin requires docker
 

.buildkite/pull-requests.json

@@ -5,7 +5,7 @@
       "pipelineSlug": "integrations",
       "allow_org_users": true,
       "allowed_repo_permissions": ["admin", "write"],
-      "allowed_list": ["dependabot[bot]", "mergify[bot]"],
+      "allowed_list": ["dependabot[bot]", "mergify[bot]", "elastic-vault-github-plugin-prod[bot]", "github-actions[bot]"],
       "set_commit_status": true,
       "build_on_commit": true,
       "build_on_comment": true,

.buildkite/scripts/backport_branch.sh

@@ -17,7 +17,7 @@ DRY_RUN="$(buildkite-agent meta-data get DRY_RUN --default ${DRY_RUN:-"true"})"
 BASE_COMMIT="$(buildkite-agent meta-data get BASE_COMMIT --default ${BASE_COMMIT:-""})"
 PACKAGE_NAME="$(buildkite-agent meta-data get PACKAGE_NAME --default ${PACKAGE_NAME:-""})"
 PACKAGE_VERSION="$(buildkite-agent meta-data get PACKAGE_VERSION --default ${PACKAGE_VERSION:-""})"
-REMOVE_OTHER_PACKAGES="$(buildkite-agent meta-data get PREMOVE_OTHER_PACKAGES --default ${REMOVE_OTHER_PACKAGES:-"false"})"
+REMOVE_OTHER_PACKAGES="$(buildkite-agent meta-data get REMOVE_OTHER_PACKAGES --default ${REMOVE_OTHER_PACKAGES:-"false"})"
 
 if [[ -z "$PACKAGE_NAME" ]] || [[ -z "$PACKAGE_VERSION" ]]; then
   buildkite-agent annotate "The variables **PACKAGE_NAME** or **PACKAGE_VERSION** aren't defined, please try again" --style "warning"

.buildkite/scripts/common.sh

@@ -503,6 +503,10 @@ prepare_stack() {
         fi
     fi
 
+    if [ "${STACK_LOGSDB_ENABLED:-false}" == "true" ]; then
+        args="${args} -U stack.logsdb_enabled=true"
+    fi
+
     echo "Boot up the Elastic stack"
     if ! ${ELASTIC_PACKAGE_BIN} stack up -d ${args} ; then
         return 1
@@ -692,7 +696,7 @@ is_pr_affected() {
 
     echo "[${package}] git-diff: check non-package files"
     commit_merge=$(git merge-base "${from}" "${to}")
-    if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|.github/CODEOWNERS|docs/)' ; then
+    if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|\.github/(CODEOWNERS|ISSUE_TEMPLATE|PULL_REQUEST_TEMPLATE)|README\.md|docs/)' ; then
         echo "[${package}] PR is affected: found non-package files"
         return 0
     fi

.buildkite/scripts/find_oldest_supported_version.py

@@ -2,14 +2,27 @@
 import argparse
 import requests
 import sys
-import yaml
 import unittest
+import yaml
 
-VERSION_URL = "https://artifacts-api.elastic.co/v1/versions?x-elastic-no-kpi=true"
+from requests.adapters import HTTPAdapter, Retry
+
+ARTIFACTS_URL = "https://artifacts-api.elastic.co"
+VERSION_URL = ARTIFACTS_URL + "/v1/versions?x-elastic-no-kpi=true"
 
 
 def fetch_version():
-    return requests.get(VERSION_URL).json()
+    # Retry forever on connection or 500 errors, assume the artifacts API
+    # will come back. If it doesn't come back we cannot continue executing
+    # jobs in any case.
+    retries = Retry(
+        total=None,
+        backoff_factor=0.5,
+        status_forcelist=[500, 502, 503, 504],
+    )
+    session = requests.Session()
+    session.mount(ARTIFACTS_URL, HTTPAdapter(max_retries=retries))
+    return session.get(VERSION_URL).json()
 
 
 def find_oldest_supported_version(kibana_version_condition: str) -> str:

.buildkite/scripts/trigger_integrations_in_parallel.sh

@@ -50,6 +50,7 @@ for package in ${PACKAGE_LIST}; do
     - label: "Check integrations ${package}"
       key: "test-integrations-${package}"
       command: ".buildkite/scripts/test_one_package.sh ${package} ${from} ${to}"
+      timeout_in_minutes: 240
       agents:
         provider: gcp
         image: ${IMAGE_UBUNTU_X86_64}

.github/CODEOWNERS

@@ -8,6 +8,7 @@
 # Package owners below.
 # Please keep the list sorted.
 /packages/1password @elastic/security-service-integrations
+/packages/abnormal_security @elastic/security-service-integrations
 /packages/activemq @elastic/obs-infraobs-integrations
 /packages/airflow @elastic/obs-infraobs-integrations
 /packages/akamai @elastic/security-service-integrations
@@ -23,6 +24,7 @@
 /packages/auditd @elastic/sec-linux-platform
 /packages/auditd_manager @elastic/sec-linux-platform
 /packages/auth0 @elastic/security-service-integrations
+/packages/authentik @elastic/security-service-integrations
 /packages/aws @elastic/obs-infraobs-integrations @elastic/obs-ds-hosted-services @elastic/security-service-integrations
 /packages/aws/changelog.yml @elastic/obs-ds-hosted-services @elastic/security-service-integrations @elastic/obs-infraobs-integrations
 /packages/aws/data_stream/apigateway_logs @elastic/obs-infraobs-integrations
@@ -118,6 +120,7 @@
 /packages/bitwarden @elastic/security-service-integrations
 /packages/bluecoat @elastic/sec-deployment-and-devices
 /packages/box_events @elastic/security-service-integrations
+/packages/canva @elastic/security-service-integrations
 /packages/carbon_black_cloud @elastic/security-service-integrations
 /packages/carbonblack_edr @elastic/security-service-integrations
 /packages/cassandra @elastic/obs-infraobs-integrations
@@ -139,6 +142,7 @@
 /packages/cisco_umbrella @elastic/security-service-integrations
 /packages/citrix_adc @elastic/obs-infraobs-integrations
 /packages/citrix_waf @elastic/sec-deployment-and-devices
+/packages/claroty_ctd @elastic/security-service-integrations
 /packages/cloud_defend @elastic/sec-linux-platform
 /packages/cloud_security_posture @elastic/cloud-security-posture
 /packages/cloud_asset_inventory @elastic/cloud-security-posture
@@ -158,6 +162,7 @@
 /packages/darktrace @elastic/security-service-integrations
 /packages/ded @elastic/ml-ui @elastic/sec-applied-ml
 /packages/dga @elastic/ml-ui @elastic/sec-applied-ml
+/packages/digital_guardian @elastic/security-service-integrations
 /packages/docker @elastic/obs-cloudnative-monitoring
 /packages/elastic_agent @elastic/elastic-agent
 /packages/elastic_package_registry @elastic/ecosystem
@@ -170,8 +175,10 @@
 /packages/etcd @elastic/obs-infraobs-integrations
 /packages/f5 @elastic/security-service-integrations
 /packages/f5_bigip @elastic/security-service-integrations
+/packages/falco @elastic/security-service-integrations
 /packages/fim @elastic/sec-linux-platform
 /packages/fireeye @elastic/security-service-integrations
+/packages/first_epss @elastic/security-service-integrations
 /packages/fleet_server @elastic/fleet
 /packages/forcepoint_web @elastic/security-service-integrations
 /packages/forgerock @elastic/security-service-integrations
@@ -202,9 +209,11 @@
 /packages/gcp/data_stream/vpcflow @elastic/security-service-integrations
 /packages/gcp_metrics @elastic/obs-ds-hosted-services
 /packages/gcp_pubsub @elastic/security-service-integrations
+/packages/gigamon @elastic/security-service-integrations
 /packages/github @elastic/security-service-integrations
 /packages/gitlab @elastic/security-service-integrations
 /packages/golang @elastic/obs-infraobs-integrations
+/packages/goflow2 @elastic/sec-deployment-and-devices
 /packages/google_cloud_storage @elastic/security-service-integrations
 /packages/google_scc @elastic/security-service-integrations
 /packages/google_workspace @elastic/security-service-integrations
@@ -225,6 +234,8 @@
 /packages/iptables @elastic/sec-deployment-and-devices
 /packages/istio @elastic/obs-cloudnative-monitoring
 /packages/jamf_compliance_reporter @elastic/security-service-integrations
+/packages/jamf_pro @elastic/security-service-integrations
+/packages/jamf_protect @elastic/security-service-integrations
 /packages/jolokia_input @elastic/obs-infraobs-integrations
 /packages/journald @elastic/elastic-agent-data-plane
 /packages/jumpcloud @elastic/security-service-integrations
@@ -284,13 +295,15 @@
 /packages/platform_observability @elastic/stack-monitoring
 /packages/postgresql @elastic/obs-infraobs-integrations
 /packages/pps @elastic/security-service-integrations
+/packages/prisma_access @elastic/security-service-integrations
 /packages/prisma_cloud @elastic/security-service-integrations
 /packages/problemchild @elastic/ml-ui @elastic/sec-applied-ml
 /packages/prometheus @elastic/obs-infraobs-integrations
 /packages/prometheus/data_stream/remote_write @elastic/obs-cloudnative-monitoring
 /packages/prometheus/data_stream/collector @elastic/obs-infraobs-integrations
 /packages/prometheus/data_stream/query @elastic/obs-infraobs-integrations
 /packages/prometheus_input @elastic/obs-infraobs-integrations
+/packages/proofpoint_on_demand @elastic/security-service-integrations
 /packages/proofpoint_tap @elastic/security-service-integrations
 /packages/pulse_connect_secure @elastic/security-service-integrations
 /packages/qualys_vmdr @elastic/security-service-integrations
@@ -312,11 +325,13 @@
 /packages/sophos @elastic/sec-deployment-and-devices
 /packages/sophos_central @elastic/security-service-integrations
 /packages/spring_boot @elastic/obs-infraobs-integrations
+/packages/spycloud @elastic/security-service-integrations
 /packages/sql_input @elastic/obs-infraobs-integrations
 /packages/squid @elastic/sec-deployment-and-devices
 /packages/stan @elastic/obs-infraobs-integrations
 /packages/statsd_input @elastic/obs-infraobs-integrations
 /packages/stormshield @elastic/sec-deployment-and-devices
+/packages/sublime_security @elastic/security-service-integrations
 /packages/suricata @elastic/sec-deployment-and-devices
 /packages/symantec_edr_cloud @elastic/security-service-integrations
 /packages/symantec_endpoint @elastic/security-service-integrations
@@ -326,7 +341,7 @@
 /packages/sysmon_linux @elastic/sec-linux-platform
 /packages/system @elastic/obs-infraobs-integrations
 /packages/system/changelog.yml @elastic/obs-infraobs-integrations @elastic/sec-linux-platform @elastic/sec-windows-platform
-/packages/system/data_stream/auth @elastic/sec-windows-platform
+/packages/system/data_stream/auth @elastic/sec-linux-platform
 /packages/system/data_stream/security @elastic/sec-windows-platform
 /packages/system/data_stream/application @elastic/sec-windows-platform
 /packages/system/data_stream/core @elastic/obs-infraobs-integrations
@@ -381,6 +396,7 @@
 /packages/universal_profiling_symbolizer @elastic/obs-ds-intake-services
 /packages/vectra_detect @elastic/security-service-integrations
 /packages/vsphere @elastic/obs-infraobs-integrations
+/packages/websocket @elastic/security-service-integrations
 /packages/watchguard_firebox @elastic/sec-deployment-and-devices
 /packages/websphere_application_server @elastic/obs-infraobs-integrations
 /packages/windows @elastic/elastic-agent-data-plane @elastic/sec-windows-platform
@@ -405,5 +421,4 @@
 /packages/zoom @elastic/security-service-integrations
 /packages/zscaler_zia @elastic/security-service-integrations
 /packages/zscaler_zpa @elastic/security-service-integrations
-/packages/jamf_protect @elastic/security-service-integrations
 /packages/panw_metrics @elastic/obs-infraobs-integrations