From 51589eba266d9dcd498b0e418f2c5527c564cedd Mon Sep 17 00:00:00 2001 From: Jakub Stejskal Date: Fri, 12 Jul 2024 13:03:52 +0200 Subject: [PATCH] Bump Kafka to 3.7.1, remove several Kafka CR warnings, and fix exporter dashboard Signed-off-by: Jakub Stejskal --- debezium/connector/040-Connect.yaml | 2 +- .../00-kafka-exporter-dashboard.yaml | 2 +- strimzi/kraft-mode/kafka/00-kafka-route.yaml | 49 +------------------ .../00-kafka-mirror-maker-2-tls.yaml | 2 +- .../zookeeper-mode/kafka/00-kafka-route.yaml | 12 ++--- .../00-kafka-mirror-maker-2-tls.yaml | 2 +- 6 files changed, 8 insertions(+), 61 deletions(-) diff --git a/debezium/connector/040-Connect.yaml b/debezium/connector/040-Connect.yaml index 4581cbc8..502ec472 100755 --- a/debezium/connector/040-Connect.yaml +++ b/debezium/connector/040-Connect.yaml @@ -8,7 +8,7 @@ metadata: annotations: strimzi.io/use-connector-resources: "true" spec: - version: 3.7.0 + version: 3.7.1 replicas: 1 bootstrapServers: anubis-kafka-bootstrap.strimzi-kafka.svc:9093 metricsConfig: diff --git a/metrics/grafana/dashboards/00-kafka-exporter-dashboard.yaml b/metrics/grafana/dashboards/00-kafka-exporter-dashboard.yaml index 55bc4614..85bf144b 100755 --- a/metrics/grafana/dashboards/00-kafka-exporter-dashboard.yaml +++ b/metrics/grafana/dashboards/00-kafka-exporter-dashboard.yaml @@ -813,7 +813,7 @@ spec: "targets": [ { "datasource": "${DS_THANOS}", - "expr": "sum(delta(kafka_consumergroup_current_offset{consumergroup=~\"$consumergroup\",topic=~\"$topic\", namespace=~\"$kubernetes_namespace\"}[5m])/60) by (consumergroup, topic)", + "expr": "sum(delta(kafka_consumergroup_current_offset{consumergroup=~\"$consumergroup\",topic=~\"$topic\", namespace=~\"$kubernetes_namespace\"}[5m])/360) by (consumergroup, topic)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{consumergroup}} (topic: {{topic}})", diff --git a/strimzi/kraft-mode/kafka/00-kafka-route.yaml b/strimzi/kraft-mode/kafka/00-kafka-route.yaml index 4345b905..62d76c5c 100755 --- a/strimzi/kraft-mode/kafka/00-kafka-route.yaml +++ b/strimzi/kraft-mode/kafka/00-kafka-route.yaml @@ -66,9 +66,8 @@ metadata: strimzi.io/kraft: enabled spec: kafka: - version: 3.7.0 + version: 3.7.1 # ignored by nodepools, needed just for schema validation - replicas: 5 listeners: - name: plain port: 9092 @@ -94,13 +93,7 @@ spec: log.retention.hours: -1 # i.e., not time limit is applied log.retention.bytes: 128000000 # 128 MB log.segment.bytes: 64000000 # 64 MB - log.message.format.version: "3.6" - inter.broker.protocol.version: "3.6" min.insync.replicas: 2 - # ignored by nodepools, needed just for schema validation - storage: - type: persistent-claim - size: 100Gi authorization: type: simple superUsers: @@ -155,46 +148,6 @@ spec: matchLabels: # label on Kafka pods strimzi.io/name: horus-kafka - zookeeper: - replicas: 3 - storage: - type: persistent-claim - size: 10Gi - class: standard-csi - metricsConfig: - type: jmxPrometheusExporter - valueFrom: - configMapKeyRef: - key: horus-zookeeper-jmx-exporter-configuration.yaml - name: horus-zookeeper-jmx-exporter-configuration - template: - podDisruptionBudget: - maxUnavailable: 0 - pod: - tolerations: - - key: "nodetype" - operator: "Equal" - value: "kafka" - effect: "NoSchedule" - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nodetype - operator: In - values: - - kafka - topologySpreadConstraints: - # Deploy Zookeeper pods with spread with difference max 2 pod - - maxSkew: 4 - # All nodes with same key are considered as one group, so we should pick a key, which has different values on the nodes - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: DoNotSchedule - labelSelector: - matchLabels: - # label on Zookeper pods - strimzi.io/kind: Kafka entityOperator: template: diff --git a/strimzi/kraft-mode/mirror-maker2/00-kafka-mirror-maker-2-tls.yaml b/strimzi/kraft-mode/mirror-maker2/00-kafka-mirror-maker-2-tls.yaml index 9ccc7788..33838820 100755 --- a/strimzi/kraft-mode/mirror-maker2/00-kafka-mirror-maker-2-tls.yaml +++ b/strimzi/kraft-mode/mirror-maker2/00-kafka-mirror-maker-2-tls.yaml @@ -5,7 +5,7 @@ metadata: labels: mode: kraft spec: - version: 3.7.0 + version: 3.7.1 replicas: 3 resources: requests: diff --git a/strimzi/zookeeper-mode/kafka/00-kafka-route.yaml b/strimzi/zookeeper-mode/kafka/00-kafka-route.yaml index cbe94e92..db36721e 100755 --- a/strimzi/zookeeper-mode/kafka/00-kafka-route.yaml +++ b/strimzi/zookeeper-mode/kafka/00-kafka-route.yaml @@ -41,9 +41,7 @@ metadata: strimzi.io/node-pools: enabled spec: kafka: - version: 3.7.0 - # ignored by nodepools, needed just for schema validation - replicas: 7 + version: 3.7.1 listeners: - name: plain port: 9092 @@ -69,13 +67,9 @@ spec: log.retention.hours: -1 # i.e., not time limit is applied log.retention.bytes: 128000000 # 128 MB log.segment.bytes: 64000000 - log.message.format.version: "3.6" - inter.broker.protocol.version: "3.6" + log.message.format.version: "3.7" + inter.broker.protocol.version: "3.7" min.insync.replicas: 2 - # ignored by nodepools, needed just for schema validation - storage: - type: persistent-claim - size: 300Gi authorization: type: simple superUsers: diff --git a/strimzi/zookeeper-mode/mirror-maker2/00-kafka-mirror-maker-2-tls.yaml b/strimzi/zookeeper-mode/mirror-maker2/00-kafka-mirror-maker-2-tls.yaml index 1eb7d189..a99f7af4 100755 --- a/strimzi/zookeeper-mode/mirror-maker2/00-kafka-mirror-maker-2-tls.yaml +++ b/strimzi/zookeeper-mode/mirror-maker2/00-kafka-mirror-maker-2-tls.yaml @@ -5,7 +5,7 @@ metadata: labels: mode: zookeeper spec: - version: 3.7.0 + version: 3.7.1 replicas: 3 resources: requests: